@xyo-network/diviner-temporal-indexing-memory 3.5.2 → 3.6.0-rc.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/diviner-temporal-indexing-memory",
-  "version": "3.5.2",
+  "version": "3.6.0-rc.10",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -29,45 +29,47 @@
   "module": "dist/neutral/index.mjs",
   "types": "dist/neutral/index.d.ts",
   "dependencies": {
-    "@xylabs/array": "^4.4.9",
-    "@xylabs/assert": "^4.4.9",
-    "@xylabs/exists": "^4.4.9",
-    "@xylabs/hex": "^4.4.9",
-    "@xyo-network/archivist-model": "^3.5.2",
-    "@xyo-network/archivist-wrapper": "^3.5.2",
-    "@xyo-network/boundwitness-model": "^3.5.2",
-    "@xyo-network/diviner-abstract": "^3.5.2",
-    "@xyo-network/diviner-boundwitness-abstract": "^3.5.2",
-    "@xyo-network/diviner-boundwitness-model": "^3.5.2",
-    "@xyo-network/diviner-indexing-memory": "^3.5.2",
-    "@xyo-network/diviner-indexing-model": "^3.5.2",
-    "@xyo-network/diviner-jsonpath-aggregate-memory": "^3.5.2",
-    "@xyo-network/diviner-jsonpath-model": "^3.5.2",
-    "@xyo-network/diviner-model": "^3.5.2",
-    "@xyo-network/diviner-payload-model": "^3.5.2",
-    "@xyo-network/diviner-temporal-indexing-model": "^3.5.2",
-    "@xyo-network/diviner-wrapper": "^3.5.2",
-    "@xyo-network/module-model": "^3.5.2",
-    "@xyo-network/payload-builder": "^3.5.2",
-    "@xyo-network/payload-model": "^3.5.2",
-    "@xyo-network/payload-utils": "^3.5.2",
-    "@xyo-network/witness-timestamp": "^3.5.2"
+    "@xylabs/array": "^4.4.21",
+    "@xylabs/assert": "^4.4.21",
+    "@xylabs/exists": "^4.4.21",
+    "@xylabs/hex": "^4.4.21",
+    "@xyo-network/archivist-model": "^3.6.0-rc.10",
+    "@xyo-network/archivist-wrapper": "^3.6.0-rc.10",
+    "@xyo-network/boundwitness-model": "^3.6.0-rc.10",
+    "@xyo-network/boundwitness-validator": "^3.6.0-rc.10",
+    "@xyo-network/diviner-abstract": "^3.6.0-rc.10",
+    "@xyo-network/diviner-boundwitness-abstract": "^3.6.0-rc.10",
+    "@xyo-network/diviner-boundwitness-model": "^3.6.0-rc.10",
+    "@xyo-network/diviner-indexing-memory": "^3.6.0-rc.10",
+    "@xyo-network/diviner-indexing-model": "^3.6.0-rc.10",
+    "@xyo-network/diviner-jsonpath-aggregate-memory": "^3.6.0-rc.10",
+    "@xyo-network/diviner-jsonpath-model": "^3.6.0-rc.10",
+    "@xyo-network/diviner-model": "^3.6.0-rc.10",
+    "@xyo-network/diviner-payload-model": "^3.6.0-rc.10",
+    "@xyo-network/diviner-temporal-indexing-model": "^3.6.0-rc.10",
+    "@xyo-network/diviner-wrapper": "^3.6.0-rc.10",
+    "@xyo-network/module-model": "^3.6.0-rc.10",
+    "@xyo-network/payload-builder": "^3.6.0-rc.10",
+    "@xyo-network/payload-model": "^3.6.0-rc.10",
+    "@xyo-network/payload-utils": "^3.6.0-rc.10",
+    "@xyo-network/witness-timestamp": "^3.6.0-rc.10"
   },
   "devDependencies": {
-    "@xylabs/delay": "^4.4.9",
-    "@xylabs/ts-scripts-yarn3": "^4.2.4",
-    "@xylabs/tsconfig": "^4.2.4",
-    "@xylabs/vitest-extended": "^4.4.9",
-    "@xyo-network/account": "^3.5.2",
-    "@xyo-network/archivist-memory": "^3.5.2",
-    "@xyo-network/boundwitness-builder": "^3.5.2",
-    "@xyo-network/manifest": "^3.5.2",
-    "@xyo-network/module-factory-locator": "^3.5.2",
-    "@xyo-network/node-memory": "^3.5.2",
+    "@xylabs/delay": "^4.4.21",
+    "@xylabs/ts-scripts-yarn3": "^4.2.6",
+    "@xylabs/tsconfig": "^4.2.6",
+    "@xylabs/vitest-extended": "^4.4.21",
+    "@xyo-network/account": "^3.6.0-rc.10",
+    "@xyo-network/archivist-memory": "^3.6.0-rc.10",
+    "@xyo-network/boundwitness-builder": "^3.6.0-rc.10",
+    "@xyo-network/manifest": "^3.6.0-rc.10",
+    "@xyo-network/module-factory-locator": "^3.6.0-rc.10",
+    "@xyo-network/node-memory": "^3.6.0-rc.10",
     "typescript": "^5.7.2",
-    "vitest": "^2.1.5"
+    "vitest": "^2.1.8"
   },
   "publishConfig": {
     "access": "public"
-  }
+  },
+  "stableVersion": "3.5.2"
 }

package/src/DivinerQueryToIndexQueryDiviner/Diviner.ts

@@ -82,9 +82,9 @@ export class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner<
             sourcePathExpression: '$.limit',
           },
           {
-            defaultValue: 0,
-            destinationField: 'offset',
-            sourcePathExpression: '$.offset',
+            // defaultValue: 0,
+            destinationField: 'cursor',
+            sourcePathExpression: '$.cursor',
           },
           {
             defaultValue: 'desc',
@@ -111,7 +111,7 @@ export class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner<
           // TODO: Make sources not need to be deleted
           delete fields.sources
           // TODO: Add support for additional filters
-          return await new PayloadBuilder<Payload>({ schema: this.indexQuerySchema }).fields(fields).build()
+          return new PayloadBuilder<Payload>({ schema: this.indexQuerySchema }).fields(fields).build()
         }),
       )
     }

package/src/IndexCandidateToIndexDiviner/Diviner.ts

@@ -3,7 +3,7 @@ import { assertEx } from '@xylabs/assert'
 import { exists } from '@xylabs/exists'
 import type { Hash } from '@xylabs/hex'
 import type { BoundWitness } from '@xyo-network/boundwitness-model'
-import { isBoundWitnessWithMeta } from '@xyo-network/boundwitness-model'
+import { isBoundWitness } from '@xyo-network/boundwitness-model'
 import { AbstractDiviner } from '@xyo-network/diviner-abstract'
 import { jsonPathToTransformersDictionary } from '@xyo-network/diviner-jsonpath-aggregate-memory'
 import type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'
@@ -69,13 +69,12 @@ export class TemporalIndexingDivinerIndexCandidateToIndexDiviner<
   }
 
   protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {
-    const builtPayloads = await Promise.all(payloads.map(payload => PayloadBuilder.build(payload)))
     // If the Bound Witness does not contain all the required schemas do not index it
-    const indexableBoundWitnesses = builtPayloads
-      .filter(isBoundWitnessWithMeta)
+    const indexableBoundWitnesses = payloads
+      .filter(isBoundWitness)
       .filter(bw => containsAll(bw.payload_schemas, this.indexableSchemas))
     // If the Payload is not one of the indexable schemas do not index it
-    const indexablePayloads = builtPayloads.filter(p => this.isIndexablePayload(p))
+    const indexablePayloads = payloads.filter(p => this.isIndexablePayload(p))
     // If there is nothing to index, return an empty array
     if (indexableBoundWitnesses.length === 0 || indexablePayloads.length === 0) return []
     // Hash all the indexable data once
@@ -105,24 +104,22 @@ export class TemporalIndexingDivinerIndexCandidateToIndexDiviner<
     }
 
     // Create the indexes from the tuples
-    const indexes = await Promise.all(
-      validIndexableTuples.map<Promise<TemporalIndexingDivinerResultIndex>>(async ([bwHash, ...sourcePayloadHashes]) => {
-        const sourcePayloads = sourcePayloadHashes.map(hash => payloadDictionary[hash])
-        // Use the payload transformers to convert the fields from the source payloads to the destination fields
-        const indexFields = sourcePayloads.flatMap((payload) => {
-          // Find the transformers for this payload
-          const transformers = this.payloadTransformers[payload.schema]
-          // If transformers exist, apply them to the payload otherwise return an empty array
-          return transformers ? transformers.map(transform => transform(payload)) : []
-        })
-        // Include all the sources for reference
-        const sources: string[] = [bwHash, ...sourcePayloadHashes]
-        // Build and return the index
-        return await new PayloadBuilder<TemporalIndexingDivinerResultIndex>({ schema: TemporalIndexingDivinerResultIndexSchema })
-          .fields(Object.assign({ sources }, ...indexFields))
-          .build()
-      }),
-    )
+    const indexes = validIndexableTuples.map<TemporalIndexingDivinerResultIndex>(([bwHash, ...sourcePayloadHashes]) => {
+      const sourcePayloads = sourcePayloadHashes.map(hash => payloadDictionary[hash])
+      // Use the payload transformers to convert the fields from the source payloads to the destination fields
+      const indexFields = sourcePayloads.flatMap((payload) => {
+        // Find the transformers for this payload
+        const transformers = this.payloadTransformers[payload.schema]
+        // If transformers exist, apply them to the payload otherwise return an empty array
+        return transformers ? transformers.map(transform => transform(payload)) : []
+      })
+      // Include all the sources for reference
+      const sources: Hash[] = [bwHash, ...sourcePayloadHashes]
+      // Build and return the index
+      return new PayloadBuilder<TemporalIndexingDivinerResultIndex>({ schema: TemporalIndexingDivinerResultIndexSchema })
+        .fields(Object.assign({ sources }, ...indexFields))
+        .build()
+    })
     return indexes.flat()
   }
 

package/src/StateToIndexCandidateDiviner/Diviner.ts

@@ -1,13 +1,14 @@
+import { filterAs } from '@xylabs/array'
 import { assertEx } from '@xylabs/assert'
 import { exists } from '@xylabs/exists'
-import type { ArchivistInstance } from '@xyo-network/archivist-model'
+import type { ArchivistInstance, ArchivistNextOptions } from '@xyo-network/archivist-model'
 import { ArchivistWrapper } from '@xyo-network/archivist-wrapper'
 import type { BoundWitness } from '@xyo-network/boundwitness-model'
-import { isBoundWitnessWithMeta } from '@xyo-network/boundwitness-model'
+import { asBoundWitness, isBoundWitness } from '@xyo-network/boundwitness-model'
+import { payloadSchemasContainsAll } from '@xyo-network/boundwitness-validator'
 import { AbstractDiviner } from '@xyo-network/diviner-abstract'
 import type { BoundWitnessDiviner } from '@xyo-network/diviner-boundwitness-abstract'
 import type { BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'
-import { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'
 import type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model'
 import type { TemporalIndexingDivinerStateToIndexCandidateDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'
 import { TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'
@@ -16,8 +17,11 @@ import type {
   Labels, ModuleIdentifier, ModuleState,
 } from '@xyo-network/module-model'
 import { isModuleState, ModuleStateSchema } from '@xyo-network/module-model'
-import { PayloadBuilder } from '@xyo-network/payload-builder'
-import type { Payload, Schema } from '@xyo-network/payload-model'
+import type {
+  Payload, Schema,
+  WithStorageMeta,
+} from '@xyo-network/payload-model'
+import { SequenceConstants } from '@xyo-network/payload-model'
 import { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'
 import type { TimeStamp } from '@xyo-network/witness-timestamp'
 import { TimestampSchema } from '@xyo-network/witness-timestamp'
@@ -76,31 +80,35 @@ export class TemporalIndexingDivinerStateToIndexCandidateDiviner<
     return [TimestampSchema, ...(schemas ?? [])]
   }
 
-  protected override async divineHandler(payloads: Payload[] = []): Promise<[ModuleState, ...IndexCandidate[]]> {
+  protected override async divineHandler(payloads: Payload[] = []): Promise<TemporalStateToIndexCandidateDivinerResponse> {
     // Retrieve the last state from what was passed in
     const lastState = payloads.find(isModuleState<IndexingDivinerState>)
     // If there is no last state, start from the beginning
-    if (!lastState) return [{ schema: ModuleStateSchema, state: { offset: 0 } }]
-    // Otherwise, get the last offset
-    const { offset } = lastState.state
-    // Get next batch of results starting from the offset
-    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore()
-    if (!boundWitnessDiviner) return [lastState]
-    const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
-      .fields({
-        limit: this.payloadDivinerLimit, offset, order, payload_schemas: this.payload_schemas,
-      })
-      .build()
-    const batch = await boundWitnessDiviner.divine([query])
-    if (batch.length === 0) return [lastState]
-    // Get source data
+      ?? { schema: ModuleStateSchema, state: { cursor: SequenceConstants.minLocalSequence } }
+
+    // Get the last cursor
+    const cursor = lastState?.state?.cursor
+    // Get the archivist for the store
     const sourceArchivist = await this.getArchivistForStore()
     if (!sourceArchivist) return [lastState]
-    const bws = batch.filter(isBoundWitnessWithMeta)
+
+    // Get the next batch of results
+    const nextOffset: ArchivistNextOptions = { limit: this.payloadDivinerLimit, order }
+    // Only use the cursor if it's a valid offset
+    if (cursor !== SequenceConstants.minLocalSequence) nextOffset.cursor = cursor
+    // Get next batch of results starting from the offset
+    const next = await sourceArchivist.next(nextOffset)
+    if (next.length === 0) return [lastState]
+    const batch = filterAs(next, asBoundWitness)
+      .filter(exists)
+      .filter(bw => payloadSchemasContainsAll(bw, this.payload_schemas))
+    // Get source data
+    const bws = batch.filter(isBoundWitness)
     const indexCandidates: IndexCandidate[] = (await Promise.all(bws.map(bw => this.getPayloadsInBoundWitness(bw, sourceArchivist))))
       .filter(exists)
       .flat()
-    const nextState = { schema: ModuleStateSchema, state: { ...lastState.state, offset: offset + batch.length } }
+    const nextCursor = assertEx(next.at(-1)?._sequence, () => `${moduleName}: Expected next to have a sequence`)
+    const nextState: ModuleState<IndexingDivinerState> = { schema: ModuleStateSchema, state: { ...lastState.state, cursor: nextCursor } }
     return [nextState, ...indexCandidates]
   }
 
@@ -139,7 +147,7 @@ export class TemporalIndexingDivinerStateToIndexCandidateDiviner<
       DivinerWrapper<
         BoundWitnessDiviner<BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload, BoundWitness>,
         BoundWitnessDivinerQueryPayload,
-        BoundWitness
+        WithStorageMeta<BoundWitness>
       >
     >(mod, this.account)
   }