@xyo-network/diviner-temporal-indexing-memory 2.80.12 → 2.80.14

package/src/DivinerQueryToIndexQueryDiviner/Diviner.ts

@@ -0,0 +1,120 @@
+import { AbstractDiviner } from '@xyo-network/abstract-diviner'
+import { DivinerConfigSchema } from '@xyo-network/diviner-model'
+import { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
+import {
+  SchemaToJsonPathTransformExpressionsDictionary,
+  SchemaToPayloadTransformersDictionary,
+  TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema,
+  TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams,
+  TemporalIndexingDivinerResultIndexSchema,
+} from '@xyo-network/diviner-temporal-indexing-model'
+import { Labels } from '@xyo-network/module-model'
+import { PayloadBuilder } from '@xyo-network/payload-builder'
+import { isPayloadOfSchemaType, Payload } from '@xyo-network/payload-model'
+
+import { jsonPathToTransformersDictionary, reducePayloads } from '../jsonpath'
+
+/**
+ * A diviner that converts diviner query to index query
+ */
+export class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner<
+  TParams extends TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams,
+> extends AbstractDiviner<TParams> {
+  static override readonly configSchema = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema
+  static override configSchemas = [DivinerConfigSchema, TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema]
+  static labels: Labels = {
+    'network.xyo.diviner.stage': 'divinerQueryToIndexQueryDiviner',
+  }
+
+  private _indexableSchemas: string[] | undefined
+  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined
+
+  /**
+   * The schema of the diviner query payloads
+   */
+  protected get divinerQuerySchema(): string {
+    return this.config.divinerQuerySchema ?? PayloadDivinerQuerySchema
+  }
+
+  /**
+   * The schema of the index query payloads
+   */
+  protected get indexQuerySchema(): string {
+    return this.config.indexQuerySchema ?? PayloadDivinerQuerySchema
+  }
+
+  /**
+   * The schema of the index payloads
+   */
+  protected get indexSchema(): string {
+    return this.config.indexSchema ?? TemporalIndexingDivinerResultIndexSchema
+  }
+
+  /**
+   * List of indexable schemas for this diviner
+   */
+  protected get indexableSchemas(): string[] {
+    if (!this._indexableSchemas) this._indexableSchemas = [...Object.keys(this.schemaTransforms)]
+    return this._indexableSchemas
+  }
+
+  /**
+   * Dictionary of schemas to payload transformers for creating indexes
+   * from the payloads within a Bound Witness
+   */
+  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {
+    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)
+    return this._payloadTransformers
+  }
+
+  /**
+   * The dictionary of schemas to JSON Path transform expressions for creating indexes
+   * from the payloads within a Bound Witness
+   */
+  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {
+    return (
+      this.config?.schemaTransforms ?? {
+        [this.divinerQuerySchema]: [
+          {
+            defaultValue: 1,
+            destinationField: 'limit',
+            sourcePathExpression: '$.limit',
+          },
+          {
+            defaultValue: 0,
+            destinationField: 'offset',
+            sourcePathExpression: '$.offset',
+          },
+          {
+            defaultValue: 'desc',
+            destinationField: 'order',
+            sourcePathExpression: '$.order',
+          },
+        ],
+      }
+    )
+  }
+
+  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {
+    const queries = payloads.filter(isPayloadOfSchemaType<PayloadDivinerQueryPayload>(this.divinerQuerySchema))
+    if (queries.length) {
+      const results = await Promise.all(
+        queries.map(async (query) => {
+          const fields = await reducePayloads<PayloadDivinerQueryPayload & { sources?: string[] }>(
+            [query],
+            this.payloadTransformers,
+            this.indexQuerySchema,
+          )
+          // TODO: Make index schema configurable
+          fields.schemas = [this.indexSchema]
+          // TODO: Make sources not need to be deleted
+          delete fields.sources
+          // TODO: Add support for additional filters
+          return new PayloadBuilder({ schema: this.indexQuerySchema }).fields(fields).build()
+        }),
+      )
+      return results
+    }
+    return Promise.resolve([])
+  }
+}

package/src/DivinerQueryToIndexQueryDiviner/index.ts

@@ -0,0 +1 @@
+export * from './Diviner'

package/src/IndexCandidateToIndexDiviner/Diviner.ts

@@ -5,26 +5,32 @@ import { AbstractDiviner } from '@xyo-network/abstract-diviner'
 import { BoundWitness, isBoundWitness } from '@xyo-network/boundwitness-model'
 import { PayloadHasher } from '@xyo-network/core'
 import { DivinerConfigSchema } from '@xyo-network/diviner-model'
-import { TemporalIndexingDivinerResultIndexSchema } from '@xyo-network/diviner-temporal-indexing-model'
+import {
+  SchemaToJsonPathTransformExpressionsDictionary,
+  SchemaToPayloadTransformersDictionary,
+  TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema,
+  TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,
+  TemporalIndexingDivinerResultIndex,
+  TemporalIndexingDivinerResultIndexSchema,
+} from '@xyo-network/diviner-temporal-indexing-model'
 import { Labels } from '@xyo-network/module-model'
-import { Payload } from '@xyo-network/payload-model'
-import { isTimestamp, TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'
-import jsonpath from 'jsonpath'
+import { PayloadBuilder } from '@xyo-network/payload-builder'
+import { Payload, PayloadFields } from '@xyo-network/payload-model'
 
-import { PayloadTransformer, StringToJsonPathTransformExpressionsDictionary, StringToPayloadTransformersDictionary } from '../lib'
-import { TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema } from './Config'
-import { TemporalIndexCandidateToIndexDivinerParams } from './Params'
+import { jsonPathToTransformersDictionary } from '../jsonpath'
 
-export type IndexablePayloads = [BoundWitness, TimeStamp, ...Payload[]]
+export type IndexablePayloads = [BoundWitness, ...Payload[]]
 
 const moduleName = 'TemporalIndexingDivinerIndexCandidateToIndexDiviner'
 
 /**
- * Transforms candidates for image thumbnail indexing into their indexed representation
+ * Diviner which transforms index candidates to indexes using JSON Path to map
+ * source properties in the supplied payloads to destination fields in the
+ * resultant index
  */
 export class TemporalIndexingDivinerIndexCandidateToIndexDiviner<
-  TParams extends TemporalIndexCandidateToIndexDivinerParams = TemporalIndexCandidateToIndexDivinerParams,
-> extends AbstractDiviner<TParams, Payload, Payload> {
+  TParams extends TemporalIndexingDivinerIndexCandidateToIndexDivinerParams = TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,
+> extends AbstractDiviner<TParams> {
   static override configSchema = TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema
   static override configSchemas = [DivinerConfigSchema, TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema]
   static labels: Labels = {
@@ -32,16 +38,13 @@ export class TemporalIndexingDivinerIndexCandidateToIndexDiviner<
   }
 
   private _indexableSchemas: string[] | undefined
-  private _schemaToPayloadTransformersDictionary: StringToPayloadTransformersDictionary | undefined
+  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined
 
   /**
    * List of indexable schemas for this diviner
    */
-  protected get indexableSchemas() {
-    // Return the computed result if we've calculated it before as the config
-    // shouldn't change after initialization
-    if (this._indexableSchemas) return this._indexableSchemas
-    this._indexableSchemas = [...Object.keys(this.schemaTransforms)]
+  protected get indexableSchemas(): string[] {
+    if (!this._indexableSchemas) this._indexableSchemas = [...Object.keys(this.schemaTransforms)]
     return this._indexableSchemas
   }
 
@@ -49,74 +52,54 @@ export class TemporalIndexingDivinerIndexCandidateToIndexDiviner<
    * Dictionary of schemas to payload transformers for creating indexes
    * from the payloads within a Bound Witness
    */
-  protected get schemaToPayloadTransformersDictionary(): StringToPayloadTransformersDictionary {
-    // Return the computed result if we've calculated it before as the config
-    // shouldn't change after initialization
-    if (this._schemaToPayloadTransformersDictionary) return this._schemaToPayloadTransformersDictionary
-    this._schemaToPayloadTransformersDictionary = Object.fromEntries(
-      Object.entries(this.schemaTransforms).map(([schema, jsonPathTransformerExpressions]) => {
-        const transformers = jsonPathTransformerExpressions.map((transformExpression) => {
-          const { sourcePathExpression, destinationField } = transformExpression
-          const transformer: PayloadTransformer = (x: Payload) => {
-            // eslint-disable-next-line import/no-named-as-default-member
-            const source = jsonpath.value(x, sourcePathExpression)
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            const transformed = {} as { [key: string]: any }
-            transformed[destinationField] = source
-            return transformed
-          }
-          return transformer
-        })
-        return [schema, transformers]
-      }),
-    )
-    return this._schemaToPayloadTransformersDictionary
+  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {
+    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)
+    return this._payloadTransformers
   }
 
   /**
    * The dictionary of schemas to JSON Path transform expressions for creating indexes
    * from the payloads within a Bound Witness
    */
-  protected get schemaTransforms(): StringToJsonPathTransformExpressionsDictionary {
+  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {
     return assertEx(this.config?.schemaTransforms, () => `${moduleName}: Missing config.schemaTransforms section`)
   }
 
   protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {
     const bws: BoundWitness[] = payloads.filter(isBoundWitness)
-    const timestampPayloads: TimeStamp[] = payloads.filter(isTimestamp)
     const indexablePayloads: Payload[] = payloads.filter((p) => this.isIndexablePayload(p))
-    if (bws.length && timestampPayloads.length && indexablePayloads.length) {
+    if (bws.length && indexablePayloads.length) {
       const payloadDictionary = await PayloadHasher.toMap(payloads)
       const validIndexableTuples: IndexablePayloads[] = bws.reduce<IndexablePayloads[]>((indexableTuples, bw) => {
         // If this Bound Witness doesn't contain all the required schemas don't index it
         if (!containsAll(bw.payload_schemas, this.indexableSchemas)) return indexableTuples
-        // Find the timestamp
-        const timestampPosition = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)
-        const timestampHash = bw.payload_hashes?.[timestampPosition]
-        const timestamp = [payloadDictionary[timestampHash]].find(isTimestamp)
         // Find the remaining indexable payloads
         const indexablePayloadPositions = this.indexableSchemas.map((schema) => bw.payload_schemas.indexOf(schema))
         const indexablePayloadHashes = indexablePayloadPositions.map((index) => bw.payload_hashes?.[index])
         const indexablePayloads = indexablePayloadHashes.map((hash) => payloadDictionary[hash]).filter(exists)
         // If we found a timestamp and the right amount of indexable payloads (of the
         // correct schema as checked above) in this BW, then index it
-        if (timestamp && indexablePayloads.length === this.indexableSchemas.length) indexableTuples.push([bw, timestamp, ...indexablePayloads])
+        if (indexablePayloads.length === this.indexableSchemas.length) indexableTuples.push([bw, ...indexablePayloads])
         return indexableTuples
       }, [])
       // Create the indexes from the tuples
       const indexes = await Promise.all(
-        validIndexableTuples.map(async ([bw, timestampPayload, ...remainingPayloads]) => {
-          const partials = remainingPayloads
-            .map((payload) => {
-              const transforms = this.schemaToPayloadTransformersDictionary[payload.schema]
-              const transformed: Partial<Payload>[] = transforms.map((transform) => transform(payload))
-              return transformed
+        validIndexableTuples.map<Promise<TemporalIndexingDivinerResultIndex>>(async ([bw, ...sourcePayloads]) => {
+          // Use the payload transformers to convert the fields from the source payloads to the destination fields
+          const indexFields = sourcePayloads
+            .map<PayloadFields[]>((payload) => {
+              // Find the transformers for this payload
+              const transformers = this.payloadTransformers[payload.schema]
+              // If transformers exist, apply them to the payload otherwise return an empty array
+              return transformers ? transformers.map((transform) => transform(payload)) : []
             })
             .flat()
-          const transformed = Object.assign({}, ...partials, { schema: TemporalIndexingDivinerResultIndexSchema })
-          const { timestamp } = timestampPayload
-          const sources = (await PayloadHasher.hashPairs([bw, timestampPayload, ...remainingPayloads])).map(([, hash]) => hash)
-          return [{ ...transformed, sources, timestamp }]
+          // Include all the sources for reference
+          const sources = Object.keys(await PayloadHasher.toMap([bw, ...sourcePayloads]))
+          // Build and return the index
+          return new PayloadBuilder<TemporalIndexingDivinerResultIndex>({ schema: TemporalIndexingDivinerResultIndexSchema })
+            .fields(Object.assign({ sources }, ...indexFields))
+            .build()
         }),
       )
       return indexes.flat()

package/src/IndexCandidateToIndexDiviner/_Diviner.ts

@@ -0,0 +1,135 @@
+import { containsAll } from '@xylabs/array'
+import { assertEx } from '@xylabs/assert'
+import { exists } from '@xylabs/exists'
+import { AbstractDiviner } from '@xyo-network/abstract-diviner'
+import { BoundWitness, isBoundWitness } from '@xyo-network/boundwitness-model'
+import { PayloadHasher } from '@xyo-network/core'
+import { DivinerConfigSchema } from '@xyo-network/diviner-model'
+import {
+  SchemaToJsonPathTransformExpressionsDictionary,
+  SchemaToPayloadTransformersDictionary,
+  TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema,
+  TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,
+  TemporalIndexingDivinerResultIndex,
+  TemporalIndexingDivinerResultIndexSchema,
+} from '@xyo-network/diviner-temporal-indexing-model'
+import { Labels } from '@xyo-network/module-model'
+import { PayloadBuilder } from '@xyo-network/payload-builder'
+import { Payload, PayloadFields } from '@xyo-network/payload-model'
+import { isTimestamp, TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'
+
+import { jsonPathToTransformersDictionary } from '../jsonpath'
+
+export type IndexablePayloads = [BoundWitness, TimeStamp, ...Payload[]]
+
+const moduleName = 'TemporalIndexingDivinerIndexCandidateToIndexDiviner'
+
+/**
+ * Diviner which transforms index candidates to indexes using JSON Path to map
+ * source properties in the supplied payloads to destination fields in the
+ * resultant index
+ */
+export class TemporalIndexingDivinerIndexCandidateToIndexDiviner<
+  TParams extends TemporalIndexingDivinerIndexCandidateToIndexDivinerParams = TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,
+> extends AbstractDiviner<TParams> {
+  static override configSchema = TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema
+  static override configSchemas = [DivinerConfigSchema, TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema]
+  static labels: Labels = {
+    'network.xyo.diviner.stage': 'indexCandidateToIndexDiviner',
+  }
+
+  private _indexableSchemas: string[] | undefined
+  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined
+
+  /**
+   * List of indexable schemas for this diviner
+   */
+  protected get indexableSchemas(): string[] {
+    if (!this._indexableSchemas) this._indexableSchemas = [...Object.keys(this.schemaTransforms)]
+    return this._indexableSchemas
+  }
+
+  /**
+   * Dictionary of schemas to payload transformers for creating indexes
+   * from the payloads within a Bound Witness
+   */
+  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {
+    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)
+    return this._payloadTransformers
+  }
+
+  /**
+   * The dictionary of schemas to JSON Path transform expressions for creating indexes
+   * from the payloads within a Bound Witness
+   */
+  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {
+    return assertEx(this.config?.schemaTransforms, () => `${moduleName}: Missing config.schemaTransforms section`)
+  }
+
+  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {
+    const bws: BoundWitness[] = payloads.filter(isBoundWitness)
+    const timestampPayloads: TimeStamp[] = payloads.filter(isTimestamp)
+    const indexablePayloads: Payload[] = payloads.filter((p) => this.isIndexablePayload(p))
+    if (bws.length && timestampPayloads.length && indexablePayloads.length) {
+      const payloadDictionary = await PayloadHasher.toMap(payloads)
+      const validIndexableTuples: IndexablePayloads[] = bws.reduce<IndexablePayloads[]>((indexableTuples, bw) => {
+        // If this Bound Witness doesn't contain all the required schemas don't index it
+        if (!containsAll(bw.payload_schemas, this.indexableSchemas)) return indexableTuples
+        // Find the timestamp
+        const timestampPosition = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)
+        const timestampHash = bw.payload_hashes?.[timestampPosition]
+        const timestamp = [payloadDictionary[timestampHash]].find(isTimestamp)
+        // Find the remaining indexable payloads
+        const indexablePayloadPositions = this.indexableSchemas.map((schema) => bw.payload_schemas.indexOf(schema))
+        const indexablePayloadHashes = indexablePayloadPositions.map((index) => bw.payload_hashes?.[index])
+        const indexablePayloads = indexablePayloadHashes.map((hash) => payloadDictionary[hash]).filter(exists)
+        // If we found a timestamp and the right amount of indexable payloads (of the
+        // correct schema as checked above) in this BW, then index it
+        if (timestamp && indexablePayloads.length === this.indexableSchemas.length) indexableTuples.push([bw, timestamp, ...indexablePayloads])
+        return indexableTuples
+      }, [])
+      // Create the indexes from the tuples
+      const indexes = await Promise.all(
+        validIndexableTuples.map<Promise<TemporalIndexingDivinerResultIndex>>(async ([bw, timestampPayload, ...sourcePayloads]) => {
+          // Use the payload transformers to convert the fields from the source payloads to the destination fields
+          const indexFields = sourcePayloads
+            .map<PayloadFields[]>((payload) => {
+              // Find the transformers for this payload
+              const transformers = this.payloadTransformers[payload.schema]
+              // If transformers exist, apply them to the payload otherwise return an empty array
+              return transformers ? transformers.map((transform) => transform(payload)) : []
+            })
+            .flat()
+          // Extract the timestamp from the timestamp payload
+          const { timestamp } = timestampPayload
+          // Include all the sources for reference
+          const sources = Object.keys(await PayloadHasher.toMap([bw, timestampPayload, ...sourcePayloads]))
+          // Build and return the index
+          return new PayloadBuilder<TemporalIndexingDivinerResultIndex>({ schema: TemporalIndexingDivinerResultIndexSchema })
+            .fields(Object.assign({ sources, timestamp }, ...indexFields))
+            .build()
+        }),
+      )
+      return indexes.flat()
+    }
+    return Promise.resolve([])
+  }
+
+  /**
+   * Identifies if a payload is one that is indexed by this diviner
+   * @param x The candidate payload
+   * @returns True if the payload is one indexed by this diviner, false otherwise
+   */
+  protected isIndexablePayload = (x: Payload) => {
+    return this.indexableSchemas.includes(x?.schema)
+  }
+
+  /**
+   * Identifies if a schema is one that is indexed by this diviner
+   * @param schema The candidate schema
+   * @returns True if this schema is one indexed by this diviner, false otherwise
+   */
+  protected isIndexableSchema = (schema?: string | null) => {
+    return this.indexableSchemas.some((s) => s === schema)
+  }
+}

package/src/IndexCandidateToIndexDiviner/index.ts

@@ -1,4 +1 @@
-export * from './Config'
 export * from './Diviner'
-export * from './Params'
-export * from './Schema'

package/src/IndexQueryResponseToDivinerQueryResponseDiviner/Diviner.ts

@@ -0,0 +1,26 @@
+import { AbstractDiviner } from '@xyo-network/abstract-diviner'
+import { DivinerConfigSchema } from '@xyo-network/diviner-model'
+import { isPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'
+import { TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'
+import { Labels } from '@xyo-network/module-model'
+import { Payload } from '@xyo-network/payload-model'
+
+/**
+ * Transforms an ImageThumbnailIndex response into an ImageThumbnailResponse response
+ */
+export class TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner extends AbstractDiviner {
+  static override readonly configSchema = TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema
+  static override configSchemas = [DivinerConfigSchema, TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema]
+  static labels: Labels = {
+    'network.xyo.diviner.stage': 'indexQueryResponseToDivinerQueryResponseDiviner',
+  }
+
+  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {
+    // NOTE: We're not doing anything with the query payloads but some diviners
+    // might want to use this to transform from the query to the response (for example
+    // if we use a plaintext value in the query to generate a hash key in the index)
+    // const queries = payloads.filter(isPayloadDivinerQueryPayload)
+    const responses = payloads.filter((p) => !isPayloadDivinerQueryPayload(p))
+    return await Promise.resolve(responses)
+  }
+}

package/src/IndexQueryResponseToDivinerQueryResponseDiviner/index.ts

@@ -0,0 +1 @@
+export * from './Diviner'

package/src/StateToIndexCandidateDiviner/Diviner.ts

@@ -7,15 +7,16 @@ import { BoundWitness, isBoundWitness } from '@xyo-network/boundwitness-model'
 import { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'
 import { IndexingDivinerState } from '@xyo-network/diviner-indexing-model'
 import { DivinerConfigSchema } from '@xyo-network/diviner-model'
+import {
+  TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema,
+  TemporalIndexingDivinerStateToIndexCandidateDivinerParams,
+} from '@xyo-network/diviner-temporal-indexing-model'
 import { DivinerWrapper } from '@xyo-network/diviner-wrapper'
 import { isModuleState, Labels, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
 import { isPayloadOfSchemaType, Payload } from '@xyo-network/payload-model'
 import { TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'
 
-import { TemporalStateToIndexCandidateDivinerConfigSchema } from './Config'
-import { TemporalStateToIndexCandidateDivinerParams as TemporalIndexingDivinerStateToIndexCandidateDiviner } from './Params'
-
 /**
  * All Payload types involved in index candidates for indexing
  */
@@ -43,15 +44,16 @@ const order = 'asc'
 /**
  * The name of the module (for logging purposes)
  */
-const moduleName = 'TemporalStateToIndexCandidateDiviner'
+const moduleName = 'TemporalIndexingDivinerStateToIndexCandidateDiviner'
 
 /**
  * Transforms candidates for image thumbnail indexing into their indexed representation
  */
-export class TemporalStateToIndexCandidateDiviner<
-  TParams extends TemporalIndexingDivinerStateToIndexCandidateDiviner = TemporalIndexingDivinerStateToIndexCandidateDiviner,
+export class TemporalIndexingDivinerStateToIndexCandidateDiviner<
+  TParams extends TemporalIndexingDivinerStateToIndexCandidateDivinerParams = TemporalIndexingDivinerStateToIndexCandidateDivinerParams,
 > extends AbstractDiviner<TParams> {
-  static override configSchemas = [DivinerConfigSchema, TemporalStateToIndexCandidateDivinerConfigSchema]
+  static override readonly configSchema = TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema
+  static override configSchemas = [DivinerConfigSchema, TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema]
   static labels: Labels = {
     'network.xyo.diviner.stage': 'stateToIndexCandidateDiviner',
   }
@@ -77,6 +79,7 @@ export class TemporalStateToIndexCandidateDiviner<
     const { offset } = lastState.state
     // Get next batch of results starting from the offset
     const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore()
+    if (!boundWitnessDiviner) return [lastState]
     const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
       .fields({ limit: this.payloadDivinerLimit, offset, order, payload_schemas: this.payload_schemas })
       .build()
@@ -84,6 +87,7 @@ export class TemporalStateToIndexCandidateDiviner<
     if (batch.length === 0) return [lastState]
     // Get source data
     const sourceArchivist = await this.getArchivistForStore()
+    if (!sourceArchivist) return [lastState]
     const indexCandidates: IndexCandidate[] = (
       await Promise.all(batch.filter(isBoundWitness).map((bw) => this.getPayloadsInBoundWitness(bw, sourceArchivist)))
     )
@@ -94,24 +98,32 @@ export class TemporalStateToIndexCandidateDiviner<
   }
   /**
    * Retrieves the archivist for the payloadStore
-   * @returns The archivist for the payloadStore
+   * @returns The archivist for the payloadStore or undefined if not resolvable
    */
-  protected async getArchivistForStore() {
+  protected async getArchivistForStore(): Promise<ArchivistWrapper | undefined> {
+    // It should be defined, so we'll error if it's not
     const name: string = assertEx(this.config?.payloadStore?.archivist, () => `${moduleName}: Config for payloadStore.archivist not specified`)
-    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve payloadStore.archivist`)
+    // It might not be resolvable (yet), so we'll return undefined if it's not
+    const mod = await this.resolve(name)
+    if (!mod) return undefined
+    // Return the wrapped archivist
     return ArchivistWrapper.wrap(mod, this.account)
   }
 
   /**
    * Retrieves the BoundWitness Diviner for the payloadStore
-   * @returns The BoundWitness Diviner for the payloadStore
+   * @returns The BoundWitness Diviner for the payloadStore or undefined if not resolvable
    */
-  protected async getBoundWitnessDivinerForStore() {
+  protected async getBoundWitnessDivinerForStore(): Promise<DivinerWrapper | undefined> {
+    // It should be defined, so we'll error if it's not
     const name: string = assertEx(
       this.config?.payloadStore?.boundWitnessDiviner,
       () => `${moduleName}: Config for payloadStore.boundWitnessDiviner not specified`,
     )
-    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve payloadStore.boundWitnessDiviner`)
+    // It might not be resolvable (yet), so we'll return undefined if it's not
+    const mod = await this.resolve(name)
+    if (!mod) return undefined
+    // Return the wrapped diviner
     return DivinerWrapper.wrap(mod, this.account)
   }
 

package/src/index.ts

@@ -1 +1,5 @@
 export * from './Diviner'
+export * from './DivinerQueryToIndexQueryDiviner'
+export * from './IndexCandidateToIndexDiviner'
+export * from './IndexQueryResponseToDivinerQueryResponseDiviner'
+export * from './StateToIndexCandidateDiviner'

package/src/jsonpath/index.ts

@@ -0,0 +1,3 @@
+export * from './jsonPathToTransformersDictionary'
+export * from './reducePayloads'
+export * from './toPayloadTransformer'

package/src/jsonpath/jsonPathToTransformersDictionary.ts

@@ -0,0 +1,21 @@
+import { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-temporal-indexing-model'
+
+import { toPayloadTransformer } from './toPayloadTransformer'
+
+/**
+ * Materializes the JSON-path expressions into memoized functions by converting a
+ * dictionary of schema to JSON Path transform expressions to a dictionary
+ * of schema to payload transformers
+ * @param schemaTransforms The schema transforms to convert
+ * @returns A dictionary of schema to payload transformers
+ */
+export const jsonPathToTransformersDictionary = (
+  schemaTransforms: SchemaToJsonPathTransformExpressionsDictionary,
+): SchemaToPayloadTransformersDictionary => {
+  return Object.fromEntries(
+    Object.entries(schemaTransforms).map(([schema, jsonPathTransformerExpressions]) => {
+      const transformers = jsonPathTransformerExpressions.map(toPayloadTransformer)
+      return [schema, transformers]
+    }),
+  )
+}

package/src/jsonpath/reducePayloads.ts

@@ -0,0 +1,50 @@
+import { PayloadHasher } from '@xyo-network/core'
+import { SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-temporal-indexing-model'
+import { PayloadBuilder } from '@xyo-network/payload-builder'
+import { Payload, PayloadFields } from '@xyo-network/payload-model'
+
+/**
+ * Reduces the payloads to a single payload using the supplied transformers
+ * @param payloads The payloads to reduce
+ * @param payloadTransformers The transformers to use to convert the payloads to the destination payload
+ * @param destinationSchema The schema of the destination payload
+ * @returns The reduced payload
+ */
+export const reducePayloads = async <T extends Payload = Payload>(
+  payloads: Payload[],
+  payloadTransformers: SchemaToPayloadTransformersDictionary,
+  destinationSchema: string,
+): Promise<T> => {
+  // Use the payload transformers to convert the fields from the source payloads to the destination fields
+  const indexFields = payloads
+    .map<PayloadFields[]>((payload) => {
+      // Find the transformers for this payload
+      const transformers = payloadTransformers[payload.schema]
+      // If transformers exist, apply them to the payload otherwise return an empty array
+      return transformers ? transformers.map((transform) => transform(payload)) : []
+    })
+    .flat()
+  // Include all the sources for reference
+  const sources = (await PayloadHasher.hashPairs([...payloads])).map(([, hash]) => hash)
+  // Build and return the index
+  return new PayloadBuilder<T>({ schema: destinationSchema }).fields(Object.assign({ sources }, ...indexFields)).build()
+}
+
+/**
+ * Reduces the arrays of payload arrays to an array of payloads using the supplied transformers
+ * @param payloadsArray The arrays of payloads to reduce
+ * @param payloadTransformers The transformers to use to convert the payloads to the destination payloads
+ * @param destinationSchema The schema of the destination payloads
+ * @returns The reduced payloads
+ */
+export const reducePayloadsArray = async <T extends Payload = Payload>(
+  payloadsArray: Payload[][],
+  payloadTransformers: SchemaToPayloadTransformersDictionary,
+  destinationSchema: string,
+): Promise<T[]> => {
+  return await Promise.all(
+    payloadsArray.map(async (payloads) => {
+      return await reducePayloads<T>(payloads, payloadTransformers, destinationSchema)
+    }),
+  )
+}