@xyo-network/diviner-temporal-indexing-memory 5.3.22 → 5.3.25

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/diviner-temporal-indexing-memory",
-  "version": "5.3.22",
+  "version": "5.3.25",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -30,80 +30,76 @@
   "types": "dist/neutral/index.d.ts",
   "files": [
     "dist",
-    "src",
     "!**/*.bench.*",
     "!**/*.spec.*",
     "!**/*.test.*",
     "README.md"
   ],
   "dependencies": {
-    "@xyo-network/archivist-model": "~5.3.22",
-    "@xyo-network/archivist-wrapper": "~5.3.22",
-    "@xyo-network/boundwitness-model": "~5.3.22",
-    "@xyo-network/boundwitness-validator": "~5.3.22",
-    "@xyo-network/diviner-abstract": "~5.3.22",
-    "@xyo-network/diviner-boundwitness-abstract": "~5.3.22",
-    "@xyo-network/diviner-boundwitness-model": "~5.3.22",
-    "@xyo-network/diviner-indexing-memory": "~5.3.22",
-    "@xyo-network/diviner-indexing-model": "~5.3.22",
-    "@xyo-network/diviner-jsonpath-aggregate-memory": "~5.3.22",
-    "@xyo-network/diviner-jsonpath-model": "~5.3.22",
-    "@xyo-network/diviner-model": "~5.3.22",
-    "@xyo-network/diviner-payload-model": "~5.3.22",
-    "@xyo-network/diviner-temporal-indexing-model": "~5.3.22",
-    "@xyo-network/diviner-wrapper": "~5.3.22",
-    "@xyo-network/module-model": "~5.3.22",
-    "@xyo-network/payload-builder": "~5.3.22",
-    "@xyo-network/payload-model": "~5.3.22",
-    "@xyo-network/payload-utils": "~5.3.22",
-    "@xyo-network/witness-timestamp": "~5.3.22"
+    "@xyo-network/archivist-model": "~5.3.25",
+    "@xyo-network/boundwitness-model": "~5.3.25",
+    "@xyo-network/archivist-wrapper": "~5.3.25",
+    "@xyo-network/diviner-boundwitness-abstract": "~5.3.25",
+    "@xyo-network/diviner-indexing-memory": "~5.3.25",
+    "@xyo-network/diviner-boundwitness-model": "~5.3.25",
+    "@xyo-network/diviner-abstract": "~5.3.25",
+    "@xyo-network/diviner-model": "~5.3.25",
+    "@xyo-network/diviner-indexing-model": "~5.3.25",
+    "@xyo-network/boundwitness-validator": "~5.3.25",
+    "@xyo-network/diviner-payload-model": "~5.3.25",
+    "@xyo-network/diviner-jsonpath-aggregate-memory": "~5.3.25",
+    "@xyo-network/diviner-jsonpath-model": "~5.3.25",
+    "@xyo-network/module-model": "~5.3.25",
+    "@xyo-network/diviner-temporal-indexing-model": "~5.3.25",
+    "@xyo-network/diviner-wrapper": "~5.3.25",
+    "@xyo-network/payload-builder": "~5.3.25",
+    "@xyo-network/payload-model": "~5.3.25",
+    "@xyo-network/witness-timestamp": "~5.3.25",
+    "@xyo-network/payload-utils": "~5.3.25"
   },
   "devDependencies": {
     "@opentelemetry/api": "^1.9.1",
     "@types/node": "^25.5.0",
-    "@xylabs/sdk-js": "^5.0.91",
-    "@xylabs/ts-scripts-common": "~7.6.8",
-    "@xylabs/ts-scripts-yarn3": "~7.6.8",
-    "@xylabs/tsconfig": "~7.6.8",
-    "@xylabs/vitest-extended": "~5.0.91",
-    "@xyo-network/archivist-memory": "~5.3.22",
-    "@xyo-network/archivist-model": "~5.3.22",
-    "@xyo-network/archivist-wrapper": "~5.3.22",
-    "@xyo-network/boundwitness-builder": "~5.3.22",
-    "@xyo-network/boundwitness-model": "~5.3.22",
-    "@xyo-network/boundwitness-validator": "~5.3.22",
-    "@xyo-network/diviner-abstract": "~5.3.22",
-    "@xyo-network/diviner-boundwitness-abstract": "~5.3.22",
-    "@xyo-network/diviner-boundwitness-model": "~5.3.22",
-    "@xyo-network/diviner-indexing-memory": "~5.3.22",
-    "@xyo-network/diviner-indexing-model": "~5.3.22",
-    "@xyo-network/diviner-jsonpath-aggregate-memory": "~5.3.22",
-    "@xyo-network/diviner-jsonpath-model": "~5.3.22",
-    "@xyo-network/diviner-model": "~5.3.22",
-    "@xyo-network/diviner-payload-model": "~5.3.22",
-    "@xyo-network/diviner-temporal-indexing-model": "~5.3.22",
-    "@xyo-network/diviner-wrapper": "~5.3.22",
-    "@xyo-network/manifest": "~5.3.22",
-    "@xyo-network/module-factory-locator": "~5.3.22",
-    "@xyo-network/module-model": "~5.3.22",
-    "@xyo-network/node-memory": "~5.3.22",
-    "@xyo-network/payload-builder": "~5.3.22",
-    "@xyo-network/payload-model": "~5.3.22",
-    "@xyo-network/payload-utils": "~5.3.22",
-    "@xyo-network/wallet": "~5.3.22",
-    "@xyo-network/witness-timestamp": "~5.3.22",
+    "@xylabs/sdk-js": "^5.0.93",
+    "@xylabs/ts-scripts-common": "~7.6.16",
+    "@xylabs/ts-scripts-pnpm": "~7.6.16",
+    "@xylabs/tsconfig": "~7.6.16",
+    "@xylabs/vitest-extended": "~5.0.93",
     "acorn": "^8.16.0",
     "axios": "^1.14.0",
-    "cosmiconfig": "^9.0.1",
-    "esbuild": "^0.27.4",
-    "eslint": "^10.1.0",
+    "esbuild": "^0.28.0",
     "ethers": "^6.16.0",
-    "rollup": "^4.60.1",
     "tslib": "^2.8.1",
     "typescript": "~5.9.3",
     "vite": "^8.0.3",
     "vitest": "~4.1.2",
-    "zod": "^4.3.6"
+    "zod": "^4.3.6",
+    "@xyo-network/archivist-memory": "~5.3.25",
+    "@xyo-network/archivist-model": "~5.3.25",
+    "@xyo-network/boundwitness-builder": "~5.3.25",
+    "@xyo-network/boundwitness-validator": "~5.3.25",
+    "@xyo-network/archivist-wrapper": "~5.3.25",
+    "@xyo-network/boundwitness-model": "~5.3.25",
+    "@xyo-network/diviner-abstract": "~5.3.25",
+    "@xyo-network/diviner-boundwitness-model": "~5.3.25",
+    "@xyo-network/diviner-jsonpath-aggregate-memory": "~5.3.25",
+    "@xyo-network/diviner-indexing-model": "~5.3.25",
+    "@xyo-network/diviner-indexing-memory": "~5.3.25",
+    "@xyo-network/diviner-boundwitness-abstract": "~5.3.25",
+    "@xyo-network/diviner-jsonpath-model": "~5.3.25",
+    "@xyo-network/diviner-model": "~5.3.25",
+    "@xyo-network/diviner-payload-model": "~5.3.25",
+    "@xyo-network/diviner-temporal-indexing-model": "~5.3.25",
+    "@xyo-network/diviner-wrapper": "~5.3.25",
+    "@xyo-network/module-model": "~5.3.25",
+    "@xyo-network/manifest": "~5.3.25",
+    "@xyo-network/module-factory-locator": "~5.3.25",
+    "@xyo-network/node-memory": "~5.3.25",
+    "@xyo-network/payload-utils": "~5.3.25",
+    "@xyo-network/payload-builder": "~5.3.25",
+    "@xyo-network/payload-model": "~5.3.25",
+    "@xyo-network/witness-timestamp": "~5.3.25",
+    "@xyo-network/wallet": "~5.3.25"
   },
   "peerDependencies": {
     "@xylabs/sdk-js": "^5",
@@ -114,4 +110,4 @@
   "publishConfig": {
     "access": "public"
   }
-}
+}

package/src/Diviner.ts

@@ -1,23 +0,0 @@
-import { IndexingDiviner } from '@xyo-network/diviner-indexing-memory'
-import type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'
-import type { TemporalIndexingDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'
-import { TemporalIndexingDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'
-import type { Payload, Schema } from '@xyo-network/payload-model'
-
-export class TemporalIndexingDiviner<
-  TParams extends TemporalIndexingDivinerParams = TemporalIndexingDivinerParams,
-  TIn extends Payload = Payload,
-  TOut extends Payload = Payload,
-  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<
-    DivinerInstance<TParams, TIn, TOut>,
-    TIn,
-    TOut
-  >,
-> extends IndexingDiviner<TParams, TIn, TOut, TEventData> {
-  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerConfigSchema]
-  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerConfigSchema
-
-  protected override async startHandler() {
-    await super.startHandler()
-  }
-}

package/src/DivinerQueryToIndexQueryDiviner/Diviner.ts

@@ -1,122 +0,0 @@
-import type { Hash } from '@xylabs/sdk-js'
-import { AbstractDiviner } from '@xyo-network/diviner-abstract'
-import { jsonPathToTransformersDictionary, reducePayloads } from '@xyo-network/diviner-jsonpath-aggregate-memory'
-import type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'
-import type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'
-import { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
-import type { TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'
-import {
-  TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema,
-  TemporalIndexingDivinerResultIndexSchema,
-} from '@xyo-network/diviner-temporal-indexing-model'
-import type { Labels } from '@xyo-network/module-model'
-import { PayloadBuilder } from '@xyo-network/payload-builder'
-import type { Payload, Schema } from '@xyo-network/payload-model'
-import { isPayloadOfSchemaType } from '@xyo-network/payload-model'
-// TODO: Inherit from JsonPathAggregateDiviner
-/**
- * A diviner that converts diviner query to index query
- */
-export class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner<
-  TParams extends TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerParams,
-> extends AbstractDiviner<TParams> {
-  static override readonly configSchemas = [...super.configSchemas, TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema]
-  static override readonly defaultConfigSchema = TemporalIndexingDivinerDivinerQueryToIndexQueryDivinerConfigSchema
-  static override readonly labels: Labels = {
-    ...super.labels,
-    'network.xyo.diviner.stage': 'divinerQueryToIndexQueryDiviner',
-  }
-
-  private _indexableSchemas: Schema[] | undefined
-  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined
-
-  /**
-   * The schema of the diviner query payloads
-   */
-  protected get divinerQuerySchema(): Schema {
-    return this.config.divinerQuerySchema ?? PayloadDivinerQuerySchema
-  }
-
-  /**
-   * The schema of the index query payloads
-   */
-  protected get indexQuerySchema(): Schema {
-    return this.config.indexQuerySchema ?? PayloadDivinerQuerySchema
-  }
-
-  /**
-   * The schema of the index payloads
-   */
-  protected get indexSchema(): Schema {
-    return this.config.indexSchema ?? TemporalIndexingDivinerResultIndexSchema
-  }
-
-  /**
-   * List of indexable schemas for this diviner
-   */
-  protected get indexableSchemas(): Schema[] {
-    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms) as Schema[]
-    return this._indexableSchemas
-  }
-
-  /**
-   * Dictionary of schemas to payload transformers for creating indexes
-   * from the payloads within a Bound Witness
-   */
-  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {
-    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)
-    return this._payloadTransformers
-  }
-
-  /**
-   * The dictionary of schemas to JSON Path transform expressions for creating indexes
-   * from the payloads within a Bound Witness
-   */
-  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {
-    return (
-      this.config?.schemaTransforms ?? {
-        [this.divinerQuerySchema]: [
-          {
-            defaultValue: 1,
-            destinationField: 'limit',
-            sourcePathExpression: '$.limit',
-          },
-          {
-            // defaultValue: 0,
-            destinationField: 'cursor',
-            sourcePathExpression: '$.cursor',
-          },
-          {
-            defaultValue: 'desc',
-            destinationField: 'order',
-            sourcePathExpression: '$.order',
-          },
-        ],
-      }
-    )
-  }
-
-  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const queries = payloads.filter(isPayloadOfSchemaType<PayloadDivinerQueryPayload>(this.divinerQuerySchema as any))
-    if (queries.length > 0) {
-      return await Promise.all(
-        queries.map(async (query) => {
-          const fields = await reducePayloads<PayloadDivinerQueryPayload & { $sources?: Hash[]; sources?: Hash[] }>(
-            [query],
-            this.payloadTransformers,
-            this.indexQuerySchema,
-          )
-          // TODO: Make index schema configurable
-          fields.schemas = [this.indexSchema]
-          // TODO: Make sources not need to be deleted
-          delete fields.sources
-          delete fields?.$sources
-          // TODO: Add support for additional filters
-          return new PayloadBuilder<Payload>({ schema: this.indexQuerySchema }).fields(fields).build()
-        }),
-      )
-    }
-    return []
-  }
-}

package/src/DivinerQueryToIndexQueryDiviner/index.ts

@@ -1 +0,0 @@
-export * from './Diviner.ts'

package/src/IndexCandidateToIndexDiviner/Diviner.ts

@@ -1,144 +0,0 @@
-import type { Hash } from '@xylabs/sdk-js'
-import {
-  assertEx, containsAll, exists,
-} from '@xylabs/sdk-js'
-import type { BoundWitness } from '@xyo-network/boundwitness-model'
-import { isBoundWitness } from '@xyo-network/boundwitness-model'
-import { AbstractDiviner } from '@xyo-network/diviner-abstract'
-import { jsonPathToTransformersDictionary } from '@xyo-network/diviner-jsonpath-aggregate-memory'
-import type { SchemaToJsonPathTransformExpressionsDictionary, SchemaToPayloadTransformersDictionary } from '@xyo-network/diviner-jsonpath-model'
-import type {
-  TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,
-  TemporalIndexingDivinerResultIndex,
-} from '@xyo-network/diviner-temporal-indexing-model'
-import {
-  TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema,
-  TemporalIndexingDivinerResultIndexSchema,
-} from '@xyo-network/diviner-temporal-indexing-model'
-import type { Labels } from '@xyo-network/module-model'
-import { PayloadBuilder } from '@xyo-network/payload-builder'
-import type { Payload, Schema } from '@xyo-network/payload-model'
-import { isAnyPayload } from '@xyo-network/payload-model'
-import { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'
-
-type IndexableHashes = [Hash, ...Hash[]]
-
-const moduleName = 'TemporalIndexingDivinerIndexCandidateToIndexDiviner'
-
-/**
- * Diviner which transforms index candidates to indexes using JSON Path to map
- * source properties in the supplied payloads to destination fields in the
- * resultant index
- */
-export class TemporalIndexingDivinerIndexCandidateToIndexDiviner<
-  TParams extends TemporalIndexingDivinerIndexCandidateToIndexDivinerParams = TemporalIndexingDivinerIndexCandidateToIndexDivinerParams,
-> extends AbstractDiviner<TParams> {
-  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema]
-  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexCandidateToIndexDivinerConfigSchema
-  static override readonly labels: Labels = {
-    ...super.labels,
-    'network.xyo.diviner.stage': 'indexCandidateToIndexDiviner',
-  }
-
-  private _indexableSchemas: Schema[] | undefined
-  private _payloadTransformers: SchemaToPayloadTransformersDictionary | undefined
-
-  /**
-   * List of indexable schemas for this diviner
-   */
-  protected get indexableSchemas(): Schema[] {
-    if (!this._indexableSchemas) this._indexableSchemas = Object.keys(this.schemaTransforms) as Schema[]
-    return this._indexableSchemas
-  }
-
-  /**
-   * Dictionary of schemas to payload transformers for creating indexes
-   * from the payloads within a Bound Witness
-   */
-  protected get payloadTransformers(): SchemaToPayloadTransformersDictionary {
-    if (!this._payloadTransformers) this._payloadTransformers = jsonPathToTransformersDictionary(this.schemaTransforms)
-    return this._payloadTransformers
-  }
-
-  /**
-   * The dictionary of schemas to JSON Path transform expressions for creating indexes
-   * from the payloads within a Bound Witness
-   */
-  protected get schemaTransforms(): SchemaToJsonPathTransformExpressionsDictionary {
-    return assertEx(this.config?.schemaTransforms, () => `${moduleName}: Missing config.schemaTransforms section`)
-  }
-
-  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {
-    // If the Bound Witness does not contain all the required schemas do not index it
-    const indexableBoundWitnesses = payloads
-      .filter(isBoundWitness)
-      .filter(bw => containsAll(bw.payload_schemas, this.indexableSchemas))
-    // If the Payload is not one of the indexable schemas do not index it
-    const indexablePayloads = payloads.filter(p => this.isIndexablePayload(p))
-    // If there is nothing to index, return an empty array
-    if (indexableBoundWitnesses.length === 0 || indexablePayloads.length === 0) return []
-    // Hash all the indexable data once
-    const [bwDictionary, payloadDictionary] = await Promise.all([
-      PayloadBuilder.toHashMap(indexableBoundWitnesses),
-      PayloadBuilder.toAllHashMap(indexablePayloads),
-    ])
-
-    // Initialize the array for validIndexableTuples outside of the loop
-    const validIndexableTuples: IndexableHashes[] = []
-
-    // Iterate over each entry in bwDictionary
-    for (const [bwHash, bw] of Object.entries(bwDictionary) as [Hash, BoundWitness][]) {
-      // Find the combinations of payloads that satisfy the required schemas
-      const combinations = intraBoundwitnessSchemaCombinations(bw, this.indexableSchemas)
-
-      // Iterate over each combination
-      for (const combination of combinations) {
-        const indexablePayloads = combination.map(hash => payloadDictionary[hash]).filter(exists)
-
-        // If we found the right amount of indexable payloads (of the correct schema as checked
-        // above) in this BW, then index it
-        if (indexablePayloads.length === this.indexableSchemas.length) {
-          validIndexableTuples.push([bwHash, ...combination])
-        }
-      }
-    }
-
-    // Create the indexes from the tuples
-    const indexes = validIndexableTuples.map<TemporalIndexingDivinerResultIndex>(([bwHash, ...sourcePayloadHashes]) => {
-      const sourcePayloads = sourcePayloadHashes.map(hash => payloadDictionary[hash])
-      // Use the payload transformers to convert the fields from the source payloads to the destination fields
-      const indexFields = sourcePayloads.flatMap((payload) => {
-        // Find the transformers for this payload
-        const transformers = this.payloadTransformers[payload.schema]
-        // If transformers exist, apply them to the payload otherwise return an empty array
-        return transformers ? transformers.map(transform => transform(payload)) : []
-      })
-      // Include all the sources for reference
-      const $sources: Hash[] = [bwHash, ...sourcePayloadHashes]
-      // Build and return the index
-      return new PayloadBuilder<TemporalIndexingDivinerResultIndex>({ schema: TemporalIndexingDivinerResultIndexSchema })
-        .fields(Object.assign({}, ...indexFields))
-        .meta({ $sources })
-        .build()
-    })
-    return indexes.flat()
-  }
-
-  /**
-   * Identifies if a payload is one that is indexed by this diviner
-   * @param x The candidate payload
-   * @returns True if the payload is one indexed by this diviner, false otherwise
-   */
-  protected isIndexablePayload = (x: unknown) => {
-    return isAnyPayload(x) && this.indexableSchemas.includes(x?.schema)
-  }
-
-  /**
-   * Identifies if a schema is one that is indexed by this diviner
-   * @param schema The candidate schema
-   * @returns True if this schema is one indexed by this diviner, false otherwise
-   */
-  protected isIndexableSchema = (schema?: Schema | null) => {
-    return typeof schema === 'string' ? this.indexableSchemas.includes(schema) : false
-  }
-}

package/src/IndexCandidateToIndexDiviner/index.ts

@@ -1 +0,0 @@
-export * from './Diviner.ts'

package/src/IndexQueryResponseToDivinerQueryResponseDiviner/Diviner.ts

@@ -1,30 +0,0 @@
-import { AbstractDiviner } from '@xyo-network/diviner-abstract'
-import { isPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'
-import { TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'
-import type { Labels } from '@xyo-network/module-model'
-import type { Payload, Schema } from '@xyo-network/payload-model'
-
-/**
- * Transforms an ImageThumbnailIndex response into an ImageThumbnailResponse response
- */
-export class TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner extends AbstractDiviner {
-  static override readonly configSchemas: Schema[] = [
-    ...super.configSchemas,
-    TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema,
-  ]
-
-  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDivinerConfigSchema
-  static override readonly labels: Labels = {
-    ...super.labels,
-    'network.xyo.diviner.stage': 'indexQueryResponseToDivinerQueryResponseDiviner',
-  }
-
-  protected override async divineHandler(payloads: Payload[] = []): Promise<Payload[]> {
-    // We're not doing anything with the query payloads but some diviners
-    // might want to use this to transform from the query to the response (for example
-    // if we use a plaintext value in the query to generate a hash key in the index)
-    // const queries = payloads.filter(isPayloadDivinerQueryPayload)
-    const responses = payloads.filter(p => !isPayloadDivinerQueryPayload(p))
-    return await Promise.resolve(responses)
-  }
-}

package/src/IndexQueryResponseToDivinerQueryResponseDiviner/index.ts

@@ -1 +0,0 @@
-export * from './Diviner.ts'

package/src/StateToIndexCandidateDiviner/Diviner.ts

@@ -1,160 +0,0 @@
-import { assertEx, exists } from '@xylabs/sdk-js'
-import type { ArchivistInstance, ArchivistNextOptions } from '@xyo-network/archivist-model'
-import { ArchivistWrapper } from '@xyo-network/archivist-wrapper'
-import type { BoundWitness } from '@xyo-network/boundwitness-model'
-import { isBoundWitness } from '@xyo-network/boundwitness-model'
-import { payloadSchemasContainsAll } from '@xyo-network/boundwitness-validator'
-import { AbstractDiviner } from '@xyo-network/diviner-abstract'
-import type { BoundWitnessDiviner } from '@xyo-network/diviner-boundwitness-abstract'
-import type { BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'
-import type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model'
-import type { TemporalIndexingDivinerStateToIndexCandidateDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'
-import { TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'
-import { DivinerWrapper } from '@xyo-network/diviner-wrapper'
-import type {
-  Labels, ModuleIdentifier, ModuleState,
-} from '@xyo-network/module-model'
-import { isModuleState, ModuleStateSchema } from '@xyo-network/module-model'
-import type {
-  Payload, Schema,
-  WithStorageMeta,
-} from '@xyo-network/payload-model'
-import { SequenceConstants } from '@xyo-network/payload-model'
-import { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'
-import type { TimeStamp } from '@xyo-network/witness-timestamp'
-import { TimestampSchema } from '@xyo-network/witness-timestamp'
-
-/**
- * All Payload types involved in index candidates for indexing
- */
-export type IndexCandidate = BoundWitness | Payload | TimeStamp
-
-/**
- * The response from the TemporalStateToIndexCandidateDiviner
- */
-export type TemporalStateToIndexCandidateDivinerResponse = [
-  /**
-   * The next state of the diviner
-   */
-  nextState: ModuleState<IndexingDivinerState>,
-  /**
-   * The index candidates
-   */
-  ...IndexCandidate[],
-]
-
-/**
- * The default order to search Bound Witnesses to identify index candidates
- */
-const order = 'asc'
-
-/**
- * The name of the module (for logging purposes)
- */
-const moduleName = 'TemporalIndexingDivinerStateToIndexCandidateDiviner'
-
-/**
- * Transforms candidates for image thumbnail indexing into their indexed representation
- */
-export class TemporalIndexingDivinerStateToIndexCandidateDiviner<
-  TParams extends TemporalIndexingDivinerStateToIndexCandidateDivinerParams = TemporalIndexingDivinerStateToIndexCandidateDivinerParams,
-> extends AbstractDiviner<TParams, Payload, ModuleState | IndexCandidate> {
-  static override readonly configSchemas: Schema[] = [...super.configSchemas, TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema]
-  static override readonly defaultConfigSchema: Schema = TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema
-  static override readonly labels: Labels = {
-    ...super.labels,
-    'network.xyo.diviner.stage': 'stateToIndexCandidateDiviner',
-  }
-
-  get payloadDivinerLimit() {
-    return this.config.payloadDivinerLimit ?? 1000
-  }
-
-  /**
-   * The required payload_schemas within BoundWitnesses to identify index candidates
-   */
-  protected get payload_schemas(): Schema[] {
-    const schemas = this.config.filter?.payload_schemas
-    return [TimestampSchema, ...(schemas ?? [])]
-  }
-
-  protected override async divineHandler(payloads: Payload[] = []): Promise<TemporalStateToIndexCandidateDivinerResponse> {
-    // Retrieve the last state from what was passed in
-    const lastState = payloads.find(isModuleState<IndexingDivinerState>)
-    // If there is no last state, start from the beginning
-      ?? { schema: ModuleStateSchema, state: { cursor: SequenceConstants.minLocalSequence } }
-
-    // Get the last cursor
-    const cursor = lastState?.state?.cursor
-    // Get the archivist for the store
-    const sourceArchivist = await this.getArchivistForStore()
-    if (!sourceArchivist) return [lastState]
-
-    // Get the next batch of results
-    const nextOffset: ArchivistNextOptions = { limit: this.payloadDivinerLimit, order }
-    // Only use the cursor if it's a valid offset
-    if (cursor !== SequenceConstants.minLocalSequence) nextOffset.cursor = cursor
-    // Get next batch of results starting from the offset
-    const next = await sourceArchivist.next(nextOffset)
-    if (next.length === 0) return [lastState]
-
-    const batch = next.filter(x => isBoundWitness(x))
-      .filter(exists)
-      .filter(bw => payloadSchemasContainsAll(bw, this.payload_schemas))
-    // Get source data
-    const indexCandidates: IndexCandidate[] = (await Promise.all(batch.map(bw => this.getPayloadsInBoundWitness(bw, sourceArchivist))))
-      .filter(exists)
-      .flat()
-    const nextCursor = assertEx(next.at(-1)?._sequence, () => `${moduleName}: Expected next to have a sequence`)
-    const nextState: ModuleState<IndexingDivinerState> = { schema: ModuleStateSchema, state: { ...lastState.state, cursor: nextCursor } }
-    return [nextState, ...indexCandidates]
-  }
-
-  /**
-   * Retrieves the archivist for the payloadStore
-   * @returns The archivist for the payloadStore or undefined if not resolvable
-   */
-  protected async getArchivistForStore(): Promise<ArchivistWrapper | undefined> {
-    // It should be defined, so we'll error if it's not
-    const name: ModuleIdentifier = assertEx(
-      this.config?.payloadStore?.archivist,
-      () => `${moduleName}: Config for payloadStore.archivist not specified`,
-    )
-    // It might not be resolvable (yet), so we'll return undefined if it's not
-    const mod = await this.resolve(name)
-    if (!mod) return undefined
-    // Return the wrapped archivist
-    return ArchivistWrapper.wrap(mod, this.account)
-  }
-
-  /**
-   * Retrieves the BoundWitness Diviner for the payloadStore
-   * @returns The BoundWitness Diviner for the payloadStore or undefined if not resolvable
-   */
-  protected async getBoundWitnessDivinerForStore() {
-    // It should be defined, so we'll error if it's not
-    const name: ModuleIdentifier = assertEx(
-      this.config?.payloadStore?.boundWitnessDiviner,
-      () => `${moduleName}: Config for payloadStore.boundWitnessDiviner not specified`,
-    )
-    // It might not be resolvable (yet), so we'll return undefined if it's not
-    const mod = await this.resolve(name)
-    if (!mod) return
-    // Return the wrapped diviner
-    return DivinerWrapper.wrap<
-      DivinerWrapper<
-        BoundWitnessDiviner<BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload, BoundWitness>,
-        BoundWitnessDivinerQueryPayload,
-        WithStorageMeta<BoundWitness>
-      >
-    >(mod, this.account)
-  }
-
-  protected async getPayloadsInBoundWitness(bw: BoundWitness, archivist: ArchivistInstance): Promise<IndexCandidate[] | undefined> {
-    const combinations = intraBoundwitnessSchemaCombinations(bw, this.payload_schemas).flat()
-    if (combinations.length === 0) return undefined
-    const hashes = new Set(combinations)
-    const indexCandidates = await archivist.get([...hashes])
-    return [bw, ...indexCandidates]
-  }
-}

package/src/StateToIndexCandidateDiviner/index.ts

@@ -1 +0,0 @@
-export * from './Diviner.ts'

package/src/StateToIndexCandidateDiviner/spec/TemporalStateToIndexCandidateDiviner.json

@@ -1,59 +0,0 @@
-{
-  "$schema": "https://raw.githubusercontent.com/XYOracleNetwork/sdk-xyo-client-js/main/packages/manifest/src/compilations/schema.json",
-  "nodes": [
-    {
-      "config": {
-        "name": "DivinerTest",
-        "schema": "network.xyo.node.config"
-      },
-      "modules": {
-        "public": [
-          {
-            "config": {
-              "language": "javascript",
-              "name": "ImageThumbnailArchivist",
-              "schema": "network.xyo.archivist.config"
-            }
-          },
-          {
-            "config": {
-              "archivist": "ImageThumbnailArchivist",
-              "language": "javascript",
-              "name": "ImageThumbnailBoundWitnessDiviner",
-              "schema": "network.xyo.diviner.boundwitness.config"
-            }
-          },
-          {
-            "config": {
-              "archivist": "ImageThumbnailArchivist",
-              "language": "javascript",
-              "name": "ImageThumbnailPayloadDiviner",
-              "schema": "network.xyo.diviner.payload.config"
-            }
-          },
-          {
-            "config": {
-              "filter": {
-                "payload_schemas": [
-                  "network.xyo.image.thumbnail"
-                ]
-              },
-              "labels": {
-                "network.xyo.diviner.stage": "stateToIndexCandidateDiviner"
-              },
-              "language": "javascript",
-              "name": "TemporalStateToIndexCandidateDiviner",
-              "payloadStore": {
-                "archivist": "ImageThumbnailArchivist",
-                "boundWitnessDiviner": "ImageThumbnailBoundWitnessDiviner",
-                "payloadDiviner": "ImageThumbnailPayloadDiviner"
-              },
-              "schema": "network.xyo.diviner.config"
-            }
-          }
-        ]
-      }
-    }
-  ],
-  "schema": "network.xyo.manifest"
-}

package/src/index.ts

@@ -1,5 +0,0 @@
-export * from './Diviner.ts'
-export * from './DivinerQueryToIndexQueryDiviner/index.ts'
-export * from './IndexCandidateToIndexDiviner/index.ts'
-export * from './IndexQueryResponseToDivinerQueryResponseDiviner/index.ts'
-export * from './StateToIndexCandidateDiviner/index.ts'

package/src/spec/TemporalDiviner.json

@@ -1,206 +0,0 @@
-{
-  "nodes": [
-    {
-      "config": {
-        "name": "DivinerTest",
-        "schema": "network.xyo.node.config"
-      },
-      "modules": {
-        "public": [
-          {
-            "config": {
-              "language": "javascript",
-              "name": "AddressStateArchivist",
-              "schema": "network.xyo.archivist.config",
-              "storeQueries": false
-            }
-          },
-          {
-            "config": {
-              "archivist": "AddressStateArchivist",
-              "language": "javascript",
-              "name": "AddressStateBoundWitnessDiviner",
-              "schema": "network.xyo.diviner.boundwitness.config"
-            }
-          },
-          {
-            "config": {
-              "archivist": "AddressStateArchivist",
-              "language": "javascript",
-              "name": "AddressStatePayloadDiviner",
-              "schema": "network.xyo.diviner.payload.config"
-            }
-          },
-          {
-            "config": {
-              "language": "javascript",
-              "name": "ImageThumbnailArchivist",
-              "schema": "network.xyo.archivist.config"
-            }
-          },
-          {
-            "config": {
-              "archivist": "ImageThumbnailArchivist",
-              "language": "javascript",
-              "name": "ImageThumbnailBoundWitnessDiviner",
-              "schema": "network.xyo.diviner.boundwitness.config"
-            }
-          },
-          {
-            "config": {
-              "archivist": "ImageThumbnailArchivist",
-              "language": "javascript",
-              "name": "ImageThumbnailPayloadDiviner",
-              "schema": "network.xyo.diviner.payload.config"
-            }
-          },
-          {
-            "config": {
-              "language": "javascript",
-              "name": "ImageThumbnailDivinerIndexArchivist",
-              "schema": "network.xyo.archivist.config"
-            }
-          },
-          {
-            "config": {
-              "archivist": "ImageThumbnailDivinerIndexArchivist",
-              "language": "javascript",
-              "name": "ImageThumbnailDivinerIndexBoundWitnessDiviner",
-              "schema": "network.xyo.diviner.boundwitness.config"
-            }
-          },
-          {
-            "config": {
-              "archivist": "ImageThumbnailDivinerIndexArchivist",
-              "language": "javascript",
-              "name": "ImageThumbnailDivinerIndexPayloadDiviner",
-              "schema": "network.xyo.diviner.payload.config"
-            }
-          },
-          {
-            "config": {
-              "filter": {
-                "payload_schemas": [
-                  "network.xyo.image.thumbnail"
-                ]
-              },
-              "labels": {
-                "network.xyo.diviner.stage": "stateToIndexCandidateDiviner",
-                "network.xyo.image.thumbnail": "diviner"
-              },
-              "language": "javascript",
-              "name": "ImageThumbnailStateToIndexCandidateDiviner",
-              "payloadStore": {
-                "archivist": "ImageThumbnailArchivist",
-                "boundWitnessDiviner": "ImageThumbnailBoundWitnessDiviner",
-                "payloadDiviner": "ImageThumbnailPayloadDiviner"
-              },
-              "schema": "network.xyo.diviner.config"
-            }
-          },
-          {
-            "config": {
-              "labels": {
-                "network.xyo.diviner.stage": "indexCandidateToIndexDiviner",
-                "network.xyo.image.thumbnail": "diviner"
-              },
-              "language": "javascript",
-              "name": "ImageThumbnailIndexCandidateToImageThumbnailIndexDiviner",
-              "schema": "network.xyo.diviner.config",
-              "schemaTransforms": {
-                "network.xyo.image.thumbnail": [
-                  {
-                    "destinationField": "url",
-                    "sourcePathExpression": "$.sourceUrl"
-                  },
-                  {
-                    "destinationField": "status",
-                    "sourcePathExpression": "$.http.status"
-                  }
-                ],
-                "network.xyo.timestamp": [
-                  {
-                    "destinationField": "timestamp",
-                    "sourcePathExpression": "$.timestamp"
-                  }
-                ]
-              }
-            }
-          },
-          {
-            "config": {
-              "divinerQuerySchema": "network.xyo.diviner.payload.query",
-              "indexQuerySchema": "network.xyo.diviner.payload.query",
-              "indexSchema": "network.xyo.diviner.indexing.temporal.result.index",
-              "labels": {
-                "network.xyo.diviner.stage": "divinerQueryToIndexQueryDiviner",
-                "network.xyo.image.thumbnail": "diviner"
-              },
-              "language": "javascript",
-              "name": "ImageThumbnailQueryToImageThumbnailIndexQueryDiviner",
-              "schema": "network.xyo.diviner.config",
-              "schemaTransforms": {
-                "network.xyo.diviner.payload.query": [
-                  {
-                    "destinationField": "url",
-                    "sourcePathExpression": "$.url"
-                  },
-                  {
-                    "defaultValue": 1,
-                    "destinationField": "limit",
-                    "sourcePathExpression": "$.limit"
-                  },
-                  {
-                    "defaultValue": "desc",
-                    "destinationField": "order",
-                    "sourcePathExpression": "$.order"
-                  },
-                  {
-                    "destinationField": "status",
-                    "sourcePathExpression": "$.status"
-                  }
-                ]
-              }
-            }
-          },
-          {
-            "config": {
-              "labels": {
-                "network.xyo.diviner.stage": "indexQueryResponseToDivinerQueryResponseDiviner",
-                "network.xyo.image.thumbnail": "diviner"
-              },
-              "language": "javascript",
-              "name": "ImageThumbnailIndexQueryResponseToImageThumbnailQueryResponseDiviner",
-              "schema": "network.xyo.diviner.config"
-            }
-          },
-          {
-            "config": {
-              "indexStore": {
-                "archivist": "ImageThumbnailDivinerIndexArchivist",
-                "boundWitnessDiviner": "ImageThumbnailDivinerIndexBoundWitnessDiviner",
-                "payloadDiviner": "ImageThumbnailDivinerIndexPayloadDiviner"
-              },
-              "indexingDivinerStages": {
-                "divinerQueryToIndexQueryDiviner": "ImageThumbnailQueryToImageThumbnailIndexQueryDiviner",
-                "indexCandidateToIndexDiviner": "ImageThumbnailIndexCandidateToImageThumbnailIndexDiviner",
-                "indexQueryResponseToDivinerQueryResponseDiviner": "ImageThumbnailIndexQueryResponseToImageThumbnailQueryResponseDiviner",
-                "stateToIndexCandidateDiviner": "ImageThumbnailStateToIndexCandidateDiviner"
-              },
-              "language": "javascript",
-              "name": "ImageThumbnailDiviner",
-              "pollFrequency": 100,
-              "schema": "network.xyo.diviner.indexing.config",
-              "stateStore": {
-                "archivist": "AddressStateArchivist",
-                "boundWitnessDiviner": "AddressStateBoundWitnessDiviner",
-                "payloadDiviner": "AddressStatePayloadDiviner"
-              }
-            }
-          }
-        ]
-      }
-    }
-  ],
-  "schema": "network.xyo.manifest"
-}