@xyo-network/diviner-image-thumbnail 2.78.5 → 2.79.0

package/package.json

@@ -12,31 +12,34 @@
   "dependencies": {
     "@xylabs/assert": "^2.13.3",
     "@xylabs/exists": "^2.13.3",
-    "@xyo-network/abstract-diviner": "~2.78.5",
-    "@xyo-network/archivist-model": "~2.78.5",
-    "@xyo-network/archivist-wrapper": "~2.78.5",
-    "@xyo-network/boundwitness-model": "~2.78.5",
-    "@xyo-network/core": "~2.78.5",
-    "@xyo-network/diviner-boundwitness-model": "~2.78.5",
-    "@xyo-network/diviner-model": "~2.78.5",
-    "@xyo-network/diviner-payload-model": "~2.78.5",
-    "@xyo-network/diviner-wrapper": "~2.78.5",
-    "@xyo-network/image-thumbnail-payload-plugin": "~2.78.5",
-    "@xyo-network/module-model": "~2.78.5",
-    "@xyo-network/payload-builder": "~2.78.5",
-    "@xyo-network/payload-model": "~2.78.5",
-    "@xyo-network/witness-timestamp": "~2.78.5"
+    "@xyo-network/abstract-diviner": "~2.79.0",
+    "@xyo-network/archivist-model": "~2.79.0",
+    "@xyo-network/archivist-wrapper": "~2.79.0",
+    "@xyo-network/boundwitness-model": "~2.79.0",
+    "@xyo-network/core": "~2.79.0",
+    "@xyo-network/diviner-boundwitness-model": "~2.79.0",
+    "@xyo-network/diviner-indexing": "~2.79.0",
+    "@xyo-network/diviner-model": "~2.79.0",
+    "@xyo-network/diviner-payload-model": "~2.79.0",
+    "@xyo-network/diviner-wrapper": "~2.79.0",
+    "@xyo-network/image-thumbnail-payload-plugin": "~2.79.0",
+    "@xyo-network/module-model": "~2.79.0",
+    "@xyo-network/payload-builder": "~2.79.0",
+    "@xyo-network/payload-model": "~2.79.0",
+    "@xyo-network/url-payload-plugin": "~2.79.0",
+    "@xyo-network/witness-timestamp": "~2.79.0"
   },
   "devDependencies": {
     "@xylabs/delay": "^2.13.3",
     "@xylabs/ts-scripts-yarn3": "^3.1.21",
     "@xylabs/tsconfig": "^3.1.21",
-    "@xyo-network/account": "~2.78.5",
-    "@xyo-network/boundwitness-builder": "~2.78.5",
-    "@xyo-network/diviner-boundwitness-memory": "~2.78.5",
-    "@xyo-network/diviner-payload-memory": "~2.78.5",
-    "@xyo-network/memory-archivist": "~2.78.5",
-    "@xyo-network/node-memory": "~2.78.5",
+    "@xyo-network/account": "~2.79.0",
+    "@xyo-network/boundwitness-builder": "~2.79.0",
+    "@xyo-network/diviner-boundwitness-memory": "~2.79.0",
+    "@xyo-network/diviner-payload-memory": "~2.79.0",
+    "@xyo-network/manifest": "~2.79.0",
+    "@xyo-network/memory-archivist": "~2.79.0",
+    "@xyo-network/node-memory": "~2.79.0",
     "typescript": "^5.2.2"
   },
   "description": "Primary SDK for using XYO Protocol 2.0",
@@ -79,5 +82,5 @@
     "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js.git"
   },
   "sideEffects": false,
-  "version": "2.78.5"
+  "version": "2.79.0"
 }

package/src/Diviner/Diviner.ts

@@ -1,303 +1,7 @@
-import { assertEx } from '@xylabs/assert'
-import { exists } from '@xylabs/exists'
-import { AbstractDiviner } from '@xyo-network/abstract-diviner'
-import { ArchivistInstance } from '@xyo-network/archivist-model'
-import { ArchivistWrapper } from '@xyo-network/archivist-wrapper'
-import { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'
-import { isBoundWitness } from '@xyo-network/boundwitness-model'
-import { PayloadHasher } from '@xyo-network/core'
-import { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'
-import { DivinerConfigSchema } from '@xyo-network/diviner-model'
-import { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
-import { DivinerWrapper } from '@xyo-network/diviner-wrapper'
-import {
-  ImageThumbnail,
-  ImageThumbnailDivinerConfig,
-  ImageThumbnailDivinerConfigSchema,
-  ImageThumbnailDivinerParams,
-  ImageThumbnailResult,
-  ImageThumbnailResultIndexSchema,
-  ImageThumbnailResultInfo,
-  ImageThumbnailSchema,
-  isImageThumbnail,
-  isImageThumbnailDivinerQuery,
-  isImageThumbnailResult,
-} from '@xyo-network/image-thumbnail-payload-plugin'
-import { isModuleState, ModuleState, ModuleStateSchema, StateDictionary } from '@xyo-network/module-model'
-import { PayloadBuilder } from '@xyo-network/payload-builder'
-import { Payload } from '@xyo-network/payload-model'
-import { isTimestamp, TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'
+import { IndexingDiviner } from '@xyo-network/diviner-indexing'
 
-export type ImageThumbnailDivinerState = StateDictionary & {
-  offset: number
-}
-
-type ConfigStoreKey = 'indexStore' | 'stateStore' | 'thumbnailStore'
-
-type ConfigStore = Extract<keyof ImageThumbnailDivinerConfig, ConfigStoreKey>
-
-/**
- * The fields that will need to be indexed on in the underlying store
- */
-type QueryableImageThumbnailResultProperties = Extract<keyof ImageThumbnailResultInfo, 'status' | 'success' | 'timestamp' | 'url'>
-
-/**
- * The query that will be used to retrieve the results from the underlying store
- */
-type ImageThumbnailResultQuery = PayloadDivinerQueryPayload & { schemas: [ImageThumbnailSchema] } & Pick<
-    ImageThumbnailResult,
-    QueryableImageThumbnailResultProperties
-  >
-
-type IndexableHashes = Readonly<[boundWitnessHash: string, imageThumbnailHash: string, timestampHash: string]>
-
-const moduleName = 'ImageThumbnailDiviner'
-
-export class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams = ImageThumbnailDivinerParams> extends AbstractDiviner<TParams> {
-  static override configSchemas = [ImageThumbnailDivinerConfigSchema, DivinerConfigSchema]
-
-  private _lastState?: ImageThumbnailDivinerState
-  private _pollId?: string | number | NodeJS.Timeout
-
-  get payloadDivinerLimit() {
-    return this.config.payloadDivinerLimit ?? 1_0000
-  }
-
-  get pollFrequency() {
-    return this.config.pollFrequency ?? 10_000
-  }
-
-  private static async indexableData(indexableHashes: IndexableHashes[], archivist: ArchivistInstance) {
-    // Find all the indexable data associated with the indexable hashes
-    type IndexableData = Readonly<
-      [
-        boundWitnessHash: string,
-        imageThumbnailHash: string,
-        imageThumbnailPayload: ImageThumbnail,
-        timestampHash: string,
-        timestampPayload: TimeStamp,
-      ]
-    >
-    const indexableData: IndexableData[] = (
-      await Promise.all(
-        indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {
-          const results = await archivist.get([imageThumbnailHash, timestampHash])
-          const imageThumbnailPayload = results.find(isImageThumbnail)
-          if (!imageThumbnailPayload) {
-            console.log(
-              `${moduleName}: Could not find ${ImageThumbnailSchema} Payload (${imageThumbnailHash}) from BoundWitness (${boundWitnessHash})`,
-            )
-            return undefined
-          }
-          const timestampPayload = results.find(isTimestamp)
-          if (!timestampPayload) {
-            console.log(`${moduleName}: Could not find ${TimestampSchema} Payload (${timestampHash}) from BoundWitness (${boundWitnessHash})`)
-            return undefined
-          }
-          const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload)
-          const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload)
-          if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash) return undefined
-          return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload] as const
-        }),
-      )
-    ).filter(exists)
-    return indexableData
-  }
-
-  private static async indexableHashes(batch: Payload[]) {
-    // Find all the indexable hashes in this batch
-    const indexableHashes: IndexableHashes[] = (
-      await Promise.all(
-        batch.filter(isBoundWitness).map(async (bw) => {
-          const imageThumbnailIndexes = bw.payload_schemas
-            ?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined))
-            .filter(exists)
-          const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)
-          if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined
-          const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)
-          const timestampHash = bw.payload_hashes?.[timestampIndex]
-          const boundWitnessHash = await PayloadHasher.hashAsync(bw)
-          return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash] as const)
-        }),
-      )
-    )
-      .flat()
-      .filter(exists)
-    return indexableHashes
-  }
-
-  /**
-   * Works in the background to populate index for the Diviner
-   * @returns
-   */
-  protected backgroundDivine = async (): Promise<void> => {
-    // Load last state
-    const lastState = (await this.retrieveState()) ?? { offset: 0 }
-    const { offset } = lastState
-    // Get next batch of results
-    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore('thumbnailStore')
-    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
-      .fields({
-        limit: this.payloadDivinerLimit,
-        offset,
-        order: 'asc',
-        payload_schemas: [ImageThumbnailSchema, TimestampSchema],
-      })
-      .build()
-    const batch = await boundWitnessDiviner.divine([query])
-    if (batch.length === 0) return
-    const indexableHashes = await ImageThumbnailDiviner.indexableHashes(batch)
-    const archivist = await this.getArchivistForStore('thumbnailStore')
-    const indexableData = await ImageThumbnailDiviner.indexableData(indexableHashes, archivist)
-    // Build index results from the indexable data
-    const indexes: ImageThumbnailResult[] = indexableData.map(
-      ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {
-        const { sourceUrl: url } = thumbnailPayload
-        const { timestamp } = timestampPayload
-        const status = thumbnailPayload.http?.status
-        //call anything with a thumbnail url a success
-        const success = !!thumbnailPayload.url
-        const sources = [boundWitnessHash, thumbnailHash, timestampHash]
-        const fields = { sources, status, success, timestamp, url }
-        const result = new PayloadBuilder<ImageThumbnailResult>({ schema: ImageThumbnailResultIndexSchema }).fields(fields).build()
-        return result
-      },
-    )
-    // Insert index results
-    const indexArchivist = await this.getArchivistForStore('indexStore')
-    await indexArchivist.insert(indexes)
-    // Update state
-    const nextOffset = offset + batch.length
-    const currentState = { ...lastState, offset: nextOffset }
-    await this.commitState(currentState)
-  }
-
-  /**
-   * Commit the internal state of the Diviner process. This is similar
-   * to a transaction completion in a database and should only be called
-   * when results have been successfully persisted to the appropriate
-   * external stores.
-   */
-  protected async commitState(state: ImageThumbnailDivinerState) {
-    this._lastState = state
-    const archivist = await this.getArchivistForStore('stateStore')
-    const payload = new PayloadBuilder<ModuleState<ImageThumbnailDivinerState>>({ schema: ModuleStateSchema }).fields({ state }).build()
-    const [bw] = await new BoundWitnessBuilder().payloads([payload]).witness(this.account).build()
-    await archivist.insert([bw, payload])
-  }
-
-  protected override async divineHandler(payloads: Payload[] = []): Promise<ImageThumbnailResult[]> {
-    const urls = payloads.filter(isImageThumbnailDivinerQuery)
-    const diviner = await this.getPayloadDivinerForStore('indexStore')
-    const results = (
-      await Promise.all(
-        urls.map(async (payload) => {
-          const { limit: payloadLimit, offset: payloadOffset, order: payloadOrder, status: payloadStatus, success: payloadSuccess, url } = payload
-          const limit = payloadLimit ?? 1
-          const order = payloadOrder ?? 'desc'
-          const offset = payloadOffset ?? 0
-          const fields: Partial<ImageThumbnailResultQuery> = { limit, offset, order, url }
-          if (payloadSuccess !== undefined) fields.success = payloadSuccess
-          if (payloadStatus !== undefined) fields.status = payloadStatus
-          const query = new PayloadBuilder<ImageThumbnailResultQuery>({ schema: PayloadDivinerQuerySchema }).fields(fields).build()
-          return await diviner.divine([query])
-        }),
-      )
-    )
-      .flat()
-      .filter(isImageThumbnailResult)
-    return results
-  }
-
-  protected async getArchivistForStore(store: ConfigStore) {
-    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)
-    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)
-    return ArchivistWrapper.wrap(mod, this.account)
-  }
-
-  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {
-    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)
-    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)
-    return DivinerWrapper.wrap(mod, this.account)
-  }
-
-  protected async getPayloadDivinerForStore(store: ConfigStore) {
-    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)
-    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)
-    return DivinerWrapper.wrap(mod, this.account)
-  }
-
-  /**
-   * Retrieves the last state of the Diviner process. Used to recover state after
-   * preemptions, reboots, etc.
-   */
-  protected async retrieveState(): Promise<ImageThumbnailDivinerState | undefined> {
-    if (this._lastState) return this._lastState
-    let hash: string = ''
-    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')
-    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
-      .fields({
-        address: this.account.address,
-        limit: 1,
-        offset: 0,
-        order: 'desc',
-        payload_schemas: [ModuleStateSchema],
-      })
-      .build()
-    const boundWitnesses = await diviner.divine([query])
-    if (boundWitnesses.length > 0) {
-      const boundWitness = boundWitnesses[0]
-      if (isBoundWitness(boundWitness)) {
-        // Find the index for this address in the BoundWitness that is a ModuleState
-        hash = boundWitness.addresses
-          .map((address, index) => ({ address, index }))
-          .filter(({ address }) => address === this.account.address)
-          .reduce(
-            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),
-            '',
-          )
-      }
-    }
-
-    // If we able to located the last state
-    if (hash) {
-      // Get last state
-      const archivist = await this.getArchivistForStore('stateStore')
-      const payload = (await archivist.get([hash])).find(isModuleState)
-      if (payload) {
-        return payload.state as ImageThumbnailDivinerState
-      }
-    } else {
-      console.log('Failed to locate last state')
-    }
-    return undefined
-  }
-
-  protected override async startHandler(): Promise<boolean> {
-    await super.startHandler()
-    this.poll()
-    return true
-  }
-
-  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {
-    if (this._pollId) {
-      clearTimeout(this._pollId)
-      this._pollId = undefined
-    }
-    return await super.stopHandler()
-  }
+import { ImageThumbnailDivinerLabels } from './ImageThumbnailDivinerLabels'
 
-  private poll() {
-    this._pollId = setTimeout(async () => {
-      try {
-        await this.backgroundDivine()
-      } catch (e) {
-        console.log(e)
-      } finally {
-        if (this._pollId) clearTimeout(this._pollId)
-        this._pollId = undefined
-        this.poll()
-      }
-    }, this.pollFrequency)
-  }
+export class ImageThumbnailDiviner extends IndexingDiviner {
+  static labels: ImageThumbnailDivinerLabels = ImageThumbnailDivinerLabels
 }

package/src/Diviner/ImageThumbnailDivinerLabels.ts

@@ -0,0 +1,26 @@
+import { DivinerStageSchema, IndexingDivinerStage } from '@xyo-network/diviner-indexing'
+import { Labels } from '@xyo-network/module-model'
+
+/**
+ * Labels for Image Thumbnail Diviner components
+ */
+export interface ImageThumbnailDivinerLabels extends Labels {
+  'network.xyo.image.thumbnail': 'diviner'
+}
+
+/**
+ * Labels for Image Thumbnail Diviner components
+ */
+export const ImageThumbnailDivinerLabels: ImageThumbnailDivinerLabels = {
+  'network.xyo.image.thumbnail': 'diviner',
+}
+
+/**
+ * Labels for Image Thumbnail Diviner Stage Diviners
+ */
+export type ImageThumbnailDivinerStageLabels = ImageThumbnailDivinerLabels & {
+  /**
+   * Labels for the stage of the Image Thumbnail Diviner Stage Diviner
+   */
+  [key in DivinerStageSchema]: IndexingDivinerStage
+}

package/src/Diviner/ImageThumbnailDivinerState.ts

@@ -0,0 +1,5 @@
+import { StateDictionary } from '@xyo-network/module-model'
+
+export type ImageThumbnailDivinerState = StateDictionary & {
+  offset: number
+}

package/src/Diviner/ImageThumbnailIndexCandidateToImageThumbnailIndexDiviner.ts

@@ -0,0 +1,70 @@
+import { AbstractDiviner } from '@xyo-network/abstract-diviner'
+import { BoundWitness, isBoundWitness } from '@xyo-network/boundwitness-model'
+import { PayloadHasher } from '@xyo-network/core'
+import { DivinerConfigSchema } from '@xyo-network/diviner-model'
+import {
+  ImageThumbnail,
+  ImageThumbnailResultIndex,
+  ImageThumbnailResultIndexFields,
+  ImageThumbnailResultIndexSchema,
+  ImageThumbnailSchema,
+  isImageThumbnail,
+} from '@xyo-network/image-thumbnail-payload-plugin'
+import { PayloadBuilder } from '@xyo-network/payload-builder'
+import { Payload } from '@xyo-network/payload-model'
+import { UrlSchema } from '@xyo-network/url-payload-plugin'
+import { isTimestamp, TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'
+
+import { ImageThumbnailDivinerLabels, ImageThumbnailDivinerStageLabels } from './ImageThumbnailDivinerLabels'
+
+/**
+ * Transforms candidates for image thumbnail indexing into their indexed representation
+ */
+export class ImageThumbnailIndexCandidateToImageThumbnailIndexDiviner extends AbstractDiviner {
+  static override configSchemas = [DivinerConfigSchema]
+  static labels: ImageThumbnailDivinerStageLabels = {
+    ...ImageThumbnailDivinerLabels,
+    'network.xyo.diviner.stage': 'indexCandidateToIndexDiviner',
+  }
+
+  protected override async divineHandler(payloads: Payload[] = []): Promise<ImageThumbnailResultIndex[]> {
+    const bws: BoundWitness[] = payloads.filter(isBoundWitness)
+    const imageThumbnailPayloads: ImageThumbnail[] = payloads.filter(isImageThumbnail)
+    const timestampPayloads: TimeStamp[] = payloads.filter(isTimestamp)
+    if (bws.length && imageThumbnailPayloads.length && timestampPayloads.length) {
+      const payloadDictionary = await PayloadHasher.toMap(payloads)
+      const tuples: [BoundWitness, ImageThumbnail, TimeStamp][] = bws.reduce<[BoundWitness, ImageThumbnail, TimeStamp][]>(
+        (acc, curr) => {
+          const imageThumbnailIndex = curr.payload_schemas?.findIndex((schema) => schema === ImageThumbnailSchema)
+          const timestampIndex = curr.payload_schemas?.findIndex((schema) => schema === TimestampSchema)
+          const imageThumbnailHash = curr.payload_hashes?.[imageThumbnailIndex]
+          const timestampHash = curr.payload_hashes?.[timestampIndex]
+          const imageThumbnailPayload = [payloadDictionary[imageThumbnailHash]].find(isImageThumbnail)
+          const timestampPayload = [payloadDictionary[timestampHash]].find(isTimestamp)
+          if (imageThumbnailPayload && timestampPayload) acc.push([curr, imageThumbnailPayload, timestampPayload])
+          return acc
+        },
+        [] as [BoundWitness, ImageThumbnail, TimeStamp][],
+      )
+      const indexes = await Promise.all(
+        tuples.map(async ([bw, imageThumbnailPayload, timestampPayload]) => {
+          const { sourceUrl: url } = imageThumbnailPayload
+          const { timestamp } = timestampPayload
+          const status = imageThumbnailPayload.http?.status
+          const success = !!imageThumbnailPayload.url // Call anything with a thumbnail url a success
+          const sources = (await PayloadHasher.hashPairs([bw, imageThumbnailPayload, timestampPayload])).map(([, hash]) => hash)
+          const urlPayload = { schema: UrlSchema, url }
+          const key = await PayloadHasher.hashAsync(urlPayload)
+          const fields: ImageThumbnailResultIndexFields = { key, sources, success, timestamp }
+          if (status) fields.status = status
+          const result: ImageThumbnailResultIndex = new PayloadBuilder<ImageThumbnailResultIndex>({ schema: ImageThumbnailResultIndexSchema })
+            .fields(fields)
+            .build()
+          return [result]
+        }),
+      )
+      return indexes.flat()
+    }
+    return Promise.resolve([])
+  }
+}

package/src/Diviner/ImageThumbnailIndexQueryResponseToImageThumbnailQueryResponseDiviner.ts

@@ -0,0 +1,61 @@
+import { exists } from '@xylabs/exists'
+import { AbstractDiviner } from '@xyo-network/abstract-diviner'
+import { PayloadHasher } from '@xyo-network/core'
+import { DivinerConfigSchema } from '@xyo-network/diviner-model'
+import {
+  ImageThumbnailResult,
+  ImageThumbnailResultFields,
+  ImageThumbnailResultSchema,
+  isImageThumbnailDivinerQuery,
+  isImageThumbnailResultIndex,
+} from '@xyo-network/image-thumbnail-payload-plugin'
+import { PayloadBuilder } from '@xyo-network/payload-builder'
+import { Payload } from '@xyo-network/payload-model'
+import { UrlSchema } from '@xyo-network/url-payload-plugin'
+
+import { ImageThumbnailDivinerLabels, ImageThumbnailDivinerStageLabels } from './ImageThumbnailDivinerLabels'
+
+/**
+ * Transforms an ImageThumbnailIndex response into an ImageThumbnailResponse response
+ */
+export class ImageThumbnailIndexQueryResponseToImageThumbnailQueryResponseDiviner extends AbstractDiviner {
+  static override configSchemas = [DivinerConfigSchema]
+  static labels: ImageThumbnailDivinerStageLabels = {
+    ...ImageThumbnailDivinerLabels,
+    'network.xyo.diviner.stage': 'indexQueryResponseToDivinerQueryResponseDiviner',
+  }
+
+  protected override async divineHandler(payloads: Payload[] = []): Promise<ImageThumbnailResult[]> {
+    // Filter out the two operands
+    const imageThumbnailDivinerQueries = payloads.filter(isImageThumbnailDivinerQuery)
+    const imageThumbnailResultIndexes = payloads.filter(isImageThumbnailResultIndex)
+
+    // If we have operands
+    if (imageThumbnailDivinerQueries.length && imageThumbnailResultIndexes.length) {
+      // Create a dictionary to translate index keys to the urls that represent them
+      const keyToUrlDictionary = Object.fromEntries(
+        await Promise.all(
+          imageThumbnailDivinerQueries.map(async (imageThumbnailDivinerQuery) => {
+            const { url } = imageThumbnailDivinerQuery
+            const urlPayload = new PayloadBuilder<ImageThumbnailResult>({ schema: UrlSchema }).fields({ url }).build()
+            const key = await PayloadHasher.hashAsync(urlPayload)
+            return [key, url] as const
+          }),
+        ),
+      )
+      // Map the indexes to responses using the dictionary
+      return imageThumbnailResultIndexes
+        .map((imageThumbnailResultIndex) => {
+          // eslint-disable-next-line @typescript-eslint/no-unused-vars
+          const { key, schema, ...commonFields } = imageThumbnailResultIndex
+          const url = keyToUrlDictionary?.[key]
+          if (url) {
+            const fields: ImageThumbnailResultFields = { ...commonFields, url }
+            return new PayloadBuilder<ImageThumbnailResult>({ schema: ImageThumbnailResultSchema }).fields(fields).build()
+          }
+        })
+        .filter(exists)
+    }
+    return []
+  }
+}

package/src/Diviner/ImageThumbnailQueryToImageThumbnailIndexQueryDiviner.ts

@@ -0,0 +1,43 @@
+import { AbstractDiviner } from '@xyo-network/abstract-diviner'
+import { PayloadHasher } from '@xyo-network/core'
+import { DivinerConfigSchema } from '@xyo-network/diviner-model'
+import { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
+import { isImageThumbnailDivinerQuery } from '@xyo-network/image-thumbnail-payload-plugin'
+import { PayloadBuilder } from '@xyo-network/payload-builder'
+import { Payload } from '@xyo-network/payload-model'
+import { UrlSchema } from '@xyo-network/url-payload-plugin'
+
+import { ImageThumbnailDivinerLabels, ImageThumbnailDivinerStageLabels } from './ImageThumbnailDivinerLabels'
+import { ImageThumbnailResultQuery } from './ImageThumbnailResultQuery'
+
+/**
+ * A diviner that converts ImageThumbnailDivinerQuery to ImageThumbnailResultQuery
+ */
+export class ImageThumbnailQueryToImageThumbnailIndexQueryDiviner extends AbstractDiviner {
+  static override configSchemas = [DivinerConfigSchema]
+  static labels: ImageThumbnailDivinerStageLabels = {
+    ...ImageThumbnailDivinerLabels,
+    'network.xyo.diviner.stage': 'divinerQueryToIndexQueryDiviner',
+  }
+  protected override async divineHandler(payloads: Payload[] = []): Promise<ImageThumbnailResultQuery[]> {
+    const queries = payloads.filter(isImageThumbnailDivinerQuery)
+    if (queries.length) {
+      const results = await Promise.all(
+        queries.map(async (query) => {
+          const { limit: payloadLimit, offset: payloadOffset, order: payloadOrder, status: payloadStatus, success: payloadSuccess, url } = query
+          const limit = payloadLimit ?? 1
+          const order = payloadOrder ?? 'desc'
+          const offset = payloadOffset ?? 0
+          const urlPayload = { schema: UrlSchema, url }
+          const key = await PayloadHasher.hashAsync(urlPayload)
+          const fields: Partial<ImageThumbnailResultQuery> = { key, limit, offset, order }
+          if (payloadSuccess !== undefined) fields.success = payloadSuccess
+          if (payloadStatus !== undefined) fields.status = payloadStatus
+          return new PayloadBuilder<ImageThumbnailResultQuery>({ schema: PayloadDivinerQuerySchema }).fields(fields).build()
+        }),
+      )
+      return results
+    }
+    return Promise.resolve([])
+  }
+}

package/src/Diviner/ImageThumbnailResultQuery.ts

@@ -0,0 +1,19 @@
+import { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
+import { ImageThumbnailResultIndex } from '@xyo-network/image-thumbnail-payload-plugin'
+import { isPayloadOfSchemaType } from '@xyo-network/payload-model'
+
+/**
+ * The fields that will need to be indexed on in the underlying store
+ */
+export type QueryableImageThumbnailResultProperties = Extract<keyof ImageThumbnailResultIndex, 'status' | 'success' | 'timestamp' | 'key'>
+
+/**
+ * The query that will be used to retrieve the results from the underlying store
+ */
+export type ImageThumbnailResultQuery = PayloadDivinerQueryPayload & Pick<ImageThumbnailResultIndex, QueryableImageThumbnailResultProperties>
+
+/**
+ * A type guard for ImageThumbnailResultQuery
+ */
+// TODO: Use a more derived schema than PayloadDivinerQuerySchema
+export const isImageThumbnailResultQuery = isPayloadOfSchemaType<ImageThumbnailResultQuery>(PayloadDivinerQuerySchema)

package/src/Diviner/ImageThumbnailStateToIndexCandidateDiviner/Config.ts

@@ -0,0 +1,16 @@
+import { DivinerConfig } from '@xyo-network/diviner-model'
+import { SearchableStorage } from '@xyo-network/image-thumbnail-payload-plugin'
+
+import { ImageThumbnailStateToIndexCandidateDivinerSchema } from './Schema'
+
+export type ImageThumbnailStateToIndexCandidateDivinerConfigSchema = `${ImageThumbnailStateToIndexCandidateDivinerSchema}.config`
+export const ImageThumbnailStateToIndexCandidateDivinerConfigSchema: ImageThumbnailStateToIndexCandidateDivinerConfigSchema = `${ImageThumbnailStateToIndexCandidateDivinerSchema}.config`
+
+export type ImageThumbnailStateToIndexCandidateDivinerConfig = DivinerConfig<{
+  payloadDivinerLimit?: number
+  /**
+   * Where the diviner should look for stored thumbnails
+   */
+  payloadStore?: SearchableStorage
+  schema: ImageThumbnailStateToIndexCandidateDivinerConfigSchema
+}>