@xyo-network/diviner-image-thumbnail 2.75.9 → 2.75.11

package/src/Diviner/Diviner.ts

@@ -7,9 +7,10 @@ import { isBoundWitness } from '@xyo-network/boundwitness-model'
 import { PayloadHasher } from '@xyo-network/core'
 import { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'
 import { asDivinerInstance, DivinerConfigSchema } from '@xyo-network/diviner-model'
-import { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
+import { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema, SortDirection } from '@xyo-network/diviner-payload-model'
 import { DivinerWrapper } from '@xyo-network/diviner-wrapper'
 import {
+  ImageThumbnail,
   ImageThumbnailDivinerConfig,
   ImageThumbnailDivinerConfigSchema,
   ImageThumbnailDivinerParams,
@@ -23,7 +24,7 @@ import { isModuleState, ModuleState, ModuleStateSchema, StateDictionary } from '
 import { PayloadBuilder } from '@xyo-network/payload-builder'
 import { Payload } from '@xyo-network/payload-model'
 import { isUrlPayload, UrlPayload } from '@xyo-network/url-payload-plugin'
-import { isTimestamp, TimestampSchema } from '@xyo-network/witness-timestamp'
+import { isTimestamp, TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'
 
 export type ImageThumbnailDivinerState = StateDictionary & {
   offset: number
@@ -61,6 +62,10 @@ export class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams =
     return this.config.pollFrequency ?? 10_000
   }
 
+  /**
+   * Works in the background to populate index for the Diviner
+   * @returns
+   */
   protected backgroundDivine = async (): Promise<void> => {
     // Load last state
     const lastState = (await this.retrieveState()) ?? { offset: 0 }
@@ -75,22 +80,39 @@ export class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams =
     })
     const batch = await boundWitnessDiviner.divine([query])
     if (batch.length === 0) return
-    const imageThumbnailTimestampTuples = batch
-      .filter(isBoundWitness)
-      .map((bw) => {
-        const imageThumbnailIndexes = bw.payload_schemas?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined)).filter(exists)
-        const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)
-        if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined
-        const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)
-        const timestamp = bw.payload_hashes?.[timestampIndex]
-        return imageThumbnails.map((imageThumbnail) => [imageThumbnail, timestamp] as const)
-      })
+    // Find all the indexable hashes in this batch
+    type IndexableHashes = Readonly<[boundWitnessHash: string, imageThumbnailHash: string, timestampHash: string]>
+    const indexableHashes: IndexableHashes[] = (
+      await Promise.all(
+        batch.filter(isBoundWitness).map(async (bw) => {
+          const imageThumbnailIndexes = bw.payload_schemas
+            ?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined))
+            .filter(exists)
+          const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)
+          if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined
+          const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)
+          const timestampHash = bw.payload_hashes?.[timestampIndex]
+          const boundWitnessHash = await PayloadHasher.hashAsync(bw)
+          return imageThumbnails.map((imageThumbnailHash) => [boundWitnessHash, imageThumbnailHash, timestampHash] as const)
+        }),
+      )
+    )
       .flat()
       .filter(exists)
     const archivist = await this.getArchivistForStore('thumbnailStore')
-    const payloadTuples = (
+    // Find all the indexable data associated with the indexable hashes
+    type IndexableData = Readonly<
+      [
+        boundWitnessHash: string,
+        imageThumbnailHash: string,
+        imageThumbnailPayload: ImageThumbnail,
+        timestampHash: string,
+        timestampPayload: TimeStamp,
+      ]
+    >
+    const indexableData: IndexableData[] = (
       await Promise.all(
-        imageThumbnailTimestampTuples.map(async ([imageThumbnailHash, timestampHash]) => {
+        indexableHashes.map(async ([boundWitnessHash, imageThumbnailHash, timestampHash]) => {
           const results = await archivist.get([imageThumbnailHash, timestampHash])
           const imageThumbnailPayload = results.find(isImageThumbnail)
           const timestampPayload = results.find(isTimestamp)
@@ -98,24 +120,26 @@ export class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams =
           const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload)
           const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload)
           if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash) return undefined
-          return [imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload] as const
+          return [boundWitnessHash, imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload] as const
         }),
       )
     ).filter(exists)
-    // Build index results
-    const indexedResults = payloadTuples.map(([thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {
-      const { sourceUrl: url } = thumbnailPayload
-      const { timestamp } = timestampPayload
-      const status = thumbnailPayload.http?.status ?? -1
-      const sources = [thumbnailHash, timestampHash]
-      const result = new PayloadBuilder<ImageThumbnailResult>({ schema: ImageThumbnailResultIndexSchema })
-        .fields({ sources, status, timestamp, url })
-        .build()
-      return result
-    })
+    // Build index results from the indexable data
+    const indexes: ImageThumbnailResult[] = indexableData.map(
+      ([boundWitnessHash, thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {
+        const { sourceUrl: url } = thumbnailPayload
+        const { timestamp } = timestampPayload
+        const status = thumbnailPayload.http?.status ? true : false
+        const sources = [boundWitnessHash, thumbnailHash, timestampHash]
+        const result = new PayloadBuilder<ImageThumbnailResult>({ schema: ImageThumbnailResultIndexSchema })
+          .fields({ sources, status, timestamp, url })
+          .build()
+        return result
+      },
+    )
     // Insert index results
     const indexArchivist = await this.getArchivistForStore('indexStore')
-    await indexArchivist.insert(indexedResults)
+    await indexArchivist.insert(indexes)
     // Update state
     const nextOffset = offset + batch.length + 1
     const currentState = { ...lastState, offset: nextOffset }
@@ -144,11 +168,20 @@ export class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams =
     const results = (
       await Promise.all(
         urls.map(async (payload) => {
-          const { url, status: payloadStatus } = payload as UrlPayload & { status: number }
-          const status = payloadStatus ?? 200
+          const {
+            limit: payloadLimit,
+            offset: payloadOffset,
+            order: payloadOrder,
+            status: payloadStatus,
+            url,
+          } = payload as UrlPayload & { limit: number; offset: number; order: SortDirection; status: boolean }
+          // TODO: Expose status, limit (and possibly offset) to caller.  Currently only exposing URL
+          const limit = payloadLimit ?? 1
+          const order = payloadOrder ?? 'desc'
+          const offset = payloadOffset ?? 0
+          const status = payloadStatus ?? true
           const query = new PayloadBuilder<ImageThumbnailResultQuery>({ schema: PayloadDivinerQuerySchema })
-            // TODO: Expose status, limit (and possibly offset) to caller.  Currently only exposing URL
-            .fields({ limit: 1, offset: 0, order: 'desc', status, url })
+            .fields({ limit, offset, order, status, url })
             .build()
           return await diviner.divine([query])
         }),