@xyo-network/image-thumbnail-plugin 2.75.5 → 2.75.7

package/src/Diviner/Config.ts

@@ -4,10 +4,34 @@ import { ImageThumbnailSchema } from '@xyo-network/image-thumbnail-payload-plugi
 export const ImageThumbnailDivinerConfigSchema = `${ImageThumbnailSchema}.diviner.config` as const
 export type ImageThumbnailDivinerConfigSchema = typeof ImageThumbnailDivinerConfigSchema
 
+/**
+ * Describes an Archivist/Diviner combination
+ * that enables searching signed payloads
+ */
+export interface SearchableStorage {
+  archivist: string
+  boundWitnessDiviner: string
+  payloadDiviner: string
+}
+
 export type ImageThumbnailDivinerConfig = DivinerConfig<{
+  /** @deprecated Use appropriate Storage */
   archivist?: string
+  /**
+   * Where the diviner should store it's index
+   */
+  indexStore?: SearchableStorage
+  /** @deprecated Use appropriate Storage */
   payloadDiviner?: string
   payloadDivinerLimit?: number
   pollFrequency?: number
   schema: ImageThumbnailDivinerConfigSchema
+  /**
+   * Where the diviner should persist its internal state
+   */
+  stateStore?: SearchableStorage
+  /**
+   * Where the diviner should look for stored thumbnails
+   */
+  thumbnailStore?: SearchableStorage
 }>

package/src/Diviner/Diviner.ts

@@ -1,159 +1,259 @@
 import { assertEx } from '@xylabs/assert'
-import { delay } from '@xylabs/delay'
-import { compact } from '@xylabs/lodash'
+import { exists } from '@xylabs/exists'
 import { AbstractDiviner } from '@xyo-network/abstract-diviner'
-import { ArchivistInstance, asArchivistInstance } from '@xyo-network/archivist-model'
+import { asArchivistInstance, withArchivistModule } from '@xyo-network/archivist-model'
+import { ArchivistWrapper } from '@xyo-network/archivist-wrapper'
+import { isBoundWitness } from '@xyo-network/boundwitness-model'
 import { PayloadHasher } from '@xyo-network/core'
-import { asDivinerInstance, DivinerConfigSchema, DivinerInstance } from '@xyo-network/diviner-model'
+import { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'
+import { asDivinerInstance, DivinerConfigSchema } from '@xyo-network/diviner-model'
 import { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
-import { ImageThumbnail, ImageThumbnailSchema } from '@xyo-network/image-thumbnail-payload-plugin'
-import { UrlPayload } from '@xyo-network/url-payload-plugin'
+import { DivinerWrapper } from '@xyo-network/diviner-wrapper'
+import { ImageThumbnailSchema, isImageThumbnail } from '@xyo-network/image-thumbnail-payload-plugin'
+import { PayloadBuilder } from '@xyo-network/payload-builder'
+import { isPayloadOfSchemaType, Payload } from '@xyo-network/payload-model'
+import { isUrlPayload } from '@xyo-network/url-payload-plugin'
+import { isTimestamp, TimestampSchema } from '@xyo-network/witness-timestamp'
 
-import { ImageThumbnailDivinerConfigSchema } from './Config'
+import { ImageThumbnailDivinerConfig, ImageThumbnailDivinerConfigSchema } from './Config'
 import { ImageThumbnailDivinerParams } from './Params'
 
+/**
+ * TODO: Once the shape settles, make a generic payload so that it
+ * can be used for other modules
+ */
+interface State<T> {
+  state: T
+}
+
+interface ImageThumbnailDivinerState {
+  offset: number
+}
+
+const ModuleStateSchema = 'network.xyo.module.state' as const
+type ModuleStateSchema = typeof ModuleStateSchema
+
+type ModuleState = Payload<State<ImageThumbnailDivinerState>, ModuleStateSchema>
+
+const isModuleState = isPayloadOfSchemaType<ModuleState>(ModuleStateSchema)
+
+type ConfigStoreKey = 'indexStore' | 'stateStore' | 'thumbnailStore'
+
+type ConfigStore = Extract<keyof ImageThumbnailDivinerConfig, ConfigStoreKey>
+
+const ImageThumbnailResultIndexSchema = `${ImageThumbnailSchema}.index` as const
+type ImageThumbnailResultIndexSchema = typeof ImageThumbnailResultIndexSchema
+
+interface ImageThumbnailResultInfo {
+  sources: string[]
+  // TODO: Something richer than HTTP status code that allows for info about failure modes
+  status: number
+  timestamp: number
+  url: string
+}
+
+type ImageThumbnailResult = Payload<ImageThumbnailResultInfo, ImageThumbnailResultIndexSchema>
+
+const isImageThumbnailResult = isPayloadOfSchemaType<ImageThumbnailResult>(ImageThumbnailResultIndexSchema)
+
+/**
+ * The fields that will need to be indexed on in the underlying store
+ */
+type QueryableImageThumbnailResultProperties = Extract<keyof ImageThumbnailResult, 'url' | 'timestamp' | 'status'>
+
+/**
+ * The query that will be used to retrieve the results from the underlying store
+ */
+type ImageThumbnailResultQuery = PayloadDivinerQueryPayload & { schemas: [ImageThumbnailSchema] } & Pick<
+    ImageThumbnailResult,
+    QueryableImageThumbnailResultProperties
+  >
+
+const moduleName = 'ImageThumbnailDiviner'
+
 export class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams = ImageThumbnailDivinerParams> extends AbstractDiviner<TParams> {
   static override configSchemas = [ImageThumbnailDivinerConfigSchema, DivinerConfigSchema]
 
-  private _archivistInstance: Promise<ArchivistInstance> | undefined
-  private _initializeArchivistConnectionIfNeededPromise: Promise<void> | undefined
-  private _map: Record<string, string> | undefined
-  private _payloadDivinerInstance: Promise<DivinerInstance> | undefined
   private _pollId?: string | number | NodeJS.Timeout
 
-  // static override get configSchema() {
-  //   return ImageThumbnailDivinerConfigSchema
-  // }
-
-  get archivist() {
-    return this.config.archivist
+  get payloadDivinerLimit() {
+    return this.config.payloadDivinerLimit ?? 1_0000
   }
 
-  get payloadDiviner() {
-    return this.config.payloadDiviner
+  get pollFrequency() {
+    return this.config.pollFrequency ?? 10_000
   }
 
-  get payloadDivinerLimit() {
-    return this.config.payloadDivinerLimit ?? 10000
+  protected backgroundDivine = async (): Promise<void> => {
+    // Load last state
+    const lastState = (await this.retrieveState()) ?? { offset: 0 }
+    const { offset } = lastState
+    // Get next batch of results
+    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore('thumbnailStore')
+    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema }).fields({
+      limit: this.payloadDivinerLimit,
+      offset,
+      order: 'asc',
+      payload_schemas: [ImageThumbnailSchema, TimestampSchema],
+    })
+    const batch = await boundWitnessDiviner.divine([query])
+    if (batch.length === 0) return
+    const imageThumbnailTimestampTuples = batch
+      .filter(isBoundWitness)
+      .map((bw) => {
+        const imageThumbnailIndexes = bw.payload_schemas?.map((schema, index) => (schema === ImageThumbnailSchema ? index : undefined)).filter(exists)
+        const timestampIndex = bw.payload_schemas?.findIndex((schema) => schema === TimestampSchema)
+        if (!imageThumbnailIndexes.length || timestampIndex === -1) return undefined
+        const imageThumbnails = bw.payload_hashes.map((hash, index) => (imageThumbnailIndexes.includes(index) ? hash : undefined)).filter(exists)
+        const timestamp = bw.payload_hashes?.[timestampIndex]
+        return imageThumbnails.map((imageThumbnail) => [imageThumbnail, timestamp] as const)
+      })
+      .flat()
+      .filter(exists)
+    const archivist = await this.getArchivistForStore('thumbnailStore')
+    const payloadTuples = (
+      await Promise.all(
+        imageThumbnailTimestampTuples.map(async ([imageThumbnailHash, timestampHash]) => {
+          const results = await archivist.get([imageThumbnailHash, timestampHash])
+          const imageThumbnailPayload = results.find(isImageThumbnail)
+          const timestampPayload = results.find(isTimestamp)
+          if (!imageThumbnailPayload || !timestampPayload) return undefined
+          const calculatedImageThumbnailHash = await PayloadHasher.hashAsync(imageThumbnailPayload)
+          const calculatedTimestampHash = await PayloadHasher.hashAsync(timestampPayload)
+          if (imageThumbnailHash !== calculatedImageThumbnailHash || timestampHash !== calculatedTimestampHash) return undefined
+          return [imageThumbnailHash, imageThumbnailPayload, timestampHash, timestampPayload] as const
+        }),
+      )
+    ).filter(exists)
+    // Build index results
+    const indexedResults = payloadTuples.map(([thumbnailHash, thumbnailPayload, timestampHash, timestampPayload]) => {
+      const { sourceUrl: url } = thumbnailPayload
+      const { timestamp } = timestampPayload
+      const status = thumbnailPayload.http?.status ?? -1
+      const sources = [thumbnailHash, timestampHash]
+      const result = new PayloadBuilder<ImageThumbnailResult>({ schema: ImageThumbnailResultIndexSchema })
+        .fields({ sources, status, timestamp, url })
+        .build()
+      return result
+    })
+    // Insert index results
+    const indexArchivist = await this.getArchivistForStore('indexStore')
+    await indexArchivist.insert(indexedResults)
+    // Update state
+    const nextOffset = offset + batch.length + 1
+    const currentState = { ...lastState, offset: nextOffset }
+    await this.commitState(currentState)
   }
 
-  get pollFrequency() {
-    return this.config.pollFrequency
+  /**
+   * Commit the internal state of the Diviner process. This is similar
+   * to a transaction completion in a database and should only be called
+   * when results have been successfully persisted to the appropriate
+   * external stores.
+   */
+  protected async commitState(state: ImageThumbnailDivinerState) {
+    const stateStore = assertEx(this.config.stateStore?.archivist, `${moduleName}: No stateStore configured`)
+    const module = assertEx(await this.resolve(stateStore), `${moduleName}: Failed to resolve stateStore`)
+    await withArchivistModule(module, async (archivist) => {
+      const mod = ArchivistWrapper.wrap(archivist, this.account)
+      const payload = new PayloadBuilder<ModuleState>({ schema: ModuleStateSchema }).fields({ state }).build()
+      await mod.insert([payload])
+    })
   }
 
-  //using promise as mutex
-  async getArchivistInstance(): Promise<ArchivistInstance> {
-    //if previously checked, but not found, clear promise
-    if (this._archivistInstance && !(await this._archivistInstance)) {
-      this._archivistInstance = undefined
-    }
-    this._archivistInstance =
-      this._archivistInstance ??
-      (async () => {
-        const module = this.archivist ? await this.resolve(this.archivist) : undefined
-        return asArchivistInstance(module, 'Provided archivist address did not resolve to an Archivist')
-      })()
-    return this._archivistInstance
+  protected override async divineHandler(payloads: Payload[] = []): Promise<ImageThumbnailResult[]> {
+    const urls = payloads.filter(isUrlPayload).map((urlPayload) => urlPayload.url)
+    const diviner = await this.getPayloadDivinerForStore('indexStore')
+    const results = (
+      await Promise.all(
+        urls.map(async (url) => {
+          const query = new PayloadBuilder<ImageThumbnailResultQuery>({ schema: PayloadDivinerQuerySchema })
+            // TODO: Expose status, limit (and possibly offset) to caller.  Currently only exposing URL
+            .fields({ limit: 1, offset: 0, order: 'desc', url })
+            .build()
+          return await diviner.divine([query])
+        }),
+      )
+    )
+      .flat()
+      .filter(isImageThumbnailResult)
+    return results
   }
 
-  //using promise as mutex
-  async getPayloadDivinerInstance(): Promise<DivinerInstance | undefined> {
-    const payloadDivinerAddress = this.payloadDiviner
-    if (payloadDivinerAddress) {
-      //if previously checked, but not found, clear promise
-      if (this._payloadDivinerInstance && !(await this._payloadDivinerInstance)) {
-        this._payloadDivinerInstance = undefined
-      }
-      this._payloadDivinerInstance =
-        this._payloadDivinerInstance ??
-        (async () => {
-          const module = await this.resolve(payloadDivinerAddress)
-          return asDivinerInstance(module, 'Provided payload diviner address did not resolve to a Diviner')
-        })()
-
-      return this._payloadDivinerInstance
-    }
+  protected async getArchivistForStore(store: ConfigStore, wrap?: boolean) {
+    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)
+    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)
+    return wrap ? ArchivistWrapper.wrap(mod, this.account) : asArchivistInstance(mod, () => `${moduleName}: ${store}.archivist is not an Archivist`)
   }
 
-  protected override async divineHandler(payloads: UrlPayload[] = []): Promise<ImageThumbnail[]> {
-    await this.initializeArchivistConnectionIfNeeded()
-    const urls = payloads.map((urlPayload) => urlPayload.url)
-    const map = await this.getSafeMap()
-    const archivist = await this.getArchivistInstance()
-    const hashes = compact(urls.map((url) => map?.[url]))
-    return (await archivist.get(hashes)).filter((payload): payload is ImageThumbnail => payload.schema === ImageThumbnailSchema)
+  protected async getBoundWitnessDivinerForStore(store: ConfigStore, wrap?: boolean) {
+    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)
+    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)
+    return wrap
+      ? DivinerWrapper.wrap(mod, this.account)
+      : asDivinerInstance(mod, () => `${moduleName}: ${store}.boundWitnessDiviner is not a Diviner`)
   }
 
-  //using promise as mutex
-  protected initializeArchivistConnectionIfNeeded() {
-    this._initializeArchivistConnectionIfNeededPromise =
-      this._initializeArchivistConnectionIfNeededPromise ??
-      (async () => {
-        if (!this._map) {
-          await this.attachArchivistEvents()
-          console.log('initializeArchivistConnectionIfNeeded: attachArchivistEvents done')
-          await this.poll()
-          console.log('initializeArchivistConnectionIfNeeded: poll done')
-        }
-      })()
-    return this._initializeArchivistConnectionIfNeededPromise
+  protected async getPayloadDivinerForStore(store: ConfigStore, wrap?: boolean) {
+    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)
+    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)
+    return wrap ? DivinerWrapper.wrap(mod, this.account) : asDivinerInstance(mod, () => `${moduleName}: ${store}.payloadDiviner is not a Diviner`)
   }
 
-  protected async loadMap() {
-    if (this.payloadDiviner) {
-      return await this.loadMapWithPayloadDiviner()
-    } else {
-      return await this.loadMapWithAll()
+  /**
+   * Retrieves the last state of the Diviner process. Used to recover state after
+   * preemptions, reboots, etc.
+   */
+  protected async retrieveState(): Promise<ImageThumbnailDivinerState | undefined> {
+    let hash: string = ''
+    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')
+    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema }).fields({
+      address: this.account.address,
+      limit: 1,
+      offset: 0,
+      order: 'desc',
+      payload_schemas: [ModuleStateSchema],
+    })
+    const boundWitnesses = await diviner.divine([query])
+    if (boundWitnesses.length > 0) {
+      const boundWitness = boundWitnesses[0]
+      if (isBoundWitness(boundWitness)) {
+        // Find the index for this address in the BoundWitness that is a ModuleState
+        hash = boundWitness.addresses
+          .map((address, index) => ({ address, index }))
+          .filter(({ address }) => address === this.account.address)
+          .reduce(
+            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),
+            '',
+          )
+      }
     }
-  }
 
-  protected async loadMapWithAll() {
-    if (await this.started()) {
-      const archivist = await this.getArchivistInstance()
-      assertEx(archivist.all, "Archivist does not support 'all'")
-      const allPayloads = (await archivist.all?.()) ?? []
-      const imagePayloadPairs = await Promise.all(
-        allPayloads
-          .filter((payload): payload is ImageThumbnail => payload.schema === ImageThumbnailSchema)
-          .map<Promise<[string, ImageThumbnail]>>(async (payload) => [await PayloadHasher.hashAsync(payload), payload]),
+    // If we able to located the last state
+    if (hash) {
+      // Get last state
+      const stateStoreArchivist = assertEx(this.config.stateStore?.archivist, `${moduleName}: No stateStore archivist configured`)
+      await withArchivistModule(
+        assertEx(await this.resolve(stateStoreArchivist), `${moduleName}: Failed to resolve stateStore archivist`),
+        async (mod) => {
+          const archivist = ArchivistWrapper.wrap(mod, this.account)
+          const payloads = await archivist.get([hash])
+          if (payloads.length > 0) {
+            const payload = payloads[0]
+            if (isModuleState(payload)) {
+              return payload.state
+            }
+          }
+        },
       )
-      this._map = imagePayloadPairs.reduce<Record<string, string>>((prev, [hash, payload]) => {
-        prev[payload.sourceUrl] = hash
-        return prev
-      }, {})
     }
+    return undefined
   }
 
-  protected async loadMapWithPayloadDiviner() {
-    console.log('loadMapWithPayloadDiviner: started')
-    if (await this.started()) {
-      const diviner = await this.getPayloadDivinerInstance()
-      let offset: number | undefined = undefined
-      let moreAvailable = true
-      if (diviner) {
-        const newMap: Record<string, string> = {}
-        while (moreAvailable) {
-          const payloadDivinerQuery: PayloadDivinerQueryPayload = {
-            limit: this.payloadDivinerLimit,
-            offset,
-            schema: PayloadDivinerQuerySchema,
-            schemas: [ImageThumbnailSchema],
-          }
-          const payloads = await diviner.divine([payloadDivinerQuery])
-          offset = (offset ?? 0) + payloads.length
-          moreAvailable = payloads.length > 0
-          console.log(`loadMapWithPayloadDiviner.offset: ${offset}`)
-          console.log(`loadMapWithPayloadDiviner.moreAvailable: ${moreAvailable}`)
-          const imagePayloadPairs = await Promise.all(
-            payloads
-              .filter((payload): payload is ImageThumbnail => payload.schema === ImageThumbnailSchema)
-              .map<Promise<[string, ImageThumbnail]>>(async (payload) => [await PayloadHasher.hashAsync(payload), payload]),
-          )
-          imagePayloadPairs.forEach(([hash, payload]) => (newMap[payload.sourceUrl] = hash))
-        }
-        this._map = newMap
-      }
-    }
+  protected override async startHandler(): Promise<boolean> {
+    await super.startHandler()
+    this.poll()
+    return true
   }
 
   protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {
@@ -164,42 +264,17 @@ export class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams =
     return await super.stopHandler()
   }
 
-  private async attachArchivistEvents() {
-    const archivist = await this.getArchivistInstance()
-    const mapPromise = this.getSafeMap()
-    archivist.on('inserted', async ({ payloads }) => {
-      const map = await mapPromise
-      const thumbnails = compact(payloads.filter((payload): payload is ImageThumbnail => payload.schema === ImageThumbnailSchema))
-      await Promise.all(thumbnails.map(async (payload) => (map[payload.sourceUrl] = await PayloadHasher.hashAsync(payload))))
-    })
-  }
-
-  private async getSafeMap() {
-    let mapRetry = 100 //10 seconds max
-    let map = this._map
-    while (!map) {
-      await delay(100)
-      mapRetry = mapRetry - 1
-      if (mapRetry === 0) {
-        throw Error('Map Not Loaded')
+  private poll() {
+    this._pollId = setTimeout(async () => {
+      try {
+        await this.backgroundDivine()
+      } catch (e) {
+        console.log(e)
+      } finally {
+        if (this._pollId) clearTimeout(this._pollId)
+        this._pollId = undefined
+        this.poll()
       }
-      map = this._map
-    }
-    return map
-  }
-
-  private async poll() {
-    if (await this.started()) {
-      const pollFrequency = this.pollFrequency
-      if (pollFrequency) {
-        this._pollId = setTimeout(async () => {
-          this._pollId = undefined
-          await this.loadMap()
-          await this.poll()
-        }, pollFrequency)
-      } else {
-        await this.loadMap()
-      }
-    }
+    }, this.pollFrequency)
   }
 }