@xyo-network/image-thumbnail-plugin 2.71.10 → 2.71.12

package/src/Diviner/Diviner.ts

@@ -0,0 +1,197 @@
+import { assertEx } from '@xylabs/assert'
+import { delay } from '@xylabs/delay'
+import { AbstractDiviner } from '@xyo-network/abstract-diviner'
+import { ArchivistInstance, asArchivistInstance } from '@xyo-network/archivist-model'
+import { PayloadHasher } from '@xyo-network/core'
+import { asDivinerInstance, DivinerConfigSchema, DivinerInstance } from '@xyo-network/diviner-model'
+import { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
+import { ImageThumbnail, ImageThumbnailSchema } from '@xyo-network/image-thumbnail-payload-plugin'
+import { UrlPayload } from '@xyo-network/url-payload-plugin'
+import compact from 'lodash/compact'
+
+import { ImageThumbnailDivinerConfigSchema } from './Config'
+import { ImageThumbnailDivinerParams } from './Params'
+
+export class ImageThumbnailDiviner<TParams extends ImageThumbnailDivinerParams = ImageThumbnailDivinerParams> extends AbstractDiviner<TParams> {
+  static override configSchemas = [ImageThumbnailDivinerConfigSchema, DivinerConfigSchema]
+
+  private _archivistInstance: Promise<ArchivistInstance> | undefined
+  private _initializeArchivistConnectionIfNeededPromise: Promise<void> | undefined
+  private _map: Record<string, string> | undefined
+  private _payloadDivinerInstance: Promise<DivinerInstance> | undefined
+  private _pollId?: string | number | NodeJS.Timeout
+
+  get archivist() {
+    return this.config.archivist
+  }
+
+  get payloadDiviner() {
+    return this.config.payloadDiviner
+  }
+
+  get pollFrequency() {
+    return this.config.pollFrequency
+  }
+
+  //using promise as mutex
+  async getArchivistInstance(): Promise<ArchivistInstance> {
+    //if previously checked, but not found, clear promise
+    if (this._archivistInstance && !(await this._archivistInstance)) {
+      this._archivistInstance = undefined
+    }
+    this._archivistInstance =
+      this._archivistInstance ??
+      (async () => {
+        const module = await this.resolve(this.archivist)
+        return asArchivistInstance(module, 'Provided archivist address did not resolve to an Archivist')
+      })()
+    return this._archivistInstance
+  }
+
+  //using promise as mutex
+  async getPayloadDivinerInstance(): Promise<DivinerInstance | undefined> {
+    const payloadDivinerAddress = this.payloadDiviner
+    if (payloadDivinerAddress) {
+      //if previously checked, but not found, clear promise
+      if (this._payloadDivinerInstance && !(await this._payloadDivinerInstance)) {
+        this._payloadDivinerInstance = undefined
+      }
+      this._payloadDivinerInstance =
+        this._payloadDivinerInstance ??
+        (async () => {
+          const module = await this.resolve(payloadDivinerAddress)
+          return asDivinerInstance(module, 'Provided payload diviner address did not resolve to a Diviner')
+        })()
+
+      return this._payloadDivinerInstance
+    }
+  }
+
+  protected override async divineHandler(payloads: UrlPayload[] = []): Promise<ImageThumbnail[]> {
+    await this.initializeArchivistConnectionIfNeeded()
+    const urls = payloads.map((urlPayload) => urlPayload.url)
+    const map = await this.getSafeMap()
+    const archivist = await this.getArchivistInstance()
+    const hashes = compact(urls.map((url) => map?.[url]))
+    return (await archivist.get(hashes)).filter((payload): payload is ImageThumbnail => payload.schema === ImageThumbnailSchema)
+  }
+
+  //using promise as mutex
+  protected initializeArchivistConnectionIfNeeded() {
+    this._initializeArchivistConnectionIfNeededPromise =
+      this._initializeArchivistConnectionIfNeededPromise ??
+      (async () => {
+        if (!this._map) {
+          await this.attachArchivistEvents()
+          console.log('initializeArchivistConnectionIfNeeded: attachArchivistEvents done')
+          await this.poll()
+          console.log('initializeArchivistConnectionIfNeeded: poll done')
+        }
+      })()
+    return this._initializeArchivistConnectionIfNeededPromise
+  }
+
+  protected async loadMap() {
+    if (this.payloadDiviner) {
+      return await this.loadMapWithPayloadDiviner()
+    } else {
+      return await this.loadMapWithAll()
+    }
+  }
+
+  protected async loadMapWithAll() {
+    if (await this.started()) {
+      const archivist = await this.getArchivistInstance()
+      assertEx(archivist.all, "Archivist does not support 'all'")
+      const allPayloads = (await archivist.all?.()) ?? []
+      const imagePayloadPairs = await Promise.all(
+        allPayloads
+          .filter((payload): payload is ImageThumbnail => payload.schema === ImageThumbnailSchema)
+          .map<Promise<[string, ImageThumbnail]>>(async (payload) => [await PayloadHasher.hashAsync(payload), payload]),
+      )
+      this._map = imagePayloadPairs.reduce<Record<string, string>>((prev, [hash, payload]) => {
+        prev[payload.sourceUrl] = hash
+        return prev
+      }, {})
+    }
+  }
+
+  protected async loadMapWithPayloadDiviner() {
+    console.log('loadMapWithPayloadDiviner: started')
+    if (await this.started()) {
+      const diviner = await this.getPayloadDivinerInstance()
+      let offsetHash: string | undefined = undefined
+      let moreAvailable = true
+      if (diviner) {
+        const newMap: Record<string, string> = {}
+        while (moreAvailable) {
+          const payloadDivinerQuery: PayloadDivinerQueryPayload = {
+            hash: offsetHash,
+            schema: PayloadDivinerQuerySchema,
+            schemas: [ImageThumbnailSchema],
+          }
+          const payloads = await diviner.divine([payloadDivinerQuery])
+          const lastPayload = payloads[payloads.length - 1]
+          offsetHash = lastPayload ? await PayloadHasher.hashAsync(lastPayload) : undefined
+          moreAvailable = payloads.length > 0
+          console.log(`loadMapWithPayloadDiviner.offsetHash: ${offsetHash}`)
+          console.log(`loadMapWithPayloadDiviner.moreAvailable: ${moreAvailable}`)
+          const imagePayloadPairs = await Promise.all(
+            payloads
+              .filter((payload): payload is ImageThumbnail => payload.schema === ImageThumbnailSchema)
+              .map<Promise<[string, ImageThumbnail]>>(async (payload) => [await PayloadHasher.hashAsync(payload), payload]),
+          )
+          imagePayloadPairs.forEach(([hash, payload]) => (newMap[payload.sourceUrl] = hash))
+        }
+        this._map = newMap
+      }
+    }
+  }
+
+  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {
+    if (this._pollId) {
+      clearTimeout(this._pollId)
+      this._pollId = undefined
+    }
+    return await super.stopHandler()
+  }
+
+  private async attachArchivistEvents() {
+    const archivist = await this.getArchivistInstance()
+    const mapPromise = this.getSafeMap()
+    archivist.on('inserted', async ({ payloads }) => {
+      const map = await mapPromise
+      const thumbnails = compact(payloads.filter((payload): payload is ImageThumbnail => payload.schema === ImageThumbnailSchema))
+      await Promise.all(thumbnails.map(async (payload) => (map[payload.sourceUrl] = await PayloadHasher.hashAsync(payload))))
+    })
+  }
+
+  private async getSafeMap() {
+    let mapRetry = 100 //10 seconds max
+    let map = this._map
+    while (!map) {
+      await delay(100)
+      mapRetry = mapRetry - 1
+      if (mapRetry === 0) {
+        throw Error('Map Not Loaded')
+      }
+      map = this._map
+    }
+    return map
+  }
+
+  private async poll() {
+    if (await this.started()) {
+      const pollFrequency = this.pollFrequency
+      if (pollFrequency) {
+        this._pollId = setTimeout(async () => {
+          this._pollId = undefined
+          await this.loadMap()
+          await this.poll()
+        }, pollFrequency)
+      } else {
+        await this.loadMap()
+      }
+    }
+  }
+}

package/src/Diviner/Params.ts

@@ -0,0 +1,6 @@
+import { DivinerParams } from '@xyo-network/diviner-model'
+import { AnyConfigSchema } from '@xyo-network/module'
+
+import { ImageThumbnailDivinerConfig } from './Config'
+
+export type ImageThumbnailDivinerParams = DivinerParams<AnyConfigSchema<ImageThumbnailDivinerConfig>>

package/src/Diviner/index.ts

@@ -0,0 +1,3 @@
+export * from './Config'
+export * from './Diviner'
+export * from './Params'

package/src/Plugin.ts

@@ -1,13 +1,18 @@
 import { ImageThumbnailSchema } from '@xyo-network/image-thumbnail-payload-plugin'
 import { PayloadSetSchema } from '@xyo-network/payload-model'
-import { createPayloadSetWitnessPlugin } from '@xyo-network/payloadset-plugin'
+import { createPayloadSetDualPlugin } from '@xyo-network/payloadset-plugin'
 
+import { ImageThumbnailDiviner } from './Diviner'
 import { ImageThumbnailWitness } from './Witness'
 
 export const ImageThumbnailPlugin = () =>
-  createPayloadSetWitnessPlugin<ImageThumbnailWitness>(
+  createPayloadSetDualPlugin<ImageThumbnailWitness, ImageThumbnailDiviner>(
     { required: { [ImageThumbnailSchema]: 1 }, schema: PayloadSetSchema },
     {
+      diviner: async (params) => {
+        const result = await ImageThumbnailDiviner.create(params)
+        return result
+      },
       witness: async (params) => {
         const result = await ImageThumbnailWitness.create(params)
         return result

package/src/Witness/Witness.ts

@@ -58,15 +58,15 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
   }
 
   get maxAsyncProcesses() {
-    return this.config.maxAsyncProcesses ?? 4
+    return this.config.maxAsyncProcesses ?? 2
   }
 
   get maxCacheBytes() {
-    return this.config.maxCacheBytes ?? 1024 * 1024 * 64 //64MB max size
+    return this.config.maxCacheBytes ?? 1024 * 1024 * 16 //64MB max size
   }
 
   get maxCacheEntries() {
-    return this.config.maxCacheEntries ?? 5000
+    return this.config.maxCacheEntries ?? 500
   }
 
   get quality() {

package/src/index.ts

@@ -1,5 +1,6 @@
 import { ImageThumbnailPlugin } from './Plugin'
 
+export * from './Diviner'
 export * from './Witness'
 
 export { ImageThumbnailPlugin }