@xyo-network/image-thumbnail-plugin 2.70.10 → 2.70.12

package/src/Witness/Witness.ts

@@ -1,13 +1,14 @@
+import { promises as dnsPromises } from 'node:dns'
+
 import { axios, AxiosError, AxiosResponse } from '@xyo-network/axios'
 import { PayloadHasher } from '@xyo-network/core'
-import { ImageThumbnailErrorPayload, ImageThumbnailPayload, ImageThumbnailSchema } from '@xyo-network/image-thumbnail-payload-plugin'
-import { isPayload, ModuleError, ModuleErrorSchema } from '@xyo-network/payload-model'
+import { ImageThumbnail, ImageThumbnailSchema } from '@xyo-network/image-thumbnail-payload-plugin'
 import { UrlPayload } from '@xyo-network/url-payload-plugin'
 import { AbstractWitness } from '@xyo-network/witness'
 import { subClass } from 'gm'
+import { sync as hasbin } from 'hasbin'
 import { sha256 } from 'hash-wasm'
 import compact from 'lodash/compact'
-import isBuffer from 'lodash/isBuffer'
 import { LRUCache } from 'lru-cache'
 import shajs from 'sha.js'
 import Url from 'url-parse'
@@ -19,63 +20,22 @@ import { ImageThumbnailWitnessParams } from './Params'
 
 const gm = subClass({ imageMagick: '7+' })
 
-export const binaryToSha256 = async (data: Uint8Array) => {
-  await PayloadHasher.wasmInitialized
-  if (PayloadHasher.wasmSupport.canUseWasm) {
-    try {
-      return await sha256(data)
-    } catch (ex) {
-      PayloadHasher.wasmSupport.allowWasm = false
-    }
-  }
-  // eslint-disable-next-line deprecation/deprecation
-  return shajs('sha256').update(data).digest().toString()
+export interface ImageThumbnailWitnessError extends Error {
+  name: 'ImageThumbnailWitnessError'
+  url: string
 }
 
-const checkIpfsUrl = (urlToCheck: string) => {
-  const url = new Url(urlToCheck)
-  let protocol = url.protocol
-  let host = url.host
-  let path = url.pathname
-  const query = url.query
-  if (protocol === 'ipfs:') {
-    protocol = 'https:'
-    host = 'cloudflare-ipfs.com'
-    path = url.host === 'ipfs' ? `ipfs${path}` : `ipfs/${url.host}${path}`
-    const root = `${protocol}//${host}/${path}`
-    return query?.length > 0 ? `root?${query}` : root
-  } else {
-    return urlToCheck
-  }
-}
-
-const checkDataUrl = (url: string): [Buffer | undefined, ImageThumbnailErrorPayload | undefined] => {
-  if (url.startsWith('data:image')) {
-    const data = url.split(',')[1]
-    if (data) {
-      const buffer = Buffer.from(Uint8Array.from(atob(data), (c) => c.charCodeAt(0)))
-      console.log(`data buffer: ${buffer.length}`)
-      return [buffer, undefined]
-    } else {
-      const error: ImageThumbnailErrorPayload = {
-        message: 'Invalid data Url',
-        schema: ModuleErrorSchema,
-        url,
-      }
-      return [undefined, error]
-    }
-  } else {
-    return [undefined, undefined]
-  }
+export interface DnsError extends Error {
+  code: string
 }
 
 export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams = ImageThumbnailWitnessParams> extends AbstractWitness<TParams> {
   static override configSchemas = [ImageThumbnailWitnessConfigSchema]
 
-  private _cache?: LRUCache<string, ImageThumbnailPayload | ImageThumbnailErrorPayload>
+  private _cache?: LRUCache<string, ImageThumbnail>
 
   get cache() {
-    this._cache = this._cache ?? new LRUCache<string, ImageThumbnailPayload | ImageThumbnailErrorPayload>({ max: this.maxCacheEntries })
+    this._cache = this._cache ?? new LRUCache<string, ImageThumbnail>({ max: this.maxCacheEntries })
     return this._cache
   }
 
@@ -99,124 +59,165 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
     return this.config.width ?? 128
   }
 
-  protected override async observeHandler(payloads: UrlPayload[] = []): Promise<(ImageThumbnailPayload | ImageThumbnailErrorPayload)[]> {
-    const responsePairs = compact(
-      await Promise.all(
-        payloads.map<Promise<[string, ImageThumbnailPayload | ImageThumbnailErrorPayload | AxiosResponse | Buffer]>>(async ({ url }) => {
-          const cachedResult = this.cache.get(url)
-          if (cachedResult) {
-            return [url, cachedResult]
-          }
-          //if it is a data URL, return a Buffer
-          const [dataBuffer, dataError] = checkDataUrl(url)
-
-          if (dataBuffer) {
-            return [url, dataBuffer]
-          }
+  private static async binaryToSha256(data: Uint8Array) {
+    await PayloadHasher.wasmInitialized
+    if (PayloadHasher.wasmSupport.canUseWasm) {
+      try {
+        return await sha256(data)
+      } catch (ex) {
+        PayloadHasher.wasmSupport.allowWasm = false
+      }
+    }
+    // eslint-disable-next-line deprecation/deprecation
+    return shajs('sha256').update(data).digest().toString()
+  }
 
-          if (dataError) {
-            return [url, dataError]
-          }
+  private static bufferFromDataUrl(url: string): Buffer | undefined {
+    if (url.startsWith('data:image')) {
+      const data = url.split(',')[1]
+      if (data) {
+        const buffer = Buffer.from(Uint8Array.from(atob(data), (c) => c.charCodeAt(0)))
+        console.log(`data buffer: ${buffer.length}`)
+        return buffer
+      } else {
+        const error: ImageThumbnailWitnessError = {
+          message: 'Invalid data Url',
+          name: 'ImageThumbnailWitnessError',
+          url,
+        }
+        throw error
+      }
+    }
+  }
 
-          //if it is ipfs, go through cloud flair
-          const mutatedUrl = checkIpfsUrl(url)
+  private static checkIpfsUrl(urlToCheck: string) {
+    const url = new Url(urlToCheck)
+    let protocol = url.protocol
+    let host = url.host
+    let path = url.pathname
+    const query = url.query
+    if (protocol === 'ipfs:') {
+      protocol = 'https:'
+      host = 'cloudflare-ipfs.com'
+      path = url.host === 'ipfs' ? `ipfs${path}` : `ipfs/${url.host}${path}`
+      const root = `${protocol}//${host}/${path}`
+      return query?.length > 0 ? `root?${query}` : root
+    } else {
+      return urlToCheck
+    }
+  }
 
-          try {
-            return [
-              url,
-              await axios.get(mutatedUrl, {
-                responseType: 'arraybuffer',
-              }),
-            ]
-          } catch (ex) {
-            const axiosError = ex as AxiosError
-            if (axiosError.isAxiosError) {
-              //selectively pick fields from AxiosError
-              const errorPayload: ImageThumbnailErrorPayload = {
-                code: axiosError.code,
-                message: axiosError.message,
-                schema: ModuleErrorSchema,
-                status: axiosError.status,
-                url,
-              }
-              return [url, errorPayload]
-            } else {
-              throw ex
-            }
-          }
-        }),
-      ),
-    )
+  protected override async observeHandler(payloads: UrlPayload[] = []): Promise<ImageThumbnail[]> {
+    if (!hasbin('magick')) {
+      throw Error('ImageMagick is required for this witness')
+    }
     return compact(
       await Promise.all(
-        responsePairs.map(async ([url, urlResult]) => {
-          if (isPayload(urlResult)) {
-            this.cache.set(url, urlResult)
-            return urlResult
+        payloads.map<Promise<ImageThumbnail>>(async ({ url }) => {
+          const cachedResult = this.cache.get(url)
+          if (cachedResult) {
+            return cachedResult
           }
+          let result: ImageThumbnail
 
-          let sourceBuffer: Buffer
+          //if it is a data URL, return a Buffer
+          const dataBuffer = ImageThumbnailWitness.bufferFromDataUrl(url)
 
-          if (isBuffer(urlResult)) {
-            sourceBuffer = urlResult as Buffer
-          } else {
-            const response = urlResult as AxiosResponse
-
-            if (response.status >= 200 && response.status < 300) {
-              const contentType = response.headers['content-type']?.toString()
-              if (contentType.split('/')[0] !== 'image') {
-                const error: ImageThumbnailErrorPayload = {
-                  message: `Invalid file type: ${contentType}`,
-                  schema: ModuleErrorSchema,
-                  url,
-                }
-                this.cache.set(url, error)
-                return error
-              }
-              sourceBuffer = Buffer.from(response.data, 'binary')
-            } else {
-              const error: ImageThumbnailErrorPayload = {
-                schema: ModuleErrorSchema,
-                status: response.status,
-                url,
-              }
-              this.cache.set(url, error)
-              return error
-            }
-          }
-          try {
-            const thumb = await new Promise<Buffer>((resolve, reject) => {
-              gm(sourceBuffer)
-                .quality(this.quality)
-                .resize(this.width, this.height)
-                .flatten()
-                .toBuffer(this.encoding, (error, buffer) => {
-                  if (error) {
-                    reject(error)
-                  } else {
-                    resolve(buffer)
-                  }
-                })
-            })
-            const result: ImageThumbnailPayload = {
+          if (dataBuffer) {
+            result = {
               schema: ImageThumbnailSchema,
-              sourceHash: await binaryToSha256(sourceBuffer),
+              sourceHash: await ImageThumbnailWitness.binaryToSha256(dataBuffer),
               sourceUrl: url,
-              url: `data:image/png;base64,${thumb.toString('base64')}`,
-            }
-            this.cache.set(url, result)
-            return result
-          } catch (ex) {
-            const error: ImageThumbnailErrorPayload = {
-              message: 'Failed to resize image',
-              schema: ModuleErrorSchema,
               url,
             }
-            this.cache.set(url, error)
-            return error
+          } else {
+            //if it is ipfs, go through cloud flair
+            const mutatedUrl = ImageThumbnailWitness.checkIpfsUrl(url)
+            result = await this.fromHttp(mutatedUrl)
           }
+          this.cache.set(url, result)
+          return result
         }),
       ),
     )
   }
+
+  private async createThumbnail(sourceBuffer: Buffer) {
+    const thumb = await new Promise<Buffer>((resolve, reject) => {
+      gm(sourceBuffer)
+        .quality(this.quality)
+        .resize(this.width, this.height)
+        .flatten()
+        .toBuffer(this.encoding, (error, buffer) => {
+          if (error) {
+            reject(error)
+          } else {
+            resolve(buffer)
+          }
+        })
+    })
+    return `data:image/png;base64,${thumb.toString('base64')}`
+  }
+
+  private async fromHttp(url: string): Promise<ImageThumbnail> {
+    let response: AxiosResponse
+    let dnsResult: string[]
+    try {
+      const urlObj = new Url(url)
+      dnsResult = await dnsPromises.resolve(urlObj.host)
+      console.log(`dnsResult: ${JSON.stringify(dnsResult, null, 2)}`)
+    } catch (ex) {
+      const error = ex as DnsError
+      const result: ImageThumbnail = {
+        http: {
+          dnsError: error.code,
+        },
+        schema: ImageThumbnailSchema,
+        sourceUrl: url,
+      }
+      return result
+    }
+    try {
+      response = await axios.get(url, {
+        responseType: 'arraybuffer',
+      })
+    } catch (ex) {
+      const axiosError = ex as AxiosError
+      if (axiosError.isAxiosError) {
+        //selectively pick fields from AxiosError
+        const result: ImageThumbnail = {
+          http: {
+            ipAddress: dnsResult[0],
+            status: axiosError.status,
+          },
+          schema: ImageThumbnailSchema,
+          sourceUrl: url,
+        }
+        return result
+      } else {
+        throw ex
+      }
+    }
+
+    const result: ImageThumbnail = {
+      http: {
+        status: response.status,
+      },
+      schema: ImageThumbnailSchema,
+      sourceUrl: url,
+    }
+
+    if (response.status >= 200 && response.status < 300) {
+      const contentType = response.headers['content-type']?.toString()
+      if (contentType.split('/')[0] !== 'image') {
+        result.mime = result.mime ?? {}
+        result.mime.invalid = true
+      } else {
+        const sourceBuffer = Buffer.from(response.data, 'binary')
+        result.sourceHash = await ImageThumbnailWitness.binaryToSha256(sourceBuffer)
+        result.url = await this.createThumbnail(sourceBuffer)
+      }
+    }
+    return result
+  }
 }