@xyo-network/image-thumbnail-plugin 2.72.1 → 2.72.3

package/src/Witness/Witness.ts

@@ -1,3 +1,4 @@
+/* eslint-disable max-statements */
 import { promises as dnsPromises } from 'node:dns'
 
 import { URL } from '@xylabs/url'
@@ -7,6 +8,7 @@ import { ImageThumbnail, ImageThumbnailSchema } from '@xyo-network/image-thumbna
 import { UrlPayload } from '@xyo-network/url-payload-plugin'
 import { AbstractWitness } from '@xyo-network/witness'
 import { Semaphore } from 'async-mutex'
+import FileType from 'file-type'
 import { subClass } from 'gm'
 import { sync as hasbin } from 'hasbin'
 import { sha256 } from 'hash-wasm'
@@ -16,10 +18,13 @@ import shajs from 'sha.js'
 import Url from 'url-parse'
 
 import { ImageThumbnailWitnessConfigSchema } from './Config'
+import { getVideoFrameAsImageFluent } from './ffmpeg'
 import { ImageThumbnailWitnessParams } from './Params'
 
 //TODO: Break this into two Witnesses?
 
+// setFfmpegPath(ffmpegPath)
+
 const gm = subClass({ imageMagick: '7+' })
 
 export interface ImageThumbnailWitnessError extends Error {
@@ -57,6 +62,10 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
     return this.config.height ?? 128
   }
 
+  get ipfGateway() {
+    return this.config.ipfsGateway ?? 'cloudflare-ipfs.com'
+  }
+
   get maxAsyncProcesses() {
     return this.config.maxAsyncProcesses ?? 2
   }
@@ -77,29 +86,6 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
     return this.config.width ?? 128
   }
 
-  /**
-   * Returns the equivalent IPFS gateway URL for the supplied URL.
-   * @param urlToCheck The URL to check
-   * @returns If the supplied URL is an IPFS URL, it converts the URL to the
-   * equivalent IPFS gateway URL. Otherwise, returns the original URL.
-   */
-  static checkIpfsUrl(urlToCheck: string) {
-    const url = new URL(urlToCheck)
-    let protocol = url.protocol
-    let host = url.host
-    let path = url.pathname
-    const query = url.search
-    if (protocol === 'ipfs:') {
-      protocol = 'https:'
-      host = 'cloudflare-ipfs.com'
-      path = url.host === 'ipfs' ? `ipfs${path}` : `ipfs/${url.host}${path}`
-      const root = `${protocol}//${host}/${path}`
-      return query?.length > 0 ? `${root}?${query}` : root
-    } else {
-      return urlToCheck
-    }
-  }
-
   private static async binaryToSha256(data: Uint8Array) {
     await PayloadHasher.wasmInitialized
     if (PayloadHasher.wasmSupport.canUseWasm) {
@@ -131,6 +117,29 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
     }
   }
 
+  /**
+   * Returns the equivalent IPFS gateway URL for the supplied URL.
+   * @param urlToCheck The URL to check
+   * @returns If the supplied URL is an IPFS URL, it converts the URL to the
+   * equivalent IPFS gateway URL. Otherwise, returns the original URL.
+   */
+  checkIpfsUrl(urlToCheck: string) {
+    const url = new URL(urlToCheck)
+    let protocol = url.protocol
+    let host = url.host
+    let path = url.pathname
+    const query = url.search
+    if (protocol === 'ipfs:') {
+      protocol = 'https:'
+      host = this.ipfGateway
+      path = url.host === 'ipfs' ? `ipfs${path}` : `ipfs/${url.host}${path}`
+      const root = `${protocol}//${host}/${path}`
+      return query?.length > 0 ? `${root}?${query}` : root
+    } else {
+      return urlToCheck
+    }
+  }
+
   protected override async observeHandler(payloads: UrlPayload[] = []): Promise<ImageThumbnail[]> {
     if (!hasbin('magick')) {
       throw Error('ImageMagick is required for this witness')
@@ -157,7 +166,7 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
               }
             } else {
               //if it is ipfs, go through cloud flair
-              const mutatedUrl = ImageThumbnailWitness.checkIpfsUrl(url)
+              const mutatedUrl = this.checkIpfsUrl(url)
               result = await this.fromHttp(mutatedUrl)
             }
             this.cache.set(url, result)
@@ -185,6 +194,16 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
     return `data:image/png;base64,${thumb.toString('base64')}`
   }
 
+  /**
+   * Creates an image thumbnail from a video.
+   * @param videoBuffer The input video buffer.
+   * @returns An buffer containing an image thumbnail for the video.
+   */
+  private async createThumbnailFromVideo(videoBuffer: Buffer) {
+    const imageBuffer = await getVideoFrameAsImageFluent(videoBuffer)
+    return this.createThumbnailDataUrl(imageBuffer)
+  }
+
   private async fromHttp(url: string): Promise<ImageThumbnail> {
     let response: AxiosResponse
     let dnsResult: string[]
@@ -235,17 +254,64 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
 
     if (response.status >= 200 && response.status < 300) {
       const contentType = response.headers['content-type']?.toString()
-      if (contentType.split('/')[0] !== 'image') {
-        // TODO: Use FFMPEG to create a thumbnail for videos
-        // ffmpeg -i input.mp4 -ss 00:00:05 -vframes 1 output.png
-        // Then resize thumbnail using
-        result.mime = result.mime ?? {}
-        result.mime.invalid = true
-      } else {
-        const sourceBuffer = Buffer.from(response.data, 'binary')
+      const mediaType = contentType.split('/')[0]
+      result.mime = result.mime ?? {}
+      result.mime.returned = mediaType
+      const sourceBuffer = Buffer.from(response.data, 'binary')
+      try {
+        result.mime.detected = await FileType.fromBuffer(sourceBuffer)
+      } catch (ex) {
+        const error = ex as Error
+        this.logger?.error(`FileType error: ${error.message}`)
+      }
+
+      const processImage = async () => {
         result.sourceHash = await ImageThumbnailWitness.binaryToSha256(sourceBuffer)
         result.url = await this.createThumbnailDataUrl(sourceBuffer)
       }
+
+      const processVideo = async () => {
+        // Gracefully handle the case where ffmpeg is not installed.
+        if (hasbin('ffmpeg')) {
+          result.sourceHash = await ImageThumbnailWitness.binaryToSha256(sourceBuffer)
+          result.url = await this.createThumbnailFromVideo(sourceBuffer)
+        } else {
+          result.mime = result.mime ?? {}
+          result.mime.invalid = true
+        }
+      }
+
+      switch (mediaType) {
+        case 'image': {
+          await processImage()
+          result.mime.type = mediaType
+          break
+        }
+        case 'video': {
+          await processVideo()
+          result.mime.type = mediaType
+          break
+        }
+        default: {
+          switch (result.mime.detected?.mime) {
+            case 'image': {
+              await processImage()
+              result.mime.type = result.mime.detected?.mime
+              break
+            }
+            case 'video': {
+              await processVideo()
+              result.mime.type = result.mime.detected?.mime
+              break
+            }
+            default: {
+              result.mime.invalid = true
+              break
+            }
+          }
+          break
+        }
+      }
     }
     return result
   }

package/src/Witness/ffmpeg/fluent/getVideoFrameAsImageFluent.ts

@@ -0,0 +1,53 @@
+import ffmpeg from 'fluent-ffmpeg'
+import { Readable, Writable } from 'stream'
+
+/**
+ * Execute FFmpeg using fluent API with provided input buffer and video thumbnail image.
+ * @param videoBuffer Input video buffer.
+ * @returns Output buffer containing the video thumbnail image.
+ */
+export const getVideoFrameAsImageFluent = async (videoBuffer: Buffer) => {
+  const imageBuffer = await new Promise<Buffer>((resolve, reject) => {
+    // const videoStream = new PassThrough()
+    // videoStream.end(videoBuffer)
+
+    const videoStream = new Readable()
+    videoStream._read = () => {} // _read is required but you can noop it
+    videoStream.push(videoBuffer)
+    videoStream.push(null)
+
+    // Initialize empty array to collect PNG chunks
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const pngChunks: any[] = []
+
+    // Create a Writable stream to collect PNG output from ffmpeg
+    const writableStream = new Writable({
+      write(chunk, encoding, callback) {
+        pngChunks.push(chunk)
+        callback()
+      },
+    })
+
+    const command = ffmpeg()
+      // Uncomment to debug CLI args to ffmpeg
+      // .on('start', (commandLine) => console.log('Spawned Ffmpeg with command: ' + commandLine))
+      .input(videoStream)
+      .takeFrames(1)
+      .withNoAudio()
+      // Exactly as their docs but does not work
+      // .setStartTime('00:00:00')
+      // .seekInput(0)
+      .on('error', (err) => reject(err.message))
+      // Listen for the 'end' event to combine the chunks and create a PNG buffer
+      .on('end', () => resolve(Buffer.concat(pngChunks)))
+      .on('data', (chunk) => pngChunks.push(chunk))
+      // .toFormat('png')
+      .outputOptions('-f image2pipe')
+    // .outputOptions('-vcodec png')
+    // .output(writableStream, { end: true })
+
+    // Start processing
+    command.pipe(writableStream)
+  })
+  return imageBuffer
+}

package/src/Witness/ffmpeg/fluent/index.ts

@@ -0,0 +1 @@
+export * from './getVideoFrameAsImageFluent'

package/src/Witness/ffmpeg/index.ts

@@ -0,0 +1,2 @@
+export * from './fluent'
+export * from './spawn'

package/src/Witness/ffmpeg/spawn/executeFfmpeg.ts

@@ -0,0 +1,31 @@
+import { spawn } from 'child_process'
+import { PassThrough } from 'stream'
+
+/**
+ * Execute FFmpeg with the provided arguments.
+ * @param videoBuffer Input video buffer.
+ * @param ffmpegArgs FFmpeg arguments.
+ * @returns Output buffer containing the video thumbnail image.
+ */
+export const executeFFmpeg = (videoBuffer: Buffer, ffmpegArgs: string[]): Promise<Buffer> => {
+  return new Promise((resolve, reject) => {
+    const ffmpeg = spawn('ffmpeg', ffmpegArgs)
+
+    // Create a readable stream from the input buffer
+    const videoStream = new PassThrough().end(videoBuffer)
+
+    // Pipe the input stream to ffmpeg's stdin
+    videoStream.pipe(ffmpeg.stdin)
+    const chunks: Buffer[] = []
+    ffmpeg.stdout.on('data', (chunk: Buffer) => chunks.push(chunk))
+    // TODO: This is required as we're seeing errors thrown due to
+    // how we're piping the data to ffmpeg. Works perfectly though.
+    ffmpeg.stdin.on('error', () => {})
+    ffmpeg.on('close', (code) => {
+      if (code !== 0) {
+        return reject(new Error(`FFmpeg exited with code ${code}`))
+      }
+      resolve(Buffer.concat(chunks))
+    })
+  })
+}

package/src/Witness/ffmpeg/spawn/getVideoFrameAsImage.ts

@@ -0,0 +1,11 @@
+import { executeFFmpeg } from './executeFfmpeg'
+
+/**
+ * Execute FFmpeg with provided input buffer and return video thumbnail image.
+ * @param videoBuffer Input video buffer.
+ * @returns Output buffer containing the video thumbnail image.
+ */
+export const getVideoFrameAsImage = async (videoBuffer: Buffer) => {
+  const imageBuffer = await executeFFmpeg(videoBuffer, ['-i', 'pipe:', '-ss', '00:00:00', '-vframes', '1', '-f', 'image2pipe', '-'])
+  return imageBuffer
+}

package/src/Witness/ffmpeg/spawn/index.ts

@@ -0,0 +1,2 @@
+export * from './executeFfmpeg'
+export * from './getVideoFrameAsImage'