@xyo-network/image-thumbnail-plugin 2.70.8 → 2.70.9

package/dist/esm/Witness/Witness.js

@@ -1,12 +1,17 @@
 import { axios } from '@xyo-network/axios';
 import { PayloadHasher } from '@xyo-network/core';
 import { ImageThumbnailSchema } from '@xyo-network/image-thumbnail-payload-plugin';
+import { isPayload, ModuleErrorSchema } from '@xyo-network/payload-model';
 import { AbstractWitness } from '@xyo-network/witness';
 import { subClass } from 'gm';
 import { sha256 } from 'hash-wasm';
 import compact from 'lodash/compact';
+import isBuffer from 'lodash/isBuffer';
+import { LRUCache } from 'lru-cache';
 import shajs from 'sha.js';
+import Url from 'url-parse';
 import { ImageThumbnailWitnessConfigSchema } from './Config';
+//TODO: Break this into two Witnesses?
 const gm = subClass({ imageMagick: '7+' });
 export const binaryToSha256 = async (data) => {
     await PayloadHasher.wasmInitialized;
@@ -21,23 +26,149 @@ export const binaryToSha256 = async (data) => {
     // eslint-disable-next-line deprecation/deprecation
     return shajs('sha256').update(data).digest().toString();
 };
+const checkIpfsUrl = (urlToCheck) => {
+    const url = new Url(urlToCheck);
+    let protocol = url.protocol;
+    let host = url.host;
+    let path = url.pathname;
+    const query = url.query;
+    if (protocol === 'ipfs:') {
+        protocol = 'https:';
+        host = 'cloudflare-ipfs.com';
+        path = url.host === 'ipfs' ? `ipfs${path}` : `ipfs/${url.host}${path}`;
+        const root = `${protocol}//${host}/${path}`;
+        return query?.length > 0 ? `root?${query}` : root;
+    }
+    else {
+        return urlToCheck;
+    }
+};
+const checkDataUrl = (url) => {
+    if (url.startsWith('data:image')) {
+        const data = url.split(',')[1];
+        if (data) {
+            const buffer = Buffer.from(Uint8Array.from(atob(data), (c) => c.charCodeAt(0)));
+            console.log(`data buffer: ${buffer.length}`);
+            return [buffer, undefined];
+        }
+        else {
+            const error = {
+                message: 'Invalid data Url',
+                schema: ModuleErrorSchema,
+                url,
+            };
+            return [undefined, error];
+        }
+    }
+    else {
+        return [undefined, undefined];
+    }
+};
 export class ImageThumbnailWitness extends AbstractWitness {
     static configSchemas = [ImageThumbnailWitnessConfigSchema];
+    _cache;
+    get cache() {
+        this._cache = this._cache ?? new LRUCache({ max: this.maxCacheEntries });
+        return this._cache;
+    }
+    get encoding() {
+        return this.config.encoding ?? 'PNG';
+    }
+    get height() {
+        return this.config.height ?? 128;
+    }
+    get maxCacheEntries() {
+        return this.config.maxCacheEntries ?? 5000;
+    }
+    get quality() {
+        return this.config.quality ?? 50;
+    }
+    get width() {
+        return this.config.width ?? 128;
+    }
     async observeHandler(payloads = []) {
-        return compact(await Promise.all(payloads.map(async ({ url }) => {
+        const responsePairs = compact(await Promise.all(payloads.map(async ({ url }) => {
+            const cachedResult = this.cache.get(url);
+            if (cachedResult) {
+                return [url, cachedResult];
+            }
+            //if it is a data URL, return a Buffer
+            const [dataBuffer, dataError] = checkDataUrl(url);
+            if (dataBuffer) {
+                return [url, dataBuffer];
+            }
+            if (dataError) {
+                return [url, dataError];
+            }
             //if it is ipfs, go through cloud flair
-            const mutatedUrl = url.replace('ipfs://', 'https://cloudflare-ipfs.com/');
-            const response = await axios.get(mutatedUrl, {
-                responseType: 'arraybuffer',
-            });
-            if (response.status >= 200 && response.status < 300) {
-                const bytes = Buffer.from(response.data, 'binary');
+            const mutatedUrl = checkIpfsUrl(url);
+            try {
+                return [
+                    url,
+                    await axios.get(mutatedUrl, {
+                        responseType: 'arraybuffer',
+                    }),
+                ];
+            }
+            catch (ex) {
+                const axiosError = ex;
+                if (axiosError.isAxiosError) {
+                    //selectively pick fields from AxiosError
+                    const errorPayload = {
+                        code: axiosError.code,
+                        message: axiosError.message,
+                        schema: ModuleErrorSchema,
+                        status: axiosError.status,
+                        url,
+                    };
+                    return [url, errorPayload];
+                }
+                else {
+                    throw ex;
+                }
+            }
+        })));
+        return compact(await Promise.all(responsePairs.map(async ([url, urlResult]) => {
+            if (isPayload(urlResult)) {
+                this.cache.set(url, urlResult);
+                return urlResult;
+            }
+            let sourceBuffer;
+            if (isBuffer(urlResult)) {
+                sourceBuffer = urlResult;
+            }
+            else {
+                const response = urlResult;
+                if (response.status >= 200 && response.status < 300) {
+                    const contentType = response.headers['content-type']?.toString();
+                    if (contentType.split('/')[0] !== 'image') {
+                        const error = {
+                            message: `Invalid file type: ${contentType}`,
+                            schema: ModuleErrorSchema,
+                            url,
+                        };
+                        this.cache.set(url, error);
+                        return error;
+                    }
+                    sourceBuffer = Buffer.from(response.data, 'binary');
+                }
+                else {
+                    const error = {
+                        schema: ModuleErrorSchema,
+                        status: response.status,
+                        url,
+                    };
+                    this.cache.set(url, error);
+                    return error;
+                }
+            }
+            try {
                 const thumb = await new Promise((resolve, reject) => {
-                    gm(bytes)
-                        .quality(50)
-                        .resize(128, 128)
+                    gm(sourceBuffer)
+                        .quality(this.quality)
+                        .resize(this.width, this.height)
                         .flatten()
-                        .toBuffer('PNG', (error, buffer) => {
+                        .toBuffer(this.encoding, (error, buffer) => {
                         if (error) {
                             reject(error);
                         }
@@ -48,12 +179,22 @@ export class ImageThumbnailWitness extends AbstractWitness {
                 });
                 const result = {
                     schema: ImageThumbnailSchema,
-                    sourceHash: await binaryToSha256(bytes),
+                    sourceHash: await binaryToSha256(sourceBuffer),
                     sourceUrl: url,
                     url: `data:image/png;base64,${thumb.toString('base64')}`,
                 };
+                this.cache.set(url, result);
                 return result;
             }
+            catch (ex) {
+                const error = {
+                    message: 'Failed to resize image',
+                    schema: ModuleErrorSchema,
+                    url,
+                };
+                this.cache.set(url, error);
+                return error;
+            }
         })));
     }
 }

package/dist/esm/Witness/Witness.js.map

@@ -1 +1 @@
-{"version":3,"file":"Witness.js","sourceRoot":"","sources":["../../../src/Witness/Witness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AACjD,OAAO,EAAyB,oBAAoB,EAAE,MAAM,6CAA6C,CAAA;AAEzG,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAA;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAClC,OAAO,OAAO,MAAM,gBAAgB,CAAA;AACpC,OAAO,KAAK,MAAM,QAAQ,CAAA;AAE1B,OAAO,EAAE,iCAAiC,EAAE,MAAM,UAAU,CAAA;AAG5D,MAAM,EAAE,GAAG,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;AAE1C,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EAAE,IAAgB,EAAE,EAAE;IACvD,MAAM,aAAa,CAAC,eAAe,CAAA;IACnC,IAAI,aAAa,CAAC,WAAW,CAAC,UAAU,EAAE;QACxC,IAAI;YACF,OAAO,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;SAC1B;QAAC,OAAO,EAAE,EAAE;YACX,aAAa,CAAC,WAAW,CAAC,SAAS,GAAG,KAAK,CAAA;SAC5C;KACF;IACD,mDAAmD;IACnD,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAA;AACzD,CAAC,CAAA;AAED,MAAM,OAAO,qBAAiG,SAAQ,eAAwB;IAC5I,MAAM,CAAU,aAAa,GAAG,CAAC,iCAAiC,CAAC,CAAA;IAEhD,KAAK,CAAC,cAAc,CAAC,WAAyB,EAAE;QACjE,OAAO,OAAO,CACZ,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;YAC7B,uCAAuC;YACvC,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,8BAA8B,CAAC,CAAA;YACzE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE;gBAC3C,YAAY,EAAE,aAAa;aAC5B,CAAC,CAAA;YACF,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE;gBACnD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;gBAClD,MAAM,KAAK,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBAC1D,EAAE,CAAC,KAAK,CAAC;yBACN,OAAO,CAAC,EAAE,CAAC;yBACX,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC;yBAChB,OAAO,EAAE;yBACT,QAAQ,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;wBACjC,IAAI,KAAK,EAAE;4BACT,MAAM,CAAC,KAAK,CAAC,CAAA;yBACd;6BAAM;4BACL,OAAO,CAAC,MAAM,CAAC,CAAA;yBAChB;oBACH,CAAC,CAAC,CAAA;gBACN,CAAC,CAAC,CAAA;gBACF,MAAM,MAAM,GAA0B;oBACpC,MAAM,EAAE,oBAAoB;oBAC5B,UAAU,EAAE,MAAM,cAAc,CAAC,KAAK,CAAC;oBACvC,SAAS,EAAE,GAAG;oBACd,GAAG,EAAE,yBAAyB,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;iBACzD,CAAA;gBACD,OAAO,MAAM,CAAA;aACd;QACH,CAAC,CAAC,CACH,CACF,CAAA;IACH,CAAC"}
+{"version":3,"file":"Witness.js","sourceRoot":"","sources":["../../../src/Witness/Witness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAA6B,MAAM,oBAAoB,CAAA;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AACjD,OAAO,EAAqD,oBAAoB,EAAE,MAAM,6CAA6C,CAAA;AACrI,OAAO,EAAE,SAAS,EAAe,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAEtF,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAA;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAClC,OAAO,OAAO,MAAM,gBAAgB,CAAA;AACpC,OAAO,QAAQ,MAAM,iBAAiB,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,KAAK,MAAM,QAAQ,CAAA;AAC1B,OAAO,GAAG,MAAM,WAAW,CAAA;AAE3B,OAAO,EAAE,iCAAiC,EAAE,MAAM,UAAU,CAAA;AAG5D,sCAAsC;AAEtC,MAAM,EAAE,GAAG,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;AAE1C,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EAAE,IAAgB,EAAE,EAAE;IACvD,MAAM,aAAa,CAAC,eAAe,CAAA;IACnC,IAAI,aAAa,CAAC,WAAW,CAAC,UAAU,EAAE;QACxC,IAAI;YACF,OAAO,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;SAC1B;QAAC,OAAO,EAAE,EAAE;YACX,aAAa,CAAC,WAAW,CAAC,SAAS,GAAG,KAAK,CAAA;SAC5C;KACF;IACD,mDAAmD;IACnD,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAA;AACzD,CAAC,CAAA;AAED,MAAM,YAAY,GAAG,CAAC,UAAkB,EAAE,EAAE;IAC1C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAA;IAC/B,IAAI,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAA;IAC3B,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,CAAA;IACnB,IAAI,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAA;IACvB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAA;IACvB,IAAI,QAAQ,KAAK,OAAO,EAAE;QACxB,QAAQ,GAAG,QAAQ,CAAA;QACnB,IAAI,GAAG,qBAAqB,CAAA;QAC5B,IAAI,GAAG,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,IAAI,GAAG,IAAI,EAAE,CAAA;QACtE,MAAM,IAAI,GAAG,GAAG,QAAQ,KAAK,IAAI,IAAI,IAAI,EAAE,CAAA;QAC3C,OAAO,KAAK,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;KAClD;SAAM;QACL,OAAO,UAAU,CAAA;KAClB;AACH,CAAC,CAAA;AAED,MAAM,YAAY,GAAG,CAAC,GAAW,EAAgE,EAAE;IACjG,IAAI,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QAChC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;QAC9B,IAAI,IAAI,EAAE;YACR,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAC/E,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;YAC5C,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;SAC3B;aAAM;YACL,MAAM,KAAK,GAA+B;gBACxC,OAAO,EAAE,kBAAkB;gBAC3B,MAAM,EAAE,iBAAiB;gBACzB,GAAG;aACJ,CAAA;YACD,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;SAC1B;KACF;SAAM;QACL,OAAO,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;KAC9B;AACH,CAAC,CAAA;AAED,MAAM,OAAO,qBAAiG,SAAQ,eAAwB;IAC5I,MAAM,CAAU,aAAa,GAAG,CAAC,iCAAiC,CAAC,CAAA;IAE3D,MAAM,CAAuE;IAErF,IAAI,KAAK;QACP,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,QAAQ,CAA6D,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAA;QACpI,OAAO,IAAI,CAAC,MAAM,CAAA;IACpB,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,KAAK,CAAA;IACtC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,GAAG,CAAA;IAClC,CAAC;IAED,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,IAAI,CAAA;IAC5C,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAA;IAClC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,GAAG,CAAA;IACjC,CAAC;IAEkB,KAAK,CAAC,cAAc,CAAC,WAAyB,EAAE;QACjE,MAAM,aAAa,GAAG,OAAO,CAC3B,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAiG,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;YAC7H,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACxC,IAAI,YAAY,EAAE;gBAChB,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAA;aAC3B;YACD,sCAAsC;YACtC,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;YAEjD,IAAI,UAAU,EAAE;gBACd,OAAO,CAAC,GAAG,EAAE,UAAU,CAAC,CAAA;aACzB;YAED,IAAI,SAAS,EAAE;gBACb,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;aACxB;YAED,uCAAuC;YACvC,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;YAEpC,IAAI;gBACF,OAAO;oBACL,GAAG;oBACH,MAAM,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE;wBAC1B,YAAY,EAAE,aAAa;qBAC5B,CAAC;iBACH,CAAA;aACF;YAAC,OAAO,EAAE,EAAE;gBACX,MAAM,UAAU,GAAG,EAAgB,CAAA;gBACnC,IAAI,UAAU,CAAC,YAAY,EAAE;oBAC3B,yCAAyC;oBACzC,MAAM,YAAY,GAA+B;wBAC/C,IAAI,EAAE,UAAU,CAAC,IAAI;wBACrB,OAAO,EAAE,UAAU,CAAC,OAAO;wBAC3B,MAAM,EAAE,iBAAiB;wBACzB,MAAM,EAAE,UAAU,CAAC,MAAM;wBACzB,GAAG;qBACJ,CAAA;oBACD,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAA;iBAC3B;qBAAM;oBACL,MAAM,EAAE,CAAA;iBACT;aACF;QACH,CAAC,CAAC,CACH,CACF,CAAA;QACD,OAAO,OAAO,CACZ,MAAM,OAAO,CAAC,GAAG,CACf,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE,EAAE;YAC3C,IAAI,SAAS,CAAC,SAAS,CAAC,EAAE;gBACxB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;gBAC9B,OAAO,SAAS,CAAA;aACjB;YAED,IAAI,YAAoB,CAAA;YAExB,IAAI,QAAQ,CAAC,SAAS,CAAC,EAAE;gBACvB,YAAY,GAAG,SAAmB,CAAA;aACnC;iBAAM;gBACL,MAAM,QAAQ,GAAG,SAA0B,CAAA;gBAE3C,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE;oBACnD,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,QAAQ,EAAE,CAAA;oBAChE,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE;wBACzC,MAAM,KAAK,GAA+B;4BACxC,OAAO,EAAE,sBAAsB,WAAW,EAAE;4BAC5C,MAAM,EAAE,iBAAiB;4BACzB,GAAG;yBACJ,CAAA;wBACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;wBAC1B,OAAO,KAAK,CAAA;qBACb;oBACD,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;iBACpD;qBAAM;oBACL,MAAM,KAAK,GAA+B;wBACxC,MAAM,EAAE,iBAAiB;wBACzB,MAAM,EAAE,QAAQ,CAAC,MAAM;wBACvB,GAAG;qBACJ,CAAA;oBACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;oBAC1B,OAAO,KAAK,CAAA;iBACb;aACF;YACD,IAAI;gBACF,MAAM,KAAK,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBAC1D,EAAE,CAAC,YAAY,CAAC;yBACb,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;yBACrB,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;yBAC/B,OAAO,EAAE;yBACT,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;wBACzC,IAAI,KAAK,EAAE;4BACT,MAAM,CAAC,KAAK,CAAC,CAAA;yBACd;6BAAM;4BACL,OAAO,CAAC,MAAM,CAAC,CAAA;yBAChB;oBACH,CAAC,CAAC,CAAA;gBACN,CAAC,CAAC,CAAA;gBACF,MAAM,MAAM,GAA0B;oBACpC,MAAM,EAAE,oBAAoB;oBAC5B,UAAU,EAAE,MAAM,cAAc,CAAC,YAAY,CAAC;oBAC9C,SAAS,EAAE,GAAG;oBACd,GAAG,EAAE,yBAAyB,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;iBACzD,CAAA;gBACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;gBAC3B,OAAO,MAAM,CAAA;aACd;YAAC,OAAO,EAAE,EAAE;gBACX,MAAM,KAAK,GAA+B;oBACxC,OAAO,EAAE,wBAAwB;oBACjC,MAAM,EAAE,iBAAiB;oBACzB,GAAG;iBACJ,CAAA;gBACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;gBAC1B,OAAO,KAAK,CAAA;aACb;QACH,CAAC,CAAC,CACH,CACF,CAAA;IACH,CAAC"}

package/dist/types/Plugin.d.ts

@@ -36,7 +36,12 @@ export declare const ImageThumbnailPlugin: () => import("@xyo-network/payloadset
             schema: "network.xyo.payload.set";
         }) | undefined;
     } & {
+        encoding?: "PNG" | undefined;
+        height?: number | undefined;
+        maxCacheEntries?: number | undefined;
+        quality?: number | undefined;
         schema: "network.xyo.image.thumbnail.witness.config";
+        width?: number | undefined;
     }, "schema"> & {
         schema: "network.xyo.image.thumbnail.witness.config";
     }, "schema"> & {

package/dist/types/Plugin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Plugin.d.ts","sourceRoot":"","sources":["../../src/Plugin.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAA;AAEjD,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAS9B,CAAA"}
+{"version":3,"file":"Plugin.d.ts","sourceRoot":"","sources":["../../src/Plugin.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAA;AAEjD,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAS9B,CAAA"}

package/dist/types/Witness/Config.d.ts

@@ -2,6 +2,11 @@ import { WitnessConfig } from '@xyo-network/witness';
 export declare const ImageThumbnailWitnessConfigSchema: "network.xyo.image.thumbnail.witness.config";
 export type ImageThumbnailWitnessConfigSchema = typeof ImageThumbnailWitnessConfigSchema;
 export type ImageThumbnailWitnessConfig = WitnessConfig<{
+    encoding?: 'PNG';
+    height?: number;
+    maxCacheEntries?: number;
+    quality?: number;
     schema: ImageThumbnailWitnessConfigSchema;
+    width?: number;
 }>;
 //# sourceMappingURL=Config.d.ts.map

package/dist/types/Witness/Config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Config.d.ts","sourceRoot":"","sources":["../../../src/Witness/Config.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAEpD,eAAO,MAAM,iCAAiC,8CAAoD,CAAA;AAClG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,MAAM,MAAM,2BAA2B,GAAG,aAAa,CAAC;IACtD,MAAM,EAAE,iCAAiC,CAAA;CAC1C,CAAC,CAAA"}
+{"version":3,"file":"Config.d.ts","sourceRoot":"","sources":["../../../src/Witness/Config.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAEpD,eAAO,MAAM,iCAAiC,8CAAoD,CAAA;AAClG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,MAAM,MAAM,2BAA2B,GAAG,aAAa,CAAC;IACtD,QAAQ,CAAC,EAAE,KAAK,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,iCAAiC,CAAA;IACzC,KAAK,CAAC,EAAE,MAAM,CAAA;CACf,CAAC,CAAA"}

package/dist/types/Witness/Witness.d.ts

@@ -1,10 +1,25 @@
-import { ImageThumbnailPayload } from '@xyo-network/image-thumbnail-payload-plugin';
+import { ImageThumbnailErrorPayload, ImageThumbnailPayload } from '@xyo-network/image-thumbnail-payload-plugin';
 import { UrlPayload } from '@xyo-network/url-payload-plugin';
 import { AbstractWitness } from '@xyo-network/witness';
+import { LRUCache } from 'lru-cache';
 import { ImageThumbnailWitnessParams } from './Params';
 export declare const binaryToSha256: (data: Uint8Array) => Promise<string>;
 export declare class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams = ImageThumbnailWitnessParams> extends AbstractWitness<TParams> {
     static configSchemas: "network.xyo.image.thumbnail.witness.config"[];
-    protected observeHandler(payloads?: UrlPayload[]): Promise<ImageThumbnailPayload[]>;
+    private _cache?;
+    get cache(): LRUCache<string, ImageThumbnailErrorPayload | (import("@xyo-network/payload-model").SchemaFields & import("@xyo-network/payload-model").PayloadFields & {
+        schema: "network.xyo.image.thumbnail";
+        sourceHash: string;
+        sourceUrl: string;
+        url: string;
+    } & {
+        schema: "network.xyo.image.thumbnail";
+    }), unknown>;
+    get encoding(): "PNG";
+    get height(): number;
+    get maxCacheEntries(): number;
+    get quality(): number;
+    get width(): number;
+    protected observeHandler(payloads?: UrlPayload[]): Promise<(ImageThumbnailPayload | ImageThumbnailErrorPayload)[]>;
 }
 //# sourceMappingURL=Witness.d.ts.map

package/dist/types/Witness/Witness.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Witness.d.ts","sourceRoot":"","sources":["../../../src/Witness/Witness.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,qBAAqB,EAAwB,MAAM,6CAA6C,CAAA;AACzG,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAOtD,OAAO,EAAE,2BAA2B,EAAE,MAAM,UAAU,CAAA;AAItD,eAAO,MAAM,cAAc,SAAgB,UAAU,oBAWpD,CAAA;AAED,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,iDAAsC;cAE1C,cAAc,CAAC,QAAQ,GAAE,UAAU,EAAO,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAC;CAoCvG"}
+{"version":3,"file":"Witness.d.ts","sourceRoot":"","sources":["../../../src/Witness/Witness.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAwB,MAAM,6CAA6C,CAAA;AAErI,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAKtD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAKpC,OAAO,EAAE,2BAA2B,EAAE,MAAM,UAAU,CAAA;AAMtD,eAAO,MAAM,cAAc,SAAgB,UAAU,oBAWpD,CAAA;AAuCD,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,iDAAsC;IAEnE,OAAO,CAAC,MAAM,CAAC,CAAsE;IAErF,IAAI,KAAK;;;;;;;iBAGR;IAED,IAAI,QAAQ,UAEX;IAED,IAAI,MAAM,WAET;IAED,IAAI,eAAe,WAElB;IAED,IAAI,OAAO,WAEV;IAED,IAAI,KAAK,WAER;cAEwB,cAAc,CAAC,QAAQ,GAAE,UAAU,EAAO,GAAG,OAAO,CAAC,CAAC,qBAAqB,GAAG,0BAA0B,CAAC,EAAE,CAAC;CAwHtI"}

package/package.json

@@ -10,17 +10,18 @@
     "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js/issues"
   },
   "dependencies": {
-    "@xyo-network/axios": "~2.70.10",
-    "@xyo-network/core": "~2.70.10",
-    "@xyo-network/image-thumbnail-payload-plugin": "~2.70.8",
-    "@xyo-network/module": "~2.70.10",
-    "@xyo-network/payload-model": "~2.70.10",
-    "@xyo-network/payloadset-plugin": "~2.70.10",
-    "@xyo-network/url-payload-plugin": "~2.70.10",
-    "@xyo-network/witness": "~2.70.10",
+    "@xyo-network/axios": "~2.70.11",
+    "@xyo-network/core": "~2.70.11",
+    "@xyo-network/image-thumbnail-payload-plugin": "~2.70.9",
+    "@xyo-network/module": "~2.70.11",
+    "@xyo-network/payload-model": "~2.70.11",
+    "@xyo-network/payloadset-plugin": "~2.70.11",
+    "@xyo-network/url-payload-plugin": "~2.70.11",
+    "@xyo-network/witness": "~2.70.11",
     "gm": "^1.25.0",
     "hash-wasm": "^4.9.0",
     "lodash": "^4.17.21",
+    "lru-cache": "^10.0.0",
     "sha.js": "^2.4.11"
   },
   "devDependencies": {
@@ -63,5 +64,5 @@
   },
   "sideEffects": false,
   "types": "dist/types/index.d.ts",
-  "version": "2.70.8"
+  "version": "2.70.9"
 }

package/src/Witness/Config.ts

@@ -5,5 +5,10 @@ export const ImageThumbnailWitnessConfigSchema = `${ImageThumbnailSchema}.witnes
 export type ImageThumbnailWitnessConfigSchema = typeof ImageThumbnailWitnessConfigSchema
 
 export type ImageThumbnailWitnessConfig = WitnessConfig<{
+  encoding?: 'PNG'
+  height?: number
+  maxCacheEntries?: number
+  quality?: number
   schema: ImageThumbnailWitnessConfigSchema
+  width?: number
 }>

package/src/Witness/Witness.ts

@@ -1,16 +1,22 @@
-import { axios } from '@xyo-network/axios'
+import { axios, AxiosError, AxiosResponse } from '@xyo-network/axios'
 import { PayloadHasher } from '@xyo-network/core'
-import { ImageThumbnailPayload, ImageThumbnailSchema } from '@xyo-network/image-thumbnail-payload-plugin'
+import { ImageThumbnailErrorPayload, ImageThumbnailPayload, ImageThumbnailSchema } from '@xyo-network/image-thumbnail-payload-plugin'
+import { isPayload, ModuleError, ModuleErrorSchema } from '@xyo-network/payload-model'
 import { UrlPayload } from '@xyo-network/url-payload-plugin'
 import { AbstractWitness } from '@xyo-network/witness'
 import { subClass } from 'gm'
 import { sha256 } from 'hash-wasm'
 import compact from 'lodash/compact'
+import isBuffer from 'lodash/isBuffer'
+import { LRUCache } from 'lru-cache'
 import shajs from 'sha.js'
+import Url from 'url-parse'
 
 import { ImageThumbnailWitnessConfigSchema } from './Config'
 import { ImageThumbnailWitnessParams } from './Params'
 
+//TODO: Break this into two Witnesses?
+
 const gm = subClass({ imageMagick: '7+' })
 
 export const binaryToSha256 = async (data: Uint8Array) => {
@@ -26,26 +32,165 @@ export const binaryToSha256 = async (data: Uint8Array) => {
   return shajs('sha256').update(data).digest().toString()
 }
 
+const checkIpfsUrl = (urlToCheck: string) => {
+  const url = new Url(urlToCheck)
+  let protocol = url.protocol
+  let host = url.host
+  let path = url.pathname
+  const query = url.query
+  if (protocol === 'ipfs:') {
+    protocol = 'https:'
+    host = 'cloudflare-ipfs.com'
+    path = url.host === 'ipfs' ? `ipfs${path}` : `ipfs/${url.host}${path}`
+    const root = `${protocol}//${host}/${path}`
+    return query?.length > 0 ? `root?${query}` : root
+  } else {
+    return urlToCheck
+  }
+}
+
+const checkDataUrl = (url: string): [Buffer | undefined, ImageThumbnailErrorPayload | undefined] => {
+  if (url.startsWith('data:image')) {
+    const data = url.split(',')[1]
+    if (data) {
+      const buffer = Buffer.from(Uint8Array.from(atob(data), (c) => c.charCodeAt(0)))
+      console.log(`data buffer: ${buffer.length}`)
+      return [buffer, undefined]
+    } else {
+      const error: ImageThumbnailErrorPayload = {
+        message: 'Invalid data Url',
+        schema: ModuleErrorSchema,
+        url,
+      }
+      return [undefined, error]
+    }
+  } else {
+    return [undefined, undefined]
+  }
+}
+
 export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams = ImageThumbnailWitnessParams> extends AbstractWitness<TParams> {
   static override configSchemas = [ImageThumbnailWitnessConfigSchema]
 
-  protected override async observeHandler(payloads: UrlPayload[] = []): Promise<ImageThumbnailPayload[]> {
-    return compact(
+  private _cache?: LRUCache<string, ImageThumbnailPayload | ImageThumbnailErrorPayload>
+
+  get cache() {
+    this._cache = this._cache ?? new LRUCache<string, ImageThumbnailPayload | ImageThumbnailErrorPayload>({ max: this.maxCacheEntries })
+    return this._cache
+  }
+
+  get encoding() {
+    return this.config.encoding ?? 'PNG'
+  }
+
+  get height() {
+    return this.config.height ?? 128
+  }
+
+  get maxCacheEntries() {
+    return this.config.maxCacheEntries ?? 5000
+  }
+
+  get quality() {
+    return this.config.quality ?? 50
+  }
+
+  get width() {
+    return this.config.width ?? 128
+  }
+
+  protected override async observeHandler(payloads: UrlPayload[] = []): Promise<(ImageThumbnailPayload | ImageThumbnailErrorPayload)[]> {
+    const responsePairs = compact(
       await Promise.all(
-        payloads.map(async ({ url }) => {
+        payloads.map<Promise<[string, ImageThumbnailPayload | ImageThumbnailErrorPayload | AxiosResponse | Buffer]>>(async ({ url }) => {
+          const cachedResult = this.cache.get(url)
+          if (cachedResult) {
+            return [url, cachedResult]
+          }
+          //if it is a data URL, return a Buffer
+          const [dataBuffer, dataError] = checkDataUrl(url)
+
+          if (dataBuffer) {
+            return [url, dataBuffer]
+          }
+
+          if (dataError) {
+            return [url, dataError]
+          }
+
           //if it is ipfs, go through cloud flair
-          const mutatedUrl = url.replace('ipfs://', 'https://cloudflare-ipfs.com/')
-          const response = await axios.get(mutatedUrl, {
-            responseType: 'arraybuffer',
-          })
-          if (response.status >= 200 && response.status < 300) {
-            const bytes = Buffer.from(response.data, 'binary')
+          const mutatedUrl = checkIpfsUrl(url)
+
+          try {
+            return [
+              url,
+              await axios.get(mutatedUrl, {
+                responseType: 'arraybuffer',
+              }),
+            ]
+          } catch (ex) {
+            const axiosError = ex as AxiosError
+            if (axiosError.isAxiosError) {
+              //selectively pick fields from AxiosError
+              const errorPayload: ImageThumbnailErrorPayload = {
+                code: axiosError.code,
+                message: axiosError.message,
+                schema: ModuleErrorSchema,
+                status: axiosError.status,
+                url,
+              }
+              return [url, errorPayload]
+            } else {
+              throw ex
+            }
+          }
+        }),
+      ),
+    )
+    return compact(
+      await Promise.all(
+        responsePairs.map(async ([url, urlResult]) => {
+          if (isPayload(urlResult)) {
+            this.cache.set(url, urlResult)
+            return urlResult
+          }
+
+          let sourceBuffer: Buffer
+
+          if (isBuffer(urlResult)) {
+            sourceBuffer = urlResult as Buffer
+          } else {
+            const response = urlResult as AxiosResponse
+
+            if (response.status >= 200 && response.status < 300) {
+              const contentType = response.headers['content-type']?.toString()
+              if (contentType.split('/')[0] !== 'image') {
+                const error: ImageThumbnailErrorPayload = {
+                  message: `Invalid file type: ${contentType}`,
+                  schema: ModuleErrorSchema,
+                  url,
+                }
+                this.cache.set(url, error)
+                return error
+              }
+              sourceBuffer = Buffer.from(response.data, 'binary')
+            } else {
+              const error: ImageThumbnailErrorPayload = {
+                schema: ModuleErrorSchema,
+                status: response.status,
+                url,
+              }
+              this.cache.set(url, error)
+              return error
+            }
+          }
+          try {
             const thumb = await new Promise<Buffer>((resolve, reject) => {
-              gm(bytes)
-                .quality(50)
-                .resize(128, 128)
+              gm(sourceBuffer)
+                .quality(this.quality)
+                .resize(this.width, this.height)
                 .flatten()
-                .toBuffer('PNG', (error, buffer) => {
+                .toBuffer(this.encoding, (error, buffer) => {
                   if (error) {
                     reject(error)
                   } else {
@@ -55,11 +200,20 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
             })
             const result: ImageThumbnailPayload = {
               schema: ImageThumbnailSchema,
-              sourceHash: await binaryToSha256(bytes),
+              sourceHash: await binaryToSha256(sourceBuffer),
               sourceUrl: url,
               url: `data:image/png;base64,${thumb.toString('base64')}`,
             }
+            this.cache.set(url, result)
             return result
+          } catch (ex) {
+            const error: ImageThumbnailErrorPayload = {
+              message: 'Failed to resize image',
+              schema: ModuleErrorSchema,
+              url,
+            }
+            this.cache.set(url, error)
+            return error
           }
         }),
       ),