@xyo-network/image-thumbnail-plugin 2.74.5 → 2.75.2

package/dist/node/Plugin.js

@@ -0,0 +1,556 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/Plugin.ts
+var Plugin_exports = {};
+__export(Plugin_exports, {
+  ImageThumbnailPlugin: () => ImageThumbnailPlugin
+});
+module.exports = __toCommonJS(Plugin_exports);
+var import_image_thumbnail_payload_plugin5 = require("@xyo-network/image-thumbnail-payload-plugin");
+var import_payload_model = require("@xyo-network/payload-model");
+var import_payloadset_plugin = require("@xyo-network/payloadset-plugin");
+
+// src/Diviner/Config.ts
+var import_image_thumbnail_payload_plugin = require("@xyo-network/image-thumbnail-payload-plugin");
+var ImageThumbnailDivinerConfigSchema = `${import_image_thumbnail_payload_plugin.ImageThumbnailSchema}.diviner.config`;
+
+// src/Diviner/Diviner.ts
+var import_assert = require("@xylabs/assert");
+var import_delay = require("@xylabs/delay");
+var import_lodash = require("@xylabs/lodash");
+var import_abstract_diviner = require("@xyo-network/abstract-diviner");
+var import_archivist_model = require("@xyo-network/archivist-model");
+var import_core = require("@xyo-network/core");
+var import_diviner_model = require("@xyo-network/diviner-model");
+var import_diviner_payload_model = require("@xyo-network/diviner-payload-model");
+var import_image_thumbnail_payload_plugin2 = require("@xyo-network/image-thumbnail-payload-plugin");
+var ImageThumbnailDiviner = class extends import_abstract_diviner.AbstractDiviner {
+  static configSchemas = [ImageThumbnailDivinerConfigSchema, import_diviner_model.DivinerConfigSchema];
+  _archivistInstance;
+  _initializeArchivistConnectionIfNeededPromise;
+  _map;
+  _payloadDivinerInstance;
+  _pollId;
+  // static override get configSchema() {
+  //   return ImageThumbnailDivinerConfigSchema
+  // }
+  get archivist() {
+    return this.config.archivist;
+  }
+  get payloadDiviner() {
+    return this.config.payloadDiviner;
+  }
+  get payloadDivinerLimit() {
+    return this.config.payloadDivinerLimit ?? 1e4;
+  }
+  get pollFrequency() {
+    return this.config.pollFrequency;
+  }
+  //using promise as mutex
+  async getArchivistInstance() {
+    if (this._archivistInstance && !await this._archivistInstance) {
+      this._archivistInstance = void 0;
+    }
+    this._archivistInstance = this._archivistInstance ?? (async () => {
+      const module2 = this.archivist ? await this.resolve(this.archivist) : void 0;
+      return (0, import_archivist_model.asArchivistInstance)(module2, "Provided archivist address did not resolve to an Archivist");
+    })();
+    return this._archivistInstance;
+  }
+  //using promise as mutex
+  async getPayloadDivinerInstance() {
+    const payloadDivinerAddress = this.payloadDiviner;
+    if (payloadDivinerAddress) {
+      if (this._payloadDivinerInstance && !await this._payloadDivinerInstance) {
+        this._payloadDivinerInstance = void 0;
+      }
+      this._payloadDivinerInstance = this._payloadDivinerInstance ?? (async () => {
+        const module2 = await this.resolve(payloadDivinerAddress);
+        return (0, import_diviner_model.asDivinerInstance)(module2, "Provided payload diviner address did not resolve to a Diviner");
+      })();
+      return this._payloadDivinerInstance;
+    }
+  }
+  async divineHandler(payloads = []) {
+    await this.initializeArchivistConnectionIfNeeded();
+    const urls = payloads.map((urlPayload) => urlPayload.url);
+    const map = await this.getSafeMap();
+    const archivist = await this.getArchivistInstance();
+    const hashes = (0, import_lodash.compact)(urls.map((url) => map == null ? void 0 : map[url]));
+    return (await archivist.get(hashes)).filter((payload) => payload.schema === import_image_thumbnail_payload_plugin2.ImageThumbnailSchema);
+  }
+  //using promise as mutex
+  initializeArchivistConnectionIfNeeded() {
+    this._initializeArchivistConnectionIfNeededPromise = this._initializeArchivistConnectionIfNeededPromise ?? (async () => {
+      if (!this._map) {
+        await this.attachArchivistEvents();
+        console.log("initializeArchivistConnectionIfNeeded: attachArchivistEvents done");
+        await this.poll();
+        console.log("initializeArchivistConnectionIfNeeded: poll done");
+      }
+    })();
+    return this._initializeArchivistConnectionIfNeededPromise;
+  }
+  async loadMap() {
+    if (this.payloadDiviner) {
+      return await this.loadMapWithPayloadDiviner();
+    } else {
+      return await this.loadMapWithAll();
+    }
+  }
+  async loadMapWithAll() {
+    var _a;
+    if (await this.started()) {
+      const archivist = await this.getArchivistInstance();
+      (0, import_assert.assertEx)(archivist.all, "Archivist does not support 'all'");
+      const allPayloads = await ((_a = archivist.all) == null ? void 0 : _a.call(archivist)) ?? [];
+      const imagePayloadPairs = await Promise.all(
+        allPayloads.filter((payload) => payload.schema === import_image_thumbnail_payload_plugin2.ImageThumbnailSchema).map(async (payload) => [await import_core.PayloadHasher.hashAsync(payload), payload])
+      );
+      this._map = imagePayloadPairs.reduce((prev, [hash, payload]) => {
+        prev[payload.sourceUrl] = hash;
+        return prev;
+      }, {});
+    }
+  }
+  async loadMapWithPayloadDiviner() {
+    console.log("loadMapWithPayloadDiviner: started");
+    if (await this.started()) {
+      const diviner = await this.getPayloadDivinerInstance();
+      let offset = void 0;
+      let moreAvailable = true;
+      if (diviner) {
+        const newMap = {};
+        while (moreAvailable) {
+          const payloadDivinerQuery = {
+            limit: this.payloadDivinerLimit,
+            offset,
+            schema: import_diviner_payload_model.PayloadDivinerQuerySchema,
+            schemas: [import_image_thumbnail_payload_plugin2.ImageThumbnailSchema]
+          };
+          const payloads = await diviner.divine([payloadDivinerQuery]);
+          offset = (offset ?? 0) + payloads.length;
+          moreAvailable = payloads.length > 0;
+          console.log(`loadMapWithPayloadDiviner.offset: ${offset}`);
+          console.log(`loadMapWithPayloadDiviner.moreAvailable: ${moreAvailable}`);
+          const imagePayloadPairs = await Promise.all(
+            payloads.filter((payload) => payload.schema === import_image_thumbnail_payload_plugin2.ImageThumbnailSchema).map(async (payload) => [await import_core.PayloadHasher.hashAsync(payload), payload])
+          );
+          imagePayloadPairs.forEach(([hash, payload]) => newMap[payload.sourceUrl] = hash);
+        }
+        this._map = newMap;
+      }
+    }
+  }
+  async stopHandler(_timeout) {
+    if (this._pollId) {
+      clearTimeout(this._pollId);
+      this._pollId = void 0;
+    }
+    return await super.stopHandler();
+  }
+  async attachArchivistEvents() {
+    const archivist = await this.getArchivistInstance();
+    const mapPromise = this.getSafeMap();
+    archivist.on("inserted", async ({ payloads }) => {
+      const map = await mapPromise;
+      const thumbnails = (0, import_lodash.compact)(payloads.filter((payload) => payload.schema === import_image_thumbnail_payload_plugin2.ImageThumbnailSchema));
+      await Promise.all(thumbnails.map(async (payload) => map[payload.sourceUrl] = await import_core.PayloadHasher.hashAsync(payload)));
+    });
+  }
+  async getSafeMap() {
+    let mapRetry = 100;
+    let map = this._map;
+    while (!map) {
+      await (0, import_delay.delay)(100);
+      mapRetry = mapRetry - 1;
+      if (mapRetry === 0) {
+        throw Error("Map Not Loaded");
+      }
+      map = this._map;
+    }
+    return map;
+  }
+  async poll() {
+    if (await this.started()) {
+      const pollFrequency = this.pollFrequency;
+      if (pollFrequency) {
+        this._pollId = setTimeout(async () => {
+          this._pollId = void 0;
+          await this.loadMap();
+          await this.poll();
+        }, pollFrequency);
+      } else {
+        await this.loadMap();
+      }
+    }
+  }
+};
+
+// src/Witness/Config.ts
+var import_image_thumbnail_payload_plugin3 = require("@xyo-network/image-thumbnail-payload-plugin");
+var ImageThumbnailWitnessConfigSchema = `${import_image_thumbnail_payload_plugin3.ImageThumbnailSchema}.witness.config`;
+
+// src/Witness/Witness.ts
+var import_node_dns = require("dns");
+var import_lodash2 = require("@xylabs/lodash");
+var import_url = require("@xylabs/url");
+var import_axios = require("@xyo-network/axios");
+var import_core3 = require("@xyo-network/core");
+var import_image_thumbnail_payload_plugin4 = require("@xyo-network/image-thumbnail-payload-plugin");
+var import_url_payload_plugin = require("@xyo-network/url-payload-plugin");
+var import_witness = require("@xyo-network/witness");
+var import_async_mutex = require("async-mutex");
+var import_file_type = __toESM(require("file-type"));
+var import_gm = __toESM(require("gm"));
+var import_hasbin = __toESM(require("hasbin"));
+var import_hash_wasm = require("hash-wasm");
+var import_lru_cache = require("lru-cache");
+var import_sha = __toESM(require("sha.js"));
+var import_url_parse = __toESM(require("url-parse"));
+
+// src/Witness/ffmpeg/fluent/getVideoFrameAsImageFluent.ts
+var import_core2 = require("@xyo-network/core");
+var import_fluent_ffmpeg = __toESM(require("fluent-ffmpeg"));
+var import_promises = require("fs/promises");
+var import_os = require("os");
+var import_stream = require("stream");
+var FfmpegOutputStream = class extends import_stream.Writable {
+  chunks = [];
+  constructor(options) {
+    super(options);
+  }
+  _write(chunk, _encoding, callback) {
+    this.chunks.push(chunk);
+    callback();
+  }
+  /**
+   * Collects the output from ffmpeg into a buffer.
+   * @returns A buffer containing the concatenated
+   * output from ffmpeg.
+   */
+  toBuffer = () => Buffer.concat(this.chunks);
+};
+var getVideoFrameAsImageFluent = async (videoBuffer) => {
+  const tmpFile = `/${(0, import_os.tmpdir)()}/${(0, import_core2.uuid)()}`;
+  try {
+    await (0, import_promises.writeFile)(tmpFile, videoBuffer, { encoding: "binary" });
+    const imageBuffer = await new Promise((resolve, reject) => {
+      const ffmpegOutput = new FfmpegOutputStream();
+      (0, import_fluent_ffmpeg.default)().on("error", (err) => reject(err.message)).on("end", () => resolve(ffmpegOutput.toBuffer())).input(tmpFile).takeFrames(1).withNoAudio().outputOptions("-f image2pipe").videoCodec("png").pipe(ffmpegOutput);
+    });
+    return imageBuffer;
+  } finally {
+    try {
+      await (0, import_promises.unlink)(tmpFile);
+    } catch {
+    }
+  }
+};
+
+// src/Witness/Witness.ts
+var gm = import_gm.default.subClass({ imageMagick: "7+" });
+var ImageThumbnailWitness = class _ImageThumbnailWitness extends import_witness.AbstractWitness {
+  static configSchemas = [ImageThumbnailWitnessConfigSchema];
+  _cache;
+  _semaphore = new import_async_mutex.Semaphore(this.maxAsyncProcesses);
+  get cache() {
+    this._cache = this._cache ?? new import_lru_cache.LRUCache({
+      max: this.maxCacheEntries,
+      maxSize: this.maxCacheBytes,
+      //just returning the size of the data
+      sizeCalculation: (value) => {
+        var _a;
+        return ((_a = value.url) == null ? void 0 : _a.length) ?? 1;
+      }
+    });
+    return this._cache;
+  }
+  get encoding() {
+    return this.config.encoding ?? "PNG";
+  }
+  get height() {
+    return this.config.height ?? 128;
+  }
+  get ipfGateway() {
+    return this.config.ipfsGateway ?? "5d7b6582.beta.decentralnetworkservices.com";
+  }
+  get maxAsyncProcesses() {
+    return this.config.maxAsyncProcesses ?? 2;
+  }
+  get maxCacheBytes() {
+    return this.config.maxCacheBytes ?? 1024 * 1024 * 16;
+  }
+  get maxCacheEntries() {
+    return this.config.maxCacheEntries ?? 500;
+  }
+  get quality() {
+    return this.config.quality ?? 50;
+  }
+  get width() {
+    return this.config.width ?? 128;
+  }
+  static async binaryToSha256(data) {
+    await import_core3.PayloadHasher.wasmInitialized;
+    if (import_core3.PayloadHasher.wasmSupport.canUseWasm) {
+      try {
+        return await (0, import_hash_wasm.sha256)(data);
+      } catch (ex) {
+        import_core3.PayloadHasher.wasmSupport.allowWasm = false;
+      }
+    }
+    return (0, import_sha.default)("sha256").update(data).digest().toString();
+  }
+  static bufferFromDataUrl(url) {
+    if (url.startsWith("data:image")) {
+      const data = url.split(",")[1];
+      if (data) {
+        return Buffer.from(Uint8Array.from(atob(data), (c) => c.charCodeAt(0)));
+      } else {
+        const error = {
+          message: "Invalid data Url",
+          name: "ImageThumbnailWitnessError",
+          url
+        };
+        throw error;
+      }
+    }
+  }
+  /**
+   * Returns the equivalent IPFS gateway URL for the supplied URL.
+   * @param urlToCheck The URL to check
+   * @returns If the supplied URL is an IPFS URL, it converts the URL to the
+   * equivalent IPFS gateway URL. Otherwise, returns the original URL.
+   */
+  checkIpfsUrl(urlToCheck) {
+    const url = new import_url.URL(urlToCheck);
+    let protocol = url.protocol;
+    let host = url.host;
+    let path = url.pathname;
+    const query = url.search;
+    if (protocol === "ipfs:") {
+      protocol = "https:";
+      host = this.ipfGateway;
+      path = url.host === "ipfs" ? `ipfs${path}` : `ipfs/${url.host}${path}`;
+      const root = `${protocol}//${host}/${path}`;
+      return (query == null ? void 0 : query.length) > 0 ? `${root}?${query}` : root;
+    } else {
+      return urlToCheck;
+    }
+  }
+  async observeHandler(payloads = []) {
+    if (!import_hasbin.default.sync("magick")) {
+      throw Error("ImageMagick is required for this witness");
+    }
+    const urlPayloads = payloads.filter((payload) => payload.schema === import_url_payload_plugin.UrlSchema);
+    return await this._semaphore.runExclusive(
+      async () => (0, import_lodash2.compact)(
+        await Promise.all(
+          urlPayloads.map(async ({ url }) => {
+            const cachedResult = this.cache.get(url);
+            if (cachedResult) {
+              return cachedResult;
+            }
+            let result;
+            const dataBuffer = _ImageThumbnailWitness.bufferFromDataUrl(url);
+            if (dataBuffer) {
+              result = {
+                schema: import_image_thumbnail_payload_plugin4.ImageThumbnailSchema,
+                sourceHash: await _ImageThumbnailWitness.binaryToSha256(dataBuffer),
+                sourceUrl: url,
+                url
+              };
+            } else {
+              const mutatedUrl = this.checkIpfsUrl(url);
+              result = await this.fromHttp(mutatedUrl, url);
+            }
+            this.cache.set(url, result);
+            return result;
+          })
+        )
+      )
+    );
+  }
+  async createThumbnailDataUrl(sourceBuffer, encoding) {
+    const thumb = await new Promise((resolve, reject) => {
+      gm(sourceBuffer).quality(this.quality).resize(this.width, this.height).flatten().toBuffer(encoding ?? this.encoding, (error, buffer) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve(buffer);
+        }
+      });
+    });
+    return `data:image/png;base64,${thumb.toString("base64")}`;
+  }
+  /**
+   * Creates an image thumbnail from a video.
+   * @param videoBuffer The input video buffer.
+   * @returns An buffer containing an image thumbnail for the video.
+   */
+  async createThumbnailFromVideo(videoBuffer) {
+    const imageBuffer = await getVideoFrameAsImageFluent(videoBuffer);
+    return this.createThumbnailDataUrl(imageBuffer);
+  }
+  async fromHttp(url, sourceUrl) {
+    var _a, _b, _c, _d, _e, _f;
+    let response;
+    let dnsResult;
+    try {
+      const urlObj = new import_url_parse.default(url);
+      dnsResult = await import_node_dns.promises.resolve(urlObj.host);
+    } catch (ex) {
+      const error = ex;
+      const result2 = {
+        http: {
+          dnsError: error.code
+        },
+        schema: import_image_thumbnail_payload_plugin4.ImageThumbnailSchema,
+        sourceUrl: sourceUrl ?? url
+      };
+      return result2;
+    }
+    try {
+      response = await import_axios.axios.get(url, {
+        responseType: "arraybuffer"
+      });
+    } catch (ex) {
+      const axiosError = ex;
+      if (axiosError.isAxiosError) {
+        const result2 = {
+          http: {
+            ipAddress: dnsResult[0],
+            status: (_a = axiosError == null ? void 0 : axiosError.response) == null ? void 0 : _a.status
+          },
+          schema: import_image_thumbnail_payload_plugin4.ImageThumbnailSchema,
+          sourceUrl: sourceUrl ?? url
+        };
+        return result2;
+      } else {
+        throw ex;
+      }
+    }
+    const result = {
+      http: {
+        status: response.status
+      },
+      schema: import_image_thumbnail_payload_plugin4.ImageThumbnailSchema,
+      sourceUrl: sourceUrl ?? url
+    };
+    if (response.status >= 200 && response.status < 300) {
+      const contentType = (_b = response.headers["content-type"]) == null ? void 0 : _b.toString();
+      const [mediaType, fileType] = contentType.split("/");
+      result.mime = result.mime ?? {};
+      result.mime.returned = mediaType;
+      const sourceBuffer = Buffer.from(response.data, "binary");
+      try {
+        result.mime.detected = await import_file_type.default.fromBuffer(sourceBuffer);
+      } catch (ex) {
+        const error = ex;
+        (_c = this.logger) == null ? void 0 : _c.error(`FileType error: ${error.message}`);
+      }
+      const processImage = async (encoding2) => {
+        result.sourceHash = await _ImageThumbnailWitness.binaryToSha256(sourceBuffer);
+        result.url = await this.createThumbnailDataUrl(sourceBuffer, encoding2);
+      };
+      const processVideo = async () => {
+        if (import_hasbin.default.sync("ffmpeg")) {
+          result.sourceHash = await _ImageThumbnailWitness.binaryToSha256(sourceBuffer);
+          result.url = await this.createThumbnailFromVideo(sourceBuffer);
+        } else {
+          result.mime = result.mime ?? {};
+          result.mime.invalid = true;
+        }
+      };
+      let encoding = "PNG";
+      switch (fileType.toUpperCase()) {
+        case "GIF":
+          encoding = "GIF";
+          break;
+        case "JPG":
+        case "JPEG":
+          encoding = "JPG";
+          break;
+      }
+      switch (mediaType) {
+        case "image": {
+          await processImage(encoding);
+          result.mime.type = mediaType;
+          break;
+        }
+        case "video": {
+          await processVideo();
+          result.mime.type = mediaType;
+          break;
+        }
+        default: {
+          switch ((_d = result.mime.detected) == null ? void 0 : _d.mime) {
+            case "image": {
+              await processImage();
+              result.mime.type = (_e = result.mime.detected) == null ? void 0 : _e.mime;
+              break;
+            }
+            case "video": {
+              await processVideo();
+              result.mime.type = (_f = result.mime.detected) == null ? void 0 : _f.mime;
+              break;
+            }
+            default: {
+              result.mime.invalid = true;
+              break;
+            }
+          }
+          break;
+        }
+      }
+    }
+    return result;
+  }
+};
+
+// src/Plugin.ts
+var ImageThumbnailPlugin = () => (0, import_payloadset_plugin.createPayloadSetDualPlugin)(
+  { required: { [import_image_thumbnail_payload_plugin5.ImageThumbnailSchema]: 1 }, schema: import_payload_model.PayloadSetSchema },
+  {
+    diviner: async (params) => {
+      const result = await ImageThumbnailDiviner.create(params);
+      return result;
+    },
+    witness: async (params) => {
+      const result = await ImageThumbnailWitness.create(params);
+      return result;
+    }
+  }
+);
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ImageThumbnailPlugin
+});
+//# sourceMappingURL=Plugin.js.map