@iconify/tools 4.0.0-beta.2 → 4.0.0-beta.3

package/lib/import/figma/query.cjs

@@ -2,6 +2,9 @@
 
 const download_api_index = require('../../download/api/index.cjs');
 const download_api_cache = require('../../download/api/cache.cjs');
+const download_api_queue = require('../../download/api/queue.cjs');
+require('axios');
+require('../../download/api/config.cjs');
 require('fs');
 require('crypto');
 require('../../misc/scan.cjs');
@@ -113,78 +116,92 @@ async function figmaImagesQuery(options, nodes, cache) {
   const uri = "https://api.figma.com/v1/images/" + options.file;
   const maxLength = 2048 - uri.length;
   const svgOptions = options.svgOptions || {};
-  let ids = [];
-  let idsLength = 0;
-  let lastError;
-  let found = 0;
-  const query = async () => {
-    const params = new URLSearchParams({
-      ids: ids.join(","),
-      format: "svg"
-    });
-    if (options.version) {
-      params.set("version", options.version);
-    }
-    if (svgOptions.includeID) {
-      params.set("svg_include_id", "true");
-    }
-    if (svgOptions.simplifyStroke) {
-      params.set("svg_simplify_stroke", "true");
-    }
-    if (svgOptions.useAbsoluteBounds) {
-      params.set("use_absolute_bounds", "true");
-    }
-    const data = await download_api_index.sendAPIQuery(
-      {
-        uri,
-        params,
-        headers: {
-          "X-FIGMA-TOKEN": options.token
+  const query = (ids2) => {
+    return new Promise((resolve, reject) => {
+      const params = new URLSearchParams({
+        ids: ids2.join(","),
+        format: "svg"
+      });
+      if (options.version) {
+        params.set("version", options.version);
+      }
+      if (svgOptions.includeID) {
+        params.set("svg_include_id", "true");
+      }
+      if (svgOptions.simplifyStroke) {
+        params.set("svg_simplify_stroke", "true");
+      }
+      if (svgOptions.useAbsoluteBounds) {
+        params.set("use_absolute_bounds", "true");
+      }
+      download_api_index.sendAPIQuery(
+        {
+          uri,
+          params,
+          headers: {
+            "X-FIGMA-TOKEN": options.token
+          }
+        },
+        cache
+      ).then((data) => {
+        if (typeof data === "number") {
+          reject(data);
+          return;
         }
-      },
-      cache
-    );
-    if (typeof data === "number") {
-      lastError = data;
-      return;
-    }
-    try {
-      const parsedData = JSON.parse(data);
-      const images = parsedData.images;
-      for (const id in images) {
-        const node = nodes.icons[id];
-        const target = images[id];
-        if (node && target) {
-          node.url = target;
-          found++;
+        let parsedData;
+        try {
+          parsedData = JSON.parse(data);
+        } catch {
+          reject("Bad API response");
+          return;
         }
-      }
-    } catch (err) {
-      return;
-    }
+        resolve(parsedData);
+      }).catch(reject);
+    });
   };
+  let ids = [];
+  let idsLength = 0;
   const allKeys = Object.keys(nodes.icons);
+  const queue = [];
   for (let i = 0; i < allKeys.length; i++) {
     const id = allKeys[i];
     ids.push(id);
     idsLength += id.length + 1;
     if (idsLength >= maxLength) {
-      await query();
+      queue.push(ids.slice(0));
       ids = [];
       idsLength = 0;
     }
   }
   if (idsLength) {
-    await query();
+    queue.push(ids.slice(0));
   }
-  if (!found) {
-    if (lastError) {
-      throw new Error(
-        `Error retrieving image data from API${lastError ? ": " + lastError.toString() : ""}`
-      );
-    } else {
-      throw new Error("No valid icon layers were found");
+  const queryParams = {
+    // Params
+    total: queue.length,
+    callback: (index) => query(queue[index]),
+    // Payload to identify failed items in onfail callback
+    function: "figmaImagesQuery",
+    payload: queue
+  };
+  const results = await download_api_queue.runConcurrentQueries(queryParams);
+  let found = 0;
+  results.forEach((data) => {
+    if (!data) {
+      return;
     }
+    const images = data.images;
+    for (const id in images) {
+      const node = nodes.icons[id];
+      const target = images[id];
+      if (node && target) {
+        node.url = target;
+        found++;
+      }
+    }
+  });
+  if (!found) {
+    throw new Error("No valid icon layers were found");
   }
   nodes.generatedIconsCount = found;
   return nodes;
@@ -193,32 +210,43 @@ async function figmaDownloadImages(nodes, cache) {
   const icons = nodes.icons;
   const ids = Object.keys(icons);
   let count = 0;
-  let lastError;
+  const filtered = [];
   for (let i = 0; i < ids.length; i++) {
     const id = ids[i];
     const item = icons[id];
-    if (!item.url) {
-      continue;
-    }
-    const result = await download_api_index.sendAPIQuery(
-      {
-        uri: item.url
-      },
-      cache
-    );
-    if (typeof result === "number") {
-      lastError = result;
-      continue;
-    }
-    if (typeof result === "string") {
-      count++;
-      item.content = result;
+    if (item.url) {
+      filtered.push(item);
     }
   }
+  const params = {
+    // Params
+    total: filtered.length,
+    callback: (index) => {
+      return new Promise((resolve, reject) => {
+        const item = filtered[index];
+        download_api_index.sendAPIQuery(
+          {
+            uri: item.url
+          },
+          cache
+        ).then((data) => {
+          if (typeof data === "string") {
+            count++;
+            item.content = data;
+            resolve(void 0);
+          } else {
+            reject(data);
+          }
+        }).catch(reject);
+      });
+    },
+    // Payload to identify failed items in onfail callback
+    function: "figmaDownloadImages",
+    payload: filtered
+  };
+  await download_api_queue.runConcurrentQueries(params);
   if (!count) {
-    throw new Error(
-      `Error retrieving images${lastError ? ": " + lastError.toString() : ""}`
-    );
+    throw new Error("Error retrieving images");
   }
   nodes.downloadedIconsCount = count;
   return nodes;

package/lib/import/figma/query.d.ts

@@ -2,7 +2,7 @@ import { APICacheOptions } from '../../download/api/types.js';
 import { DocumentNotModified } from '../../download/types/modified.js';
 import { FigmaDocument } from './types/api.js';
 import { FigmaIfModifiedSinceOption, FigmaFilesQueryOptions, FigmaImagesQueryOptions } from './types/options.js';
-import { FigmaNodesImportResult } from './types/result.js';
+import { FigmaNodesImportResult, FigmaIconNode } from './types/result.js';
 import '../../icon-set/index.js';
 import '@iconify/types';
 import '@iconify/utils/lib/customisations/defaults';
@@ -12,6 +12,19 @@ import 'cheerio';
 import '@iconify/utils/lib/icon-set/tree';
 import './types/nodes.js';
 
+/**
+ * Extra parameters added to runConcurrentQueries()
+ *
+ * Can be used to identify failed items in onfail callback
+ */
+interface FigmaIconNodeWithURL extends FigmaIconNode {
+    url: string;
+}
+type FigmaConcurrentQueriesParamsFunction = 'figmaImagesQuery' | 'figmaDownloadImages';
+interface FigmaConcurrentQueriesParams<T extends FigmaConcurrentQueriesParamsFunction> {
+    function: T;
+    payload: T extends 'figmaImagesQuery' ? string[][] : FigmaIconNodeWithURL[];
+}
 /**
  * Get Figma files
  */
@@ -26,4 +39,4 @@ declare function figmaImagesQuery(options: FigmaImagesQueryOptions, nodes: Figma
  */
 declare function figmaDownloadImages(nodes: FigmaNodesImportResult, cache?: APICacheOptions): Promise<FigmaNodesImportResult>;
 
-export { figmaDownloadImages, figmaFilesQuery, figmaImagesQuery };
+export { FigmaConcurrentQueriesParams, FigmaConcurrentQueriesParamsFunction, figmaDownloadImages, figmaFilesQuery, figmaImagesQuery };

package/lib/import/figma/query.mjs

@@ -1,5 +1,8 @@
 import { sendAPIQuery } from '../../download/api/index.mjs';
 import { apiCacheKey, getAPICache, clearAPICache } from '../../download/api/cache.mjs';
+import { runConcurrentQueries } from '../../download/api/queue.mjs';
+import 'axios';
+import '../../download/api/config.mjs';
 import 'fs';
 import 'crypto';
 import '../../misc/scan.mjs';
@@ -111,78 +114,92 @@ async function figmaImagesQuery(options, nodes, cache) {
   const uri = "https://api.figma.com/v1/images/" + options.file;
   const maxLength = 2048 - uri.length;
   const svgOptions = options.svgOptions || {};
-  let ids = [];
-  let idsLength = 0;
-  let lastError;
-  let found = 0;
-  const query = async () => {
-    const params = new URLSearchParams({
-      ids: ids.join(","),
-      format: "svg"
-    });
-    if (options.version) {
-      params.set("version", options.version);
-    }
-    if (svgOptions.includeID) {
-      params.set("svg_include_id", "true");
-    }
-    if (svgOptions.simplifyStroke) {
-      params.set("svg_simplify_stroke", "true");
-    }
-    if (svgOptions.useAbsoluteBounds) {
-      params.set("use_absolute_bounds", "true");
-    }
-    const data = await sendAPIQuery(
-      {
-        uri,
-        params,
-        headers: {
-          "X-FIGMA-TOKEN": options.token
+  const query = (ids2) => {
+    return new Promise((resolve, reject) => {
+      const params = new URLSearchParams({
+        ids: ids2.join(","),
+        format: "svg"
+      });
+      if (options.version) {
+        params.set("version", options.version);
+      }
+      if (svgOptions.includeID) {
+        params.set("svg_include_id", "true");
+      }
+      if (svgOptions.simplifyStroke) {
+        params.set("svg_simplify_stroke", "true");
+      }
+      if (svgOptions.useAbsoluteBounds) {
+        params.set("use_absolute_bounds", "true");
+      }
+      sendAPIQuery(
+        {
+          uri,
+          params,
+          headers: {
+            "X-FIGMA-TOKEN": options.token
+          }
+        },
+        cache
+      ).then((data) => {
+        if (typeof data === "number") {
+          reject(data);
+          return;
         }
-      },
-      cache
-    );
-    if (typeof data === "number") {
-      lastError = data;
-      return;
-    }
-    try {
-      const parsedData = JSON.parse(data);
-      const images = parsedData.images;
-      for (const id in images) {
-        const node = nodes.icons[id];
-        const target = images[id];
-        if (node && target) {
-          node.url = target;
-          found++;
+        let parsedData;
+        try {
+          parsedData = JSON.parse(data);
+        } catch {
+          reject("Bad API response");
+          return;
         }
-      }
-    } catch (err) {
-      return;
-    }
+        resolve(parsedData);
+      }).catch(reject);
+    });
   };
+  let ids = [];
+  let idsLength = 0;
   const allKeys = Object.keys(nodes.icons);
+  const queue = [];
   for (let i = 0; i < allKeys.length; i++) {
     const id = allKeys[i];
     ids.push(id);
     idsLength += id.length + 1;
     if (idsLength >= maxLength) {
-      await query();
+      queue.push(ids.slice(0));
       ids = [];
       idsLength = 0;
     }
   }
   if (idsLength) {
-    await query();
+    queue.push(ids.slice(0));
   }
-  if (!found) {
-    if (lastError) {
-      throw new Error(
-        `Error retrieving image data from API${lastError ? ": " + lastError.toString() : ""}`
-      );
-    } else {
-      throw new Error("No valid icon layers were found");
+  const queryParams = {
+    // Params
+    total: queue.length,
+    callback: (index) => query(queue[index]),
+    // Payload to identify failed items in onfail callback
+    function: "figmaImagesQuery",
+    payload: queue
+  };
+  const results = await runConcurrentQueries(queryParams);
+  let found = 0;
+  results.forEach((data) => {
+    if (!data) {
+      return;
+    }
+    const images = data.images;
+    for (const id in images) {
+      const node = nodes.icons[id];
+      const target = images[id];
+      if (node && target) {
+        node.url = target;
+        found++;
+      }
     }
+  });
+  if (!found) {
+    throw new Error("No valid icon layers were found");
   }
   nodes.generatedIconsCount = found;
   return nodes;
@@ -191,32 +208,43 @@ async function figmaDownloadImages(nodes, cache) {
   const icons = nodes.icons;
   const ids = Object.keys(icons);
   let count = 0;
-  let lastError;
+  const filtered = [];
   for (let i = 0; i < ids.length; i++) {
     const id = ids[i];
     const item = icons[id];
-    if (!item.url) {
-      continue;
-    }
-    const result = await sendAPIQuery(
-      {
-        uri: item.url
-      },
-      cache
-    );
-    if (typeof result === "number") {
-      lastError = result;
-      continue;
-    }
-    if (typeof result === "string") {
-      count++;
-      item.content = result;
-    }
-  }
+    if (item.url) {
+      filtered.push(item);
+    }
+  }
+  const params = {
+    // Params
+    total: filtered.length,
+    callback: (index) => {
+      return new Promise((resolve, reject) => {
+        const item = filtered[index];
+        sendAPIQuery(
+          {
+            uri: item.url
+          },
+          cache
+        ).then((data) => {
+          if (typeof data === "string") {
+            count++;
+            item.content = data;
+            resolve(void 0);
+          } else {
+            reject(data);
+          }
+        }).catch(reject);
+      });
+    },
+    // Payload to identify failed items in onfail callback
+    function: "figmaDownloadImages",
+    payload: filtered
+  };
+  await runConcurrentQueries(params);
   if (!count) {
-    throw new Error(
-      `Error retrieving images${lastError ? ": " + lastError.toString() : ""}`
-    );
+    throw new Error("Error retrieving images");
   }
   nodes.downloadedIconsCount = count;
   return nodes;

package/lib/index.cjs

@@ -49,6 +49,7 @@ const download_helpers_untar = require('./download/helpers/untar.cjs');
 const misc_exec = require('./misc/exec.cjs');
 const misc_keyword = require('./misc/keyword.cjs');
 const misc_bumpVersion = require('./misc/bump-version.cjs');
+const download_api_config = require('./download/api/config.cjs');
 const download_api_index = require('./download/api/index.cjs');
 require('cheerio');
 require('@iconify/utils');
@@ -79,10 +80,10 @@ require('./import/figma/query.cjs');
 require('./download/api/cache.cjs');
 require('fs');
 require('crypto');
+require('./download/api/queue.cjs');
+require('axios');
 require('pathe');
 require('child_process');
-require('util');
-require('stream');
 require('fs/promises');
 require('extract-zip');
 require('tar');
@@ -145,4 +146,5 @@ exports.untar = download_helpers_untar.untar;
 exports.execAsync = misc_exec.execAsync;
 exports.cleanupIconKeyword = misc_keyword.cleanupIconKeyword;
 exports.bumpVersion = misc_bumpVersion.bumpVersion;
+exports.axiosConfig = download_api_config.axiosConfig;
 exports.sendAPIQuery = download_api_index.sendAPIQuery;

package/lib/index.d.ts

@@ -47,6 +47,7 @@ export { untar } from './download/helpers/untar.js';
 export { execAsync } from './misc/exec.js';
 export { cleanupIconKeyword } from './misc/keyword.js';
 export { bumpVersion } from './misc/bump-version.js';
+export { axiosConfig } from './download/api/config.js';
 export { sendAPIQuery } from './download/api/index.js';
 import 'cheerio';
 import '@iconify/types';
@@ -72,3 +73,4 @@ import 'svgo';
 import './export/helpers/custom-files.js';
 import 'fs';
 import 'child_process';
+import 'axios';

package/lib/index.mjs

@@ -47,6 +47,7 @@ export { untar } from './download/helpers/untar.mjs';
 export { execAsync } from './misc/exec.mjs';
 export { cleanupIconKeyword } from './misc/keyword.mjs';
 export { bumpVersion } from './misc/bump-version.mjs';
+export { axiosConfig } from './download/api/config.mjs';
 export { sendAPIQuery } from './download/api/index.mjs';
 import 'cheerio';
 import '@iconify/utils';
@@ -77,10 +78,10 @@ import './import/figma/query.mjs';
 import './download/api/cache.mjs';
 import 'fs';
 import 'crypto';
+import './download/api/queue.mjs';
+import 'axios';
 import 'pathe';
 import 'child_process';
-import 'util';
-import 'stream';
 import 'fs/promises';
 import 'extract-zip';
 import 'tar';

package/lib/optimise/svgo.cjs

@@ -30,7 +30,7 @@ function getSVGOPlugins(options) {
     ...options.animated ? [] : ["removeUselessStrokeAndFill"],
     // Plugins that modify shapes or are bugged when using animations
     ...options.animated || options.keepShapes ? [] : [
-      // 'removeHiddenElems', // bugged: not working before 3.0.4, created broken icons since 3.0.4
+      // 'removeHiddenElems', // bugged: not working before 3.0.4, breaks some icons since 3.0.4
       "convertShapeToPath",
       "convertEllipseToCircle",
       {

package/lib/optimise/svgo.mjs

@@ -28,7 +28,7 @@ function getSVGOPlugins(options) {
     ...options.animated ? [] : ["removeUselessStrokeAndFill"],
     // Plugins that modify shapes or are bugged when using animations
     ...options.animated || options.keepShapes ? [] : [
-      // 'removeHiddenElems', // bugged: not working before 3.0.4, created broken icons since 3.0.4
+      // 'removeHiddenElems', // bugged: not working before 3.0.4, breaks some icons since 3.0.4
       "convertShapeToPath",
       "convertEllipseToCircle",
       {

package/package.json

@@ -3,7 +3,7 @@
   "type": "module",
   "description": "Collection of functions for cleaning up and parsing SVG for Iconify project",
   "author": "Vjacheslav Trushkin",
-  "version": "4.0.0-beta.2",
+  "version": "4.0.0-beta.3",
   "publishConfig": {
     "tag": "next"
   },
@@ -21,6 +21,7 @@
     "@iconify/types": "^2.0.0",
     "@iconify/utils": "^2.1.12",
     "@types/tar": "^6.1.10",
+    "axios": "^1.6.2",
     "cheerio": "1.0.0-rc.12",
     "extract-zip": "^2.0.1",
     "local-pkg": "^0.4.3",
@@ -36,9 +37,7 @@
     "cross-env": "^7.0.3",
     "eslint": "^8.54.0",
     "eslint-config-prettier": "^8.10.0",
-    "eslint-plugin-jasmine": "^4.1.3",
     "eslint-plugin-prettier": "^4.2.1",
-    "jasmine": "^5.1.0",
     "jest": "^29.7.0",
     "prettier": "^2.8.8",
     "rimraf": "^5.0.5",
@@ -118,6 +117,11 @@
       "require": "./lib/download/api/cache.cjs",
       "import": "./lib/download/api/cache.mjs"
     },
+    "./lib/download/api/config": {
+      "types": "./lib/download/api/config.d.ts",
+      "require": "./lib/download/api/config.cjs",
+      "import": "./lib/download/api/config.mjs"
+    },
     "./lib/download/api/download": {
       "types": "./lib/download/api/download.d.ts",
       "require": "./lib/download/api/download.cjs",
@@ -133,6 +137,11 @@
       "require": "./lib/download/api/index.cjs",
       "import": "./lib/download/api/index.mjs"
     },
+    "./lib/download/api/queue": {
+      "types": "./lib/download/api/queue.d.ts",
+      "require": "./lib/download/api/queue.cjs",
+      "import": "./lib/download/api/queue.mjs"
+    },
     "./lib/download/api/types": {
       "types": "./lib/download/api/types.d.ts",
       "require": "./lib/download/api/types.cjs",
@@ -541,8 +550,7 @@
     "build": "unbuild",
     "test:jest-cjs": "jest --clearCache --config=jest.config.cjs && jest --runInBand --config=jest.config.cjs",
     "test:jest-esm": "cross-env NODE_OPTIONS=--experimental-vm-modules TEST_REMOTE=false jest --clearCache --config=jest.config.mjs && cross-env NODE_OPTIONS=--experimental-vm-modules npx jest --runInBand --config=jest.config.mjs",
-    "test:jasmine": "cross-env NODE_OPTIONS='--experimental-vm-modules --experimental-json-modules' npx jasmine",
-    "test": "npm run test:jest-cjs && npm run test:jest-esm && npm run test:jasmine",
+    "test": "npm run test:jest-cjs && npm run test:jest-esm",
     "test:ci": "cross-env TEST_REMOTE=false npm run test"
   }
 }