@iconify/tools 4.0.0-beta.2 → 4.0.0-beta.3

package/lib/download/api/config.cjs

@@ -0,0 +1,7 @@
+'use strict';
+
+const axiosConfig = {
+  // Empty by default. Add properties
+};
+
+exports.axiosConfig = axiosConfig;

package/lib/download/api/config.d.ts

@@ -0,0 +1,8 @@
+import { AxiosRequestConfig } from 'axios';
+
+/**
+ * Axios config, customisable
+ */
+declare const axiosConfig: AxiosRequestConfig;
+
+export { axiosConfig };

package/lib/download/api/config.mjs

@@ -0,0 +1,5 @@
+const axiosConfig = {
+  // Empty by default. Add properties
+};
+
+export { axiosConfig };

package/lib/download/api/download.cjs

@@ -1,21 +1,26 @@
 'use strict';
 
-const util = require('util');
-const stream = require('stream');
+const axios = require('axios');
 const promises = require('fs/promises');
+const download_api_config = require('./config.cjs');
+
+function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
+
+const axios__default = /*#__PURE__*/_interopDefaultCompat(axios);
 
-util.promisify(stream.pipeline);
 async function downloadFile(query, target) {
   const params = query.params ? query.params.toString() : "";
   const url = query.uri + (params ? "?" + params : "");
   const headers = query.headers;
-  const response = await fetch(url, {
-    headers
+  const response = await axios__default.get(url, {
+    ...download_api_config.axiosConfig,
+    headers,
+    responseType: "arraybuffer"
   });
-  if (!response.ok || !response.body) {
+  if (response.status !== 200) {
     throw new Error(`Error downloading ${url}: ${response.status}`);
   }
-  const data = await response.arrayBuffer();
+  const data = response.data;
   await promises.writeFile(target, Buffer.from(data));
 }
 

package/lib/download/api/download.mjs

@@ -1,19 +1,20 @@
-import { promisify } from 'util';
-import { pipeline } from 'stream';
+import axios from 'axios';
 import { writeFile } from 'fs/promises';
+import { axiosConfig } from './config.mjs';
 
-promisify(pipeline);
 async function downloadFile(query, target) {
   const params = query.params ? query.params.toString() : "";
   const url = query.uri + (params ? "?" + params : "");
   const headers = query.headers;
-  const response = await fetch(url, {
-    headers
+  const response = await axios.get(url, {
+    ...axiosConfig,
+    headers,
+    responseType: "arraybuffer"
   });
-  if (!response.ok || !response.body) {
+  if (response.status !== 200) {
     throw new Error(`Error downloading ${url}: ${response.status}`);
   }
-  const data = await response.arrayBuffer();
+  const data = response.data;
   await writeFile(target, Buffer.from(data));
 }
 

package/lib/download/api/index.cjs

@@ -1,10 +1,16 @@
 'use strict';
 
+const axios = require('axios');
 const download_api_cache = require('./cache.cjs');
+const download_api_config = require('./config.cjs');
 require('fs');
 require('crypto');
 require('../../misc/scan.cjs');
 
+function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
+
+const axios__default = /*#__PURE__*/_interopDefaultCompat(axios);
+
 async function sendAPIQuery(query, cache) {
   const cacheKey = cache ? download_api_cache.apiCacheKey(query) : "";
   if (cache) {
@@ -29,13 +35,18 @@ async function sendQuery(query) {
   console.log("Fetch:", url);
   const headers = query.headers;
   try {
-    const response = await fetch(url, {
-      headers
+    const response = await axios__default.get(url, {
+      ...download_api_config.axiosConfig,
+      headers,
+      responseType: "text"
     });
-    if (response.status >= 400) {
+    if (response.status !== 200) {
       return response.status;
     }
-    return await response.text();
+    if (typeof response.data !== "string") {
+      return 404;
+    }
+    return response.data;
   } catch (err) {
     return 404;
   }

package/lib/download/api/index.mjs

@@ -1,4 +1,6 @@
+import axios from 'axios';
 import { apiCacheKey, getAPICache, storeAPICache } from './cache.mjs';
+import { axiosConfig } from './config.mjs';
 import 'fs';
 import 'crypto';
 import '../../misc/scan.mjs';
@@ -27,13 +29,18 @@ async function sendQuery(query) {
   console.log("Fetch:", url);
   const headers = query.headers;
   try {
-    const response = await fetch(url, {
-      headers
+    const response = await axios.get(url, {
+      ...axiosConfig,
+      headers,
+      responseType: "text"
     });
-    if (response.status >= 400) {
+    if (response.status !== 200) {
       return response.status;
     }
-    return await response.text();
+    if (typeof response.data !== "string") {
+      return 404;
+    }
+    return response.data;
   } catch (err) {
     return 404;
   }

package/lib/download/api/queue.cjs

@@ -0,0 +1,102 @@
+'use strict';
+
+const defaultQueueParams = {
+  limit: 5,
+  retries: 3
+};
+function runConcurrentQueries(params) {
+  const allParams = {
+    ...defaultQueueParams,
+    ...params
+  };
+  const paramsWithCount = allParams;
+  const paramsWithPromises = allParams;
+  const isCallback = typeof paramsWithCount.total === "number";
+  const count = isCallback ? paramsWithCount.total : paramsWithPromises.promises.length;
+  const limit = Math.max(1, Math.min(allParams.limit || 1, count));
+  const retries = Math.max(1, allParams.retries || 1);
+  const results = Array(count).fill(void 0);
+  let nextIndex = 0;
+  const resolving = /* @__PURE__ */ new Set();
+  let rejected = false;
+  let resolved = false;
+  return new Promise((resolve, reject) => {
+    function resolvedItem() {
+      if (rejected || resolved) {
+        return;
+      }
+      if (!resolving.size && nextIndex > count) {
+        resolved = true;
+        resolve(results);
+        return;
+      }
+      if (resolving.size < limit && nextIndex <= count) {
+        startNext();
+      }
+    }
+    function fail(index, err) {
+      function done(failed) {
+        resolving.delete(index);
+        if (failed) {
+          rejected = true;
+          reject(err);
+        } else {
+          resolvedItem();
+        }
+      }
+      if (allParams.onfail) {
+        let retry;
+        try {
+          retry = allParams.onfail(index, err, params);
+        } catch (err2) {
+          err = err2;
+          done(true);
+          return;
+        }
+        if (retry instanceof Promise) {
+          retry.then(() => {
+            done(false);
+          }).catch((err2) => {
+            err = err2;
+            done(true);
+          });
+          return;
+        }
+        done(false);
+      } else {
+        done(true);
+      }
+    }
+    function run(index, retry) {
+      resolving.add(index);
+      const p = isCallback ? paramsWithCount.callback(index) : paramsWithPromises.promises[index];
+      p.then((value) => {
+        resolving.delete(index);
+        results[index] = value;
+        resolvedItem();
+      }).catch((err) => {
+        if (retry < retries) {
+          setTimeout(() => {
+            run(index, retry + 1);
+          });
+        } else if (!rejected) {
+          fail(index, err);
+        }
+      });
+    }
+    function startNext() {
+      const index = nextIndex++;
+      if (index >= count) {
+        resolvedItem();
+        return;
+      }
+      run(index, 0);
+    }
+    for (let i = 0; i < limit; i++) {
+      startNext();
+    }
+  });
+}
+
+exports.defaultQueueParams = defaultQueueParams;
+exports.runConcurrentQueries = runConcurrentQueries;

package/lib/download/api/queue.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Default parameters
+ */
+declare const defaultQueueParams: ConcurrentQueriesCommonParams<unknown>;
+/**
+ * Callback to get query
+ */
+type GetConcurrentQueryCallback<T> = (index: number) => Promise<T>;
+/**
+ * Parameters
+ */
+interface ConcurrentQueriesCommonParams<T> {
+    limit?: number;
+    retries?: number;
+    onfail?: (index: number, error: unknown, params: ConcurrentQueriesParams<T>) => void | Promise<void>;
+}
+interface ConcurrentQueriesParamsWithCount<T> extends ConcurrentQueriesCommonParams<T> {
+    total: number;
+    callback: (index: number) => Promise<T>;
+}
+interface ConcurrentQueriesParamsWithPromises<T> extends ConcurrentQueriesCommonParams<T> {
+    promises: Promise<T>[];
+}
+type ConcurrentQueriesParams<T> = ConcurrentQueriesParamsWithCount<T> | ConcurrentQueriesParamsWithPromises<T>;
+/**
+ * Runs concurrent async operations
+ */
+declare function runConcurrentQueries<T>(params: ConcurrentQueriesParams<T>): Promise<T[]>;
+
+export { ConcurrentQueriesCommonParams, ConcurrentQueriesParams, ConcurrentQueriesParamsWithCount, ConcurrentQueriesParamsWithPromises, GetConcurrentQueryCallback, defaultQueueParams, runConcurrentQueries };

package/lib/download/api/queue.mjs

@@ -0,0 +1,99 @@
+const defaultQueueParams = {
+  limit: 5,
+  retries: 3
+};
+function runConcurrentQueries(params) {
+  const allParams = {
+    ...defaultQueueParams,
+    ...params
+  };
+  const paramsWithCount = allParams;
+  const paramsWithPromises = allParams;
+  const isCallback = typeof paramsWithCount.total === "number";
+  const count = isCallback ? paramsWithCount.total : paramsWithPromises.promises.length;
+  const limit = Math.max(1, Math.min(allParams.limit || 1, count));
+  const retries = Math.max(1, allParams.retries || 1);
+  const results = Array(count).fill(void 0);
+  let nextIndex = 0;
+  const resolving = /* @__PURE__ */ new Set();
+  let rejected = false;
+  let resolved = false;
+  return new Promise((resolve, reject) => {
+    function resolvedItem() {
+      if (rejected || resolved) {
+        return;
+      }
+      if (!resolving.size && nextIndex > count) {
+        resolved = true;
+        resolve(results);
+        return;
+      }
+      if (resolving.size < limit && nextIndex <= count) {
+        startNext();
+      }
+    }
+    function fail(index, err) {
+      function done(failed) {
+        resolving.delete(index);
+        if (failed) {
+          rejected = true;
+          reject(err);
+        } else {
+          resolvedItem();
+        }
+      }
+      if (allParams.onfail) {
+        let retry;
+        try {
+          retry = allParams.onfail(index, err, params);
+        } catch (err2) {
+          err = err2;
+          done(true);
+          return;
+        }
+        if (retry instanceof Promise) {
+          retry.then(() => {
+            done(false);
+          }).catch((err2) => {
+            err = err2;
+            done(true);
+          });
+          return;
+        }
+        done(false);
+      } else {
+        done(true);
+      }
+    }
+    function run(index, retry) {
+      resolving.add(index);
+      const p = isCallback ? paramsWithCount.callback(index) : paramsWithPromises.promises[index];
+      p.then((value) => {
+        resolving.delete(index);
+        results[index] = value;
+        resolvedItem();
+      }).catch((err) => {
+        if (retry < retries) {
+          setTimeout(() => {
+            run(index, retry + 1);
+          });
+        } else if (!rejected) {
+          fail(index, err);
+        }
+      });
+    }
+    function startNext() {
+      const index = nextIndex++;
+      if (index >= count) {
+        resolvedItem();
+        return;
+      }
+      run(index, 0);
+    }
+    for (let i = 0; i < limit; i++) {
+      startNext();
+    }
+  });
+}
+
+export { defaultQueueParams, runConcurrentQueries };

package/lib/download/git/index.cjs

@@ -23,10 +23,8 @@ require('@iconify/utils/lib/icon-set/convert-info');
 require('../../icon-set/props.cjs');
 require('@iconify/utils/lib/misc/objects');
 require('@iconify/utils/lib/colors');
+require('axios');
 require('crypto');
-require('../api/download.cjs');
-require('util');
-require('stream');
 require('fs/promises');
 require('extract-zip');
 require('tar');

package/lib/download/git/index.mjs

@@ -21,10 +21,8 @@ import '@iconify/utils/lib/icon-set/convert-info';
 import '../../icon-set/props.mjs';
 import '@iconify/utils/lib/misc/objects';
 import '@iconify/utils/lib/colors';
+import 'axios';
 import 'crypto';
-import '../api/download.mjs';
-import 'util';
-import 'stream';
 import 'fs/promises';
 import 'extract-zip';
 import 'tar';

package/lib/download/git/reset.cjs

@@ -13,19 +13,17 @@ require('@iconify/utils/lib/icon-set/minify');
 require('@iconify/utils/lib/icon-set/convert-info');
 require('../../icon-set/props.cjs');
 require('@iconify/utils/lib/colors');
+require('axios');
 require('fs');
 require('crypto');
 require('pathe');
 const misc_exec = require('../../misc/exec.cjs');
-require('../api/download.cjs');
+require('fs/promises');
 require('extract-zip');
 require('tar');
 require('../../svg/parse.cjs');
 require('@iconify/utils/lib/misc/objects');
 require('child_process');
-require('util');
-require('stream');
-require('fs/promises');
 
 async function resetGitRepoContents(target) {
   await misc_exec.execAsync("git add -A", {

package/lib/download/git/reset.mjs

@@ -11,19 +11,17 @@ import '@iconify/utils/lib/icon-set/minify';
 import '@iconify/utils/lib/icon-set/convert-info';
 import '../../icon-set/props.mjs';
 import '@iconify/utils/lib/colors';
+import 'axios';
 import 'fs';
 import 'crypto';
 import 'pathe';
 import { execAsync } from '../../misc/exec.mjs';
-import '../api/download.mjs';
+import 'fs/promises';
 import 'extract-zip';
 import 'tar';
 import '../../svg/parse.mjs';
 import '@iconify/utils/lib/misc/objects';
 import 'child_process';
-import 'util';
-import 'stream';
-import 'fs/promises';
 
 async function resetGitRepoContents(target) {
   await execAsync("git add -A", {

package/lib/download/github/hash.cjs

@@ -1,10 +1,12 @@
 'use strict';
 
 const download_api_index = require('../api/index.cjs');
+require('axios');
 require('../api/cache.cjs');
 require('fs');
 require('crypto');
 require('../../misc/scan.cjs');
+require('../api/config.cjs');
 
 async function getGitHubRepoHash(options) {
   const uri = `https://api.github.com/repos/${options.user}/${options.repo}/branches/${options.branch}`;

package/lib/download/github/hash.mjs

@@ -1,8 +1,10 @@
 import { sendAPIQuery } from '../api/index.mjs';
+import 'axios';
 import '../api/cache.mjs';
 import 'fs';
 import 'crypto';
 import '../../misc/scan.mjs';
+import '../api/config.mjs';
 
 async function getGitHubRepoHash(options) {
   const uri = `https://api.github.com/repos/${options.user}/${options.repo}/branches/${options.branch}`;

package/lib/download/github/index.cjs

@@ -7,11 +7,11 @@ const download_api_download = require('../api/download.cjs');
 const download_helpers_unzip = require('../helpers/unzip.cjs');
 require('pathe');
 require('../api/index.cjs');
+require('axios');
 require('../api/cache.cjs');
 require('crypto');
 require('../../misc/scan.cjs');
-require('util');
-require('stream');
+require('../api/config.cjs');
 require('fs/promises');
 require('extract-zip');
 

package/lib/download/github/index.mjs

@@ -5,11 +5,11 @@ import { downloadFile } from '../api/download.mjs';
 import { unzip } from '../helpers/unzip.mjs';
 import 'pathe';
 import '../api/index.mjs';
+import 'axios';
 import '../api/cache.mjs';
 import 'crypto';
 import '../../misc/scan.mjs';
-import 'util';
-import 'stream';
+import '../api/config.mjs';
 import 'fs/promises';
 import 'extract-zip';
 

package/lib/download/gitlab/hash.cjs

@@ -2,10 +2,12 @@
 
 const download_api_index = require('../api/index.cjs');
 const download_gitlab_types = require('./types.cjs');
+require('axios');
 require('../api/cache.cjs');
 require('fs');
 require('crypto');
 require('../../misc/scan.cjs');
+require('../api/config.cjs');
 
 async function getGitLabRepoHash(options) {
   const uri = `${options.uri || download_gitlab_types.defaultGitLabBaseURI}/${options.project}/repository/branches/${options.branch}/`;

package/lib/download/gitlab/hash.mjs

@@ -1,9 +1,11 @@
 import { sendAPIQuery } from '../api/index.mjs';
 import { defaultGitLabBaseURI } from './types.mjs';
+import 'axios';
 import '../api/cache.mjs';
 import 'fs';
 import 'crypto';
 import '../../misc/scan.mjs';
+import '../api/config.mjs';
 
 async function getGitLabRepoHash(options) {
   const uri = `${options.uri || defaultGitLabBaseURI}/${options.project}/repository/branches/${options.branch}/`;

package/lib/download/gitlab/index.cjs

@@ -8,11 +8,11 @@ const download_api_download = require('../api/download.cjs');
 const download_helpers_unzip = require('../helpers/unzip.cjs');
 require('pathe');
 require('../api/index.cjs');
+require('axios');
 require('../api/cache.cjs');
 require('crypto');
 require('../../misc/scan.cjs');
-require('util');
-require('stream');
+require('../api/config.cjs');
 require('fs/promises');
 require('extract-zip');
 

package/lib/download/gitlab/index.mjs

@@ -6,11 +6,11 @@ import { downloadFile } from '../api/download.mjs';
 import { unzip } from '../helpers/unzip.mjs';
 import 'pathe';
 import '../api/index.mjs';
+import 'axios';
 import '../api/cache.mjs';
 import 'crypto';
 import '../../misc/scan.mjs';
-import 'util';
-import 'stream';
+import '../api/config.mjs';
 import 'fs/promises';
 import 'extract-zip';
 

package/lib/download/index.cjs

@@ -9,12 +9,12 @@ require('../export/helpers/prepare.cjs');
 require('pathe');
 require('./github/hash.cjs');
 require('./api/index.cjs');
+require('axios');
 require('./api/cache.cjs');
 require('crypto');
 require('../misc/scan.cjs');
+require('./api/config.cjs');
 require('./api/download.cjs');
-require('util');
-require('stream');
 require('fs/promises');
 require('./helpers/unzip.cjs');
 require('extract-zip');

package/lib/download/index.mjs

@@ -7,12 +7,12 @@ import '../export/helpers/prepare.mjs';
 import 'pathe';
 import './github/hash.mjs';
 import './api/index.mjs';
+import 'axios';
 import './api/cache.mjs';
 import 'crypto';
 import '../misc/scan.mjs';
+import './api/config.mjs';
 import './api/download.mjs';
-import 'util';
-import 'stream';
 import 'fs/promises';
 import './helpers/unzip.mjs';
 import 'extract-zip';

package/lib/download/npm/index.cjs

@@ -6,9 +6,9 @@ const download_api_download = require('../api/download.cjs');
 const download_helpers_untar = require('../helpers/untar.cjs');
 const download_npm_version = require('./version.cjs');
 require('pathe');
-require('util');
-require('stream');
+require('axios');
 require('fs/promises');
+require('../api/config.cjs');
 require('tar');
 require('../../misc/exec.cjs');
 require('child_process');

package/lib/download/npm/index.mjs

@@ -4,9 +4,9 @@ import { downloadFile } from '../api/download.mjs';
 import { untar } from '../helpers/untar.mjs';
 import { getNPMVersion, getPackageVersion } from './version.mjs';
 import 'pathe';
-import 'util';
-import 'stream';
+import 'axios';
 import 'fs/promises';
+import '../api/config.mjs';
 import 'tar';
 import '../../misc/exec.mjs';
 import 'child_process';

package/lib/import/figma/index.cjs

@@ -34,10 +34,13 @@ require('../../optimise/svgo.cjs');
 require('svgo');
 require('@iconify/utils/lib/svg/id');
 require('../../download/api/index.cjs');
+require('axios');
 require('../../download/api/cache.cjs');
 require('fs');
 require('crypto');
 require('../../misc/scan.cjs');
+require('../../download/api/config.cjs');
+require('../../download/api/queue.cjs');
 
 async function importFromFigma(options) {
   const cacheOptions = options.cacheDir ? {

package/lib/import/figma/index.mjs

@@ -32,10 +32,13 @@ import '../../optimise/svgo.mjs';
 import 'svgo';
 import '@iconify/utils/lib/svg/id';
 import '../../download/api/index.mjs';
+import 'axios';
 import '../../download/api/cache.mjs';
 import 'fs';
 import 'crypto';
 import '../../misc/scan.mjs';
+import '../../download/api/config.mjs';
+import '../../download/api/queue.mjs';
 
 async function importFromFigma(options) {
   const cacheOptions = options.cacheDir ? {