@iconify/tools 4.0.0-dev.1 → 4.0.0

package/lib/download/api/index.cjs

@@ -1,10 +1,16 @@
 'use strict';
 
+const axios = require('axios');
 const download_api_cache = require('./cache.cjs');
+const download_api_config = require('./config.cjs');
 require('fs');
 require('crypto');
 require('../../misc/scan.cjs');
 
+function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
+
+const axios__default = /*#__PURE__*/_interopDefaultCompat(axios);
+
 async function sendAPIQuery(query, cache) {
   const cacheKey = cache ? download_api_cache.apiCacheKey(query) : "";
   if (cache) {
@@ -26,18 +32,28 @@ async function sendAPIQuery(query, cache) {
 async function sendQuery(query) {
   const params = query.params ? query.params.toString() : "";
   const url = query.uri + (params ? "?" + params : "");
-  console.log("Fetch:", url);
   const headers = query.headers;
+  download_api_config.fetchCallbacks.onStart?.(url, query);
+  function fail(value) {
+    download_api_config.fetchCallbacks.onError?.(url, query, value);
+    return value ?? 404;
+  }
   try {
-    const response = await fetch(url, {
-      headers
+    const response = await axios__default.get(url, {
+      ...download_api_config.axiosConfig,
+      headers,
+      responseType: "text"
     });
-    if (response.status >= 400) {
-      return response.status;
+    if (response.status !== 200) {
+      return fail(response.status);
+    }
+    if (typeof response.data !== "string") {
+      return fail();
     }
-    return await response.text();
+    download_api_config.fetchCallbacks.onSuccess?.(url, query);
+    return response.data;
   } catch (err) {
-    return 404;
+    return fail();
   }
 }
 

package/lib/download/api/index.d.cts

@@ -0,0 +1,8 @@
+import { APIQueryParams, APICacheOptions } from './types.cjs';
+
+/**
+ * Send API query
+ */
+declare function sendAPIQuery(query: APIQueryParams, cache?: APICacheOptions): Promise<number | string>;
+
+export { sendAPIQuery };

package/lib/download/api/index.d.mts

@@ -0,0 +1,8 @@
+import { APIQueryParams, APICacheOptions } from './types.mjs';
+
+/**
+ * Send API query
+ */
+declare function sendAPIQuery(query: APIQueryParams, cache?: APICacheOptions): Promise<number | string>;
+
+export { sendAPIQuery };

package/lib/download/api/index.mjs

@@ -1,4 +1,6 @@
+import axios from 'axios';
 import { apiCacheKey, getAPICache, storeAPICache } from './cache.mjs';
+import { fetchCallbacks, axiosConfig } from './config.mjs';
 import 'fs';
 import 'crypto';
 import '../../misc/scan.mjs';
@@ -24,18 +26,28 @@ async function sendAPIQuery(query, cache) {
 async function sendQuery(query) {
   const params = query.params ? query.params.toString() : "";
   const url = query.uri + (params ? "?" + params : "");
-  console.log("Fetch:", url);
   const headers = query.headers;
+  fetchCallbacks.onStart?.(url, query);
+  function fail(value) {
+    fetchCallbacks.onError?.(url, query, value);
+    return value ?? 404;
+  }
   try {
-    const response = await fetch(url, {
-      headers
+    const response = await axios.get(url, {
+      ...axiosConfig,
+      headers,
+      responseType: "text"
     });
-    if (response.status >= 400) {
-      return response.status;
+    if (response.status !== 200) {
+      return fail(response.status);
+    }
+    if (typeof response.data !== "string") {
+      return fail();
     }
-    return await response.text();
+    fetchCallbacks.onSuccess?.(url, query);
+    return response.data;
   } catch (err) {
-    return 404;
+    return fail();
   }
 }
 

package/lib/download/api/queue.cjs

@@ -0,0 +1,104 @@
+'use strict';
+
+const defaultQueueParams = {
+  limit: 5,
+  retries: 3
+};
+function runConcurrentQueries(params) {
+  const allParams = {
+    ...defaultQueueParams,
+    ...params
+  };
+  const paramsWithCount = allParams;
+  const paramsWithPromises = allParams;
+  const isCallback = typeof paramsWithCount.total === "number";
+  const count = isCallback ? paramsWithCount.total : paramsWithPromises.promises.length;
+  const limit = Math.max(1, Math.min(allParams.limit || 1, count));
+  const retries = Math.max(1, allParams.retries || 1);
+  const results = Array(count).fill(void 0);
+  let nextIndex = 0;
+  const resolving = /* @__PURE__ */ new Set();
+  let rejected = false;
+  let resolved = false;
+  return new Promise((resolve, reject) => {
+    function resolvedItem() {
+      if (rejected || resolved) {
+        return;
+      }
+      if (!resolving.size && nextIndex > count) {
+        resolved = true;
+        resolve(results);
+        return;
+      }
+      if (resolving.size < limit && nextIndex <= count) {
+        startNext();
+      }
+    }
+    function fail(index, err) {
+      function done(failed) {
+        resolving.delete(index);
+        if (failed) {
+          rejected = true;
+          reject(err);
+        } else {
+          resolvedItem();
+        }
+      }
+      if (allParams.onError) {
+        let retry;
+        try {
+          retry = allParams.onError(index, err, params);
+        } catch (err2) {
+          err = err2;
+          done(true);
+          return;
+        }
+        if (retry instanceof Promise) {
+          retry.then(() => {
+            done(false);
+          }).catch((err2) => {
+            err = err2;
+            done(true);
+          });
+          return;
+        }
+        done(false);
+      } else {
+        done(true);
+      }
+    }
+    function run(index, retry) {
+      resolving.add(index);
+      const p = isCallback ? paramsWithCount.callback(index) : paramsWithPromises.promises[index];
+      allParams.onStart?.(index, params);
+      p.then((value) => {
+        resolving.delete(index);
+        results[index] = value;
+        allParams.onSuccess?.(index, params, value);
+        resolvedItem();
+      }).catch((err) => {
+        if (retry < retries) {
+          setTimeout(() => {
+            run(index, retry + 1);
+          });
+        } else if (!rejected) {
+          fail(index, err);
+        }
+      });
+    }
+    function startNext() {
+      const index = nextIndex++;
+      if (index >= count) {
+        resolvedItem();
+        return;
+      }
+      run(index, 0);
+    }
+    for (let i = 0; i < limit; i++) {
+      startNext();
+    }
+  });
+}
+
+exports.defaultQueueParams = defaultQueueParams;
+exports.runConcurrentQueries = runConcurrentQueries;

package/lib/download/api/queue.d.cts

@@ -0,0 +1,32 @@
+/**
+ * Default parameters
+ */
+declare const defaultQueueParams: ConcurrentQueriesCommonParams<unknown>;
+/**
+ * Callback to get query
+ */
+type GetConcurrentQueryCallback<T> = (index: number) => Promise<T>;
+/**
+ * Parameters
+ */
+interface ConcurrentQueriesCommonParams<T> {
+    limit?: number;
+    retries?: number;
+    onError?: (index: number, error: unknown, params: ConcurrentQueriesParams<T>) => void | Promise<void>;
+    onSuccess?: (index: number, params: ConcurrentQueriesParams<T>, result: T) => void;
+    onStart?: (index: number, params: ConcurrentQueriesParams<T>) => void;
+}
+interface ConcurrentQueriesParamsWithCount<T> extends ConcurrentQueriesCommonParams<T> {
+    total: number;
+    callback: (index: number) => Promise<T>;
+}
+interface ConcurrentQueriesParamsWithPromises<T> extends ConcurrentQueriesCommonParams<T> {
+    promises: Promise<T>[];
+}
+type ConcurrentQueriesParams<T> = ConcurrentQueriesParamsWithCount<T> | ConcurrentQueriesParamsWithPromises<T>;
+/**
+ * Runs concurrent async operations
+ */
+declare function runConcurrentQueries<T>(params: ConcurrentQueriesParams<T>): Promise<T[]>;
+
+export { type ConcurrentQueriesCommonParams, type ConcurrentQueriesParams, type ConcurrentQueriesParamsWithCount, type ConcurrentQueriesParamsWithPromises, type GetConcurrentQueryCallback, defaultQueueParams, runConcurrentQueries };

package/lib/download/api/queue.d.mts

@@ -0,0 +1,32 @@
+/**
+ * Default parameters
+ */
+declare const defaultQueueParams: ConcurrentQueriesCommonParams<unknown>;
+/**
+ * Callback to get query
+ */
+type GetConcurrentQueryCallback<T> = (index: number) => Promise<T>;
+/**
+ * Parameters
+ */
+interface ConcurrentQueriesCommonParams<T> {
+    limit?: number;
+    retries?: number;
+    onError?: (index: number, error: unknown, params: ConcurrentQueriesParams<T>) => void | Promise<void>;
+    onSuccess?: (index: number, params: ConcurrentQueriesParams<T>, result: T) => void;
+    onStart?: (index: number, params: ConcurrentQueriesParams<T>) => void;
+}
+interface ConcurrentQueriesParamsWithCount<T> extends ConcurrentQueriesCommonParams<T> {
+    total: number;
+    callback: (index: number) => Promise<T>;
+}
+interface ConcurrentQueriesParamsWithPromises<T> extends ConcurrentQueriesCommonParams<T> {
+    promises: Promise<T>[];
+}
+type ConcurrentQueriesParams<T> = ConcurrentQueriesParamsWithCount<T> | ConcurrentQueriesParamsWithPromises<T>;
+/**
+ * Runs concurrent async operations
+ */
+declare function runConcurrentQueries<T>(params: ConcurrentQueriesParams<T>): Promise<T[]>;
+
+export { type ConcurrentQueriesCommonParams, type ConcurrentQueriesParams, type ConcurrentQueriesParamsWithCount, type ConcurrentQueriesParamsWithPromises, type GetConcurrentQueryCallback, defaultQueueParams, runConcurrentQueries };

package/lib/download/api/queue.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Default parameters
+ */
+declare const defaultQueueParams: ConcurrentQueriesCommonParams<unknown>;
+/**
+ * Callback to get query
+ */
+type GetConcurrentQueryCallback<T> = (index: number) => Promise<T>;
+/**
+ * Parameters
+ */
+interface ConcurrentQueriesCommonParams<T> {
+    limit?: number;
+    retries?: number;
+    onError?: (index: number, error: unknown, params: ConcurrentQueriesParams<T>) => void | Promise<void>;
+    onSuccess?: (index: number, params: ConcurrentQueriesParams<T>, result: T) => void;
+    onStart?: (index: number, params: ConcurrentQueriesParams<T>) => void;
+}
+interface ConcurrentQueriesParamsWithCount<T> extends ConcurrentQueriesCommonParams<T> {
+    total: number;
+    callback: (index: number) => Promise<T>;
+}
+interface ConcurrentQueriesParamsWithPromises<T> extends ConcurrentQueriesCommonParams<T> {
+    promises: Promise<T>[];
+}
+type ConcurrentQueriesParams<T> = ConcurrentQueriesParamsWithCount<T> | ConcurrentQueriesParamsWithPromises<T>;
+/**
+ * Runs concurrent async operations
+ */
+declare function runConcurrentQueries<T>(params: ConcurrentQueriesParams<T>): Promise<T[]>;
+
+export { type ConcurrentQueriesCommonParams, type ConcurrentQueriesParams, type ConcurrentQueriesParamsWithCount, type ConcurrentQueriesParamsWithPromises, type GetConcurrentQueryCallback, defaultQueueParams, runConcurrentQueries };

package/lib/download/api/queue.mjs

@@ -0,0 +1,101 @@
+const defaultQueueParams = {
+  limit: 5,
+  retries: 3
+};
+function runConcurrentQueries(params) {
+  const allParams = {
+    ...defaultQueueParams,
+    ...params
+  };
+  const paramsWithCount = allParams;
+  const paramsWithPromises = allParams;
+  const isCallback = typeof paramsWithCount.total === "number";
+  const count = isCallback ? paramsWithCount.total : paramsWithPromises.promises.length;
+  const limit = Math.max(1, Math.min(allParams.limit || 1, count));
+  const retries = Math.max(1, allParams.retries || 1);
+  const results = Array(count).fill(void 0);
+  let nextIndex = 0;
+  const resolving = /* @__PURE__ */ new Set();
+  let rejected = false;
+  let resolved = false;
+  return new Promise((resolve, reject) => {
+    function resolvedItem() {
+      if (rejected || resolved) {
+        return;
+      }
+      if (!resolving.size && nextIndex > count) {
+        resolved = true;
+        resolve(results);
+        return;
+      }
+      if (resolving.size < limit && nextIndex <= count) {
+        startNext();
+      }
+    }
+    function fail(index, err) {
+      function done(failed) {
+        resolving.delete(index);
+        if (failed) {
+          rejected = true;
+          reject(err);
+        } else {
+          resolvedItem();
+        }
+      }
+      if (allParams.onError) {
+        let retry;
+        try {
+          retry = allParams.onError(index, err, params);
+        } catch (err2) {
+          err = err2;
+          done(true);
+          return;
+        }
+        if (retry instanceof Promise) {
+          retry.then(() => {
+            done(false);
+          }).catch((err2) => {
+            err = err2;
+            done(true);
+          });
+          return;
+        }
+        done(false);
+      } else {
+        done(true);
+      }
+    }
+    function run(index, retry) {
+      resolving.add(index);
+      const p = isCallback ? paramsWithCount.callback(index) : paramsWithPromises.promises[index];
+      allParams.onStart?.(index, params);
+      p.then((value) => {
+        resolving.delete(index);
+        results[index] = value;
+        allParams.onSuccess?.(index, params, value);
+        resolvedItem();
+      }).catch((err) => {
+        if (retry < retries) {
+          setTimeout(() => {
+            run(index, retry + 1);
+          });
+        } else if (!rejected) {
+          fail(index, err);
+        }
+      });
+    }
+    function startNext() {
+      const index = nextIndex++;
+      if (index >= count) {
+        resolvedItem();
+        return;
+      }
+      run(index, 0);
+    }
+    for (let i = 0; i < limit; i++) {
+      startNext();
+    }
+  });
+}
+
+export { defaultQueueParams, runConcurrentQueries };

package/lib/download/api/types.d.cts

@@ -0,0 +1,17 @@
+/**
+ * API Cache
+ */
+interface APICacheOptions {
+    dir: string;
+    ttl: number;
+}
+/**
+ * Params
+ */
+interface APIQueryParams {
+    uri: string;
+    params?: URLSearchParams;
+    headers?: Record<string, string>;
+}
+
+export type { APICacheOptions, APIQueryParams };

package/lib/download/api/types.d.mts

@@ -0,0 +1,17 @@
+/**
+ * API Cache
+ */
+interface APICacheOptions {
+    dir: string;
+    ttl: number;
+}
+/**
+ * Params
+ */
+interface APIQueryParams {
+    uri: string;
+    params?: URLSearchParams;
+    headers?: Record<string, string>;
+}
+
+export type { APICacheOptions, APIQueryParams };

package/lib/download/api/types.d.ts

@@ -14,4 +14,4 @@ interface APIQueryParams {
     headers?: Record<string, string>;
 }
 
-export { APICacheOptions, APIQueryParams };
+export type { APICacheOptions, APIQueryParams };

package/lib/download/git/branch.d.cts

@@ -0,0 +1,8 @@
+import { ExportTargetOptions } from '../../export/helpers/prepare.cjs';
+
+/**
+ * Get current branch from cloned git repo
+ */
+declare function getGitRepoBranch(options: ExportTargetOptions, checkout?: string): Promise<string>;
+
+export { getGitRepoBranch };

package/lib/download/git/branch.d.mts

@@ -0,0 +1,8 @@
+import { ExportTargetOptions } from '../../export/helpers/prepare.mjs';
+
+/**
+ * Get current branch from cloned git repo
+ */
+declare function getGitRepoBranch(options: ExportTargetOptions, checkout?: string): Promise<string>;
+
+export { getGitRepoBranch };

package/lib/download/git/hash.d.cts

@@ -0,0 +1,8 @@
+import { ExportTargetOptions } from '../../export/helpers/prepare.cjs';
+
+/**
+ * Get latest hash from cloned git repo
+ */
+declare function getGitRepoHash(options: ExportTargetOptions): Promise<string>;
+
+export { getGitRepoHash };

package/lib/download/git/hash.d.mts

@@ -0,0 +1,8 @@
+import { ExportTargetOptions } from '../../export/helpers/prepare.mjs';
+
+/**
+ * Get latest hash from cloned git repo
+ */
+declare function getGitRepoHash(options: ExportTargetOptions): Promise<string>;
+
+export { getGitRepoHash };

package/lib/download/git/index.cjs

@@ -23,17 +23,15 @@ require('@iconify/utils/lib/icon-set/convert-info');
 require('../../icon-set/props.cjs');
 require('@iconify/utils/lib/misc/objects');
 require('@iconify/utils/lib/colors');
+require('axios');
 require('crypto');
-require('../api/download.cjs');
-require('util');
-require('stream');
 require('fs/promises');
 require('extract-zip');
 require('tar');
 
 async function downloadGitRepo(options) {
   const { remote, branch } = options;
-  const hasHashInTarget = options.target.indexOf("{hash}") !== -1;
+  const hasHashInTarget = options.target.includes("{hash}");
   const ifModifiedSince = options.ifModifiedSince;
   if (ifModifiedSince || hasHashInTarget) {
     const result = await misc_exec.execAsync(

package/lib/download/git/index.d.cts

@@ -0,0 +1,29 @@
+import { ExportTargetOptions } from '../../export/helpers/prepare.cjs';
+import { DocumentNotModified } from '../types/modified.cjs';
+import { DownloadSourceMixin } from '../types/sources.cjs';
+
+interface IfModifiedSinceOption {
+    ifModifiedSince: string | true | DownloadGitRepoResult;
+}
+/**
+ * Options for downloadGitRepo()
+ */
+interface DownloadGitRepoOptions extends ExportTargetOptions, Partial<IfModifiedSinceOption> {
+    remote: string;
+    branch: string;
+    log?: boolean;
+}
+/**
+ * Result
+ */
+interface DownloadGitRepoResult extends DownloadSourceMixin<'git'> {
+    contentsDir: string;
+    hash: string;
+}
+/**
+ * Download Git repo
+ */
+declare function downloadGitRepo<T extends IfModifiedSinceOption & DownloadGitRepoOptions>(options: T): Promise<DownloadGitRepoResult | DocumentNotModified>;
+declare function downloadGitRepo(options: DownloadGitRepoOptions): Promise<DownloadGitRepoResult>;
+
+export { type DownloadGitRepoOptions, type DownloadGitRepoResult, downloadGitRepo };

package/lib/download/git/index.d.mts

@@ -0,0 +1,29 @@
+import { ExportTargetOptions } from '../../export/helpers/prepare.mjs';
+import { DocumentNotModified } from '../types/modified.mjs';
+import { DownloadSourceMixin } from '../types/sources.mjs';
+
+interface IfModifiedSinceOption {
+    ifModifiedSince: string | true | DownloadGitRepoResult;
+}
+/**
+ * Options for downloadGitRepo()
+ */
+interface DownloadGitRepoOptions extends ExportTargetOptions, Partial<IfModifiedSinceOption> {
+    remote: string;
+    branch: string;
+    log?: boolean;
+}
+/**
+ * Result
+ */
+interface DownloadGitRepoResult extends DownloadSourceMixin<'git'> {
+    contentsDir: string;
+    hash: string;
+}
+/**
+ * Download Git repo
+ */
+declare function downloadGitRepo<T extends IfModifiedSinceOption & DownloadGitRepoOptions>(options: T): Promise<DownloadGitRepoResult | DocumentNotModified>;
+declare function downloadGitRepo(options: DownloadGitRepoOptions): Promise<DownloadGitRepoResult>;
+
+export { type DownloadGitRepoOptions, type DownloadGitRepoResult, downloadGitRepo };

package/lib/download/git/index.d.ts

@@ -26,4 +26,4 @@ interface DownloadGitRepoResult extends DownloadSourceMixin<'git'> {
 declare function downloadGitRepo<T extends IfModifiedSinceOption & DownloadGitRepoOptions>(options: T): Promise<DownloadGitRepoResult | DocumentNotModified>;
 declare function downloadGitRepo(options: DownloadGitRepoOptions): Promise<DownloadGitRepoResult>;
 
-export { DownloadGitRepoOptions, DownloadGitRepoResult, downloadGitRepo };
+export { type DownloadGitRepoOptions, type DownloadGitRepoResult, downloadGitRepo };

package/lib/download/git/index.mjs

@@ -21,17 +21,15 @@ import '@iconify/utils/lib/icon-set/convert-info';
 import '../../icon-set/props.mjs';
 import '@iconify/utils/lib/misc/objects';
 import '@iconify/utils/lib/colors';
+import 'axios';
 import 'crypto';
-import '../api/download.mjs';
-import 'util';
-import 'stream';
 import 'fs/promises';
 import 'extract-zip';
 import 'tar';
 
 async function downloadGitRepo(options) {
   const { remote, branch } = options;
-  const hasHashInTarget = options.target.indexOf("{hash}") !== -1;
+  const hasHashInTarget = options.target.includes("{hash}");
   const ifModifiedSince = options.ifModifiedSince;
   if (ifModifiedSince || hasHashInTarget) {
     const result = await execAsync(

package/lib/download/git/reset.cjs

@@ -13,19 +13,17 @@ require('@iconify/utils/lib/icon-set/minify');
 require('@iconify/utils/lib/icon-set/convert-info');
 require('../../icon-set/props.cjs');
 require('@iconify/utils/lib/colors');
+require('axios');
 require('fs');
 require('crypto');
 require('pathe');
 const misc_exec = require('../../misc/exec.cjs');
-require('../api/download.cjs');
+require('fs/promises');
 require('extract-zip');
 require('tar');
 require('../../svg/parse.cjs');
 require('@iconify/utils/lib/misc/objects');
 require('child_process');
-require('util');
-require('stream');
-require('fs/promises');
 
 async function resetGitRepoContents(target) {
   await misc_exec.execAsync("git add -A", {

package/lib/download/git/reset.d.cts

@@ -0,0 +1,6 @@
+/**
+ * Reset Git repo contents
+ */
+declare function resetGitRepoContents(target: string): Promise<void>;
+
+export { resetGitRepoContents };

package/lib/download/git/reset.d.mts

@@ -0,0 +1,6 @@
+/**
+ * Reset Git repo contents
+ */
+declare function resetGitRepoContents(target: string): Promise<void>;
+
+export { resetGitRepoContents };

package/lib/download/git/reset.mjs

@@ -11,19 +11,17 @@ import '@iconify/utils/lib/icon-set/minify';
 import '@iconify/utils/lib/icon-set/convert-info';
 import '../../icon-set/props.mjs';
 import '@iconify/utils/lib/colors';
+import 'axios';
 import 'fs';
 import 'crypto';
 import 'pathe';
 import { execAsync } from '../../misc/exec.mjs';
-import '../api/download.mjs';
+import 'fs/promises';
 import 'extract-zip';
 import 'tar';
 import '../../svg/parse.mjs';
 import '@iconify/utils/lib/misc/objects';
 import 'child_process';
-import 'util';
-import 'stream';
-import 'fs/promises';
 
 async function resetGitRepoContents(target) {
   await execAsync("git add -A", {