@http-client-toolkit/core 0.0.1

package/lib/index.cjs

@@ -0,0 +1,615 @@
+'use strict';
+
+var axios = require('axios');
+var zod = require('zod');
+var crypto = require('crypto');
+
+function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
+
+var axios__default = /*#__PURE__*/_interopDefault(axios);
+
+var __async = (__this, __arguments, generator) => {
+  return new Promise((resolve, reject) => {
+    var fulfilled = (value) => {
+      try {
+        step(generator.next(value));
+      } catch (e) {
+        reject(e);
+      }
+    };
+    var rejected = (value) => {
+      try {
+        step(generator.throw(value));
+      } catch (e) {
+        reject(e);
+      }
+    };
+    var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
+    step((generator = generator.apply(__this, __arguments)).next());
+  });
+};
+
+// src/errors/http-client-error.ts
+var HttpClientError = class extends Error {
+  constructor(message, statusCode) {
+    super(message);
+    this.name = "HttpClientError";
+    this.statusCode = statusCode;
+    Object.setPrototypeOf(this, new.target.prototype);
+  }
+};
+var AdaptiveConfigSchema = zod.z.object({
+  monitoringWindowMs: zod.z.number().positive().default(15 * 60 * 1e3),
+  // 15 minutes
+  highActivityThreshold: zod.z.number().min(0).default(10),
+  // requests per window
+  moderateActivityThreshold: zod.z.number().min(0).default(3),
+  recalculationIntervalMs: zod.z.number().positive().default(3e4),
+  // 30 seconds
+  sustainedInactivityThresholdMs: zod.z.number().positive().default(30 * 60 * 1e3),
+  // 30 minutes
+  backgroundPauseOnIncreasingTrend: zod.z.boolean().default(true),
+  maxUserScaling: zod.z.number().positive().default(2),
+  // don't exceed 2x capacity
+  minUserReserved: zod.z.number().min(0).default(5)
+  // requests minimum
+}).refine(
+  (data) => {
+    return data.moderateActivityThreshold < data.highActivityThreshold;
+  },
+  {
+    message: "moderateActivityThreshold must be less than highActivityThreshold"
+  }
+);
+function hashRequest(endpoint, params = {}) {
+  const requestString = JSON.stringify({
+    endpoint,
+    params: sortObject(params)
+  });
+  return crypto.createHash("sha256").update(requestString).digest("hex");
+}
+function sortObject(obj) {
+  if (obj === null) {
+    return null;
+  }
+  const objType = typeof obj;
+  if (objType === "undefined" || objType === "string") {
+    return obj;
+  }
+  if (objType === "number" || objType === "boolean") {
+    return String(obj);
+  }
+  if (Array.isArray(obj)) {
+    return obj.map(sortObject);
+  }
+  const sorted = {};
+  const keys = Object.keys(obj).sort();
+  for (const key of keys) {
+    const value = obj[key];
+    const normalisedValue = sortObject(value);
+    if (normalisedValue !== void 0) {
+      sorted[key] = normalisedValue;
+    }
+  }
+  return sorted;
+}
+
+// src/stores/rate-limit-config.ts
+var DEFAULT_RATE_LIMIT = {
+  limit: 60,
+  windowMs: 6e4
+};
+
+// src/stores/adaptive-capacity-calculator.ts
+var AdaptiveCapacityCalculator = class {
+  constructor(config = {}) {
+    this.config = AdaptiveConfigSchema.parse(config);
+  }
+  calculateDynamicCapacity(resource, totalLimit, activityMetrics) {
+    const recentUserActivity = this.getRecentActivity(
+      activityMetrics.recentUserRequests
+    );
+    const activityTrend = this.calculateActivityTrend(
+      activityMetrics.recentUserRequests
+    );
+    if (recentUserActivity >= this.config.highActivityThreshold) {
+      const userCapacity = Math.min(
+        totalLimit * 0.9,
+        Math.floor(totalLimit * 0.5 * this.config.maxUserScaling)
+        // 50% base * scaling factor
+      );
+      return {
+        userReserved: userCapacity,
+        backgroundMax: totalLimit - userCapacity,
+        backgroundPaused: this.config.backgroundPauseOnIncreasingTrend && activityTrend === "increasing",
+        reason: `High user activity (${recentUserActivity} requests/${this.config.monitoringWindowMs / 6e4}min) - prioritizing users`
+      };
+    }
+    if (recentUserActivity >= this.config.moderateActivityThreshold) {
+      const userMultiplier = this.getUserMultiplier(
+        recentUserActivity,
+        activityTrend
+      );
+      const baseUserCapacity2 = Math.floor(totalLimit * 0.4);
+      const dynamicUserCapacity = Math.min(
+        totalLimit * 0.7,
+        baseUserCapacity2 * userMultiplier
+      );
+      return {
+        userReserved: dynamicUserCapacity,
+        backgroundMax: totalLimit - dynamicUserCapacity,
+        backgroundPaused: false,
+        reason: `Moderate user activity - dynamic scaling (${userMultiplier.toFixed(1)}x user capacity)`
+      };
+    }
+    if (recentUserActivity === 0) {
+      if (activityMetrics.recentUserRequests.length === 0 && activityMetrics.recentBackgroundRequests.length === 0) {
+        const baseUserCapacity2 = Math.floor(totalLimit * 0.3);
+        return {
+          userReserved: Math.max(baseUserCapacity2, this.config.minUserReserved),
+          backgroundMax: totalLimit - Math.max(baseUserCapacity2, this.config.minUserReserved),
+          backgroundPaused: false,
+          reason: "Initial state - default capacity allocation"
+        };
+      }
+      if (activityMetrics.recentUserRequests.length === 0) {
+        return {
+          userReserved: this.config.minUserReserved,
+          // Minimal safety buffer
+          backgroundMax: totalLimit - this.config.minUserReserved,
+          backgroundPaused: false,
+          reason: "No user activity yet - background scale up with minimal user buffer"
+        };
+      }
+      const sustainedInactivity = this.getSustainedInactivityPeriod(
+        activityMetrics.recentUserRequests
+      );
+      if (sustainedInactivity > this.config.sustainedInactivityThresholdMs) {
+        return {
+          userReserved: 0,
+          // No reservation - background gets everything!
+          backgroundMax: totalLimit,
+          // Full capacity available
+          backgroundPaused: false,
+          reason: `Sustained zero activity (${Math.floor(sustainedInactivity / 6e4)}+ min) - full capacity to background`
+        };
+      } else {
+        return {
+          userReserved: this.config.minUserReserved,
+          // Minimal safety buffer
+          backgroundMax: totalLimit - this.config.minUserReserved,
+          backgroundPaused: false,
+          reason: "Recent zero activity - background scale up with minimal user buffer"
+        };
+      }
+    }
+    const baseUserCapacity = Math.floor(totalLimit * 0.3);
+    return {
+      userReserved: Math.max(baseUserCapacity, this.config.minUserReserved),
+      backgroundMax: totalLimit - Math.max(baseUserCapacity, this.config.minUserReserved),
+      backgroundPaused: false,
+      reason: `Low user activity (${recentUserActivity} requests/${this.config.monitoringWindowMs / 6e4}min) - background scale up`
+    };
+  }
+  getRecentActivity(requests) {
+    const cutoff = Date.now() - this.config.monitoringWindowMs;
+    return requests.filter((timestamp) => timestamp > cutoff).length;
+  }
+  calculateActivityTrend(requests) {
+    const now = Date.now();
+    const windowSize = this.config.monitoringWindowMs / 3;
+    const recent = requests.filter((t) => t > now - windowSize).length;
+    const previous = requests.filter(
+      (t) => t > now - 2 * windowSize && t <= now - windowSize
+    ).length;
+    if (recent === 0 && previous === 0) return "none";
+    if (recent > previous * 1.5) return "increasing";
+    if (recent < previous * 0.5) return "decreasing";
+    return "stable";
+  }
+  getUserMultiplier(activity, trend) {
+    let base = Math.min(
+      this.config.maxUserScaling,
+      1 + activity / this.config.highActivityThreshold
+    );
+    if (trend === "increasing") base *= 1.2;
+    if (trend === "decreasing") base *= 0.8;
+    return Math.max(1, base);
+  }
+  getSustainedInactivityPeriod(requests) {
+    if (requests.length === 0) {
+      return 0;
+    }
+    const lastRequest = Math.max(...requests);
+    return Date.now() - lastRequest;
+  }
+};
+
+// src/http-client/http-client.ts
+var DEFAULT_RATE_LIMIT_HEADER_NAMES = {
+  retryAfter: ["retry-after"],
+  limit: ["ratelimit-limit", "x-ratelimit-limit", "rate-limit-limit"],
+  remaining: [
+    "ratelimit-remaining",
+    "x-ratelimit-remaining",
+    "rate-limit-remaining"
+  ],
+  reset: ["ratelimit-reset", "x-ratelimit-reset", "rate-limit-reset"],
+  combined: ["ratelimit"]
+};
+function wait(ms, signal) {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      if (signal) {
+        signal.removeEventListener("abort", onAbort);
+      }
+      resolve();
+    }, ms);
+    function onAbort() {
+      clearTimeout(timer);
+      const err = new Error("Aborted");
+      err.name = "AbortError";
+      reject(err);
+    }
+    if (signal) {
+      if (signal.aborted) {
+        onAbort();
+      } else {
+        signal.addEventListener("abort", onAbort, { once: true });
+      }
+    }
+  });
+}
+var HttpClient = class {
+  constructor(stores = {}, options = {}) {
+    this.serverCooldowns = /* @__PURE__ */ new Map();
+    var _a, _b, _c;
+    this._http = axios__default.default.create();
+    this.stores = stores;
+    this.options = {
+      defaultCacheTTL: (_a = options.defaultCacheTTL) != null ? _a : 3600,
+      throwOnRateLimit: (_b = options.throwOnRateLimit) != null ? _b : true,
+      maxWaitTime: (_c = options.maxWaitTime) != null ? _c : 6e4,
+      responseTransformer: options.responseTransformer,
+      errorHandler: options.errorHandler,
+      responseHandler: options.responseHandler,
+      rateLimitHeaders: this.normalizeRateLimitHeaders(
+        options.rateLimitHeaders
+      )
+    };
+  }
+  normalizeRateLimitHeaders(customHeaders) {
+    return {
+      retryAfter: this.normalizeHeaderNames(
+        customHeaders == null ? void 0 : customHeaders.retryAfter,
+        DEFAULT_RATE_LIMIT_HEADER_NAMES.retryAfter
+      ),
+      limit: this.normalizeHeaderNames(
+        customHeaders == null ? void 0 : customHeaders.limit,
+        DEFAULT_RATE_LIMIT_HEADER_NAMES.limit
+      ),
+      remaining: this.normalizeHeaderNames(
+        customHeaders == null ? void 0 : customHeaders.remaining,
+        DEFAULT_RATE_LIMIT_HEADER_NAMES.remaining
+      ),
+      reset: this.normalizeHeaderNames(
+        customHeaders == null ? void 0 : customHeaders.reset,
+        DEFAULT_RATE_LIMIT_HEADER_NAMES.reset
+      ),
+      combined: this.normalizeHeaderNames(
+        customHeaders == null ? void 0 : customHeaders.combined,
+        DEFAULT_RATE_LIMIT_HEADER_NAMES.combined
+      )
+    };
+  }
+  normalizeHeaderNames(providedNames, defaultNames) {
+    if (!providedNames || providedNames.length === 0) {
+      return [...defaultNames];
+    }
+    const customNames = providedNames.map((name) => name.trim().toLowerCase()).filter(Boolean);
+    if (customNames.length === 0) {
+      return [...defaultNames];
+    }
+    return [.../* @__PURE__ */ new Set([...customNames, ...defaultNames])];
+  }
+  /**
+   * Infer the resource name from the endpoint URL
+   * @param url The full URL or endpoint path
+   * @returns The resource name for rate limiting
+   */
+  inferResource(url) {
+    try {
+      const urlObj = new URL(url);
+      const segments = urlObj.pathname.split("/").filter(Boolean);
+      return segments[segments.length - 1] || "unknown";
+    } catch (e) {
+      return "unknown";
+    }
+  }
+  /**
+   * Extract endpoint and params from URL for request hashing
+   * @param url The full URL
+   * @returns Object with endpoint and params for hashing
+   */
+  parseUrlForHashing(url) {
+    const urlObj = new URL(url);
+    const endpoint = `${urlObj.origin}${urlObj.pathname}`;
+    const params = {};
+    urlObj.searchParams.forEach((value, key) => {
+      const existing = params[key];
+      if (existing === void 0) {
+        params[key] = value;
+        return;
+      }
+      if (Array.isArray(existing)) {
+        existing.push(value);
+        return;
+      }
+      params[key] = [existing, value];
+    });
+    return { endpoint, params };
+  }
+  getOriginScope(url) {
+    try {
+      return new URL(url).origin;
+    } catch (e) {
+      return "unknown";
+    }
+  }
+  getHeaderValue(headers, names) {
+    var _a;
+    if (!headers) {
+      return void 0;
+    }
+    for (const rawName of names) {
+      const name = rawName.toLowerCase();
+      const value = (_a = headers[name]) != null ? _a : headers[rawName];
+      if (typeof value === "string") {
+        return value;
+      }
+      if (Array.isArray(value) && value.length > 0) {
+        const first = value.find((entry) => typeof entry === "string");
+        if (typeof first === "string") {
+          return first;
+        }
+      }
+    }
+    return void 0;
+  }
+  parseIntegerHeader(value) {
+    if (!value) {
+      return void 0;
+    }
+    const parsed = Number.parseInt(value.trim(), 10);
+    if (!Number.isFinite(parsed) || parsed < 0) {
+      return void 0;
+    }
+    return parsed;
+  }
+  parseRetryAfterMs(value) {
+    if (!value) {
+      return void 0;
+    }
+    const numeric = Number.parseInt(value.trim(), 10);
+    if (Number.isFinite(numeric) && numeric >= 0) {
+      return numeric * 1e3;
+    }
+    const dateMs = Date.parse(value);
+    if (!Number.isFinite(dateMs)) {
+      return void 0;
+    }
+    return Math.max(0, dateMs - Date.now());
+  }
+  parseResetMs(value) {
+    const parsed = this.parseIntegerHeader(value);
+    if (parsed === void 0) {
+      return void 0;
+    }
+    if (parsed === 0) {
+      return 0;
+    }
+    const nowSeconds = Math.floor(Date.now() / 1e3);
+    if (parsed > nowSeconds + 1) {
+      return Math.max(0, (parsed - nowSeconds) * 1e3);
+    }
+    return parsed * 1e3;
+  }
+  parseCombinedRateLimitHeader(value) {
+    if (!value) {
+      return {};
+    }
+    const remainingMatch = value.match(/(?:^|[;,])\s*r\s*=\s*(\d+)/i);
+    const resetMatch = value.match(/(?:^|[;,])\s*t\s*=\s*(\d+)/i);
+    return {
+      remaining: remainingMatch ? this.parseIntegerHeader(remainingMatch[1]) : void 0,
+      resetMs: resetMatch ? this.parseResetMs(resetMatch[1]) : void 0
+    };
+  }
+  applyServerRateLimitHints(url, headers, statusCode) {
+    if (!headers) {
+      return;
+    }
+    const config = this.options.rateLimitHeaders;
+    const retryAfterRaw = this.getHeaderValue(headers, config.retryAfter);
+    const resetRaw = this.getHeaderValue(headers, config.reset);
+    const remainingRaw = this.getHeaderValue(headers, config.remaining);
+    const combinedRaw = this.getHeaderValue(headers, config.combined);
+    const retryAfterMs = this.parseRetryAfterMs(retryAfterRaw);
+    const resetMs = this.parseResetMs(resetRaw);
+    const remaining = this.parseIntegerHeader(remainingRaw);
+    const combined = this.parseCombinedRateLimitHeader(combinedRaw);
+    const effectiveRemaining = remaining != null ? remaining : combined.remaining;
+    const effectiveResetMs = resetMs != null ? resetMs : combined.resetMs;
+    const hasRateLimitErrorStatus = statusCode === 429 || statusCode === 503;
+    let waitMs;
+    if (retryAfterMs !== void 0) {
+      waitMs = retryAfterMs;
+    } else if (effectiveResetMs !== void 0 && (hasRateLimitErrorStatus || effectiveRemaining !== void 0 && effectiveRemaining <= 0)) {
+      waitMs = effectiveResetMs;
+    }
+    if (waitMs === void 0 || waitMs <= 0) {
+      return;
+    }
+    const scope = this.getOriginScope(url);
+    this.serverCooldowns.set(scope, Date.now() + waitMs);
+  }
+  enforceServerCooldown(url, signal) {
+    return __async(this, null, function* () {
+      const scope = this.getOriginScope(url);
+      const startedAt = Date.now();
+      while (true) {
+        const cooldownUntil = this.serverCooldowns.get(scope);
+        if (!cooldownUntil) {
+          return;
+        }
+        const waitMs = cooldownUntil - Date.now();
+        if (waitMs <= 0) {
+          this.serverCooldowns.delete(scope);
+          return;
+        }
+        if (this.options.throwOnRateLimit) {
+          throw new Error(
+            `Rate limit exceeded for origin '${scope}'. Wait ${waitMs}ms before retrying.`
+          );
+        }
+        const elapsedMs = Date.now() - startedAt;
+        const remainingWaitBudgetMs = this.options.maxWaitTime - elapsedMs;
+        if (remainingWaitBudgetMs <= 0) {
+          throw new Error(
+            `Rate limit wait exceeded maxWaitTime (${this.options.maxWaitTime}ms) for origin '${scope}'.`
+          );
+        }
+        yield wait(Math.min(waitMs, remainingWaitBudgetMs), signal);
+      }
+    });
+  }
+  enforceStoreRateLimit(resource, priority, signal) {
+    return __async(this, null, function* () {
+      const rateLimit = this.stores.rateLimit;
+      const startedAt = Date.now();
+      if (this.options.throwOnRateLimit) {
+        const canProceed = yield rateLimit.canProceed(resource, priority);
+        if (!canProceed) {
+          const waitTime = yield rateLimit.getWaitTime(resource, priority);
+          throw new Error(
+            `Rate limit exceeded for resource '${resource}'. Wait ${waitTime}ms before retrying.`
+          );
+        }
+        return;
+      }
+      while (!(yield rateLimit.canProceed(resource, priority))) {
+        const suggestedWaitMs = yield rateLimit.getWaitTime(resource, priority);
+        const elapsedMs = Date.now() - startedAt;
+        const remainingWaitBudgetMs = this.options.maxWaitTime - elapsedMs;
+        if (remainingWaitBudgetMs <= 0) {
+          throw new Error(
+            `Rate limit wait exceeded maxWaitTime (${this.options.maxWaitTime}ms) for resource '${resource}'.`
+          );
+        }
+        const waitTime = suggestedWaitMs > 0 ? Math.min(suggestedWaitMs, remainingWaitBudgetMs) : Math.min(25, remainingWaitBudgetMs);
+        yield wait(waitTime, signal);
+      }
+    });
+  }
+  generateClientError(err) {
+    var _a, _b, _c;
+    if (this.options.errorHandler) {
+      return this.options.errorHandler(err);
+    }
+    if (err instanceof HttpClientError) {
+      return err;
+    }
+    const error = err;
+    const statusCode = (_a = error.response) == null ? void 0 : _a.status;
+    const errorMessage = (_c = (_b = error.response) == null ? void 0 : _b.data) == null ? void 0 : _c.message;
+    const message = `${error.message}${errorMessage ? `, ${errorMessage}` : ""}`;
+    return new HttpClientError(message, statusCode);
+  }
+  get(_0) {
+    return __async(this, arguments, function* (url, options = {}) {
+      const { signal, priority = "background" } = options;
+      const { endpoint, params } = this.parseUrlForHashing(url);
+      const hash = hashRequest(endpoint, params);
+      const resource = this.inferResource(url);
+      try {
+        yield this.enforceServerCooldown(url, signal);
+        if (this.stores.cache) {
+          const cachedResult = yield this.stores.cache.get(hash);
+          if (cachedResult !== void 0) {
+            return cachedResult;
+          }
+        }
+        if (this.stores.dedupe) {
+          const existingResult = yield this.stores.dedupe.waitFor(hash);
+          if (existingResult !== void 0) {
+            return existingResult;
+          }
+          if (this.stores.dedupe.registerOrJoin) {
+            const registration = yield this.stores.dedupe.registerOrJoin(hash);
+            if (!registration.isOwner) {
+              const joinedResult = yield this.stores.dedupe.waitFor(hash);
+              if (joinedResult !== void 0) {
+                return joinedResult;
+              }
+            }
+          } else {
+            yield this.stores.dedupe.register(hash);
+          }
+        }
+        if (this.stores.rateLimit) {
+          yield this.enforceStoreRateLimit(resource, priority, signal);
+        }
+        const response = yield this._http.get(url, { signal });
+        this.applyServerRateLimitHints(
+          url,
+          response.headers,
+          response.status
+        );
+        let data = response.data;
+        if (this.options.responseTransformer && data) {
+          data = this.options.responseTransformer(data);
+        }
+        if (this.options.responseHandler) {
+          data = this.options.responseHandler(data);
+        }
+        const result = data;
+        if (this.stores.rateLimit) {
+          const rateLimit = this.stores.rateLimit;
+          yield rateLimit.record(resource, priority);
+        }
+        if (this.stores.cache) {
+          yield this.stores.cache.set(hash, result, this.options.defaultCacheTTL);
+        }
+        if (this.stores.dedupe) {
+          yield this.stores.dedupe.complete(hash, result);
+        }
+        return result;
+      } catch (error) {
+        const axiosError = error;
+        if (axiosError.response) {
+          this.applyServerRateLimitHints(
+            url,
+            axiosError.response.headers,
+            axiosError.response.status
+          );
+        }
+        if (this.stores.dedupe) {
+          yield this.stores.dedupe.fail(hash, error);
+        }
+        if (error instanceof Error && error.name === "AbortError") {
+          throw error;
+        }
+        throw this.generateClientError(error);
+      }
+    });
+  }
+};
+
+exports.AdaptiveCapacityCalculator = AdaptiveCapacityCalculator;
+exports.AdaptiveConfigSchema = AdaptiveConfigSchema;
+exports.DEFAULT_RATE_LIMIT = DEFAULT_RATE_LIMIT;
+exports.HttpClient = HttpClient;
+exports.HttpClientError = HttpClientError;
+exports.hashRequest = hashRequest;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map