@se-studio/contentful-rest-api 1.0.20 → 1.0.21

package/dist/index.d.ts

@@ -452,6 +452,8 @@ interface ContentfulQuery {
 declare class ContentfulFetchClient {
     private readonly baseUrl;
     private readonly accessToken;
+    private readonly rateLimiter;
+    private readonly isPreview;
     constructor(config: ContentfulConfig, preview?: boolean);
     getEntries<T = any>(query: ContentfulQuery, options?: FetchOptions): Promise<ContentfulResponse<T>>;
 }
@@ -613,15 +615,18 @@ declare function getRetryAfter(error: unknown): number | undefined;
 declare function calculateBackoffDelay(attempt: number, config: Required<RetryConfig>, retryAfter?: number): number;
 declare function withRetry<T>(fn: () => Promise<T>, config?: RetryConfig): Promise<T>;
 declare class RateLimiter {
-    private readonly maxTokens;
-    private readonly refillRate;
-    private tokens;
-    private lastRefill;
-    constructor(maxTokens: number, refillRate: number);
-    private refill;
-    tryConsume(): boolean;
-    consume(): Promise<void>;
-    getAvailableTokens(): number;
+    private readonly intervalMs;
+    private nextAvailableTime;
+    private readonly queue;
+    private processing;
+    private pauseUntil;
+    constructor(requestsPerSecond: number, _refillRate?: number);
+    private getWaitTime;
+    private processQueue;
+    acquire(): Promise<void>;
+    getQueueLength(): number;
+    pause(seconds: number): void;
+    getIntervalMs(): number;
 }
 
 export { AllTags, ArticleTag, ArticleTypeTag, AssetTag, AuthenticationError, BannerTag, type BaseConverterContext, type CmsError, type CmsResponse, type ContentResolverFunction, ContentfulFetchClient as ContentfulClient, type ContentfulConfig, ContentfulError, ContentfulFetchClient, type ConverterContext, CustomTypeTag, type DefaultChainModifier, EntryNotFoundError, type FetchOptions, GlobalTag, type IContentfulCollection, type IContentfulComponent, type IContentfulPerson, type IContentfulRichText, type IFetchedTemplate, type ISitemapEntry, LocationTag, NavigationTag, PageTag, PersonTag, RateLimitError, RateLimiter, type RelatedArticlesOptions, type RetryConfig, type RevalidationConfig, type SitemapChangeFrequency, type SitemapConfig, type SitemapContentTypeConfig, type SitemapEntryProvider, TagTag, TemplateTag, ValidationError, arrayOrUndefined, articleTag, articleTypeIndexTag, articleTypeTag, assetTag, basePageConverter, calculateBackoffDelay, contentfulAllArticleLinks, contentfulAllArticleTypeLinks, contentfulAllPageLinks, contentfulAllPersonLinks, contentfulAllTagLinks, contentfulArticleRest, contentfulArticleSitemapEntries, contentfulArticleTypeRest, contentfulArticleTypeSitemapEntries, contentfulCustomTypeRest, contentfulPageRest, contentfulPageSitemapEntries, contentfulPersonSitemapEntries, contentfulTagRest, contentfulTagSitemapEntries, contentfulTemplateRest, createBaseConverterContext, createContentfulClient, createContentfulPreviewClient, createResponsiveVisual, createRevalidationHandler, createSitemapProvider, customTypeTag, filterRelatedArticles, getAllSitemapEntries, getCacheTags, getCacheTagsForPreview, getCacheTagsForProduction, getContentfulClient, getRetryAfter, isContentfulError, isRateLimitError, isRetryableError, isValidDate, locationTag, lookupAsset, notEmpty, pageTag, personTag, resolveLink, resolveLinks, resolveRichTextDocument, revalidateSingleTag, revalidateTags, safeDate, tagTag, templateTag, withRetry };

package/dist/index.js

@@ -202,6 +202,154 @@ function getRetryAfter(error) {
   return void 0;
 }
 
+// src/utils/retry.ts
+var DEFAULT_RETRY_CONFIG = {
+  maxRetries: 3,
+  initialDelay: 1e3,
+  // 1 second
+  maxDelay: 3e4,
+  // 30 seconds
+  backoffMultiplier: 2
+};
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function calculateBackoffDelay(attempt, config, retryAfter) {
+  if (retryAfter !== void 0) {
+    return Math.min(retryAfter * 1e3, config.maxDelay);
+  }
+  const exponentialDelay = config.initialDelay * config.backoffMultiplier ** attempt;
+  const jitter = Math.random() * exponentialDelay;
+  return Math.min(exponentialDelay + jitter, config.maxDelay);
+}
+async function withRetry(fn, config) {
+  const retryConfig = {
+    ...DEFAULT_RETRY_CONFIG,
+    ...config
+  };
+  let lastError;
+  for (let attempt = 0; attempt <= retryConfig.maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      if (attempt === retryConfig.maxRetries) {
+        break;
+      }
+      if (!isRetryableError(error)) {
+        throw error;
+      }
+      const retryAfter = getRetryAfter(error);
+      const delay3 = calculateBackoffDelay(attempt, retryConfig, retryAfter);
+      if (typeof process !== "undefined" && process.env?.NODE_ENV !== "production") {
+        console.warn(
+          `Retry attempt ${attempt + 1}/${retryConfig.maxRetries} after ${delay3}ms`,
+          error
+        );
+      }
+      await sleep(delay3);
+    }
+  }
+  throw lastError;
+}
+var RateLimiter = class {
+  /** Minimum interval between requests in milliseconds */
+  intervalMs;
+  /** Timestamp when the next request can be made */
+  nextAvailableTime = 0;
+  /** Queue of pending requests */
+  queue = [];
+  /** Whether the queue processor is running */
+  processing = false;
+  /** Additional pause time from 429 responses (ms) */
+  pauseUntil = 0;
+  constructor(requestsPerSecond, _refillRate) {
+    this.intervalMs = Math.ceil(1e3 / (requestsPerSecond * 0.8));
+  }
+  /**
+   * Calculates how long to wait before the next request can be made
+   */
+  getWaitTime() {
+    const now = Date.now();
+    const waitForInterval = Math.max(0, this.nextAvailableTime - now);
+    const waitForPause = Math.max(0, this.pauseUntil - now);
+    return Math.max(waitForInterval, waitForPause);
+  }
+  /**
+   * Processes the queue, releasing requests one at a time with proper spacing
+   */
+  async processQueue() {
+    if (this.processing) {
+      return;
+    }
+    this.processing = true;
+    while (this.queue.length > 0) {
+      const waitTime = this.getWaitTime();
+      if (waitTime > 0) {
+        await sleep(waitTime);
+      }
+      this.nextAvailableTime = Date.now() + this.intervalMs;
+      const entry = this.queue.shift();
+      entry?.resolve();
+    }
+    this.processing = false;
+  }
+  /**
+   * Acquires permission to make a request.
+   * Returns a promise that resolves when the request can proceed.
+   * Requests are processed in FIFO order with minimum interval spacing.
+   *
+   * @returns Promise that resolves when rate limit allows the request
+   */
+  async acquire() {
+    const waitTime = this.getWaitTime();
+    if (waitTime === 0 && this.queue.length === 0) {
+      this.nextAvailableTime = Date.now() + this.intervalMs;
+      return;
+    }
+    return new Promise((resolve) => {
+      this.queue.push({ resolve });
+      this.processQueue();
+    });
+  }
+  /**
+   * Gets the number of requests currently waiting in the queue
+   */
+  getQueueLength() {
+    return this.queue.length;
+  }
+  /**
+   * Pauses the rate limiter for a specified duration.
+   * All requests (current and queued) will wait until pause expires.
+   * Does NOT block - just sets the pause time and returns.
+   *
+   * @param seconds - Duration to pause in seconds
+   */
+  pause(seconds) {
+    const pauseUntil = Date.now() + seconds * 1e3;
+    if (pauseUntil > this.pauseUntil) {
+      this.pauseUntil = pauseUntil;
+    }
+  }
+  /**
+   * Gets the current interval between requests in milliseconds
+   */
+  getIntervalMs() {
+    return this.intervalMs;
+  }
+};
+var CONTENTFUL_RATE_LIMITS = {
+  /** Content Delivery API: 55 requests/second */
+  delivery: 55,
+  /** Content Preview API: 14 requests/second */
+  preview: 14
+};
+var deliveryRateLimiter = new RateLimiter(CONTENTFUL_RATE_LIMITS.delivery);
+var previewRateLimiter = new RateLimiter(CONTENTFUL_RATE_LIMITS.preview);
+function getRateLimiter(preview) {
+  return preview ? previewRateLimiter : deliveryRateLimiter;
+}
+
 // src/client.ts
 function buildQueryString(query) {
   const params = new URLSearchParams();
@@ -212,19 +360,15 @@ function buildQueryString(query) {
   });
   return params.toString();
 }
-function sleep(ms) {
+function sleep2(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
-function calculateRetryDelay(attempt, retryAfterSeconds) {
-  let baseDelay;
-  if (retryAfterSeconds !== void 0 && retryAfterSeconds > 0) {
-    baseDelay = retryAfterSeconds * 1e3;
-  } else {
-    baseDelay = 1e3 * 2 ** attempt;
-  }
-  const jitter = Math.random() * baseDelay;
-  return Math.min(baseDelay + jitter, 15e3);
-}
+var RETRY_CONFIG = {
+  maxRetries: 10,
+  initialDelay: 1e3,
+  maxDelay: 15e3,
+  backoffMultiplier: 2
+};
 function parseRetryAfter(response) {
   const retryAfterHeader = response.headers.get("X-Contentful-RateLimit-Reset") || response.headers.get("Retry-After");
   if (retryAfterHeader) {
@@ -264,14 +408,25 @@ async function parseErrorResponse(response) {
 var ContentfulFetchClient = class {
   baseUrl;
   accessToken;
+  rateLimiter;
+  isPreview;
   constructor(config, preview = false) {
     const host = config.host || (preview ? "preview.contentful.com" : "cdn.contentful.com");
     const environment = config.environment || "master";
     this.baseUrl = `https://${host}/spaces/${config.spaceId}/environments/${environment}`;
     this.accessToken = config.accessToken;
+    this.isPreview = preview;
+    this.rateLimiter = getRateLimiter(preview);
   }
   /**
-   * Fetches entries from Contentful with automatic retry on rate limit errors
+   * Fetches entries from Contentful with proactive rate limiting
+   * and automatic retry on rate limit errors.
+   *
+   * Rate limits are applied proactively:
+   * - Preview API: 14 requests/second
+   * - Delivery API: 55 requests/second
+   *
+   * If a 429 is still received (e.g., due to other processes), retries with backoff.
    */
   async getEntries(query, options) {
     const queryString = buildQueryString(query);
@@ -285,26 +440,34 @@ var ContentfulFetchClient = class {
     if (options?.next) {
       fetchOptions.next = options.next;
     }
-    const maxRetries = 3;
-    for (let attempt = 0; attempt <= maxRetries; attempt++) {
-      const response = await fetch(url, fetchOptions);
-      if (response.ok) {
-        return response.json();
-      }
-      if (response.status === 429 && attempt < maxRetries) {
-        const retryAfter = parseRetryAfter(response);
-        const delay3 = calculateRetryDelay(attempt, retryAfter);
-        if (typeof process !== "undefined" && process.env?.NODE_ENV !== "production") {
+    for (let attempt = 0; attempt <= RETRY_CONFIG.maxRetries; attempt++) {
+      try {
+        await this.rateLimiter.acquire();
+        const response = await fetch(url, fetchOptions);
+        if (response.ok) {
+          return response.json();
+        }
+        if (response.status === 429 && attempt < RETRY_CONFIG.maxRetries) {
+          const retryAfter = parseRetryAfter(response);
+          const delay3 = calculateBackoffDelay(attempt, RETRY_CONFIG, retryAfter);
           console.warn(
-            `[Contentful] Rate limited, retrying in ${delay3}ms (attempt ${attempt + 1}/${maxRetries})`
+            `[Contentful] Rate limited (429), retrying in ${delay3}ms (attempt ${attempt + 1}/${RETRY_CONFIG.maxRetries}). Queue: ${this.rateLimiter.getQueueLength()} waiting. API: ${this.isPreview ? "preview" : "delivery"}`
           );
+          const pauseSeconds = Math.max(retryAfter ?? 2, 2);
+          this.rateLimiter.pause(pauseSeconds);
+          await sleep2(delay3);
+          continue;
+        }
+        throw await parseErrorResponse(response);
+      } catch (error) {
+        if (error instanceof ContentfulError) {
+          throw error;
         }
-        await sleep(delay3);
-        continue;
+        console.error("[Contentful] Unexpected error in request", error);
+        throw new ContentfulError("Unexpected error in request", 500);
       }
-      throw await parseErrorResponse(response);
     }
-    throw new ContentfulError("Unexpected error in retry loop", 500);
+    throw new ContentfulError("Max retries exceeded", 500);
   }
 };
 function createContentfulClient(config) {
@@ -661,113 +824,6 @@ function safeDate(date) {
   return new Date(date);
 }
 
-// src/utils/retry.ts
-var DEFAULT_RETRY_CONFIG = {
-  maxRetries: 3,
-  initialDelay: 1e3,
-  // 1 second
-  maxDelay: 3e4,
-  // 30 seconds
-  backoffMultiplier: 2
-};
-function sleep2(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-function calculateBackoffDelay(attempt, config, retryAfter) {
-  if (retryAfter !== void 0) {
-    const result2 = Math.min(retryAfter * 1e3, config.maxDelay);
-    console.log(
-      `Calculated backoff delay: ${result2}ms (attempt ${attempt + 1}), retryAfter: ${retryAfter}`
-    );
-    return result2;
-  }
-  const exponentialDelay = config.initialDelay * config.backoffMultiplier ** attempt;
-  const jitter = Math.random() * exponentialDelay;
-  const result = Math.min(exponentialDelay + jitter, config.maxDelay);
-  console.log(
-    `Calculated backoff delay: ${result}ms (attempt ${attempt + 1}), exponentialDelay: ${exponentialDelay}, jitter: ${jitter}`
-  );
-  return result;
-}
-async function withRetry(fn, config) {
-  const retryConfig = {
-    ...DEFAULT_RETRY_CONFIG,
-    ...config
-  };
-  let lastError;
-  for (let attempt = 0; attempt <= retryConfig.maxRetries; attempt++) {
-    try {
-      return await fn();
-    } catch (error) {
-      lastError = error;
-      if (attempt === retryConfig.maxRetries) {
-        break;
-      }
-      if (!isRetryableError(error)) {
-        throw error;
-      }
-      const retryAfter = getRetryAfter(error);
-      const delay3 = calculateBackoffDelay(attempt, retryConfig, retryAfter);
-      if (typeof process !== "undefined" && process.env?.NODE_ENV !== "production") {
-        console.warn(
-          `Retry attempt ${attempt + 1}/${retryConfig.maxRetries} after ${delay3}ms`,
-          error
-        );
-      }
-      await sleep2(delay3);
-    }
-  }
-  throw lastError;
-}
-var RateLimiter = class {
-  constructor(maxTokens, refillRate) {
-    this.maxTokens = maxTokens;
-    this.refillRate = refillRate;
-    this.tokens = maxTokens;
-    this.lastRefill = Date.now();
-  }
-  tokens;
-  lastRefill;
-  /**
-   * Refills tokens based on time elapsed
-   */
-  refill() {
-    const now = Date.now();
-    const timePassed = (now - this.lastRefill) / 1e3;
-    const tokensToAdd = timePassed * this.refillRate;
-    this.tokens = Math.min(this.maxTokens, this.tokens + tokensToAdd);
-    this.lastRefill = now;
-  }
-  /**
-   * Attempts to consume a token
-   * @returns true if token was consumed, false if rate limited
-   */
-  tryConsume() {
-    this.refill();
-    if (this.tokens >= 1) {
-      this.tokens -= 1;
-      return true;
-    }
-    return false;
-  }
-  /**
-   * Waits until a token is available and consumes it
-   */
-  async consume() {
-    while (!this.tryConsume()) {
-      const waitTime = 1 / this.refillRate * 1e3;
-      await sleep2(waitTime);
-    }
-  }
-  /**
-   * Gets current number of available tokens
-   */
-  getAvailableTokens() {
-    this.refill();
-    return Math.floor(this.tokens);
-  }
-};
-
 // src/api/helpers.ts
 var PAGE_LINK_FIELDS = "sys,fields.cmsLabel,fields.title,fields.slug,fields.featuredImage,fields.backgroundColour,fields.textColour,fields.indexed,fields.hidden,fields.tags";
 var ARTICLE_LINK_FIELDS = "sys,fields.cmsLabel,fields.title,fields.slug,fields.featuredImage,fields.backgroundColour,fields.textColour,fields.indexed,fields.hidden,fields.tags,fields.articleType,fields.date,fields.author";