@internetarchive/fetch-handler 1.0.1 → 1.1.0-webdev-7731.1

package/src/fetch-handler-interface.ts

@@ -1,11 +1,17 @@
+import type { FetchOptions } from './fetch-options';
+import type { RetryConfiguring } from './fetch-retry/configuration/retry-configuring';
+
 export interface FetchHandlerInterface {
   /**
    * Generic fetch function that handles retries and common IA parameters like `reCache=1`
    *
    * @param input RequestInfo
-   * @param init RequestInit
+   * @param options RequestInit | FetchOptions
    */
-  fetch(input: RequestInfo, init?: RequestInit): Promise<Response>;
+  fetch(
+    request: RequestInfo,
+    options?: RequestInit | FetchOptions,
+  ): Promise<Response>;
 
   /**
    * A helper function to fetch a response from an API and get a JSON object
@@ -20,6 +26,7 @@ export interface FetchHandlerInterface {
       method?: string;
       body?: BodyInit;
       headers?: HeadersInit;
+      retryConfig?: RetryConfiguring;
     },
   ): Promise<T>;
 
@@ -30,10 +37,10 @@ export interface FetchHandlerInterface {
    * of the full URL. If you need a full URL, use `fetchApiResponse` instead.
    *
    * @param path string
-   * @param options?: { includeCredentials?: boolean }
+   * @param options?: { includeCredentials?: boolean, retryConfig?: RetryConfiguring }
    */
   fetchIAApiResponse<T>(
     path: string,
-    options?: { includeCredentials?: boolean },
+    options?: { includeCredentials?: boolean; retryConfig?: RetryConfiguring },
   ): Promise<T>;
 }

package/src/{ia-fetch-handler.ts → fetch-handler.ts}

@@ -1,5 +1,10 @@
-import { FetchRetrier, FetchRetrierInterface } from './utils/fetch-retrier';
+import {
+  FetchRetrier,
+  FetchRetrierInterface,
+} from './fetch-retry/fetch-retrier';
 import type { FetchHandlerInterface } from './fetch-handler-interface';
+import type { FetchOptions } from './fetch-options';
+import type { RetryConfiguring } from './fetch-retry/configuration/retry-configuring';
 
 /**
  * The FetchHandler adds some common helpers:
@@ -18,6 +23,7 @@ export class IaFetchHandler implements FetchHandlerInterface {
     iaApiBaseUrl?: string;
     fetchRetrier?: FetchRetrierInterface;
     searchParams?: string;
+    defaultRetryConfiguration?: RetryConfiguring;
   }) {
     if (options?.iaApiBaseUrl) this.iaApiBaseUrl = options.iaApiBaseUrl;
     if (options?.fetchRetrier) this.fetchRetrier = options.fetchRetrier;
@@ -33,6 +39,7 @@ export class IaFetchHandler implements FetchHandlerInterface {
     path: string,
     options?: {
       includeCredentials?: boolean;
+      retryConfig?: RetryConfiguring;
     },
   ): Promise<T> {
     const url = `${this.iaApiBaseUrl}${path}`;
@@ -47,6 +54,7 @@ export class IaFetchHandler implements FetchHandlerInterface {
       method?: string;
       body?: BodyInit;
       headers?: HeadersInit;
+      retryConfig?: RetryConfiguring;
     },
   ): Promise<T> {
     const requestInit: RequestInit = {};
@@ -54,19 +62,26 @@ export class IaFetchHandler implements FetchHandlerInterface {
     if (options?.method) requestInit.method = options.method;
     if (options?.body) requestInit.body = options.body;
     if (options?.headers) requestInit.headers = options.headers;
-    const response = await this.fetch(url, requestInit);
+    const response = await this.fetch(url, {
+      requestInit: requestInit,
+      retryConfig: options?.retryConfig,
+    });
     const json = await response.json();
     return json as T;
   }
 
   /** @inheritdoc */
-  async fetch(input: RequestInfo, init?: RequestInit): Promise<Response> {
-    let finalInput = input;
+  async fetch(
+    request: RequestInfo,
+    options?: RequestInit | FetchOptions,
+  ): Promise<Response> {
+    let finalRequest = request;
     const urlParams = new URLSearchParams(this.searchParams);
     if (urlParams.get('reCache') === '1') {
-      finalInput = this.addSearchParams(input, { reCache: '1' });
+      const urlString = typeof request === 'string' ? request : request.url;
+      finalRequest = this.addSearchParams(urlString, { reCache: '1' });
     }
-    return this.fetchRetrier.fetchRetry(finalInput, init);
+    return this.fetchRetrier.fetchRetry(finalRequest, options);
   }
 
   /**
@@ -74,21 +89,15 @@ export class IaFetchHandler implements FetchHandlerInterface {
    * the way we add search params to it depending on the input.
    */
   private addSearchParams(
-    input: RequestInfo,
+    urlString: string,
     params: Record<string, string>,
-  ): RequestInfo {
-    const urlString = typeof input === 'string' ? input : input.url;
+  ): string {
     const url = new URL(urlString, window.location.href);
 
     for (const [key, value] of Object.entries(params)) {
       url.searchParams.set(key, value);
     }
 
-    if (typeof input === 'string') {
-      return url.href;
-    } else {
-      const newRequest = new Request(url.href, input);
-      return newRequest;
-    }
+    return url.href;
   }
 }

package/src/fetch-options.ts

@@ -0,0 +1,6 @@
+import type { RetryConfiguring } from './fetch-retry/configuration/retry-configuring';
+
+export type FetchOptions = {
+  requestInit?: RequestInit;
+  retryConfig?: RetryConfiguring;
+};

package/src/fetch-retry/configuration/default-retry-configuration.ts

@@ -0,0 +1,23 @@
+import type { RetryConfiguring } from './retry-configuring';
+import type { Milliseconds } from './milliseconds';
+
+export class DefaultRetryConfiguration implements RetryConfiguring {
+  private readonly maxRetries: Readonly<number> = 2;
+
+  constructor(options?: { maxRetries?: number }) {
+    if (options?.maxRetries !== undefined) {
+      this.maxRetries = options.maxRetries;
+    }
+  }
+
+  shouldRetry(response: Response | null, retryNumber: number): boolean {
+    if (response === null) return false;
+    if (retryNumber > this.maxRetries) return false;
+    return response.status >= 500 && response.status < 600;
+  }
+
+  retryDelay(retryNumber: number): Milliseconds {
+    // Exponential backoff up to 10 seconds
+    return Math.min(500 * 2 ** retryNumber, 10000);
+  }
+}

package/src/fetch-retry/configuration/milliseconds.ts

@@ -0,0 +1 @@
+export type Milliseconds = number;

package/src/fetch-retry/configuration/no-retry-configuration.ts

@@ -0,0 +1,12 @@
+import type { RetryConfiguring } from './retry-configuring';
+import type { Milliseconds } from './milliseconds';
+
+export class NoRetryConfiguration implements RetryConfiguring {
+  shouldRetry(): boolean {
+    return false;
+  }
+
+  retryDelay(): Milliseconds {
+    return 0;
+  }
+}

package/src/fetch-retry/configuration/retry-configuring.ts

@@ -0,0 +1,11 @@
+import type { Milliseconds } from './milliseconds';
+
+export interface RetryConfiguring {
+  shouldRetry(
+    response: Response | null,
+    retryNumber: number,
+    error?: unknown,
+  ): boolean;
+
+  retryDelay(retryNumber: number): Milliseconds;
+}

package/src/fetch-retry/fetch-retrier.ts

@@ -0,0 +1,146 @@
+import type { AnalyticsHandlerInterface } from '@internetarchive/analytics-manager';
+import { promisedSleep } from '../utils/promised-sleep';
+import { type FetchOptions } from '../fetch-options';
+import { legacyArgsAsFetchOptions } from './legacy-args';
+import { DefaultRetryConfiguration } from './configuration/default-retry-configuration';
+import type { RetryConfiguring } from './configuration/retry-configuring';
+
+/**
+ * A class that retries a fetch request.
+ */
+export interface FetchRetrierInterface {
+  /**
+   * Execute a fetch with retry.
+   *
+   * @param request RequestInfo
+   * @param options Optional RequestInit | FetchOptions
+   * @returns Promise<Response>
+   */
+  fetchRetry(
+    request: RequestInfo,
+    options?: RequestInit | FetchOptions,
+  ): Promise<Response>;
+}
+
+/** @inheritdoc */
+export class FetchRetrier implements FetchRetrierInterface {
+  private analyticsHandler?: AnalyticsHandlerInterface;
+
+  private retryConfiguration: RetryConfiguring =
+    new DefaultRetryConfiguration();
+
+  constructor(options?: {
+    analyticsHandler?: AnalyticsHandlerInterface;
+    retryConfiguration?: RetryConfiguring;
+  }) {
+    if (options?.analyticsHandler)
+      this.analyticsHandler = options.analyticsHandler;
+    if (options?.retryConfiguration)
+      this.retryConfiguration = options.retryConfiguration;
+  }
+
+  /** @inheritdoc */
+  public async fetchRetry(
+    request: RequestInfo,
+    options?: RequestInit | FetchOptions,
+  ): Promise<Response> {
+    const fetchOptions = legacyArgsAsFetchOptions(options);
+    return await this.fetchRetryWithOptions(request, 0, fetchOptions);
+  }
+
+  private async fetchRetryWithOptions(
+    request: RequestInfo,
+    retryNumber: number,
+    options?: FetchOptions,
+  ): Promise<Response> {
+    const urlString = typeof request === 'string' ? request : request.url;
+
+    try {
+      const response = await fetch(request, options?.requestInit);
+      if (response.ok) return response;
+
+      if (response.status >= 400 && response.status < 500) {
+        this.log4xxResponse(response);
+      }
+
+      const retryConfig = options?.retryConfig ?? this.retryConfiguration;
+      if (retryConfig.shouldRetry(response, retryNumber)) {
+        const delay = retryConfig.retryDelay(retryNumber);
+        await promisedSleep(delay);
+        this.logRetryEvent(
+          urlString,
+          retryNumber,
+          response.statusText,
+          response.status,
+        );
+        return this.fetchRetryWithOptions(request, retryNumber + 1, options);
+      }
+      this.logFailureEvent(urlString, response.status);
+      return response;
+    } catch (error) {
+      // if a content blocker is detected, log it and don't retry
+      if (this.isContentBlockerError(error)) {
+        this.logContentBlockingEvent(urlString, error);
+        throw error;
+      }
+
+      const retryConfig = options?.retryConfig ?? this.retryConfiguration;
+      if (retryConfig.shouldRetry(null, retryNumber)) {
+        const delay = retryConfig.retryDelay(retryNumber);
+        await promisedSleep(delay);
+        this.logRetryEvent(urlString, retryNumber, error, error);
+        return this.fetchRetryWithOptions(request, retryNumber + 1, options);
+      }
+      this.logFailureEvent(urlString, error);
+      throw error;
+    }
+  }
+
+  private isContentBlockerError(error: unknown): boolean {
+    // all of the content blocker errors are `TypeError`
+    if (!(error instanceof TypeError)) return false;
+    const message = error.message.toLowerCase();
+    return message.includes('content blocker');
+  }
+
+  private readonly eventCategory = 'offshootFetchRetry';
+
+  private logRetryEvent(
+    urlString: string,
+    retryNumber: number,
+    status: unknown,
+    code: unknown,
+  ) {
+    this.analyticsHandler?.sendEvent({
+      category: this.eventCategory,
+      action: 'retryingFetch',
+      label: `retryNumber: ${retryNumber}, code: ${code}, status: ${status}, url: ${urlString}`,
+    });
+  }
+
+  private logFailureEvent(urlString: string, error: unknown) {
+    this.analyticsHandler?.sendEvent({
+      category: this.eventCategory,
+      action: 'fetchFailed',
+      label: `error: ${error}, url: ${urlString}`,
+    });
+  }
+
+  private log4xxResponse(response: Response) {
+    const status = response.status;
+
+    this.analyticsHandler?.sendEvent({
+      category: this.eventCategory,
+      action: `status4xxResponse`,
+      label: `http status ${status}, url: ${response.url}`,
+    });
+  }
+
+  private logContentBlockingEvent(urlString: string, error: unknown) {
+    this.analyticsHandler?.sendEvent({
+      category: this.eventCategory,
+      action: 'contentBlockerDetectedNotRetrying',
+      label: `error: ${error}, url: ${urlString}`,
+    });
+  }
+}

package/src/fetch-retry/legacy-args.ts

@@ -0,0 +1,13 @@
+import type { FetchOptions } from '../fetch-options';
+
+export const legacyArgsAsFetchOptions = (
+  options?: RequestInit | FetchOptions,
+): FetchOptions | undefined => {
+  if (!options) return undefined;
+  // if options is already FetchOptions, return it
+  if ('requestInit' in options || 'retryConfig' in options) {
+    return options as FetchOptions;
+  }
+  // otherwise, it's RequestInit
+  return { requestInit: options as RequestInit };
+};

package/test/default-retry-config.test.ts

@@ -0,0 +1,34 @@
+import { expect } from '@open-wc/testing';
+import { DefaultRetryConfiguration } from '../src/fetch-retry/configuration/default-retry-configuration';
+
+describe('DefaultRetryConfiguration', () => {
+  it('should not retry on null response', async () => {
+    const config = new DefaultRetryConfiguration();
+    expect(config.shouldRetry(null, 1)).to.be.false;
+  });
+
+  it('should not retry after max retries exceeded', async () => {
+    const config = new DefaultRetryConfiguration({ maxRetries: 2 });
+    const mockResponse = new Response(null, { status: 500 });
+    expect(config.shouldRetry(mockResponse, 3)).to.be.false;
+  });
+
+  it('should retry on 5xx status codes', async () => {
+    const config = new DefaultRetryConfiguration();
+    const mockResponse = new Response(null, { status: 502 });
+    expect(config.shouldRetry(mockResponse, 1)).to.be.true;
+  });
+
+  it('should not retry on non-5xx status codes', async () => {
+    const config = new DefaultRetryConfiguration();
+    const mockResponse = new Response(null, { status: 404 });
+    expect(config.shouldRetry(mockResponse, 1)).to.be.false;
+  });
+
+  it('has exponential backoff delay', async () => {
+    const config = new DefaultRetryConfiguration();
+    expect(config.retryDelay(0)).to.equal(500);
+    expect(config.retryDelay(1)).to.equal(1000);
+    expect(config.retryDelay(2)).to.equal(2000);
+  });
+});

package/test/fetch-handler.test.ts

@@ -0,0 +1,99 @@
+import { expect } from '@open-wc/testing';
+import { IaFetchHandler } from '../src/fetch-handler';
+import { MockFetchRetrier } from './mocks/mock-fetch-retrier';
+import { NoRetryConfiguration } from '../src/fetch-retry/configuration/no-retry-configuration';
+
+describe('Fetch Handler', () => {
+  describe('fetch', () => {
+    it('adds reCache=1 if it is in the current url', async () => {
+      const fetchRetrier = new MockFetchRetrier();
+      const fetchHandler = new IaFetchHandler({
+        fetchRetrier: fetchRetrier,
+        searchParams: '?reCache=1',
+      });
+      await fetchHandler.fetch('https://foo.org/api/v1/snoot');
+      expect(fetchRetrier.requestInfo).to.equal(
+        'https://foo.org/api/v1/snoot?reCache=1',
+      );
+    });
+
+    it('appends reCache=1 when request is a Request object', async () => {
+      const fetchRetrier = new MockFetchRetrier();
+      const fetchHandler = new IaFetchHandler({
+        fetchRetrier,
+        searchParams: '?reCache=1',
+      });
+      const req = new Request('https://foo.org/api/v1/snoot');
+      await fetchHandler.fetch(req);
+      expect(fetchRetrier.requestInfo).to.equal(
+        'https://foo.org/api/v1/snoot?reCache=1',
+      );
+    });
+
+    it('does not append reCache when not present', async () => {
+      const fetchRetrier = new MockFetchRetrier();
+      const fetchHandler = new IaFetchHandler({
+        fetchRetrier,
+        searchParams: '?foo=bar',
+      });
+      await fetchHandler.fetch('https://foo.org/api/v1/snoot');
+      expect(fetchRetrier.requestInfo).to.equal('https://foo.org/api/v1/snoot');
+    });
+  });
+
+  describe('fetchIAApiResponse', () => {
+    it('prepends the IA basehost to the url when making a request', async () => {
+      const endpoint = '/foo/service/endpoint.php';
+      const fetchRetrier = new MockFetchRetrier();
+      const fetchHandler = new IaFetchHandler({
+        iaApiBaseUrl: 'www.example.com',
+        fetchRetrier: fetchRetrier,
+      });
+      await fetchHandler.fetchIAApiResponse(endpoint);
+      expect(fetchRetrier.requestInfo).to.equal(
+        'www.example.com/foo/service/endpoint.php',
+      );
+    });
+  });
+
+  describe('fetchApiResponse', () => {
+    it('adds credentials: include if requested', async () => {
+      const endpoint = '/foo/service/endpoint.php';
+      const fetchRetrier = new MockFetchRetrier();
+      const fetchHandler = new IaFetchHandler({
+        iaApiBaseUrl: 'www.example.com',
+        fetchRetrier: fetchRetrier,
+      });
+      await fetchHandler.fetchApiResponse(endpoint, {
+        includeCredentials: true,
+      });
+      expect(fetchRetrier.init).to.deep.equal({ credentials: 'include' });
+    });
+
+    it('passes method, body, and headers to RequestInit', async () => {
+      const fetchRetrier = new MockFetchRetrier();
+      const fetchHandler = new IaFetchHandler({ fetchRetrier });
+      const body = JSON.stringify({ hello: 'world' });
+      await fetchHandler.fetchApiResponse('https://example.org/api', {
+        method: 'POST',
+        body,
+        headers: { 'x-test': '1', 'content-type': 'application/json' },
+      });
+      expect(fetchRetrier.init).to.deep.equal({
+        method: 'POST',
+        body,
+        headers: { 'x-test': '1', 'content-type': 'application/json' },
+      });
+    });
+
+    it('passes retryConfig through to retrier', async () => {
+      const fetchRetrier = new MockFetchRetrier();
+      const retryConfig = new NoRetryConfiguration();
+      const fetchHandler = new IaFetchHandler({ fetchRetrier });
+      await fetchHandler.fetchApiResponse('https://example.org/api', {
+        retryConfig,
+      });
+      expect(fetchRetrier.retryConfig).to.equal(retryConfig);
+    });
+  });
+});