@datocms/rest-client-utils 0.1.0

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@datocms/rest-client-utils",
+  "version": "0.1.0",
+  "description": "Utilities for DatoCMS REST API clients",
+  "keywords": [
+    "datocms",
+    "client"
+  ],
+  "author": "Stefano Verna <s.verna@datocms.com>",
+  "homepage": "https://github.com/datocms/js-toolkit/tree/main/packages/rest-client-utils#readme",
+  "license": "MIT",
+  "main": "dist/cjs/index.js",
+  "module": "dist/esm/index.js",
+  "typings": "dist/types/index.d.ts",
+  "sideEffects": false,
+  "directories": {
+    "lib": "dist",
+    "test": "__tests__"
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/datocms/js-toolkit.git"
+  },
+  "scripts": {
+    "build": "tsc && tsc --project ./tsconfig.esnext.json",
+    "prebuild": "rimraf dist"
+  },
+  "bugs": {
+    "url": "https://github.com/datocms/js-toolkit/issues"
+  },
+  "dependencies": {
+    "async-scheduler": "^1.4.4",
+    "cross-fetch": "^3.1.5",
+    "qs": "^6.10.3"
+  },
+  "gitHead": "41036fc6a62b20638bf91f8c9f58ff8ebabb69bf"
+}

package/src/ApiError.ts

@@ -0,0 +1,167 @@
+export type ErrorEntity = {
+  id: string;
+  type: 'api_error';
+  attributes: {
+    code: string;
+    details: unknown;
+  };
+};
+type ErrorBody = { data: ErrorEntity[] };
+
+function isErrorBody(body: unknown): body is ErrorBody {
+  if (typeof body !== 'object' || body === null || !('data' in body)) {
+    return false;
+  }
+
+  const bodyWithData = body as { data: unknown };
+
+  if (!Array.isArray(bodyWithData.data)) {
+    return false;
+  }
+
+  const bodyWithDataList = bodyWithData as { data: unknown[] };
+
+  if (bodyWithDataList.data.length === 0) {
+    return false;
+  }
+
+  const firstEl = bodyWithDataList.data[0];
+
+  if (
+    typeof firstEl !== 'object' ||
+    firstEl === null ||
+    !('id' in firstEl) ||
+    !('type' in firstEl) ||
+    !('attributes' in firstEl) ||
+    (firstEl as ErrorEntity).type !== 'api_error'
+  ) {
+    return false;
+  }
+
+  return true;
+}
+
+const humanMessageForCode = {
+  BATCH_DATA_VALIDATION_IN_PROGRESS: `The schema of this model changed, we're re-running validations over every record in background. Please retry with this operation in a few seconds!`,
+  INSUFFICIENT_PERMISSIONS: `Your role does not permit this action`,
+  MAINTENANCE_MODE: `The project is currently in maintenance mode!`,
+  DELETE_RESTRICTION: `Sorry, but you cannot delete this resource, as it's currently used/referenced elsewhere!`,
+  INVALID_CREDENTIALS: `Credentials are incorrect!`,
+  INVALID_EMAIL: `Email address is incorrect!`,
+  INVALID_FORMAT: `The format of the parameters passed is incorrect, take a look at the details of the error to know what's wrong!`,
+  ITEM_LOCKED: `The operation cannot be completed as some other user is currently editing this record!`,
+  LINKED_FROM_PUBLISHED_ITEMS: `Couldn't unpublish the record, as some published records are linked to it!`,
+  PLAN_UPGRADE_REQUIRED: `Cannot proceed, please upgrade plan!`,
+  PUBLISHED_CHILDREN: `Couldn't unpublish the record, some children records are still published!`,
+  REQUIRED_2FA_SETUP: `This project requires every user to turn on 2-factor authentication! Please go to your Dashboard and activate it! (https://dashboard.datocms.com/account/setup-2fa)`,
+  REQUIRED_BY_ASSOCIATION: `Cannot delete the record, as it's required by other records:`,
+  STALE_ITEM_VERSION: `Someone else made a change while you were editing this record, please refresh the page!`,
+  TITLE_ALREADY_PRESENT: `There can only be one Title field per model`,
+  UNPUBLISHED_LINK: `Couldn't publish the record, as it links some unpublished records!`,
+  UNPUBLISHED_PARENT: `Couldn't publish the record, as the parent record is not published!`,
+  UPLOAD_IS_CURRENTLY_IN_USE: `Couldn't delete this asset, as it's currently used by some records!`,
+  UPLOAD_NOT_PASSING_FIELD_VALIDATIONS: `Couldn't update this asset since some records are failing to pass the validations!`,
+};
+
+const humanMessageForPlanUpgradeLimit = {
+  build_triggers: `You've reached the maximum number of build triggers your plan allows`,
+  sandbox_environments: `You've reached the maximum number of environments your plan allows`,
+  item_types: `You've reached the maximum number of models your plan allows to create`,
+  items: `You've reached the maximum number of records your plan allows to create`,
+  locales: `You've reached the maximum number of locales your plan allows`,
+  mux_encoding_seconds: `You've reached the maximum video encoding limits of your plan`,
+  otp: `Two-factor authentication cannot be on the current plan`,
+  plugins: `You've reached the maximum number of plugins your plan allows`,
+  roles: `You've reached the maximum number of roles your plan allows to create`,
+  uploadable_bytes: `You've reached the file storage limits of your plan`,
+  users: `You've reached the maximum number of collaborators your plan allows to invite to the project`,
+  access_tokens: `You've reached the maximum number of API tokens your plan allows to create`,
+};
+
+export type ApiErrorRequest = {
+  url: string;
+  method: string;
+  headers: Record<string, string>;
+  body?: unknown;
+};
+
+export type ApiErrorResponse = {
+  status: number;
+  statusText: string;
+  headers: Record<string, string>;
+  body?: unknown;
+};
+
+export type ApiErrorInitObject = {
+  request: ApiErrorRequest;
+  response: ApiErrorResponse;
+  preCallStack?: string;
+};
+
+export class ApiError extends Error {
+  request: ApiErrorRequest;
+  response: ApiErrorResponse;
+  preCallStack?: string;
+
+  constructor(initObject: ApiErrorInitObject) {
+    super('API Error!');
+    Object.setPrototypeOf(this, new.target.prototype);
+
+    if ('captureStackTrace' in Error) {
+      Error.captureStackTrace(this, ApiError);
+    } else {
+      this.stack = new Error().stack;
+    }
+
+    this.request = initObject.request;
+    this.response = initObject.response;
+    this.preCallStack = initObject.preCallStack;
+
+    let message = `${initObject.request.method} ${initObject.request.url}: ${this.response.status} ${this.response.statusText}`;
+
+    if (this.errors.length > 0) {
+      message += `\n\n${JSON.stringify(this.errors, null, 2)}`;
+    }
+
+    this.message = message;
+
+    if (this.preCallStack) {
+      this.stack += `\nCaused By:\n${this.preCallStack}`;
+    }
+  }
+
+  get errors() {
+    if (!isErrorBody(this.response.body)) {
+      return [];
+    }
+
+    return this.response.body.data;
+  }
+
+  findErrorWithCode(codeOrCodes) {
+    const codes = Array.isArray(codeOrCodes) ? codeOrCodes : [codeOrCodes];
+    return this.errors.find((error) => codes.includes(error.attributes.code));
+  }
+
+  get humanMessage() {
+    const planUpgradeError = this.findErrorWithCode('PLAN_UPGRADE_REQUIRED');
+
+    if (planUpgradeError) {
+      const { limit } = planUpgradeError.attributes.details as Record<
+        string,
+        string
+      >;
+      return `${humanMessageForPlanUpgradeLimit[limit]}. Please head over to your account dashboard (https://dashboard.datocms.com/) to upgrade the plan or, if no publicly available plan suits your needs, contact our Sales team (https://www.datocms.com/contact) to get a custom quote!`;
+    }
+
+    const errors = Object.keys(humanMessageForCode)
+      .filter((code) => this.findErrorWithCode(code))
+      .map((code) => humanMessageForCode[code]);
+
+    if (errors.length === 0) {
+      return null;
+    }
+
+    return errors.join('\n');
+  }
+}

package/src/CancelablePromise.ts

@@ -0,0 +1,41 @@
+export class CanceledPromiseError extends Error {
+  constructor() {
+    super('Promise canceled!');
+    Object.setPrototypeOf(this, new.target.prototype);
+  }
+}
+
+export interface CancelablePromise<T> extends Promise<T> {
+  cancel(): void;
+}
+
+export function makeCancelablePromise<T>(
+  promiseOrAsyncFn: Promise<T> | (() => Promise<T>),
+  onCancel: () => void,
+): CancelablePromise<T> {
+  let cancel: (() => void) | null = null;
+
+  const cancelable = <CancelablePromise<T>>new Promise((resolve, reject) => {
+    cancel = () => {
+      try {
+        onCancel();
+        reject(new CanceledPromiseError());
+      } catch (e) {
+        reject(e);
+      }
+    };
+
+    const promise =
+      typeof promiseOrAsyncFn === 'function'
+        ? promiseOrAsyncFn()
+        : promiseOrAsyncFn;
+
+    promise.then(resolve, reject);
+  });
+
+  if (cancel) {
+    cancelable.cancel = cancel;
+  }
+
+  return cancelable;
+}

package/src/deserialize.ts

@@ -0,0 +1,52 @@
+import { Rel } from './serialize';
+
+type JsonApiEntity = {
+  id?: string;
+  type?: string;
+  attributes?: object;
+  relationships?: object;
+  meta?: object;
+};
+
+type ResponseWithData = {
+  data: JsonApiEntity | JsonApiEntity[];
+};
+
+function hasData(thing: unknown): thing is ResponseWithData {
+  return typeof thing === 'object' && !!thing && 'data' in thing;
+}
+
+export function deserializeJsonEntity<S>({
+  id,
+  type,
+  attributes,
+  relationships,
+  meta,
+}: JsonApiEntity): S {
+  return {
+    ...(id ? { id } : {}),
+    ...(type ? { type } : {}),
+    ...(attributes || {}),
+    ...(relationships
+      ? Object.fromEntries(
+          Object.entries(relationships).map(([rel, value]) => [
+            rel,
+            value?.data,
+          ]),
+        )
+      : {}),
+    ...(meta ? { meta } : {}),
+  } as S;
+}
+
+export function deserializeResponseBody<T>(body: unknown): T {
+  if (!hasData(body)) {
+    throw new Error('Invalid body!');
+  }
+
+  if (Array.isArray(body.data)) {
+    return body.data.map(deserializeJsonEntity) as unknown as T;
+  }
+
+  return deserializeJsonEntity(body.data) as unknown as T;
+}

package/src/index.ts

@@ -0,0 +1,11 @@
+'use strict';
+
+export * from './ApiError';
+export * from './deserialize';
+export * from './pollJobResult';
+export * from './rawPageIterator';
+export * from './request';
+export * from './serialize';
+export * from './CancelablePromise';
+export * from './toId';
+export * from './wait';

package/src/internalTypes.ts

@@ -0,0 +1,8 @@
+export type JobResult = {
+  type: 'job_result';
+  id: string;
+  status: number;
+  payload: null | {
+    [k: string]: unknown;
+  };
+};

package/src/pollJobResult.ts

@@ -0,0 +1,24 @@
+import { ApiError } from './ApiError';
+import { JobResult } from './internalTypes';
+import { wait } from './wait';
+
+export async function pollJobResult(
+  fetcher: () => Promise<JobResult>,
+): Promise<JobResult> {
+  let jobResult: JobResult | undefined;
+  let retryCount = 0;
+
+  do {
+    try {
+      retryCount += 1;
+      await wait(retryCount * 1000);
+      jobResult = await fetcher();
+    } catch (e) {
+      if (!(e instanceof ApiError) || e.response.status !== 404) {
+        throw e;
+      }
+    }
+  } while (!jobResult);
+
+  return jobResult;
+}

package/src/rawPageIterator.ts

@@ -0,0 +1,66 @@
+import { Scheduler } from 'async-scheduler';
+
+export type IteratorOptions = {
+  perPage?: number;
+  concurrency?: number;
+};
+
+type PaginationOptions = {
+  defaultLimit: number;
+  maxLimit: number;
+};
+
+type JsonApiPage<T> = {
+  data: T[];
+  meta: { total_count: number };
+};
+
+export async function* rawPageIterator<T>(
+  pagination: PaginationOptions,
+  callPerformer: (page: {
+    limit: number;
+    offset: number;
+  }) => Promise<JsonApiPage<T>>,
+  iteratorOptions?: IteratorOptions,
+) {
+  const perPage = iteratorOptions?.perPage || pagination.defaultLimit;
+
+  if (perPage > pagination.maxLimit) {
+    throw new Error(
+      `perPage option cannot exceed maximum value of ${pagination.maxLimit}`,
+    );
+  }
+
+  const concurrency = iteratorOptions?.concurrency || 1;
+
+  if (concurrency > 10) {
+    throw new Error(`concurrency option cannot exceed maximum value of 10`);
+  }
+
+  const firstResponse = await callPerformer({ limit: perPage, offset: 0 });
+
+  for (const item of firstResponse.data) {
+    yield item;
+  }
+
+  const totalCount = firstResponse.meta.total_count;
+
+  const limiter = new Scheduler(concurrency);
+  const promises: Promise<JsonApiPage<T>>[] = [];
+
+  for (let offset = perPage; offset < totalCount; offset += perPage) {
+    promises.push(
+      limiter.enqueue<JsonApiPage<T>, void>(() =>
+        callPerformer({ limit: perPage, offset }),
+      ),
+    );
+  }
+
+  while (promises.length > 0) {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const response = await promises.shift()!;
+    for (const item of response.data) {
+      yield item;
+    }
+  }
+}

package/src/request.ts

@@ -0,0 +1,279 @@
+import qs from 'qs';
+import fetch from 'cross-fetch';
+import { ApiError, ApiErrorInitObject } from './ApiError';
+import { JobResult } from './internalTypes';
+
+export enum LogLevel {
+  /** No logging */
+  NONE = 0,
+  /** Logs HTTP requests (method, URL) and responses (status) */
+  BASIC = 1,
+  /** Logs HTTP requests (method, URL, body) and responses (status, body) */
+  BODY = 2,
+  /** Logs HTTP requests (method, URL, headers, body) and responses (status, headers, body) */
+  BODY_AND_HEADERS = 3,
+}
+
+type RequestOptions = {
+  baseUrl: string;
+  fetchJobResult: (jobId: string) => Promise<JobResult>;
+  apiToken: string | null;
+  extraHeaders?: Record<string, string>;
+  logLevel?: LogLevel;
+  autoRetry?: boolean;
+  retryCount?: number;
+  method: 'GET' | 'PUT' | 'POST' | 'DELETE';
+  url: string;
+  queryParams?: Record<string, unknown>;
+  body?: unknown;
+  preCallStack?: string;
+  userAgent?: string;
+};
+
+function headersToObject(headers: Headers): Record<string, string> {
+  const result = {};
+
+  headers.forEach((value, key) => {
+    result[key] = value;
+  });
+
+  return result;
+}
+
+function buildApiErrorInitObject(
+  method: string,
+  url: string,
+  requestHeaders: Record<string, string>,
+  requestBody: unknown,
+  response: Response,
+  responseBody: unknown,
+  preCallStack?: string,
+): ApiErrorInitObject {
+  return {
+    request: {
+      url,
+      method,
+      headers: requestHeaders,
+      body: requestBody,
+    },
+    response: {
+      status: response.status,
+      statusText: response.statusText,
+      headers: headersToObject(response.headers),
+      body: responseBody,
+    },
+    preCallStack,
+  };
+}
+
+function buildApiErrorInitObjectFromJobResult(
+  method: string,
+  url: string,
+  requestHeaders: Record<string, string>,
+  requestBody: unknown,
+  responseStatus: number,
+  responseBody: unknown,
+  preCallStack?: string,
+): ApiErrorInitObject {
+  return {
+    request: {
+      url,
+      method,
+      headers: requestHeaders,
+      body: requestBody,
+    },
+    response: {
+      status: responseStatus,
+      statusText: 'N/A',
+      headers: {},
+      body: responseBody,
+    },
+    preCallStack,
+  };
+}
+
+function wait(time) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, time);
+  });
+}
+
+function isErrorWithCode(error: unknown): error is { code: string } {
+  return typeof error === 'object' && !!error && 'code' in error;
+}
+
+let requestCount = 1;
+
+export async function request<T>(options: RequestOptions): Promise<T> {
+  const requestId = requestCount;
+  requestCount += 1;
+
+  const preCallStack = options.preCallStack;
+  const userAgent = options.userAgent || `@datocms/rest-client-utils`;
+  const retryCount = options.retryCount || 1;
+  const logLevel = options.logLevel || LogLevel.NONE;
+  const autoRetry = 'autoRetry' in options ? options.autoRetry : true;
+
+  const headers = {
+    'content-type': 'application/json',
+    accept: 'application/json',
+    authorization: `Bearer ${options.apiToken}`,
+    'user-agent': userAgent,
+    ...(options.extraHeaders || {}),
+  };
+
+  const baseUrl = options.baseUrl.replace(/\/$/, '');
+  const body = options.body ? JSON.stringify(options.body, null, 2) : undefined;
+
+  const queryString =
+    options.queryParams && Object.keys(options.queryParams).length > 0
+      ? `?${qs.stringify(options.queryParams, { arrayFormat: 'brackets' })}`
+      : '';
+
+  const url = `${baseUrl}${options.url}${queryString}`;
+
+  if (logLevel >= LogLevel.BASIC) {
+    console.log(`[${requestId}] ${options.method} ${url}`);
+    if (logLevel >= LogLevel.BODY_AND_HEADERS) {
+      for (const [key, value] of Object.entries(headers || {})) {
+        console.log(`[${requestId}] ${key}: ${value}`);
+      }
+    }
+    if (logLevel >= LogLevel.BODY && body) {
+      console.log(`[${requestId}] ${body}`);
+    }
+  }
+
+  try {
+    const response = await fetch(url, {
+      method: options.method,
+      headers,
+      body,
+    });
+
+    if (response.status === 429) {
+      if (!autoRetry) {
+        throw new ApiError(
+          buildApiErrorInitObject(
+            options.method,
+            url,
+            headers,
+            options.body,
+            response,
+            undefined,
+            preCallStack,
+          ),
+        );
+      }
+
+      const waitTimeInSecs = response.headers.has('X-RateLimit-Reset')
+        ? // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          parseInt(response.headers.get('X-RateLimit-Reset')!, 10)
+        : retryCount;
+
+      if (logLevel >= LogLevel.BASIC) {
+        console.log(
+          `[${requestId}] Rate limit exceeded, waiting ${waitTimeInSecs} seconds...`,
+        );
+      }
+
+      await wait(waitTimeInSecs * 1000);
+
+      return request({ ...options, retryCount: retryCount + 1 });
+    }
+
+    if (logLevel >= LogLevel.BASIC) {
+      console.log(
+        `[${requestId}] Status: ${response.status} (${response.statusText})`,
+      );
+      if (logLevel >= LogLevel.BODY_AND_HEADERS) {
+        [
+          'content-type',
+          'x-api-version',
+          'x-environment',
+          'x-queue-time',
+          'x-ratelimit-remaining',
+        ].forEach((key) => {
+          const value = response.headers.get(key);
+          if (value) {
+            console.log(`[${requestId}] ${key}: ${value}`);
+          }
+        });
+      }
+    }
+
+    let responseBody =
+      response.status === 204 ? undefined : await response.json();
+
+    if (logLevel >= LogLevel.BODY && responseBody) {
+      console.log(`[${requestId}] ${JSON.stringify(responseBody, null, 2)}`);
+    }
+
+    if (response.status === 202) {
+      const jobResult = await options.fetchJobResult(responseBody.data.id);
+
+      if (jobResult.status < 200 || jobResult.status >= 300) {
+        throw new ApiError(
+          buildApiErrorInitObjectFromJobResult(
+            options.method,
+            url,
+            headers,
+            options.body,
+            jobResult.status,
+            jobResult.payload,
+            preCallStack,
+          ),
+        );
+      }
+
+      responseBody = jobResult.payload;
+    }
+
+    if (response.status >= 200 && response.status < 300) {
+      return responseBody;
+    }
+
+    const error = new ApiError(
+      buildApiErrorInitObject(
+        options.method,
+        url,
+        headers,
+        options.body,
+        response,
+        responseBody,
+        preCallStack,
+      ),
+    );
+
+    if (
+      autoRetry &&
+      error.findErrorWithCode('BATCH_DATA_VALIDATION_IN_PROGRESS')
+    ) {
+      if (logLevel >= LogLevel.BASIC) {
+        console.log(
+          `[${requestId}] Data validation in progress, waiting ${retryCount} seconds...`,
+        );
+      }
+
+      await wait(retryCount * 1000);
+
+      return request({ ...options, retryCount: retryCount + 1 });
+    }
+
+    throw error;
+  } catch (error) {
+    if (isErrorWithCode(error) && error.code.includes('ETIMEDOUT')) {
+      if (logLevel >= LogLevel.BASIC) {
+        console.log(
+          `[${requestId}] Error ${error.code}, waiting ${retryCount} seconds...`,
+        );
+      }
+
+      await wait(retryCount * 1000);
+
+      return request({ ...options, retryCount: retryCount + 1 });
+    }
+
+    throw error;
+  }
+}