@clickup/rest-client 2.10.293 → 2.10.294

package/src/RestClient.ts

@@ -0,0 +1,490 @@
+import crypto from "crypto";
+import isUndefined from "lodash/isUndefined";
+import omitBy from "lodash/omitBy";
+import { Headers } from "node-fetch";
+import OAuth1 from "oauth-1.0a";
+import RestTokenInvalidError from "./errors/RestTokenInvalidError";
+import RestRangeUploader from "./internal/RestRangeUploader";
+import substituteParams from "./internal/substituteParams";
+import type RestOptions from "./RestOptions";
+import { DEFAULT_OPTIONS } from "./RestOptions";
+import RestRequest from "./RestRequest";
+import type RestResponse from "./RestResponse";
+
+/**
+ * A callback which returns access token, possibly after refreshing it, and also
+ * possibly before a retry on "invalid token" condition. I.e. it can be called
+ * once or twice (the 2nd time after the previous request error, and that error
+ * will be passed as a parameter).
+ */
+export interface TokenGetter<TData = string> {
+  (prevError: Error | null): Promise<TData>;
+}
+
+/**
+ * RestClient is an immutable object which allows to:
+ * 1. Send remote requests in different formats, in a caller-friendly manner.
+ * 2. Create a new RestClient objects deriving the current set of options and
+ *    adding new ones.
+ */
+export default class RestClient {
+  private readonly _options: RestOptions;
+
+  constructor(options: Partial<RestOptions> = {}) {
+    this._options = {
+      ...DEFAULT_OPTIONS,
+      ...omitBy(options, isUndefined), // undefined === "fallback to default"
+      ...(options.keepAlive ? { keepAlive: { ...options.keepAlive } } : {}),
+      ...(options.middlewares ? { middlewares: [...options.middlewares] } : {}),
+    };
+  }
+
+  /**
+   * Returns a new RestClient with some options updated with the passed ones.
+   */
+  withOptions(options: Partial<RestOptions>) {
+    return new RestClient({
+      ...this._options,
+      ...omitBy(options, isUndefined),
+    });
+  }
+
+  /**
+   * Returns a new RestClient with added middleware.
+   */
+  withMiddleware(
+    middleware: RestOptions["middlewares"][0],
+    method: "unshift" | "push" = "push"
+  ) {
+    const clone = new RestClient(this._options);
+    clone._options.middlewares[method](middleware);
+    return clone;
+  }
+
+  /**
+   * Returns a new RestClient with the base URL which will be prepended to all
+   * relative paths in get(), writeForm() etc. Allows to defer resolution of
+   * this base URL to the very late per-request moment. The complicated piece
+   * here is that, if we want base URL to be resolved asynchronously, we often
+   * times want to reuse the same RestClient object (to e.g. fetch some part of
+   * the base URL using already authenticated client). And a re-enterable call
+   * appears here which we must protect against in the code below.
+   */
+  withBase(base: string | (() => Promise<string>)) {
+    return this.withMiddleware(async function withBaseMiddleware(req, next) {
+      // If URL is already absolute, we don't even need to call base() lambda
+      // to resolve it. Return early.
+      if (req.url.match(/^\w+:\/\//)) {
+        return next(req);
+      }
+
+      // Re-enterable call detected, e.g. base() lambda issued a request
+      // through the same RestClient. We must just skip setting the base.
+      if (base === "") {
+        return next(req);
+      }
+
+      if (typeof base !== "string") {
+        const baseGetter = base;
+        base = ""; // disable re-enterable calls
+        try {
+          base = await baseGetter();
+        } catch (e: any) {
+          base = baseGetter;
+          throw e;
+        }
+      }
+
+      req.url = new URL(req.url, base).toString();
+      return next(req);
+    });
+  }
+
+  /**
+   * Returns a new RestClient with a custom header.
+   */
+  withHeader(name: string, value: string | (() => Promise<string>)) {
+    return this.withMiddleware(async function withHeaderMiddleware(req, next) {
+      req.headers.set(name, typeof value === "string" ? value : await value());
+      return next(req);
+    });
+  }
+
+  /**
+   * Returns a new RestClient with a bearer token authentication workflow.
+   * - RestClient supports interception of options.isTokenInvalid() signal and
+   *   conversion it into RestTokenInvalidError exception.
+   * - If a token() is a lambda with 1 argument, it may be called the 2nd time
+   *   when we get an isTokenInvalid() signal. In this case, the request is
+   *   retried.
+   * - If token() is a lambda with 0 arguments, that means it doesn't want to
+   *   watch for the isTokenInvalid() signal, so there is no sense in retrying
+   *   the request either.
+   *
+   * From the first sight, it looks like options.isTokenInvalid() signal is
+   * coupled to setBearer() auth method only. But it's not true:
+   * isTokenInvalid() makes sense for ALL authentication methods actually (even
+   * for basic auth), and setBearer() is just one of "clients" which implements
+   * refreshing/retries on top of isTokenInvalid().
+   *
+   * Passing the token as lambda allows the caller to implement some complex
+   * logic, e.g.:
+   * - oauth2 tokens refreshing
+   * - marking the token as "revoked" in the database in case the refresh fails
+   * - marking the token as "revoked" after a failed request if refresh-token is
+   *   not supported
+   */
+  withBearer(token: TokenGetter, bearerPrefix = "Bearer ") {
+    return this.withMiddleware(async function withBearerMiddleware(req, next) {
+      return tokenRetryStrategy(token, async (tokenData) => {
+        req.headers.set("Authorization", bearerPrefix + tokenData);
+        return next(req);
+      });
+    });
+  }
+
+  /**
+   * Returns a new RestClient with oauth1 authentication workflow.
+   * - In case we get an options.isTokenInvalid() signal, the token() lambda is
+   *   called the 2nd time with the error object, then the request is retries.
+   *   This gives the lambda a chance to recover or update something in the
+   *   database.
+   *
+   * We use a separate and small oauth-1.0a node library here, because the more
+   * popular one (https://www.npmjs.com/package/oauth) doesn't support signing
+   * of arbitrary requests, it can only send its own requests.
+   */
+  withOAuth1(
+    consumer: { consumerKey: string; consumerSecret: string },
+    token: TokenGetter<{ token: string; tokenSecret: string }>
+  ) {
+    const oauth = new OAuth1({
+      consumer: { key: consumer.consumerKey, secret: consumer.consumerSecret },
+      signature_method: "HMAC-SHA1",
+      hash_function: (baseString, key) =>
+        crypto.createHmac("sha1", key).update(baseString).digest("base64"),
+    });
+
+    return this.withMiddleware(async function withOAuth1Middleware(req, next) {
+      return tokenRetryStrategy(token, async (tokenData) => {
+        const requestData: OAuth1.RequestOptions = {
+          url: req.url,
+          method: req.method,
+          data: req.body,
+        };
+        const addHeaders = oauth.toHeader(
+          oauth.authorize(requestData, {
+            key: tokenData.token,
+            secret: tokenData.tokenSecret,
+          })
+        );
+        for (const [name, value] of Object.entries(addHeaders)) {
+          req.headers.set(name, value);
+        }
+
+        return next(req);
+      });
+    });
+  }
+
+  /**
+   * Returns a new RestClient with basic authorization workflow.
+   */
+  withBasic(token: TokenGetter<{ name: string; password: string }>) {
+    return this.withMiddleware(async function withBasicMiddleware(req, next) {
+      return tokenRetryStrategy(token, async ({ name, password }) => {
+        const unencodedHeader = name + ":" + password;
+        req.headers.set(
+          "Authorization",
+          "Basic " + Buffer.from(unencodedHeader).toString("base64")
+        );
+        return next(req);
+      });
+    });
+  }
+
+  /**
+   * Sends a plain GET request without body.
+   *
+   * NOTE, all args will be passed through `encodeURIComponent`.
+   */
+  get(
+    path: string,
+    args: Partial<Record<string, string | number | string[]>> = {},
+    accept: string = "application/json"
+  ) {
+    return this._noBodyRequest(path, args, "GET", accept);
+  }
+
+  /**
+   * Writes some raw string, buffer or a stream.
+   */
+  writeRaw(
+    path: string,
+    body: string | Buffer | NodeJS.ReadableStream,
+    contentType: string,
+    method: "POST" | "PUT" | "PATCH" = "POST",
+    accept?: string
+  ) {
+    const origShape = simpleShape(path);
+    return new RestRequest(
+      this._options,
+      method,
+      path,
+      new Headers({
+        Accept: accept || contentType,
+        "Content-Type": contentType,
+      }),
+      body,
+      origShape
+    );
+  }
+
+  /**
+   * A shortcut method to write JSON body.
+   */
+  writeJson(
+    path: string,
+    body: any,
+    method: "POST" | "PUT" | "PATCH" | "DELETE" = "POST",
+    accept: string = "application/json"
+  ) {
+    const origShape = simpleShape(path, body);
+    [path, body] = substituteParams(path, body);
+    return new RestRequest(
+      this._options,
+      method,
+      path,
+      new Headers({
+        Accept: accept || "application/json",
+        "Content-Type": "application/json",
+      }),
+      JSON.stringify(body),
+      origShape
+    );
+  }
+
+  /**
+   * A shortcut method to write "application/x-www-form-urlencoded" data.
+   */
+  writeForm(
+    path: string,
+    body: Partial<Record<string, string>> | string,
+    method: "POST" | "PUT" | "PATCH" = "POST",
+    accept: string = "application/json"
+  ) {
+    const origShape = simpleShape(path, body);
+    [path, body] = substituteParams(path, body);
+    return new RestRequest(
+      this._options,
+      method,
+      path,
+      new Headers({
+        Accept: accept,
+        "Content-Type": "application/x-www-form-urlencoded",
+      }),
+      typeof body === "string"
+        ? body
+        : new URLSearchParams(
+            omitBy(body, isUndefined) as Record<string, string>
+          ).toString(),
+      origShape
+    );
+  }
+
+  /**
+   * A shortcut method to write DELETE request.
+   */
+  writeDelete(
+    path: string,
+    args: Partial<Record<string, string>> = {},
+    accept: string = "application/json"
+  ) {
+    return this._noBodyRequest(path, args, "DELETE", accept);
+  }
+
+  /**
+   * Returns a RestRequest prepared for sending GraphQL operation.
+   * - Expects the response to contain no errors; throws otherwise.
+   * - In case of success, returns just the content of `data` field (this is
+   *   different with writeGraphQLNullable() which returns `data` as a separate
+   *   fields along with `error` and `errors`).
+   */
+  writeGraphQLX(query: string, variables: any = {}) {
+    const oldIsSuccessResponse = this._options.isSuccessResponse;
+    return this.withOptions({
+      isSuccessResponse: (res) => {
+        const oldIsSuccess = oldIsSuccessResponse?.(res);
+        const json: any = res.json;
+        return oldIsSuccess === "THROW" ||
+          !json?.data ||
+          json.error ||
+          (json.errors instanceof Array && json.errors.length > 0)
+          ? "THROW"
+          : "BEST_EFFORT";
+      },
+    })
+      .withMiddleware(async function writeGraphQLXMiddleware(req, next) {
+        const res = await next(req);
+        const json: any = res.json;
+        // Substitute the response json object with just its data field.
+        Object.defineProperty(res, "json", { value: json.data });
+        return res;
+      })
+
+      ._writeGraphQLImpl<any>(query, variables);
+  }
+
+  /**
+   * Same as writeGraphQLX(), but doesn't throw if GraphQL response contains
+   * non-empty `error` or `errors` fields and instead returns the full response.
+   * I.e. allows the caller to process these errors.
+   */
+  writeGraphQLNullable(query: string, variables: any = {}) {
+    return this._writeGraphQLImpl<
+      { data?: any; error?: any; errors?: any[] } | null | undefined
+    >(query, variables);
+  }
+
+  /**
+   * Performs a series of Content-Range requests with content from a sequence of
+   * Buffers.
+   */
+  async rangeUpload(
+    path: string,
+    mimeType: string,
+    stream: AsyncIterable<Buffer>,
+    method: "POST" | "PUT" = "POST",
+    chunkSize: number
+  ) {
+    return new RestRangeUploader(
+      this,
+      chunkSize,
+      method,
+      path,
+      mimeType
+    ).upload(stream);
+  }
+
+  private _writeGraphQLImpl<TAssertShape>(query: string, variables: any = {}) {
+    const origShape = query;
+    // To beautify the query, we remove leading spaces (extracted from the last line).
+    query = query.trimEnd();
+    if (query.match(/\n([ \t]+)[^\n]+$/)) {
+      const prefix = RegExp.$1;
+      query = query.replace(new RegExp("^" + prefix, "gm"), "").trimEnd();
+    }
+
+    return new RestRequest<TAssertShape>(
+      this._options,
+      "POST",
+      "",
+      new Headers({ "Content-Type": "application/json" }),
+      JSON.stringify({ variables, query }), // variables first - for truncation in logs,
+      origShape
+    );
+  }
+
+  /**
+   * Sends a plain request (with no body, like GET or DELETE).
+   */
+  private _noBodyRequest(
+    path: string,
+    args: Partial<Record<string, string | number | string[]>> = {},
+    method: "GET" | "DELETE",
+    accept: string = "application/json"
+  ) {
+    const origShape = simpleShape(path, args);
+    [path, args] = substituteParams(path, args);
+    const searchParams = new URLSearchParams();
+    for (const [k, v] of Object.entries(args)) {
+      if (Array.isArray(v)) {
+        for (const element of v) {
+          searchParams.append(k, element.toString());
+        }
+      } else if (v !== undefined) {
+        searchParams.append(k, v.toString());
+      }
+    }
+
+    const suffix = searchParams.toString();
+    if (suffix !== "") {
+      path += (path.includes("?") ? "&" : "?") + suffix;
+    }
+
+    return new RestRequest(
+      this._options,
+      method,
+      path,
+      new Headers({ Accept: accept }),
+      "",
+      origShape
+    );
+  }
+}
+
+/**
+ * @ignore
+ * Calls token(null), then runs body() passing the result there. If we get a
+ * RestTokenInvalidError exception, call token() with this error as a parameter
+ * and then passes the response to body() again (kinda retry with a new token).
+ */
+export async function tokenRetryStrategy<TData>(
+  token: TokenGetter<TData>,
+  body: (tokenData: TData) => Promise<RestResponse>
+) {
+  let tokenData = await token(null);
+  try {
+    return await body(tokenData);
+  } catch (e: any) {
+    const supportsRetry = token.length > 0; // args count
+    if (e instanceof RestTokenInvalidError && supportsRetry) {
+      tokenData = await token(e);
+      try {
+        return await body(tokenData);
+      } catch (e: any) {
+        if (e instanceof RestTokenInvalidError) {
+          e.message = `(still failed, even with updated token) ${e.message}`;
+        }
+
+        throw e;
+      }
+    }
+
+    throw e;
+  }
+}
+
+/**
+ * Returns a simple "shape" of a request useful for e.g. grouping by in logger
+ * to figure out, which requests are more frequent.
+ * - reflects args names and not values
+ * - wipes the domain
+ * - removes query string parameter values
+ */
+function simpleShape(path: string, args?: any) {
+  path = path.replace(/^\w+:\/\/[^/]+/, "").replace(/=[^&]+/g, "=");
+
+  const argsInPath: string[] = [];
+  const regex = /:([a-z0-9_]+)/gi;
+  while (true) {
+    const match = regex.exec(path);
+    if (!match) {
+      break;
+    } else {
+      argsInPath.push(match[1]);
+    }
+  }
+
+  const keys =
+    args && typeof args === "object"
+      ? Object.keys(args)
+          // Filter out args that are already mentioned in the path, e.g.
+          // /pages/:pageID/blocks.
+          .filter((arg) => !argsInPath.includes(arg))
+          .sort()
+          .join(",")
+      : "";
+
+  return path + (keys !== "" ? `:${keys}` : "");
+}

package/src/RestOptions.ts

@@ -0,0 +1,186 @@
+import { Agent as HttpAgent } from "http";
+import { Agent as HttpsAgent } from "https";
+import delay from "delay";
+import { Memoize } from "fast-typescript-memoize";
+import type RestRequest from "./RestRequest";
+import type RestResponse from "./RestResponse";
+
+/**
+ * An event which is passed to an external logger (see RestOptions).
+ */
+export interface RestLogEvent {
+  attempt: number;
+  req: RestRequest;
+  res: RestResponse | "backoff_delay" | null;
+  exception: any | null;
+  timestamp: number;
+  elapsed: number;
+  isFinalAttempt: boolean;
+  privateDataInResponse: boolean;
+  comment: string;
+}
+
+/**
+ * Middlewares allow to modify RestRequest and RestResponse objects during the
+ * request processing.
+ */
+export interface Middleware {
+  (
+    req: RestRequest,
+    next: (req: RestRequest) => Promise<RestResponse>
+  ): Promise<RestResponse>;
+}
+
+/**
+ * @ignore
+ * Parameters for Agents.
+ */
+interface AgentOptions {
+  keepAlive: boolean;
+  timeout?: number;
+  maxSockets?: number;
+  rejectUnauthorized?: boolean;
+  family?: 4 | 6 | 0;
+}
+
+/**
+ * @ignore
+ * An internal class which keeps the list of HttpAgent instances used in a
+ * particular RestClient instance.
+ */
+export class Agents {
+  @Memoize((...args: any[]) => JSON.stringify(args))
+  http(options: AgentOptions) {
+    return new HttpAgent(options);
+  }
+
+  @Memoize((...args: any[]) => JSON.stringify(args))
+  https(options: AgentOptions) {
+    return new HttpsAgent(options);
+  }
+}
+
+/**
+ * Options passed to RestClient. More options can be added by cloning an
+ * instance of RestClient, withOption() method.
+ */
+export default interface RestOptions {
+  /** Max number of retries. Default is 0, because some requests are from the
+   * web app, and we don't want to retry them. */
+  retries: number;
+  /** How much time to wait by default on the 1st retry attempt. */
+  retryDelayFirstMs: number;
+  /** How much to increase the retry delay on each retry. */
+  retryDelayFactor: number;
+  /** Use this fraction (random) of the current retry delay to jitter both ways
+   * (e.g. 0.1 means 90%...110% of the delay to be actually applied). */
+  retryDelayJitter: number;
+  /** Maximum delay between each retry. */
+  retryDelayMaxMs: number;
+  /** A logic which runs on different IO stages (delay and heartbeats). */
+  heartbeater: {
+    /* This function is called after each request with 0 passed, so it can serve
+     * the purpose of a heartbeat. */
+    heartbeat(): Promise<void>;
+    /** A function which, when runs, resolves in the provided number of ms. Can
+     * be used for several purposes, like overriding in unit tests or to pass a
+     * custom delay implementation which can throw on some external event (like
+     * when the process wants to gracefully stop). */
+    delay(ms: number): Promise<void>;
+  };
+  /** Allows to limit huge requests and throw instead. */
+  throwIfResIsBigger: number | undefined;
+  /** Passed to the logger which may decide, should it log details of the
+   * response or not. */
+  privateDataInResponse: boolean;
+  /** If true, non-public IP addresses are allowed too; otherwise, only unicast
+   * addresses are allowed. */
+  allowInternalIPs: boolean;
+  /** If true, logs request-response pairs to console. */
+  isDebug: boolean;
+  /** @ignore Holds HttpsAgent/HttpAgent instances; used internally only. */
+  agents: Agents;
+  /** Sets Keep-Alive parameters (persistent connections). */
+  keepAlive: {
+    /** How much time to keep an idle connection alive in the pool. If 0, closes
+     * the connection immediately after the response. */
+    timeoutMs: number;
+    /** How many sockets at maximum will be kept open. */
+    maxSockets?: number;
+  };
+  /** When resolving DNS, use IPv4, IPv6 or both (see dns.lookup() docs). */
+  family: 4 | 6 | 0;
+  /** Max timeout to wait for a response. */
+  timeoutMs: number;
+  /** Logger to be used for each responses (including retried) plus for backoff
+   * delay events logging. */
+  logger: (event: RestLogEvent) => void;
+  /** Middlewares to wrap requests. May alter both request and response. */
+  middlewares: Middleware[];
+  /** If set, makes decision whether the response is successful or not. The
+   * response will either be returned to the client, or an error will be thrown.
+   * This allows to treat some non-successful HTTP statuses as success if the
+   * remote API is that weird. Return values:
+   * * "SUCCESS" - the request will be considered successful, no further checks
+   *   will be performed;
+   * * "BEST_EFFORT" - inconclusive, the request may be either successful or
+   *   unsuccessful, additional tests (e.g. will check HTTP status code) will be
+   *   performed;
+   * * "THROW" - the request resulted in error. Additional tests will be
+   *   performed to determine is the error is retriable, is OAuth token good,
+   *   and etc. */
+  isSuccessResponse: (res: RestResponse) => "SUCCESS" | "THROW" | "BEST_EFFORT";
+  /** Decides whether the response is a rate-limit error or not. Returning
+   * non-zero value is treated as retry delay (if retries are set up). In case
+   * the returned value is "SOMETHING_ELSE", the response ought to be either
+   * success or some other error. Returning "BEST_EFFORT" turns on built-in
+   * heuristic (e.g. relying on HTTP status code and Retry-After header). In
+   * case we've made a decision that it's a rate limited error, the request is
+   * always retried; this covers a very common case when we have both
+   * isRateLimitError and isRetriableError handlers set up, and they return
+   * contradictory information; then isRateLimitError wins. */
+  isRateLimitError: (
+    res: RestResponse
+  ) => "SOMETHING_ELSE" | "RATE_LIMIT" | "BEST_EFFORT" | number;
+  /** Decides whether the response is a token-invalid error or not. In case it's
+   * not, the response ought to be either success or some other error. */
+  isTokenInvalidError: (res: RestResponse) => boolean;
+  /** Called only if we haven't decided earlier that it's a rate limit error.
+   * Decides whether the response is a retriable error or not. In case the
+   * returned value is "NEVER_RETRY", the response ought to be either success or
+   * some other error, but it's guaranteed that the request won't be retried.
+   * Returning "BEST_EFFORT" turns on built-in heuristics (e.g. never retry "not
+   * found" errors). Returning a number is treated as "RETRY", and the next
+   * retry will happen in not less than this number of milliseconds. */
+  isRetriableError: (
+    res: RestResponse,
+    _error: any
+  ) => "NEVER_RETRY" | "RETRY" | "BEST_EFFORT" | number;
+}
+
+/** @ignore */
+export const DEFAULT_OPTIONS: RestOptions = {
+  retries: 0,
+  retryDelayFirstMs: 1000,
+  retryDelayFactor: 2,
+  retryDelayJitter: 0.1,
+  retryDelayMaxMs: Number.MAX_SAFE_INTEGER,
+  heartbeater: {
+    heartbeat: async () => {},
+    delay,
+  },
+  throwIfResIsBigger: undefined,
+  privateDataInResponse: false,
+  allowInternalIPs: false,
+  isDebug: false,
+  agents: new Agents(),
+  keepAlive: { timeoutMs: 10000 },
+  family: 4, // we don't want to hit ~5s DNS timeout on a missing IPv6 by default
+  timeoutMs: 4 * 60 * 1000,
+  logger: () => {},
+  middlewares: [],
+  isSuccessResponse: () => "BEST_EFFORT",
+  isRateLimitError: () => "BEST_EFFORT",
+  isTokenInvalidError: () => false,
+  isRetriableError: () => "BEST_EFFORT",
+};