@clickup/rest-client 2.10.293 → 2.10.296

package/src/RestOptions.ts

@@ -0,0 +1,186 @@
+import { Agent as HttpAgent } from "http";
+import { Agent as HttpsAgent } from "https";
+import delay from "delay";
+import { Memoize } from "fast-typescript-memoize";
+import type RestRequest from "./RestRequest";
+import type RestResponse from "./RestResponse";
+
+/**
+ * An event which is passed to an external logger (see RestOptions).
+ */
+export interface RestLogEvent {
+  attempt: number;
+  req: RestRequest;
+  res: RestResponse | "backoff_delay" | null;
+  exception: any | null;
+  timestamp: number;
+  elapsed: number;
+  isFinalAttempt: boolean;
+  privateDataInResponse: boolean;
+  comment: string;
+}
+
+/**
+ * Middlewares allow to modify RestRequest and RestResponse objects during the
+ * request processing.
+ */
+export interface Middleware {
+  (
+    req: RestRequest,
+    next: (req: RestRequest) => Promise<RestResponse>,
+  ): Promise<RestResponse>;
+}
+
+/**
+ * @ignore
+ * Parameters for Agents.
+ */
+interface AgentOptions {
+  keepAlive: boolean;
+  timeout?: number;
+  maxSockets?: number;
+  rejectUnauthorized?: boolean;
+  family?: 4 | 6 | 0;
+}
+
+/**
+ * @ignore
+ * An internal class which keeps the list of HttpAgent instances used in a
+ * particular RestClient instance.
+ */
+export class Agents {
+  @Memoize((...args: any[]) => JSON.stringify(args))
+  http(options: AgentOptions) {
+    return new HttpAgent(options);
+  }
+
+  @Memoize((...args: any[]) => JSON.stringify(args))
+  https(options: AgentOptions) {
+    return new HttpsAgent(options);
+  }
+}
+
+/**
+ * Options passed to RestClient. More options can be added by cloning an
+ * instance of RestClient, withOption() method.
+ */
+export default interface RestOptions {
+  /** Max number of retries. Default is 0, because some requests are from the
+   * web app, and we don't want to retry them. */
+  retries: number;
+  /** How much time to wait by default on the 1st retry attempt. */
+  retryDelayFirstMs: number;
+  /** How much to increase the retry delay on each retry. */
+  retryDelayFactor: number;
+  /** Use this fraction (random) of the current retry delay to jitter both ways
+   * (e.g. 0.1 means 90%...110% of the delay to be actually applied). */
+  retryDelayJitter: number;
+  /** Maximum delay between each retry. */
+  retryDelayMaxMs: number;
+  /** A logic which runs on different IO stages (delay and heartbeats). */
+  heartbeater: {
+    /* This function is called after each request with 0 passed, so it can serve
+     * the purpose of a heartbeat. */
+    heartbeat(): Promise<void>;
+    /** A function which, when runs, resolves in the provided number of ms. Can
+     * be used for several purposes, like overriding in unit tests or to pass a
+     * custom delay implementation which can throw on some external event (like
+     * when the process wants to gracefully stop). */
+    delay(ms: number): Promise<void>;
+  };
+  /** Allows to limit huge requests and throw instead. */
+  throwIfResIsBigger: number | undefined;
+  /** Passed to the logger which may decide, should it log details of the
+   * response or not. */
+  privateDataInResponse: boolean;
+  /** If true, non-public IP addresses are allowed too; otherwise, only unicast
+   * addresses are allowed. */
+  allowInternalIPs: boolean;
+  /** If true, logs request-response pairs to console. */
+  isDebug: boolean;
+  /** @ignore Holds HttpsAgent/HttpAgent instances; used internally only. */
+  agents: Agents;
+  /** Sets Keep-Alive parameters (persistent connections). */
+  keepAlive: {
+    /** How much time to keep an idle connection alive in the pool. If 0, closes
+     * the connection immediately after the response. */
+    timeoutMs: number;
+    /** How many sockets at maximum will be kept open. */
+    maxSockets?: number;
+  };
+  /** When resolving DNS, use IPv4, IPv6 or both (see dns.lookup() docs). */
+  family: 4 | 6 | 0;
+  /** Max timeout to wait for a response. */
+  timeoutMs: number;
+  /** Logger to be used for each responses (including retried) plus for backoff
+   * delay events logging. */
+  logger: (event: RestLogEvent) => void;
+  /** Middlewares to wrap requests. May alter both request and response. */
+  middlewares: Middleware[];
+  /** If set, makes decision whether the response is successful or not. The
+   * response will either be returned to the client, or an error will be thrown.
+   * This allows to treat some non-successful HTTP statuses as success if the
+   * remote API is that weird. Return values:
+   * * "SUCCESS" - the request will be considered successful, no further checks
+   *   will be performed;
+   * * "BEST_EFFORT" - inconclusive, the request may be either successful or
+   *   unsuccessful, additional tests (e.g. will check HTTP status code) will be
+   *   performed;
+   * * "THROW" - the request resulted in error. Additional tests will be
+   *   performed to determine is the error is retriable, is OAuth token good,
+   *   and etc. */
+  isSuccessResponse: (res: RestResponse) => "SUCCESS" | "THROW" | "BEST_EFFORT";
+  /** Decides whether the response is a rate-limit error or not. Returning
+   * non-zero value is treated as retry delay (if retries are set up). In case
+   * the returned value is "SOMETHING_ELSE", the response ought to be either
+   * success or some other error. Returning "BEST_EFFORT" turns on built-in
+   * heuristic (e.g. relying on HTTP status code and Retry-After header). In
+   * case we've made a decision that it's a rate limited error, the request is
+   * always retried; this covers a very common case when we have both
+   * isRateLimitError and isRetriableError handlers set up, and they return
+   * contradictory information; then isRateLimitError wins. */
+  isRateLimitError: (
+    res: RestResponse,
+  ) => "SOMETHING_ELSE" | "RATE_LIMIT" | "BEST_EFFORT" | number;
+  /** Decides whether the response is a token-invalid error or not. In case it's
+   * not, the response ought to be either success or some other error. */
+  isTokenInvalidError: (res: RestResponse) => boolean;
+  /** Called only if we haven't decided earlier that it's a rate limit error.
+   * Decides whether the response is a retriable error or not. In case the
+   * returned value is "NEVER_RETRY", the response ought to be either success or
+   * some other error, but it's guaranteed that the request won't be retried.
+   * Returning "BEST_EFFORT" turns on built-in heuristics (e.g. never retry "not
+   * found" errors). Returning a number is treated as "RETRY", and the next
+   * retry will happen in not less than this number of milliseconds. */
+  isRetriableError: (
+    res: RestResponse,
+    _error: any,
+  ) => "NEVER_RETRY" | "RETRY" | "BEST_EFFORT" | number;
+}
+
+/** @ignore */
+export const DEFAULT_OPTIONS: RestOptions = {
+  retries: 0,
+  retryDelayFirstMs: 1000,
+  retryDelayFactor: 2,
+  retryDelayJitter: 0.1,
+  retryDelayMaxMs: Number.MAX_SAFE_INTEGER,
+  heartbeater: {
+    heartbeat: async () => {},
+    delay,
+  },
+  throwIfResIsBigger: undefined,
+  privateDataInResponse: false,
+  allowInternalIPs: false,
+  isDebug: false,
+  agents: new Agents(),
+  keepAlive: { timeoutMs: 10000 },
+  family: 4, // we don't want to hit ~5s DNS timeout on a missing IPv6 by default
+  timeoutMs: 4 * 60 * 1000,
+  logger: () => {},
+  middlewares: [],
+  isSuccessResponse: () => "BEST_EFFORT",
+  isRateLimitError: () => "BEST_EFFORT",
+  isTokenInvalidError: () => false,
+  isRetriableError: () => "BEST_EFFORT",
+};

package/src/RestRequest.ts

@@ -0,0 +1,441 @@
+import dns from "dns";
+import { promisify } from "util";
+import { Memoize } from "fast-typescript-memoize";
+import { parse } from "ipaddr.js";
+import random from "lodash/random";
+import type { RequestInit, RequestRedirect } from "node-fetch";
+import { Headers } from "node-fetch";
+import RestContentSizeOverLimitError from "./errors/RestContentSizeOverLimitError";
+import RestError from "./errors/RestError";
+import RestTimeoutError from "./errors/RestTimeoutError";
+import calcRetryDelay from "./internal/calcRetryDelay";
+import inspectPossibleJSON from "./internal/inspectPossibleJSON";
+import RestFetchReader from "./internal/RestFetchReader";
+import throwIfErrorResponse from "./internal/throwIfErrorResponse";
+import toFloatMs from "./internal/toFloatMs";
+import type RestOptions from "./RestOptions";
+import type { RestLogEvent } from "./RestOptions";
+import RestResponse from "./RestResponse";
+import RestStream from "./RestStream";
+
+const MAX_DEBUG_LEN = 1024 * 100;
+
+/**
+ * Type TAssertShape allows to limit json()'s assert callbacks to only those
+ * which return an object compatible with TAssertShape.
+ */
+export default class RestRequest<TAssertShape = any> {
+  readonly options: RestOptions;
+
+  constructor(
+    options: RestOptions,
+    public readonly method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE",
+    public url: string,
+    public readonly headers: Headers,
+    public readonly body: string | Buffer | NodeJS.ReadableStream,
+    public readonly shape?: string,
+  ) {
+    this.options = { ...options };
+  }
+
+  /**
+   * Modifies the request by adding a custom HTTP header.
+   */
+  setHeader(name: string, value: string) {
+    this.headers.append(name, value);
+    return this;
+  }
+
+  /**
+   * Modifies the request by adding a custom request option.
+   */
+  setOptions(options: Partial<RestOptions>) {
+    for (const [k, v] of Object.entries(options)) {
+      (this.options as any)[k] = v;
+    }
+
+    return this;
+  }
+
+  /**
+   * Forces RestClient to debug-output the request and response to console.
+   * Never use in production.
+   */
+  setDebug(flag = true) {
+    this.options.isDebug = flag;
+    return this;
+  }
+
+  /**
+   * Sends the request and reads the response a JSON. In absolute most of the
+   * cases, this method is used to reach API responses. The assert callback
+   * (typically generated by typescript-is) is intentionally made mandatory to
+   * not let people to do anti-patterns.
+   */
+  async json<TJson extends TAssertShape>(
+    assert:
+      | ((obj: any) => TJson)
+      | { mask(obj: any): TJson }
+      | { $assert(obj: any): TJson },
+    ...checkers: Array<(json: TJson, res: RestResponse) => false | Error>
+  ): Promise<TJson> {
+    const res = await this.response();
+
+    // Support Superstruct in a duck-typing way.
+    if (typeof assert !== "function") {
+      if ("mask" in assert) {
+        assert = assert.mask.bind(assert);
+      } else if ("$assert" in assert) {
+        assert = assert.$assert.bind(assert);
+      }
+    }
+
+    const json = assert(res.json);
+    for (const checker of checkers) {
+      const error = checker(json, res);
+      if (error !== false) {
+        throw error;
+      }
+    }
+
+    return json as any;
+  }
+
+  /**
+   * Sends the request and returns plaintext response.
+   */
+  async text() {
+    const res = await this.response();
+    return res.text;
+  }
+
+  /**
+   * Returns the entire RestResponse object with response status and headers
+   * information in it. Try to minimize usage of this method, because it doesn't
+   * make any assumptions on the response structure.
+   */
+  @Memoize()
+  async response(): Promise<RestResponse> {
+    const stream = await this.stream(
+      // By passing Number.MAX_SAFE_INTEGER to stream(), we ensure that the
+      // entire data will be preloaded, or the loading will fail due to
+      // throwIfResIsBigger limitation, whichever will happen faster.
+      Number.MAX_SAFE_INTEGER,
+    );
+    await stream.close();
+    return stream.res;
+  }
+
+  /**
+   * Sends the requests and returns RestStream object. You MUST iterate over
+   * this object entirely (or call its return() method), otherwise the
+   * connection will remain dangling.
+   */
+  async stream(preloadChars = 128 * 1024): Promise<RestStream> {
+    const finalAttempt = Math.max(this.options.retries + 1, 1);
+    let retryDelayMs = Math.max(this.options.retryDelayFirstMs, 10);
+
+    for (let attempt = 1; attempt <= finalAttempt; attempt++) {
+      let timeStart = process.hrtime();
+      let res = null as RestResponse | null;
+
+      try {
+        // Middlewares work with RestResponse (can alter it) and not with
+        // RestStream intentionally. So the goal here is to create a
+        // RestResponse object, and then later derive a RestStream from it.
+        let reader = null as RestFetchReader | null;
+        res = await runMiddlewares(
+          this,
+          this.options.middlewares,
+          async (req) => {
+            reader = null;
+            try {
+              timeStart = process.hrtime();
+              res = null;
+
+              try {
+                const fetchReq = await req._createFetchRequest();
+                reader = req._createFetchReader(fetchReq);
+                await reader.preload(preloadChars);
+              } finally {
+                res = reader
+                  ? req._createRestResponse(reader)
+                  : new RestResponse(req, null, 0, new Headers(), "", false);
+              }
+
+              throwIfErrorResponse(req.options, res);
+
+              req._logResponse({
+                attempt,
+                req,
+                res,
+                exception: null,
+                timestamp: Date.now(),
+                elapsed: toFloatMs(process.hrtime(timeStart)),
+                isFinalAttempt: attempt === finalAttempt,
+                privateDataInResponse: req.options.privateDataInResponse,
+                comment: "",
+              });
+
+              return res;
+            } catch (e: unknown) {
+              await reader?.close();
+              throw e;
+            }
+          },
+        );
+
+        // The only place where we return the response. Otherwise we retry or
+        // throw an exception.
+        return new RestStream(res, reader!);
+      } catch (error: unknown) {
+        this._logResponse({
+          attempt,
+          req: this,
+          res,
+          exception: error,
+          timestamp: Date.now(),
+          elapsed: toFloatMs(process.hrtime(timeStart)),
+          isFinalAttempt: attempt === finalAttempt,
+          privateDataInResponse: this.options.privateDataInResponse,
+          comment: "",
+        });
+
+        if (res === null) {
+          // An error in internal function or middleware; this must not happen.
+          throw error;
+        }
+
+        if (attempt === finalAttempt) {
+          // Last retry attempt; always throw.
+          throw error;
+        }
+
+        const newRetryDelay = calcRetryDelay(
+          error,
+          this.options,
+          res,
+          retryDelayMs,
+        );
+        if (newRetryDelay === "no_retry") {
+          throw error;
+        }
+
+        retryDelayMs = newRetryDelay;
+      }
+
+      const delayStart = process.hrtime();
+      retryDelayMs *= random(
+        1 - this.options.retryDelayJitter,
+        1 + this.options.retryDelayJitter,
+        true,
+      );
+      await this.options.heartbeater.delay(retryDelayMs);
+
+      this._logResponse({
+        attempt,
+        req: this,
+        res: "backoff_delay",
+        exception: null,
+        timestamp: Date.now(),
+        elapsed: toFloatMs(process.hrtime(delayStart)),
+        isFinalAttempt: false,
+        privateDataInResponse: this.options.privateDataInResponse,
+        comment: "",
+      });
+
+      retryDelayMs *= this.options.retryDelayFactor;
+      retryDelayMs = Math.min(this.options.retryDelayMaxMs, retryDelayMs);
+    }
+
+    throw Error("BUG: should never happen");
+  }
+
+  /**
+   * We can actually only create RequestInit. We can't create an instance of
+   * node-fetch.Request object since it doesn't allow injection of
+   * AbortController later, and we don't want to deal with AbortController here.
+   */
+  private async _createFetchRequest(): Promise<RequestInit & { url: string }> {
+    const url = new URL(this.url);
+    if (!["http:", "https:"].includes(url.protocol)) {
+      throw new RestError(`Unsupported protocol: ${url.protocol}`);
+    }
+
+    // TODO: rework DNS resolution to use a custom Agent. We'd be able to
+    // support safe redirects then.
+
+    // Resolve IP address, check it for being a public IP address and substitute
+    // it in the URL; the hostname will be passed separately via Host header.
+    const hostname = url.hostname;
+    const headers = new Headers(this.headers);
+    const addr = await promisify(dns.lookup)(hostname, {
+      family: this.options.family,
+    });
+    let redirectMode: RequestRedirect = "follow";
+    if (!this.options.allowInternalIPs) {
+      const range = parse(addr.address).range();
+      // External requests are returned as "unicast" by ipaddr.js.
+      const isInternal = range !== "unicast";
+      if (isInternal) {
+        throw new RestError(
+          `Domain ${hostname} resolves to a non-public (${range}) IP address ${addr.address}`,
+        );
+      }
+
+      url.hostname = addr.address;
+      if (!headers.get("host")) {
+        headers.set("host", hostname);
+      }
+
+      // ATTENTION: don't turn on redirects, it's a security breach when using
+      // with allowInternalIPs=false which is default!
+      redirectMode = "error";
+    }
+
+    const hasKeepAlive = this.options.keepAlive.timeoutMs > 0;
+    if (hasKeepAlive) {
+      headers.append("connection", "Keep-Alive");
+    }
+
+    // Use lazily created/cached per-RestClient Agent instance to utilize HTTP
+    // persistent connections and save on HTTPS connection re-establishment.
+    const agent = (
+      url.protocol === "https:"
+        ? this.options.agents.https.bind(this.options.agents)
+        : this.options.agents.http.bind(this.options.agents)
+    )({
+      keepAlive: hasKeepAlive,
+      timeout: hasKeepAlive ? this.options.keepAlive.timeoutMs : undefined,
+      maxSockets: this.options.keepAlive.maxSockets,
+      rejectUnauthorized: this.options.allowInternalIPs ? false : undefined,
+      family: this.options.family,
+    });
+
+    return {
+      url: url.toString(),
+      method: this.method,
+      headers,
+      body: this.body || undefined,
+      timeout: this.options.timeoutMs,
+      redirect: redirectMode,
+      agent,
+    };
+  }
+
+  /**
+   * Creates an instance of RestFetchReader.
+   */
+  private _createFetchReader(req: RequestInit & { url: string }) {
+    return new RestFetchReader(req.url, req, {
+      timeoutMs: this.options.timeoutMs,
+      heartbeat: async () => this.options.heartbeater.heartbeat(),
+      onTimeout: (reader) => {
+        throw new RestTimeoutError(
+          `Timed out while reading response body (${this.options.timeoutMs} ms)`,
+          this._createRestResponse(reader)!,
+        );
+      },
+      onAfterRead: (reader) => {
+        if (
+          this.options.throwIfResIsBigger &&
+          reader.charsRead > this.options.throwIfResIsBigger
+        ) {
+          throw new RestContentSizeOverLimitError(
+            `Content size is over limit of ${this.options.throwIfResIsBigger} characters`,
+            this._createRestResponse(reader)!,
+          );
+        }
+      },
+    });
+  }
+
+  /**
+   * Creates a RestResponse from a RestFetchReader. Assumes that
+   * RestFetchReader.preload() has already been called.
+   */
+  private _createRestResponse(reader: RestFetchReader) {
+    return new RestResponse(
+      this,
+      reader.agent,
+      reader.status,
+      reader.headers,
+      reader.textFetched,
+      reader.textIsPartial,
+    );
+  }
+
+  /**
+   * Logs a response event, an error event or a backoff event. If RestResponse
+   * is not yet known (e.g. an exception happened in a DNS resolution), res must
+   * be passed as null.
+   */
+  private _logResponse(event: RestLogEvent) {
+    this.options.logger(event);
+
+    // Debug-logging to console?
+    if (
+      this.options.isDebug ||
+      (process.env["NODE_ENV"] === "development" &&
+        event.res === "backoff_delay")
+    ) {
+      let reqMessage =
+        `${this.method} ${this.url}\n` +
+        Object.entries(this.headers.raw())
+          .map(([k, vs]) => vs.map((v) => `${k}: ${v}`))
+          .join("\n");
+      if (this.body) {
+        reqMessage +=
+          "\n" + inspectPossibleJSON(this.headers, this.body, MAX_DEBUG_LEN);
+      }
+
+      let resMessage = "";
+      if (event.res === "backoff_delay") {
+        resMessage =
+          "Previous request failed, backoff delay elapsed, retrying attempt " +
+          (event.attempt + 1) +
+          "...";
+      } else if (event.res) {
+        resMessage =
+          `HTTP ${event.res.status} ` +
+          `(took ${Math.round(event.elapsed)} ms)`;
+        if (event.res.text) {
+          resMessage +=
+            "\n" +
+            inspectPossibleJSON(
+              event.res.headers,
+              event.res.text,
+              MAX_DEBUG_LEN,
+            );
+        }
+      } else if (event.exception) {
+        resMessage = "" + event.exception;
+      }
+
+      // eslint-disable-next-line no-console
+      console.log(reqMessage.replace(/^/gm, "+++ "));
+      // eslint-disable-next-line no-console
+      console.log(resMessage.replace(/^/gm, "=== ") + "\n");
+    }
+  }
+}
+
+/**
+ * Runs the middlewares chain of responsibility. Each middleware receives a
+ * mutable RestRequest object and next() callback which, when called, triggers
+ * execution of the remaining middlewares in the chain. This allows a middleware
+ * to not only modify the request object, but also alter the response received
+ * from the subsequent middlewares.
+ */
+async function runMiddlewares(
+  req: RestRequest,
+  middlewares: RestOptions["middlewares"],
+  last: (req: RestRequest) => Promise<RestResponse>,
+): Promise<RestResponse> {
+  if (middlewares.length > 0) {
+    const [head, ...tail] = middlewares;
+    return head(req, async (req) => runMiddlewares(req, tail, last));
+  }
+
+  return last(req);
+}

package/src/RestResponse.ts

@@ -0,0 +1,49 @@
+import type { Agent as HttpAgent } from "http";
+import { Memoize } from "fast-typescript-memoize";
+import type { Headers } from "node-fetch";
+import type RestRequest from "./RestRequest";
+
+/**
+ * RestResponse is intentionally not aware of the data structure it carries, and
+ * it doesn't do any assertions/validations which is the responsibility of
+ * RestRequest helper methods.
+ *
+ * We also use a concept of "body preloading". Sometimes, e.g. on non-successful
+ * HTTP status codes, we also need to know the body content (at least its
+ * beginning), do double check whether should we retry, throw through or through
+ * a user-friendly error. To do this, we need to preload the beginning of the
+ * body and make it a part of RestResponse abstraction.
+ */
+export default class RestResponse {
+  constructor(
+    public readonly req: RestRequest,
+    public readonly agent: HttpAgent | null,
+    public readonly status: number,
+    public readonly headers: Headers,
+    public readonly text: string,
+    public readonly textIsPartial: boolean,
+  ) {}
+
+  /**
+   * A safe way to treat the response as JSON.
+   * - Never throws, i.e. we imply that the caller will verify the structure of
+   *   the response and do its own errors processing.
+   * - It's a getter, so we can use typescript-is'es is<xyz>() type guard, e.g.:
+   *   `if (is<{ errors: any[] }>(res.json) && res.json.errors.length) { ... }`
+   *
+   * Notice that there is NO `assert()` abstraction inside RestResponse class.
+   * This is because RestClient sometimes substitutes the response with some
+   * sub-field (e.g. see writeGraphQLX() method), and we still need to run the
+   * assertion in such cases. By not having strong typing here, we intentionally
+   * make the use of this method harder, so people will prefer using
+   * RestRequest.json() instead.
+   */
+  @Memoize()
+  get json(): object | string | number | boolean | null | undefined {
+    try {
+      return this.text ? JSON.parse(this.text) : undefined;
+    } catch (e: any) {
+      return undefined;
+    }
+  }
+}