@clickup/rest-client 2.10.293 → 2.10.294

package/src/RestRequest.ts

@@ -0,0 +1,441 @@
+import dns from "dns";
+import { promisify } from "util";
+import { Memoize } from "fast-typescript-memoize";
+import { parse } from "ipaddr.js";
+import random from "lodash/random";
+import type { RequestInit, RequestRedirect } from "node-fetch";
+import { Headers } from "node-fetch";
+import RestContentSizeOverLimitError from "./errors/RestContentSizeOverLimitError";
+import RestError from "./errors/RestError";
+import RestTimeoutError from "./errors/RestTimeoutError";
+import calcRetryDelay from "./internal/calcRetryDelay";
+import inspectPossibleJSON from "./internal/inspectPossibleJSON";
+import RestFetchReader from "./internal/RestFetchReader";
+import throwIfErrorResponse from "./internal/throwIfErrorResponse";
+import toFloatMs from "./internal/toFloatMs";
+import type RestOptions from "./RestOptions";
+import type { RestLogEvent } from "./RestOptions";
+import RestResponse from "./RestResponse";
+import RestStream from "./RestStream";
+
+const MAX_DEBUG_LEN = 1024 * 100;
+
+/**
+ * Type TAssertShape allows to limit json()'s assert callbacks to only those
+ * which return an object compatible with TAssertShape.
+ */
+export default class RestRequest<TAssertShape = any> {
+  readonly options: RestOptions;
+
+  constructor(
+    options: RestOptions,
+    public readonly method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE",
+    public url: string,
+    public readonly headers: Headers,
+    public readonly body: string | Buffer | NodeJS.ReadableStream,
+    public readonly shape?: string
+  ) {
+    this.options = { ...options };
+  }
+
+  /**
+   * Modifies the request by adding a custom HTTP header.
+   */
+  setHeader(name: string, value: string) {
+    this.headers.append(name, value);
+    return this;
+  }
+
+  /**
+   * Modifies the request by adding a custom request option.
+   */
+  setOptions(options: Partial<RestOptions>) {
+    for (const [k, v] of Object.entries(options)) {
+      (this.options as any)[k] = v;
+    }
+
+    return this;
+  }
+
+  /**
+   * Forces RestClient to debug-output the request and response to console.
+   * Never use in production.
+   */
+  setDebug(flag = true) {
+    this.options.isDebug = flag;
+    return this;
+  }
+
+  /**
+   * Sends the request and reads the response a JSON. In absolute most of the
+   * cases, this method is used to reach API responses. The assert callback
+   * (typically generated by typescript-is) is intentionally made mandatory to
+   * not let people to do anti-patterns.
+   */
+  async json<TJson extends TAssertShape>(
+    assert:
+      | ((obj: any) => TJson)
+      | { mask(obj: any): TJson }
+      | { $assert(obj: any): TJson },
+    ...checkers: Array<(json: TJson, res: RestResponse) => false | Error>
+  ): Promise<TJson> {
+    const res = await this.response();
+
+    // Support Superstruct in a duck-typing way.
+    if (typeof assert !== "function") {
+      if ("mask" in assert) {
+        assert = assert.mask.bind(assert);
+      } else if ("$assert" in assert) {
+        assert = assert.$assert.bind(assert);
+      }
+    }
+
+    const json = assert(res.json);
+    for (const checker of checkers) {
+      const error = checker(json, res);
+      if (error !== false) {
+        throw error;
+      }
+    }
+
+    return json as any;
+  }
+
+  /**
+   * Sends the request and returns plaintext response.
+   */
+  async text() {
+    const res = await this.response();
+    return res.text;
+  }
+
+  /**
+   * Returns the entire RestResponse object with response status and headers
+   * information in it. Try to minimize usage of this method, because it doesn't
+   * make any assumptions on the response structure.
+   */
+  @Memoize()
+  async response(): Promise<RestResponse> {
+    const stream = await this.stream(
+      // By passing Number.MAX_SAFE_INTEGER to stream(), we ensure that the
+      // entire data will be preloaded, or the loading will fail due to
+      // throwIfResIsBigger limitation, whichever will happen faster.
+      Number.MAX_SAFE_INTEGER
+    );
+    await stream.close();
+    return stream.res;
+  }
+
+  /**
+   * Sends the requests and returns RestStream object. You MUST iterate over
+   * this object entirely (or call its return() method), otherwise the
+   * connection will remain dangling.
+   */
+  async stream(preloadChars = 128 * 1024): Promise<RestStream> {
+    const finalAttempt = Math.max(this.options.retries + 1, 1);
+    let retryDelayMs = Math.max(this.options.retryDelayFirstMs, 10);
+
+    for (let attempt = 1; attempt <= finalAttempt; attempt++) {
+      let timeStart = process.hrtime();
+      let res = null as RestResponse | null;
+
+      try {
+        // Middlewares work with RestResponse (can alter it) and not with
+        // RestStream intentionally. So the goal here is to create a
+        // RestResponse object, and then later derive a RestStream from it.
+        let reader = null as RestFetchReader | null;
+        res = await runMiddlewares(
+          this,
+          this.options.middlewares,
+          async (req) => {
+            reader = null;
+            try {
+              timeStart = process.hrtime();
+              res = null;
+
+              try {
+                const fetchReq = await req._createFetchRequest();
+                reader = req._createFetchReader(fetchReq);
+                await reader.preload(preloadChars);
+              } finally {
+                res = reader
+                  ? req._createRestResponse(reader)
+                  : new RestResponse(req, null, 0, new Headers(), "", false);
+              }
+
+              throwIfErrorResponse(req.options, res);
+
+              req._logResponse({
+                attempt,
+                req,
+                res,
+                exception: null,
+                timestamp: Date.now(),
+                elapsed: toFloatMs(process.hrtime(timeStart)),
+                isFinalAttempt: attempt === finalAttempt,
+                privateDataInResponse: req.options.privateDataInResponse,
+                comment: "",
+              });
+
+              return res;
+            } catch (e: unknown) {
+              await reader?.close();
+              throw e;
+            }
+          }
+        );
+
+        // The only place where we return the response. Otherwise we retry or
+        // throw an exception.
+        return new RestStream(res, reader!);
+      } catch (error: unknown) {
+        this._logResponse({
+          attempt,
+          req: this,
+          res,
+          exception: error,
+          timestamp: Date.now(),
+          elapsed: toFloatMs(process.hrtime(timeStart)),
+          isFinalAttempt: attempt === finalAttempt,
+          privateDataInResponse: this.options.privateDataInResponse,
+          comment: "",
+        });
+
+        if (res === null) {
+          // An error in internal function or middleware; this must not happen.
+          throw error;
+        }
+
+        if (attempt === finalAttempt) {
+          // Last retry attempt; always throw.
+          throw error;
+        }
+
+        const newRetryDelay = calcRetryDelay(
+          error,
+          this.options,
+          res,
+          retryDelayMs
+        );
+        if (newRetryDelay === "no_retry") {
+          throw error;
+        }
+
+        retryDelayMs = newRetryDelay;
+      }
+
+      const delayStart = process.hrtime();
+      retryDelayMs *= random(
+        1 - this.options.retryDelayJitter,
+        1 + this.options.retryDelayJitter,
+        true
+      );
+      await this.options.heartbeater.delay(retryDelayMs);
+
+      this._logResponse({
+        attempt,
+        req: this,
+        res: "backoff_delay",
+        exception: null,
+        timestamp: Date.now(),
+        elapsed: toFloatMs(process.hrtime(delayStart)),
+        isFinalAttempt: false,
+        privateDataInResponse: this.options.privateDataInResponse,
+        comment: "",
+      });
+
+      retryDelayMs *= this.options.retryDelayFactor;
+      retryDelayMs = Math.min(this.options.retryDelayMaxMs, retryDelayMs);
+    }
+
+    throw Error("BUG: should never happen");
+  }
+
+  /**
+   * We can actually only create RequestInit. We can't create an instance of
+   * node-fetch.Request object since it doesn't allow injection of
+   * AbortController later, and we don't want to deal with AbortController here.
+   */
+  private async _createFetchRequest(): Promise<RequestInit & { url: string }> {
+    const url = new URL(this.url);
+    if (!["http:", "https:"].includes(url.protocol)) {
+      throw new RestError(`Unsupported protocol: ${url.protocol}`);
+    }
+
+    // TODO: rework DNS resolution to use a custom Agent. We'd be able to
+    // support safe redirects then.
+
+    // Resolve IP address, check it for being a public IP address and substitute
+    // it in the URL; the hostname will be passed separately via Host header.
+    const hostname = url.hostname;
+    const headers = new Headers(this.headers);
+    const addr = await promisify(dns.lookup)(hostname, {
+      family: this.options.family,
+    });
+    let redirectMode: RequestRedirect = "follow";
+    if (!this.options.allowInternalIPs) {
+      const range = parse(addr.address).range();
+      // External requests are returned as "unicast" by ipaddr.js.
+      const isInternal = range !== "unicast";
+      if (isInternal) {
+        throw new RestError(
+          `Domain ${hostname} resolves to a non-public (${range}) IP address ${addr.address}`
+        );
+      }
+
+      url.hostname = addr.address;
+      if (!headers.get("host")) {
+        headers.set("host", hostname);
+      }
+
+      // ATTENTION: don't turn on redirects, it's a security breach when using
+      // with allowInternalIPs=false which is default!
+      redirectMode = "error";
+    }
+
+    const hasKeepAlive = this.options.keepAlive.timeoutMs > 0;
+    if (hasKeepAlive) {
+      headers.append("connection", "Keep-Alive");
+    }
+
+    // Use lazily created/cached per-RestClient Agent instance to utilize HTTP
+    // persistent connections and save on HTTPS connection re-establishment.
+    const agent = (
+      url.protocol === "https:"
+        ? this.options.agents.https.bind(this.options.agents)
+        : this.options.agents.http.bind(this.options.agents)
+    )({
+      keepAlive: hasKeepAlive,
+      timeout: hasKeepAlive ? this.options.keepAlive.timeoutMs : undefined,
+      maxSockets: this.options.keepAlive.maxSockets,
+      rejectUnauthorized: this.options.allowInternalIPs ? false : undefined,
+      family: this.options.family,
+    });
+
+    return {
+      url: url.toString(),
+      method: this.method,
+      headers,
+      body: this.body || undefined,
+      timeout: this.options.timeoutMs,
+      redirect: redirectMode,
+      agent,
+    };
+  }
+
+  /**
+   * Creates an instance of RestFetchReader.
+   */
+  private _createFetchReader(req: RequestInit & { url: string }) {
+    return new RestFetchReader(req.url, req, {
+      timeoutMs: this.options.timeoutMs,
+      heartbeat: async () => this.options.heartbeater.heartbeat(),
+      onTimeout: (reader) => {
+        throw new RestTimeoutError(
+          `Timed out while reading response body (${this.options.timeoutMs} ms)`,
+          this._createRestResponse(reader)!
+        );
+      },
+      onAfterRead: (reader) => {
+        if (
+          this.options.throwIfResIsBigger &&
+          reader.charsRead > this.options.throwIfResIsBigger
+        ) {
+          throw new RestContentSizeOverLimitError(
+            `Content size is over limit of ${this.options.throwIfResIsBigger} characters`,
+            this._createRestResponse(reader)!
+          );
+        }
+      },
+    });
+  }
+
+  /**
+   * Creates a RestResponse from a RestFetchReader. Assumes that
+   * RestFetchReader.preload() has already been called.
+   */
+  private _createRestResponse(reader: RestFetchReader) {
+    return new RestResponse(
+      this,
+      reader.agent,
+      reader.status,
+      reader.headers,
+      reader.textFetched,
+      reader.textIsPartial
+    );
+  }
+
+  /**
+   * Logs a response event, an error event or a backoff event. If RestResponse
+   * is not yet known (e.g. an exception happened in a DNS resolution), res must
+   * be passed as null.
+   */
+  private _logResponse(event: RestLogEvent) {
+    this.options.logger(event);
+
+    // Debug-logging to console?
+    if (
+      this.options.isDebug ||
+      (process.env["NODE_ENV"] === "development" &&
+        event.res === "backoff_delay")
+    ) {
+      let reqMessage =
+        `${this.method} ${this.url}\n` +
+        Object.entries(this.headers.raw())
+          .map(([k, vs]) => vs.map((v) => `${k}: ${v}`))
+          .join("\n");
+      if (this.body) {
+        reqMessage +=
+          "\n" + inspectPossibleJSON(this.headers, this.body, MAX_DEBUG_LEN);
+      }
+
+      let resMessage = "";
+      if (event.res === "backoff_delay") {
+        resMessage =
+          "Previous request failed, backoff delay elapsed, retrying attempt " +
+          (event.attempt + 1) +
+          "...";
+      } else if (event.res) {
+        resMessage =
+          `HTTP ${event.res.status} ` +
+          `(took ${Math.round(event.elapsed)} ms)`;
+        if (event.res.text) {
+          resMessage +=
+            "\n" +
+            inspectPossibleJSON(
+              event.res.headers,
+              event.res.text,
+              MAX_DEBUG_LEN
+            );
+        }
+      } else if (event.exception) {
+        resMessage = "" + event.exception;
+      }
+
+      // eslint-disable-next-line no-console
+      console.log(reqMessage.replace(/^/gm, "+++ "));
+      // eslint-disable-next-line no-console
+      console.log(resMessage.replace(/^/gm, "=== ") + "\n");
+    }
+  }
+}
+
+/**
+ * Runs the middlewares chain of responsibility. Each middleware receives a
+ * mutable RestRequest object and next() callback which, when called, triggers
+ * execution of the remaining middlewares in the chain. This allows a middleware
+ * to not only modify the request object, but also alter the response received
+ * from the subsequent middlewares.
+ */
+async function runMiddlewares(
+  req: RestRequest,
+  middlewares: RestOptions["middlewares"],
+  last: (req: RestRequest) => Promise<RestResponse>
+): Promise<RestResponse> {
+  if (middlewares.length > 0) {
+    const [head, ...tail] = middlewares;
+    return head(req, async (req) => runMiddlewares(req, tail, last));
+  }
+
+  return last(req);
+}

package/src/RestResponse.ts

@@ -0,0 +1,49 @@
+import type { Agent as HttpAgent } from "http";
+import { Memoize } from "fast-typescript-memoize";
+import type { Headers } from "node-fetch";
+import type RestRequest from "./RestRequest";
+
+/**
+ * RestResponse is intentionally not aware of the data structure it carries, and
+ * it doesn't do any assertions/validations which is the responsibility of
+ * RestRequest helper methods.
+ *
+ * We also use a concept of "body preloading". Sometimes, e.g. on non-successful
+ * HTTP status codes, we also need to know the body content (at least its
+ * beginning), do double check whether should we retry, throw through or through
+ * a user-friendly error. To do this, we need to preload the beginning of the
+ * body and make it a part of RestResponse abstraction.
+ */
+export default class RestResponse {
+  constructor(
+    public readonly req: RestRequest,
+    public readonly agent: HttpAgent | null,
+    public readonly status: number,
+    public readonly headers: Headers,
+    public readonly text: string,
+    public readonly textIsPartial: boolean
+  ) {}
+
+  /**
+   * A safe way to treat the response as JSON.
+   * - Never throws, i.e. we imply that the caller will verify the structure of
+   *   the response and do its own errors processing.
+   * - It's a getter, so we can use typescript-is'es is<xyz>() type guard, e.g.:
+   *   `if (is<{ errors: any[] }>(res.json) && res.json.errors.length) { ... }`
+   *
+   * Notice that there is NO `assert()` abstraction inside RestResponse class.
+   * This is because RestClient sometimes substitutes the response with some
+   * sub-field (e.g. see writeGraphQLX() method), and we still need to run the
+   * assertion in such cases. By not having strong typing here, we intentionally
+   * make the use of this method harder, so people will prefer using
+   * RestRequest.json() instead.
+   */
+  @Memoize()
+  get json(): object | string | number | boolean | null | undefined {
+    try {
+      return this.text ? JSON.parse(this.text) : undefined;
+    } catch (e: any) {
+      return undefined;
+    }
+  }
+}

package/src/RestStream.ts

@@ -0,0 +1,89 @@
+import { Memoize } from "fast-typescript-memoize";
+import type RestResponse from "./RestResponse";
+
+/**
+ * Once created, RestStream must be iterated in full, otherwise the connection
+ * will remain dangling. Also, this class is where we hide the details of the
+ * actual stream reading using AsyncGenerator bridge abstraction.
+ *
+ * RestStream can also read binary data depending on the Content-Type response
+ * header and/or the charset provided there. The binary data is still returned
+ * as a string, one string character per each byte. To convert it to a Buffer,
+ * use something like `Buffer.from(responseText, "binary")`.
+ */
+export default class RestStream {
+  private _generator?: AsyncGenerator<string, void>;
+
+  constructor(
+    public readonly res: RestResponse,
+    readerIterable: {
+      [Symbol.asyncIterator]: () => AsyncGenerator<string, void>;
+    }
+  ) {
+    this._generator = readerIterable[Symbol.asyncIterator]();
+  }
+
+  /**
+   * Reads the prefix of the stream. Closes the connection after the read is
+   * done in all cases, so safe to be used to e.g. receive a trimmed response.
+   */
+  async consumeReturningPrefix(maxChars: number): Promise<string> {
+    const text: string[] = [];
+    let length = 0;
+    for await (const chunk of this) {
+      // According to Google, in v8 string concatenation is as efficient as
+      // array or buffer joining.
+      text.push(chunk);
+      length += chunk.length;
+      if (length >= maxChars) {
+        break;
+      }
+    }
+
+    return text.join("").substring(0, maxChars);
+  }
+
+  /**
+   * Closes the connection.
+   */
+  async close(): Promise<void> {
+    // First, try to interrupt the active iteration, if any.
+    await this[Symbol.asyncIterator]().return();
+    // It is possible that this.[Symbol.asyncIterator] has never been iterated
+    // before at all, so its `finally` got never executed. This happens when
+    // RestFetchReader#preload() consumed some small chunk of data, and then
+    // no-one iterated the remaining body in RestStream, they just called
+    // close() on it. So we recheck & close the RestFetchReader iterator if it
+    // is still there.
+    const generator = this._generator;
+    if (generator) {
+      delete this._generator;
+      await generator.return();
+    }
+  }
+
+  /**
+   * Allows to iterate over the entire stream of data. You must consume the
+   * entire iterable or at least call this.close(), otherwise the connection may
+   * remain open.
+   */
+  @Memoize()
+  async *[Symbol.asyncIterator](): AsyncGenerator<string, void> {
+    const generator = this._generator;
+    if (!generator) {
+      return;
+    }
+
+    delete this._generator;
+    try {
+      yield this.res.text;
+      for await (const chunk of generator) {
+        yield chunk;
+      }
+    } finally {
+      // The code enters here if the caller interrupted (returned) the iteration
+      // after receiving the 1st textFetched chunk.
+      await generator.return();
+    }
+  }
+}

package/src/errors/RestContentSizeOverLimitError.ts

@@ -0,0 +1,3 @@
+import RestResponseError from "./RestResponseError";
+
+export default class RestContentSizeOverLimitError extends RestResponseError {}

package/src/errors/RestError.ts

@@ -0,0 +1,6 @@
+export default class RestError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = this.constructor.name; // https://javascript.info/custom-errors#further-inheritance
+  }
+}

package/src/errors/RestRateLimitError.ts

@@ -0,0 +1,8 @@
+import type RestResponse from "../RestResponse";
+import RestResponseError from "./RestResponseError";
+
+export default class RestRateLimitError extends RestResponseError {
+  constructor(message: string, public delayMs: number, res: RestResponse) {
+    super(message, res);
+  }
+}

package/src/errors/RestResponseError.ts

@@ -0,0 +1,46 @@
+import inspectPossibleJSON from "../internal/inspectPossibleJSON";
+import prependNewlineIfMultiline from "../internal/prependNewlineIfMultiline";
+import type RestResponse from "../RestResponse";
+import RestError from "./RestError";
+
+const RESPONSE_MAX_LEN_IN_ERROR =
+  process.env["NODE_ENV"] === "development" ? Number.MAX_SAFE_INTEGER : 2048;
+
+export default class RestResponseError extends RestError {
+  readonly res!: RestResponse;
+
+  readonly method: string;
+  readonly host: string;
+  readonly pathname: string;
+  readonly requestArgs: string;
+  readonly requestBody: string;
+  readonly responseHeaders: string;
+
+  constructor(message: string, res: RestResponse) {
+    super(
+      `HTTP ${res.status}: ` +
+        (message ? `${message}: ` : "") +
+        prependNewlineIfMultiline(
+          inspectPossibleJSON(res.headers, res.text, RESPONSE_MAX_LEN_IN_ERROR)
+        )
+    );
+    Object.defineProperty(this, "res", { value: res, enumerable: false }); // hidden from inspect()
+    const url = new URL(res.req.url);
+    const search = (url.search || "").replace(/^\?/, "");
+    this.method = res.req.method;
+    this.host = res.req.headers.get("host") || url.host;
+    this.pathname = url.pathname;
+    this.requestArgs = search.replace(/(?<=[&])/g, "\n");
+    this.requestBody = inspectPossibleJSON(
+      res.headers,
+      res.req.body,
+      RESPONSE_MAX_LEN_IN_ERROR
+    );
+    this.responseHeaders = [...res.headers]
+      .map(([name, value]) => `${name}: ${value}`)
+      .join("\n");
+    // Don't expose query string parameters in the message as they may contain
+    // sensitive information.
+    this.message += `\n  ${this.method} ${url.protocol}//${url.host}${url.pathname}`;
+  }
+}

package/src/errors/RestRetriableError.ts

@@ -0,0 +1,8 @@
+import type RestResponse from "../RestResponse";
+import RestResponseError from "./RestResponseError";
+
+export default class RestRetriableError extends RestResponseError {
+  constructor(message: string, public delayMs: number, res: RestResponse) {
+    super(message, res);
+  }
+}

package/src/errors/RestTimeoutError.ts

@@ -0,0 +1,3 @@
+import RestResponseError from "./RestResponseError";
+
+export default class RestTimeoutError extends RestResponseError {}

package/src/errors/RestTokenInvalidError.ts

@@ -0,0 +1,8 @@
+import type RestResponse from "../RestResponse";
+import RestResponseError from "./RestResponseError";
+
+export default class RestTokenInvalidError extends RestResponseError {
+  constructor(public readonly humanReason: string, res: RestResponse) {
+    super(humanReason, res);
+  }
+}