@clickup/rest-client 2.10.293 → 2.10.294

package/src/helpers/depaginate.ts

@@ -0,0 +1,37 @@
+/**
+ * Keeps calling a function with an updating cursor, and depaginates all the
+ * results until the cursor returned is null or undefined.
+ *
+ * On each call, the inner function needs to return an array with two elements:
+ * 1. Array or results, which could be empty, but not null or undefined.
+ * 2. A new cursor.
+ */
+export default async function* depaginate<TItem, TCursor = string>(
+  readFunc: (
+    cursor: TCursor | undefined
+  ) => Promise<readonly [TItem[], TCursor | null | undefined]>
+): AsyncGenerator<TItem, void, undefined> {
+  let prevCursor: TCursor | null | undefined = undefined;
+  let cursor: TCursor | null | undefined = undefined;
+  for (;;) {
+    let items: TItem[];
+    [items, cursor] = await readFunc(cursor === null ? undefined : cursor);
+    yield* items;
+
+    if (cursor === null || cursor === undefined) {
+      break;
+    }
+
+    if (JSON.stringify(prevCursor) === JSON.stringify(cursor)) {
+      throw Error(
+        "Depagination got stuck: prevCursor=" +
+          JSON.stringify(prevCursor) +
+          ", cursor=" +
+          JSON.stringify(cursor) +
+          " (they must differ)"
+      );
+    }
+
+    prevCursor = cursor;
+  }
+}

package/src/index.ts

@@ -0,0 +1,50 @@
+import { Headers } from "node-fetch";
+import RestContentSizeOverLimitError from "./errors/RestContentSizeOverLimitError";
+import RestError from "./errors/RestError";
+import RestRateLimitError from "./errors/RestRateLimitError";
+import RestResponseError from "./errors/RestResponseError";
+import RestRetriableError from "./errors/RestRetriableError";
+import RestTimeoutError from "./errors/RestTimeoutError";
+import RestTokenInvalidError from "./errors/RestTokenInvalidError";
+import depaginate from "./helpers/depaginate";
+import paceRequests from "./middlewares/paceRequests";
+import Pacer from "./pacers/Pacer";
+import PacerComposite from "./pacers/PacerComposite";
+import PacerQPS, { PacerQPSBackend } from "./pacers/PacerQPS";
+import RestClient, { TokenGetter } from "./RestClient";
+import RestOptions, {
+  RestLogEvent,
+  Middleware,
+  Agents,
+  DEFAULT_OPTIONS,
+} from "./RestOptions";
+import RestRequest from "./RestRequest";
+import RestResponse from "./RestResponse";
+import RestStream from "./RestStream";
+
+export {
+  Agents,
+  DEFAULT_OPTIONS,
+  depaginate,
+  Headers,
+  Middleware,
+  Pacer,
+  PacerComposite,
+  paceRequests,
+  PacerQPS,
+  PacerQPSBackend,
+  RestClient,
+  RestContentSizeOverLimitError,
+  RestError,
+  RestLogEvent,
+  RestOptions,
+  RestRateLimitError,
+  RestRequest,
+  RestResponse,
+  RestResponseError,
+  RestRetriableError,
+  RestStream,
+  RestTimeoutError,
+  RestTokenInvalidError,
+  TokenGetter,
+};

package/src/internal/RestFetchReader.ts

@@ -0,0 +1,188 @@
+import type { Agent as HttpAgent } from "http";
+import AbortControllerPolyfilled from "abort-controller";
+import { Memoize } from "fast-typescript-memoize";
+import type { RequestInit } from "node-fetch";
+import fetch, { Headers, Request } from "node-fetch";
+import inferResBodyEncoding from "./inferResBodyEncoding";
+
+export interface RestFetchReaderOptions {
+  timeoutMs?: number;
+  heartbeat?: () => Promise<void>;
+  onTimeout?: (reader: RestFetchReader, e: any) => void;
+  onAfterRead?: (reader: RestFetchReader) => void;
+}
+
+/**
+ * A low-level stateful reader engine on top of node-fetch which implements
+ * "preload first N chars and then leave the rest ready for iteration" pattern,
+ * with global timeout for the entire fetching time.
+ *
+ * Once created, the object MUST be iterated in full to consume the rest of the
+ * stream and close the connection. In case you're not interested in its entire
+ * content, you must prematurely "return" (close) the iterator.
+ *
+ * The abstraction is intentionally kept independent on all others, to make it
+ * simple and testable separately.
+ */
+export default class RestFetchReader {
+  private _status = 0;
+  private _headers = new Headers();
+  private _textFetched = "";
+  private _textIsPartial = true;
+  private _charsRead = 0;
+
+  constructor(
+    private _url: string,
+    private _reqInit: RequestInit,
+    private _options: RestFetchReaderOptions
+  ) {}
+
+  /**
+   * Returns the number of characters read from the stream so far.
+   */
+  get charsRead() {
+    return this._charsRead;
+  }
+
+  /**
+   * Returns the Agent instance used for this request. It's implied that
+   * RestRequest#agent always points to a http.Agent object.
+   */
+  get agent() {
+    return (
+      this._reqInit.agent &&
+      typeof this._reqInit.agent === "object" &&
+      "sockets" in this._reqInit.agent
+        ? this._reqInit.agent
+        : null
+    ) as HttpAgent | null;
+  }
+
+  /**
+   * Returns HTTP status after preload() was called.
+   */
+  get status() {
+    return this._status;
+  }
+
+  /**
+   * Returns HTTP headers after preload() was called.
+   */
+  get headers() {
+    return this._headers;
+  }
+
+  /**
+   * Returns the data preloaded so far.
+   */
+  get textFetched(): string {
+    return this._textFetched;
+  }
+
+  /**
+   * If true, then there is a chance that reading more from the stream will
+   * return more data.
+   */
+  get textIsPartial() {
+    return this._textIsPartial;
+  }
+
+  /**
+   * Reads preloadChars chars or a little bit more from the response and puts
+   * them to this.textFetched. Leaves the rest of the data in res.body for
+   * future reads if there are more data to fetch (you must consume them or
+   * close the stream, otherwise the connection will remain open).
+   */
+  async preload(preloadChars: number) {
+    const generator = this[Symbol.asyncIterator]();
+    try {
+      while (this._charsRead < preloadChars) {
+        const { value, done } = await generator.next();
+        if (done) {
+          this._textIsPartial = false;
+          await generator.return();
+          return;
+        }
+
+        this._textFetched += value;
+      }
+    } catch (e: unknown) {
+      await generator.return();
+      throw e;
+    }
+  }
+
+  /**
+   * Closes the connection.
+   */
+  async close() {
+    await this[Symbol.asyncIterator]().return();
+  }
+
+  /**
+   * Returns an async generator for the rest of the data. Must be consumed
+   * entirely, otherwise the connection may remain dangling.
+   *
+   * Memoization is important here, to return the same generator when we call
+   * this method multiple times and to not start a new iteration over and over.
+   */
+  @Memoize()
+  async *[Symbol.asyncIterator]() {
+    const { timeoutMs, onTimeout, onAfterRead } = this._options;
+
+    // Some of react-client users are still on v14 node.
+    const controller =
+      typeof AbortController === "undefined"
+        ? new AbortControllerPolyfilled()
+        : new AbortController();
+
+    const timeout = timeoutMs
+      ? setTimeout(() => controller.abort(), timeoutMs)
+      : undefined;
+
+    try {
+      // DO NOT use fetch(fetchReq) with one argument! It clones the stream
+      // which just doesn't work in practice, even with file streams. I wasted
+      // 4h on debugging this: fetch(fetchReq.url, fetchReq) works and
+      // fetch(fetchReq) doesn't for e.g. Dropbox API and
+      // https://stackoverflow.com/a/44577569
+      const res = await fetch(
+        this._url,
+        new Request(this._url, {
+          ...this._reqInit,
+          signal: controller.signal as any,
+        })
+      );
+      this._status = res.status;
+      this._headers = res.headers;
+
+      // See https://nodejs.org/api/stream.html#readablesetencodingencoding on
+      // how Node streams and setEncoding() handle decoding when the returned
+      // chunks cross the boundaries of multi-byte characters (TL;DR: it works
+      // fine, that's why we work with string and not Buffer here).
+      res.body.setEncoding(inferResBodyEncoding(res));
+
+      await this._options.heartbeat?.();
+      for await (const chunk of res.body) {
+        await this._options.heartbeat?.();
+        this._charsRead += chunk.length;
+        yield chunk as string;
+        onAfterRead?.(this);
+      }
+    } catch (e: unknown) {
+      if (controller.signal.aborted && onTimeout) {
+        onTimeout(this, e);
+      } else {
+        throw e;
+      }
+    } finally {
+      timeout && clearTimeout(timeout);
+      // If someone stops iterating prematurely, we forcefully close the
+      // connection in all cases. Theoretically, stopping the iteration on
+      // res.body should've closed the connection, but in practice it doesn't
+      // happen; it looks like a bug in node-fetch, and thus, we must always use
+      // the AbortController in the end.
+      controller.abort();
+    }
+  }
+}

package/src/internal/RestRangeUploader.ts

@@ -0,0 +1,61 @@
+import type RestClient from "../RestClient";
+
+/**
+ * Sends a series of Content-Range requests to an URL.
+ * - The stream size is unknown in advance even theoretically. So we read it
+ *   with chunkSize+1 bytes chunks (+1 is to know for sure, is there something
+ *   else left in the stream or not) and then send data with chunkSize bytes
+ *   chunks.
+ * - The last chunk is a terminating one (and we know, which one is the last),
+ *   so we reflect it in "Content-Range: x-y/S" format setting S to the total
+ *   number of bytes in the stream.
+ */
+export default class RestRangeUploader {
+  private _pos = 0;
+
+  constructor(
+    private _client: RestClient,
+    private _chunkSize: number,
+    private _method: "POST" | "PUT",
+    private _path: string,
+    private _mimeType: string
+  ) {}
+
+  async upload(stream: AsyncIterable<Buffer>) {
+    let buf = Buffer.allocUnsafe(0);
+    let res: string | null = null;
+    for await (const readData of stream) {
+      buf = Buffer.concat([buf, readData]);
+      while (buf.length >= this._chunkSize + 1) {
+        res = await this._flush(buf.slice(0, this._chunkSize), false);
+        buf = Buffer.from(buf.slice(this._chunkSize));
+      }
+      // After this `while` loop finishes, there is always something left in buf
+      // (due to the +1 trick). It guarantees that we have a chance to call
+      // flush(..., true) for the very last chunk.
+    }
+
+    if (buf.length > 0) {
+      res = await this._flush(buf, true);
+    }
+
+    return res;
+  }
+
+  private async _flush(buf: Buffer, isLast: boolean) {
+    if (buf.length === 0) {
+      return null;
+    }
+
+    const totalSize = isLast ? this._pos + buf.length : "*";
+    const res = await this._client
+      .writeRaw(this._path, buf, this._mimeType, this._method, "*/*")
+      .setHeader(
+        "Content-Range",
+        `bytes ${this._pos}-${this._pos + buf.length - 1}/${totalSize}`
+      )
+      .text();
+    this._pos += buf.length;
+    return res;
+  }
+}

package/src/internal/calcRetryDelay.ts

@@ -0,0 +1,59 @@
+import RestContentSizeOverLimitError from "../errors/RestContentSizeOverLimitError";
+import RestRateLimitError from "../errors/RestRateLimitError";
+import RestRetriableError from "../errors/RestRetriableError";
+import RestTokenInvalidError from "../errors/RestTokenInvalidError";
+import type RestOptions from "../RestOptions";
+import type RestResponse from "../RestResponse";
+
+/**
+ * Returns a new retry delay of the error needs to be retried, otherwise
+ * "no_retry".
+ */
+export default function calcRetryDelay(
+  error: any,
+  options: RestOptions,
+  res: RestResponse,
+  retryDelayMs: number
+): number | "no_retry" {
+  if (
+    error instanceof RestRateLimitError ||
+    error instanceof RestRetriableError
+  ) {
+    // We've already made a decision to retry this error.
+    return Math.min(
+      options.retryDelayMaxMs,
+      Math.max(retryDelayMs, error.delayMs)
+    );
+  }
+
+  switch (options.isRetriableError(res, error)) {
+    case "RETRY":
+    default:
+      break; // number returned
+
+    case "BEST_EFFORT":
+      if (error instanceof RestTokenInvalidError) {
+        return "no_retry";
+      }
+
+      if (
+        !(error instanceof RestRateLimitError) &&
+        res.status >= 400 &&
+        res.status <= 499
+      ) {
+        return "no_retry";
+      }
+
+      if (error instanceof RestContentSizeOverLimitError) {
+        // Content size ... over limit.
+        return "no_retry";
+      }
+
+      break;
+
+    case "NEVER_RETRY":
+      return "no_retry";
+  }
+
+  return retryDelayMs;
+}

package/src/internal/inferResBodyEncoding.ts

@@ -0,0 +1,33 @@
+import type { Response } from "node-fetch";
+
+const CHARSET_RE =
+  /(?:charset|encoding)\s{0,10}=\s{0,10}['"]? {0,10}([-\w]{1,100})/i;
+const BUFFER_ENCODINGS = ["ascii", "utf8", "utf-8", "utf16le", "ucs2", "ucs-2"];
+
+/**
+ * Tries its best to infer the encoding of the Response, falling back to UTF-8
+ * as an opinionated default value on failure.
+ */
+export default function inferResBodyEncoding(res: Response): BufferEncoding {
+  const contentType = res.headers.get("content-type")?.toLowerCase();
+  const charset = contentType?.match(CHARSET_RE)
+    ? RegExp.$1.toLowerCase()
+    : undefined;
+  return contentType?.startsWith("application/octet-stream")
+    ? // It's a binary Content-Type.
+      "binary"
+    : charset && !BUFFER_ENCODINGS.includes(charset)
+    ? // The charset is provided in Content-Type, but unknown by Buffer.
+      "binary"
+    : charset && BUFFER_ENCODINGS.includes(charset)
+    ? // Charset is provided in Content-Type header, and Buffer knows
+      // how to decode it.
+      (charset as BufferEncoding)
+    : // An opinionated choice is made here to always default-decode the
+      // response stream as UTF-8. This is because JSON is by definition a UTF-8
+      // stream, and people often time respond with JSONs forgetting to provide
+      // "; charset=utf-8" part of the Content-Type header (or they forget
+      // Content-Type header at all, or put some wrong value as "text/plain"
+      // there; there is an endless list of mistake variations here).
+      "utf-8";
+}

package/src/internal/inspectPossibleJSON.ts

@@ -0,0 +1,71 @@
+import { inspect } from "util";
+import sortBy from "lodash/sortBy";
+import truncate from "lodash/truncate";
+
+export default function inspectPossibleJSON(
+  headers: { get(name: string): string | null },
+  text: string | Buffer | NodeJS.ReadableStream,
+  maxOutputLen: number
+): string {
+  const MAX_LEN_TO_TRY_PARSE = 1024 * 1024;
+
+  if (typeof text === "string" && text.length > MAX_LEN_TO_TRY_PARSE) {
+    // Don't even try to JSON-parse if the text is too long.
+    return ellipsis(text, maxOutputLen);
+  }
+
+  if (text instanceof Buffer) {
+    return `<Buffer: ${text.length} bytes>`;
+  }
+
+  if (!text || typeof text === "string") {
+    if (!(headers.get("content-type") || "").match(/json/)) {
+      return ellipsis(text, maxOutputLen);
+    }
+
+    try {
+      const json = JSON.parse(text);
+
+      if (json && typeof json === "object" && !(json instanceof Array)) {
+        // Move error/errors fields on top for better logging. This is a poor
+        // man's approach: of course not all APIs return error/errors fields at
+        // all, but it's hard to reorder at any other layer of abstraction.
+        reorderObjectProps(json, (k) =>
+          k === "error" || k === "errors" ? "" : k
+        );
+      }
+
+      return ellipsis(
+        inspect(json, { depth: 20, compact: true }),
+        maxOutputLen
+      );
+    } catch (e: any) {
+      return ellipsis(text, maxOutputLen);
+    }
+  }
+
+  return "<Stream>";
+}
+
+/**
+ * In-place-reorders keys in a given object. The important part is to do it
+ * in-place to e.g. be able to alter some @Memoized values.
+ */
+function reorderObjectProps(
+  obj: Record<string, any>,
+  ranker: (k: string, v: any) => string | number
+) {
+  const entries = Object.entries(obj);
+  for (const k in obj) {
+    delete obj[k];
+  }
+
+  Object.assign(
+    obj,
+    Object.fromEntries(sortBy(entries, ([k, v]) => ranker(k, v)))
+  );
+}
+
+function ellipsis(text: any, length: number) {
+  return truncate("" + text, { length }).trimEnd();
+}

package/src/internal/prependNewlineIfMultiline.ts

@@ -0,0 +1,3 @@
+export default function prependNewlineIfMultiline(str: string) {
+  return str.trim().includes("\n") ? "\n" + str : str;
+}

package/src/internal/substituteParams.ts

@@ -0,0 +1,25 @@
+/**
+ * Allows to use URLs like /some/:abc/other and pass { abc: "xyz" } as one of
+ * body parameters. Such body parameters will be excluded from the body before
+ * sending the request (so they're "moved" into the URL).
+ */
+export default function substituteParams<TBody>(
+  url: string,
+  body: TBody
+): [string, TBody] {
+  if (!body || typeof body !== "object") {
+    return [url, body];
+  }
+
+  url = url.replace(/:([a-z0-9_]+)/gi, (match, param) => {
+    const value = (body as any)[param];
+    if (typeof value === "string" || typeof value === "number") {
+      body = { ...body };
+      delete (body as any)[param];
+      return encodeURIComponent(value);
+    }
+
+    return match;
+  });
+  return [url, body];
+}

package/src/internal/throwIfErrorResponse.ts

@@ -0,0 +1,89 @@
+import RestRateLimitError from "../errors/RestRateLimitError";
+import RestResponseError from "../errors/RestResponseError";
+import RestRetriableError from "../errors/RestRetriableError";
+import RestTokenInvalidError from "../errors/RestTokenInvalidError";
+import type RestOptions from "../RestOptions";
+import type RestResponse from "../RestResponse";
+
+const STATUS_TOO_MANY_REQUESTS = 429;
+
+/**
+ * The general idea is that we turn all logical errors into exceptions and then
+ * deal with exceptions only. I.e. throwing an exception becomes an internal API
+ * convention for errors. This is because fetch() throws its own exceptions, and
+ * also there may be some exceptions during validation of the response, or
+ * inside a middleware etc.
+ */
+export default function throwIfErrorResponse(
+  options: RestOptions,
+  res: RestResponse
+) {
+  const isSuccessResponse = options.isSuccessResponse(res);
+  if (isSuccessResponse === "SUCCESS") {
+    return;
+  }
+
+  const rateLimitDelayMs = options.isRateLimitError(res);
+  switch (rateLimitDelayMs) {
+    case "RATE_LIMIT":
+      throw new RestRateLimitError(
+        `isRateLimitError() returned ${rateLimitDelayMs}`,
+        0,
+        res
+      );
+    case "BEST_EFFORT":
+      if (res.status === STATUS_TOO_MANY_REQUESTS) {
+        const retryAfterHeader = res.headers.get("Retry-After") || "0";
+        throw new RestRateLimitError(
+          `Rate limited by HTTP status ${STATUS_TOO_MANY_REQUESTS}`,
+          parseInt(retryAfterHeader) || 0,
+          res
+        );
+      }
+
+      break;
+    case "SOMETHING_ELSE":
+      break;
+    default:
+      throw new RestRateLimitError(
+        `isRateLimitError() returned retry delay ${rateLimitDelayMs} ms`,
+        rateLimitDelayMs,
+        res
+      );
+  }
+
+  const isTokenInvalidError = options.isTokenInvalidError(res);
+  if (isTokenInvalidError) {
+    throw new RestTokenInvalidError("Invalid app token", res);
+  }
+
+  const retryDelayMs = options.isRetriableError(res, null);
+  switch (retryDelayMs) {
+    case "RETRY":
+      throw new RestRetriableError(
+        `isRetriableError() returned ${retryDelayMs}`,
+        0,
+        res
+      );
+    case "BEST_EFFORT":
+    case "NEVER_RETRY":
+      break;
+    default:
+      throw new RestRetriableError(
+        `"isRetriableError() returned retry delay ${retryDelayMs} ms`,
+        retryDelayMs,
+        res
+      );
+  }
+
+  if (isSuccessResponse === "THROW") {
+    throw new RestResponseError(
+      `isSuccessResponse() returned ${isSuccessResponse}`,
+      res
+    );
+  }
+
+  if (res.status >= 300) {
+    throw new RestResponseError("", res);
+  }
+}

package/src/internal/toFloatMs.ts

@@ -0,0 +1,3 @@
+export default function toFloatMs(elapsed: [number, number]) {
+  return elapsed[0] * 1e3 + elapsed[1] / 1e6;
+}

package/src/middlewares/paceRequests.ts

@@ -0,0 +1,42 @@
+import type Pacer from "../pacers/Pacer";
+import type { Middleware } from "../RestOptions";
+import type RestRequest from "../RestRequest";
+
+const MIN_LOG_DELAY_MS = 10;
+
+/**
+ * Rest Client middleware that adds some delay between requests using one of
+ * Pacer implementations.
+ */
+export default function paceRequests(
+  pacer: Pacer | ((req: RestRequest) => Promise<Pacer | null>) | null
+): Middleware {
+  return async (req, next) => {
+    if (typeof pacer === "function") {
+      pacer = await pacer(req);
+    }
+
+    if (pacer) {
+      const { delayMs, reason } = await pacer.touch();
+      if (delayMs > 0) {
+        await req.options.heartbeater.delay(delayMs);
+      }
+
+      if (delayMs > MIN_LOG_DELAY_MS) {
+        req.options.logger({
+          attempt: 0,
+          req,
+          res: "backoff_delay",
+          exception: null,
+          timestamp: Date.now(),
+          elapsed: delayMs,
+          isFinalAttempt: true,
+          privateDataInResponse: false,
+          comment: reason,
+        });
+      }
+    }
+
+    return next(req);
+  };
+}

package/src/pacers/Pacer.ts

@@ -0,0 +1,22 @@
+/**
+ * A result of some Pacer work.
+ */
+export interface PacerDelay {
+  delayMs: number;
+  reason: string;
+}
+
+/**
+ * Pacer is a class which allows to pace requests on some resource identified by
+ * the instance of this class.
+ */
+export default interface Pacer {
+  /** Human readable name of the pacer, used when composing multiple pacers. */
+  readonly name: string;
+
+  /**
+   * Signals that we're about to send a request. Returns the delay we need to
+   * wait for before actually sending.
+   */
+  touch(): Promise<PacerDelay>;
+}