@clickup/rest-client 2.10.293 → 2.10.296

package/src/RestStream.ts

@@ -0,0 +1,89 @@
+import { Memoize } from "fast-typescript-memoize";
+import type RestResponse from "./RestResponse";
+
+/**
+ * Once created, RestStream must be iterated in full, otherwise the connection
+ * will remain dangling. Also, this class is where we hide the details of the
+ * actual stream reading using AsyncGenerator bridge abstraction.
+ *
+ * RestStream can also read binary data depending on the Content-Type response
+ * header and/or the charset provided there. The binary data is still returned
+ * as a string, one string character per each byte. To convert it to a Buffer,
+ * use something like `Buffer.from(responseText, "binary")`.
+ */
+export default class RestStream {
+  private _generator?: AsyncGenerator<string, void>;
+
+  constructor(
+    public readonly res: RestResponse,
+    readerIterable: {
+      [Symbol.asyncIterator]: () => AsyncGenerator<string, void>;
+    },
+  ) {
+    this._generator = readerIterable[Symbol.asyncIterator]();
+  }
+
+  /**
+   * Reads the prefix of the stream. Closes the connection after the read is
+   * done in all cases, so safe to be used to e.g. receive a trimmed response.
+   */
+  async consumeReturningPrefix(maxChars: number): Promise<string> {
+    const text: string[] = [];
+    let length = 0;
+    for await (const chunk of this) {
+      // According to Google, in v8 string concatenation is as efficient as
+      // array or buffer joining.
+      text.push(chunk);
+      length += chunk.length;
+      if (length >= maxChars) {
+        break;
+      }
+    }
+
+    return text.join("").substring(0, maxChars);
+  }
+
+  /**
+   * Closes the connection.
+   */
+  async close(): Promise<void> {
+    // First, try to interrupt the active iteration, if any.
+    await this[Symbol.asyncIterator]().return();
+    // It is possible that this.[Symbol.asyncIterator] has never been iterated
+    // before at all, so its `finally` got never executed. This happens when
+    // RestFetchReader#preload() consumed some small chunk of data, and then
+    // no-one iterated the remaining body in RestStream, they just called
+    // close() on it. So we recheck & close the RestFetchReader iterator if it
+    // is still there.
+    const generator = this._generator;
+    if (generator) {
+      delete this._generator;
+      await generator.return();
+    }
+  }
+
+  /**
+   * Allows to iterate over the entire stream of data. You must consume the
+   * entire iterable or at least call this.close(), otherwise the connection may
+   * remain open.
+   */
+  @Memoize()
+  async *[Symbol.asyncIterator](): AsyncGenerator<string, void> {
+    const generator = this._generator;
+    if (!generator) {
+      return;
+    }
+
+    delete this._generator;
+    try {
+      yield this.res.text;
+      for await (const chunk of generator) {
+        yield chunk;
+      }
+    } finally {
+      // The code enters here if the caller interrupted (returned) the iteration
+      // after receiving the 1st textFetched chunk.
+      await generator.return();
+    }
+  }
+}

package/src/errors/RestContentSizeOverLimitError.ts

@@ -0,0 +1,3 @@
+import RestResponseError from "./RestResponseError";
+
+export default class RestContentSizeOverLimitError extends RestResponseError {}

package/src/errors/RestError.ts

@@ -0,0 +1,6 @@
+export default class RestError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = this.constructor.name; // https://javascript.info/custom-errors#further-inheritance
+  }
+}

package/src/errors/RestRateLimitError.ts

@@ -0,0 +1,12 @@
+import type RestResponse from "../RestResponse";
+import RestResponseError from "./RestResponseError";
+
+export default class RestRateLimitError extends RestResponseError {
+  constructor(
+    message: string,
+    public delayMs: number,
+    res: RestResponse,
+  ) {
+    super(message, res);
+  }
+}

package/src/errors/RestResponseError.ts

@@ -0,0 +1,46 @@
+import inspectPossibleJSON from "../internal/inspectPossibleJSON";
+import prependNewlineIfMultiline from "../internal/prependNewlineIfMultiline";
+import type RestResponse from "../RestResponse";
+import RestError from "./RestError";
+
+const RESPONSE_MAX_LEN_IN_ERROR =
+  process.env["NODE_ENV"] === "development" ? Number.MAX_SAFE_INTEGER : 2048;
+
+export default class RestResponseError extends RestError {
+  readonly res!: RestResponse;
+
+  readonly method: string;
+  readonly host: string;
+  readonly pathname: string;
+  readonly requestArgs: string;
+  readonly requestBody: string;
+  readonly responseHeaders: string;
+
+  constructor(message: string, res: RestResponse) {
+    super(
+      `HTTP ${res.status}: ` +
+        (message ? `${message}: ` : "") +
+        prependNewlineIfMultiline(
+          inspectPossibleJSON(res.headers, res.text, RESPONSE_MAX_LEN_IN_ERROR),
+        ),
+    );
+    Object.defineProperty(this, "res", { value: res, enumerable: false }); // hidden from inspect()
+    const url = new URL(res.req.url);
+    const search = (url.search || "").replace(/^\?/, "");
+    this.method = res.req.method;
+    this.host = res.req.headers.get("host") || url.host;
+    this.pathname = url.pathname;
+    this.requestArgs = search.replace(/(?<=[&])/g, "\n");
+    this.requestBody = inspectPossibleJSON(
+      res.headers,
+      res.req.body,
+      RESPONSE_MAX_LEN_IN_ERROR,
+    );
+    this.responseHeaders = [...res.headers]
+      .map(([name, value]) => `${name}: ${value}`)
+      .join("\n");
+    // Don't expose query string parameters in the message as they may contain
+    // sensitive information.
+    this.message += `\n  ${this.method} ${url.protocol}//${url.host}${url.pathname}`;
+  }
+}

package/src/errors/RestRetriableError.ts

@@ -0,0 +1,12 @@
+import type RestResponse from "../RestResponse";
+import RestResponseError from "./RestResponseError";
+
+export default class RestRetriableError extends RestResponseError {
+  constructor(
+    message: string,
+    public delayMs: number,
+    res: RestResponse,
+  ) {
+    super(message, res);
+  }
+}

package/src/errors/RestTimeoutError.ts

@@ -0,0 +1,3 @@
+import RestResponseError from "./RestResponseError";
+
+export default class RestTimeoutError extends RestResponseError {}

package/src/errors/RestTokenInvalidError.ts

@@ -0,0 +1,11 @@
+import type RestResponse from "../RestResponse";
+import RestResponseError from "./RestResponseError";
+
+export default class RestTokenInvalidError extends RestResponseError {
+  constructor(
+    public readonly humanReason: string,
+    res: RestResponse,
+  ) {
+    super(humanReason, res);
+  }
+}

package/src/helpers/depaginate.ts

@@ -0,0 +1,37 @@
+/**
+ * Keeps calling a function with an updating cursor, and depaginates all the
+ * results until the cursor returned is null or undefined.
+ *
+ * On each call, the inner function needs to return an array with two elements:
+ * 1. Array or results, which could be empty, but not null or undefined.
+ * 2. A new cursor.
+ */
+export default async function* depaginate<TItem, TCursor = string>(
+  readFunc: (
+    cursor: TCursor | undefined,
+  ) => Promise<readonly [TItem[], TCursor | null | undefined]>,
+): AsyncGenerator<TItem, void, undefined> {
+  let prevCursor: TCursor | null | undefined = undefined;
+  let cursor: TCursor | null | undefined = undefined;
+  for (;;) {
+    let items: TItem[];
+    [items, cursor] = await readFunc(cursor === null ? undefined : cursor);
+    yield* items;
+
+    if (cursor === null || cursor === undefined) {
+      break;
+    }
+
+    if (JSON.stringify(prevCursor) === JSON.stringify(cursor)) {
+      throw Error(
+        "Depagination got stuck: prevCursor=" +
+          JSON.stringify(prevCursor) +
+          ", cursor=" +
+          JSON.stringify(cursor) +
+          " (they must differ)",
+      );
+    }
+
+    prevCursor = cursor;
+  }
+}

package/src/index.ts

@@ -0,0 +1,50 @@
+import { Headers } from "node-fetch";
+import RestContentSizeOverLimitError from "./errors/RestContentSizeOverLimitError";
+import RestError from "./errors/RestError";
+import RestRateLimitError from "./errors/RestRateLimitError";
+import RestResponseError from "./errors/RestResponseError";
+import RestRetriableError from "./errors/RestRetriableError";
+import RestTimeoutError from "./errors/RestTimeoutError";
+import RestTokenInvalidError from "./errors/RestTokenInvalidError";
+import depaginate from "./helpers/depaginate";
+import paceRequests from "./middlewares/paceRequests";
+import Pacer from "./pacers/Pacer";
+import PacerComposite from "./pacers/PacerComposite";
+import PacerQPS, { PacerQPSBackend } from "./pacers/PacerQPS";
+import RestClient, { TokenGetter } from "./RestClient";
+import RestOptions, {
+  RestLogEvent,
+  Middleware,
+  Agents,
+  DEFAULT_OPTIONS,
+} from "./RestOptions";
+import RestRequest from "./RestRequest";
+import RestResponse from "./RestResponse";
+import RestStream from "./RestStream";
+
+export {
+  Agents,
+  DEFAULT_OPTIONS,
+  depaginate,
+  Headers,
+  Middleware,
+  Pacer,
+  PacerComposite,
+  paceRequests,
+  PacerQPS,
+  PacerQPSBackend,
+  RestClient,
+  RestContentSizeOverLimitError,
+  RestError,
+  RestLogEvent,
+  RestOptions,
+  RestRateLimitError,
+  RestRequest,
+  RestResponse,
+  RestResponseError,
+  RestRetriableError,
+  RestStream,
+  RestTimeoutError,
+  RestTokenInvalidError,
+  TokenGetter,
+};

package/src/internal/RestFetchReader.ts

@@ -0,0 +1,188 @@
+import type { Agent as HttpAgent } from "http";
+import AbortControllerPolyfilled from "abort-controller";
+import { Memoize } from "fast-typescript-memoize";
+import type { RequestInit } from "node-fetch";
+import fetch, { Headers, Request } from "node-fetch";
+import inferResBodyEncoding from "./inferResBodyEncoding";
+
+export interface RestFetchReaderOptions {
+  timeoutMs?: number;
+  heartbeat?: () => Promise<void>;
+  onTimeout?: (reader: RestFetchReader, e: any) => void;
+  onAfterRead?: (reader: RestFetchReader) => void;
+}
+
+/**
+ * A low-level stateful reader engine on top of node-fetch which implements
+ * "preload first N chars and then leave the rest ready for iteration" pattern,
+ * with global timeout for the entire fetching time.
+ *
+ * Once created, the object MUST be iterated in full to consume the rest of the
+ * stream and close the connection. In case you're not interested in its entire
+ * content, you must prematurely "return" (close) the iterator.
+ *
+ * The abstraction is intentionally kept independent on all others, to make it
+ * simple and testable separately.
+ */
+export default class RestFetchReader {
+  private _status = 0;
+  private _headers = new Headers();
+  private _textFetched = "";
+  private _textIsPartial = true;
+  private _charsRead = 0;
+
+  constructor(
+    private _url: string,
+    private _reqInit: RequestInit,
+    private _options: RestFetchReaderOptions,
+  ) {}
+
+  /**
+   * Returns the number of characters read from the stream so far.
+   */
+  get charsRead() {
+    return this._charsRead;
+  }
+
+  /**
+   * Returns the Agent instance used for this request. It's implied that
+   * RestRequest#agent always points to a http.Agent object.
+   */
+  get agent() {
+    return (
+      this._reqInit.agent &&
+      typeof this._reqInit.agent === "object" &&
+      "sockets" in this._reqInit.agent
+        ? this._reqInit.agent
+        : null
+    ) as HttpAgent | null;
+  }
+
+  /**
+   * Returns HTTP status after preload() was called.
+   */
+  get status() {
+    return this._status;
+  }
+
+  /**
+   * Returns HTTP headers after preload() was called.
+   */
+  get headers() {
+    return this._headers;
+  }
+
+  /**
+   * Returns the data preloaded so far.
+   */
+  get textFetched(): string {
+    return this._textFetched;
+  }
+
+  /**
+   * If true, then there is a chance that reading more from the stream will
+   * return more data.
+   */
+  get textIsPartial() {
+    return this._textIsPartial;
+  }
+
+  /**
+   * Reads preloadChars chars or a little bit more from the response and puts
+   * them to this.textFetched. Leaves the rest of the data in res.body for
+   * future reads if there are more data to fetch (you must consume them or
+   * close the stream, otherwise the connection will remain open).
+   */
+  async preload(preloadChars: number) {
+    const generator = this[Symbol.asyncIterator]();
+    try {
+      while (this._charsRead < preloadChars) {
+        const { value, done } = await generator.next();
+        if (done) {
+          this._textIsPartial = false;
+          await generator.return();
+          return;
+        }
+
+        this._textFetched += value;
+      }
+    } catch (e: unknown) {
+      await generator.return();
+      throw e;
+    }
+  }
+
+  /**
+   * Closes the connection.
+   */
+  async close() {
+    await this[Symbol.asyncIterator]().return();
+  }
+
+  /**
+   * Returns an async generator for the rest of the data. Must be consumed
+   * entirely, otherwise the connection may remain dangling.
+   *
+   * Memoization is important here, to return the same generator when we call
+   * this method multiple times and to not start a new iteration over and over.
+   */
+  @Memoize()
+  async *[Symbol.asyncIterator]() {
+    const { timeoutMs, onTimeout, onAfterRead } = this._options;
+
+    // Some of react-client users are still on v14 node.
+    const controller =
+      typeof AbortController === "undefined"
+        ? new AbortControllerPolyfilled()
+        : new AbortController();
+
+    const timeout = timeoutMs
+      ? setTimeout(() => controller.abort(), timeoutMs)
+      : undefined;
+
+    try {
+      // DO NOT use fetch(fetchReq) with one argument! It clones the stream
+      // which just doesn't work in practice, even with file streams. I wasted
+      // 4h on debugging this: fetch(fetchReq.url, fetchReq) works and
+      // fetch(fetchReq) doesn't for e.g. Dropbox API and
+      // https://stackoverflow.com/a/44577569
+      const res = await fetch(
+        this._url,
+        new Request(this._url, {
+          ...this._reqInit,
+          signal: controller.signal as any,
+        }),
+      );
+      this._status = res.status;
+      this._headers = res.headers;
+
+      // See https://nodejs.org/api/stream.html#readablesetencodingencoding on
+      // how Node streams and setEncoding() handle decoding when the returned
+      // chunks cross the boundaries of multi-byte characters (TL;DR: it works
+      // fine, that's why we work with string and not Buffer here).
+      res.body.setEncoding(inferResBodyEncoding(res));
+
+      await this._options.heartbeat?.();
+      for await (const chunk of res.body) {
+        await this._options.heartbeat?.();
+        this._charsRead += chunk.length;
+        yield chunk as string;
+        onAfterRead?.(this);
+      }
+    } catch (e: unknown) {
+      if (controller.signal.aborted && onTimeout) {
+        onTimeout(this, e);
+      } else {
+        throw e;
+      }
+    } finally {
+      timeout && clearTimeout(timeout);
+      // If someone stops iterating prematurely, we forcefully close the
+      // connection in all cases. Theoretically, stopping the iteration on
+      // res.body should've closed the connection, but in practice it doesn't
+      // happen; it looks like a bug in node-fetch, and thus, we must always use
+      // the AbortController in the end.
+      controller.abort();
+    }
+  }
+}

package/src/internal/RestRangeUploader.ts

@@ -0,0 +1,61 @@
+import type RestClient from "../RestClient";
+
+/**
+ * Sends a series of Content-Range requests to an URL.
+ * - The stream size is unknown in advance even theoretically. So we read it
+ *   with chunkSize+1 bytes chunks (+1 is to know for sure, is there something
+ *   else left in the stream or not) and then send data with chunkSize bytes
+ *   chunks.
+ * - The last chunk is a terminating one (and we know, which one is the last),
+ *   so we reflect it in "Content-Range: x-y/S" format setting S to the total
+ *   number of bytes in the stream.
+ */
+export default class RestRangeUploader {
+  private _pos = 0;
+
+  constructor(
+    private _client: RestClient,
+    private _chunkSize: number,
+    private _method: "POST" | "PUT",
+    private _path: string,
+    private _mimeType: string,
+  ) {}
+
+  async upload(stream: AsyncIterable<Buffer>) {
+    let buf = Buffer.allocUnsafe(0);
+    let res: string | null = null;
+    for await (const readData of stream) {
+      buf = Buffer.concat([buf, readData]);
+      while (buf.length >= this._chunkSize + 1) {
+        res = await this._flush(buf.slice(0, this._chunkSize), false);
+        buf = Buffer.from(buf.slice(this._chunkSize));
+      }
+      // After this `while` loop finishes, there is always something left in buf
+      // (due to the +1 trick). It guarantees that we have a chance to call
+      // flush(..., true) for the very last chunk.
+    }
+
+    if (buf.length > 0) {
+      res = await this._flush(buf, true);
+    }
+
+    return res;
+  }
+
+  private async _flush(buf: Buffer, isLast: boolean) {
+    if (buf.length === 0) {
+      return null;
+    }
+
+    const totalSize = isLast ? this._pos + buf.length : "*";
+    const res = await this._client
+      .writeRaw(this._path, buf, this._mimeType, this._method, "*/*")
+      .setHeader(
+        "Content-Range",
+        `bytes ${this._pos}-${this._pos + buf.length - 1}/${totalSize}`,
+      )
+      .text();
+    this._pos += buf.length;
+    return res;
+  }
+}

package/src/internal/calcRetryDelay.ts

@@ -0,0 +1,59 @@
+import RestContentSizeOverLimitError from "../errors/RestContentSizeOverLimitError";
+import RestRateLimitError from "../errors/RestRateLimitError";
+import RestRetriableError from "../errors/RestRetriableError";
+import RestTokenInvalidError from "../errors/RestTokenInvalidError";
+import type RestOptions from "../RestOptions";
+import type RestResponse from "../RestResponse";
+
+/**
+ * Returns a new retry delay of the error needs to be retried, otherwise
+ * "no_retry".
+ */
+export default function calcRetryDelay(
+  error: any,
+  options: RestOptions,
+  res: RestResponse,
+  retryDelayMs: number,
+): number | "no_retry" {
+  if (
+    error instanceof RestRateLimitError ||
+    error instanceof RestRetriableError
+  ) {
+    // We've already made a decision to retry this error.
+    return Math.min(
+      options.retryDelayMaxMs,
+      Math.max(retryDelayMs, error.delayMs),
+    );
+  }
+
+  switch (options.isRetriableError(res, error)) {
+    case "RETRY":
+    default:
+      break; // number returned
+
+    case "BEST_EFFORT":
+      if (error instanceof RestTokenInvalidError) {
+        return "no_retry";
+      }
+
+      if (
+        !(error instanceof RestRateLimitError) &&
+        res.status >= 400 &&
+        res.status <= 499
+      ) {
+        return "no_retry";
+      }
+
+      if (error instanceof RestContentSizeOverLimitError) {
+        // Content size ... over limit.
+        return "no_retry";
+      }
+
+      break;
+
+    case "NEVER_RETRY":
+      return "no_retry";
+  }
+
+  return retryDelayMs;
+}

package/src/internal/inferResBodyEncoding.ts

@@ -0,0 +1,33 @@
+import type { Response } from "node-fetch";
+
+const CHARSET_RE =
+  /(?:charset|encoding)\s{0,10}=\s{0,10}['"]? {0,10}([-\w]{1,100})/i;
+const BUFFER_ENCODINGS = ["ascii", "utf8", "utf-8", "utf16le", "ucs2", "ucs-2"];
+
+/**
+ * Tries its best to infer the encoding of the Response, falling back to UTF-8
+ * as an opinionated default value on failure.
+ */
+export default function inferResBodyEncoding(res: Response): BufferEncoding {
+  const contentType = res.headers.get("content-type")?.toLowerCase();
+  const charset = contentType?.match(CHARSET_RE)
+    ? RegExp.$1.toLowerCase()
+    : undefined;
+  return contentType?.startsWith("application/octet-stream")
+    ? // It's a binary Content-Type.
+      "binary"
+    : charset && !BUFFER_ENCODINGS.includes(charset)
+      ? // The charset is provided in Content-Type, but unknown by Buffer.
+        "binary"
+      : charset && BUFFER_ENCODINGS.includes(charset)
+        ? // Charset is provided in Content-Type header, and Buffer knows
+          // how to decode it.
+          (charset as BufferEncoding)
+        : // An opinionated choice is made here to always default-decode the
+          // response stream as UTF-8. This is because JSON is by definition a UTF-8
+          // stream, and people often time respond with JSONs forgetting to provide
+          // "; charset=utf-8" part of the Content-Type header (or they forget
+          // Content-Type header at all, or put some wrong value as "text/plain"
+          // there; there is an endless list of mistake variations here).
+          "utf-8";
+}