@clickup/rest-client 2.10.296 → 2.11.1

package/src/__tests__/RestRequest.test.ts

@@ -0,0 +1,262 @@
+import { FetchError, Headers } from "node-fetch";
+import {
+  DEFAULT_OPTIONS,
+  RestClient,
+  RestRateLimitError,
+  RestResponseError,
+  RestTokenInvalidError,
+} from "..";
+import calcRetryDelay from "../internal/calcRetryDelay";
+import throwIfErrorResponse from "../internal/throwIfErrorResponse";
+import RestResponse from "../RestResponse";
+
+const REQUEST = new RestClient().get("https://example.com");
+const SUCCESS_RESPONSE = new RestResponse(
+  REQUEST,
+  null,
+  200,
+  new Headers(),
+  "",
+  false,
+);
+const ERROR_RESPONSE = new RestResponse(
+  REQUEST,
+  null,
+  500,
+  new Headers(),
+  "",
+  false,
+);
+const NOT_FOUND_RESPONSE = new RestResponse(
+  REQUEST,
+  null,
+  404,
+  new Headers(),
+  "",
+  false,
+);
+const TOO_MANY_REQUESTS_RESPONSE = new RestResponse(
+  REQUEST,
+  null,
+  429,
+  new Headers(),
+  "",
+  false,
+);
+
+test("isSuccessResponse", () => {
+  expect(
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isSuccessResponse: () => "SUCCESS" },
+      ERROR_RESPONSE,
+    ),
+  ).toBeUndefined();
+
+  expect(() =>
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isSuccessResponse: () => "BEST_EFFORT" },
+      NOT_FOUND_RESPONSE,
+    ),
+  ).toThrow(/404/);
+
+  expect(
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isSuccessResponse: () => "BEST_EFFORT" },
+      SUCCESS_RESPONSE,
+    ),
+  ).toBeUndefined();
+
+  expect(() =>
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isSuccessResponse: () => "THROW" },
+      SUCCESS_RESPONSE,
+    ),
+  ).toThrow(/isSuccessResponse/);
+});
+
+test("isRateLimitError", () => {
+  expect(() =>
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isRateLimitError: () => "RATE_LIMIT" },
+      SUCCESS_RESPONSE,
+    ),
+  ).toThrow(RestRateLimitError);
+
+  expect(
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isRateLimitError: () => "BEST_EFFORT" },
+      SUCCESS_RESPONSE,
+    ),
+  ).toBeUndefined();
+
+  expect(() =>
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isRateLimitError: () => "BEST_EFFORT" },
+      TOO_MANY_REQUESTS_RESPONSE,
+    ),
+  ).toThrow(RestRateLimitError);
+
+  expect(
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isRateLimitError: () => "SOMETHING_ELSE" },
+      SUCCESS_RESPONSE,
+    ),
+  ).toBeUndefined();
+});
+
+test("isTokenInvalidError", () => {
+  expect(() =>
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isTokenInvalidError: () => true },
+      SUCCESS_RESPONSE,
+    ),
+  ).toThrow(RestTokenInvalidError);
+
+  expect(
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isTokenInvalidError: () => false },
+      SUCCESS_RESPONSE,
+    ),
+  ).toBeUndefined();
+});
+
+test("isRetriableError", () => {
+  expect(() =>
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isRetriableError: () => "RETRY" },
+      SUCCESS_RESPONSE,
+    ),
+  ).toThrow(RestResponseError);
+
+  expect(
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isRetriableError: () => "BEST_EFFORT" },
+      SUCCESS_RESPONSE,
+    ),
+  ).toBeUndefined();
+
+  expect(() =>
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isRetriableError: () => "BEST_EFFORT" },
+      NOT_FOUND_RESPONSE,
+    ),
+  ).toThrow(RestResponseError);
+
+  expect(
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isRateLimitError: () => "SOMETHING_ELSE" },
+      SUCCESS_RESPONSE,
+    ),
+  ).toBeUndefined();
+
+  expect(() =>
+    throwIfErrorResponse(
+      { ...DEFAULT_OPTIONS, isRetriableError: () => "NEVER_RETRY" },
+      NOT_FOUND_RESPONSE,
+    ),
+  ).toThrow(RestResponseError);
+});
+
+test("calcRetryDelay", () => {
+  expect(
+    calcRetryDelay(
+      new RestRateLimitError("", 42, SUCCESS_RESPONSE),
+      DEFAULT_OPTIONS,
+      SUCCESS_RESPONSE,
+      0,
+    ),
+  ).toEqual(42);
+
+  expect(
+    calcRetryDelay(
+      new RestTokenInvalidError("", ERROR_RESPONSE),
+      DEFAULT_OPTIONS,
+      ERROR_RESPONSE,
+      0,
+    ),
+  ).toEqual("no_retry");
+
+  expect(
+    calcRetryDelay(
+      new RestTokenInvalidError("", ERROR_RESPONSE),
+      { ...DEFAULT_OPTIONS, isRetriableError: () => "RETRY" },
+      ERROR_RESPONSE,
+      42,
+    ),
+  ).toEqual(42);
+
+  expect(
+    calcRetryDelay(
+      new RestResponseError("", ERROR_RESPONSE),
+      { ...DEFAULT_OPTIONS, isRetriableError: () => "BEST_EFFORT" },
+      ERROR_RESPONSE,
+      42,
+    ),
+  ).toEqual(42);
+
+  expect(
+    calcRetryDelay(
+      new RestResponseError("", NOT_FOUND_RESPONSE),
+      { ...DEFAULT_OPTIONS, isRetriableError: () => "BEST_EFFORT" },
+      NOT_FOUND_RESPONSE,
+      42,
+    ),
+  ).toEqual("no_retry");
+
+  expect(
+    calcRetryDelay(
+      new RestRateLimitError("", 42, NOT_FOUND_RESPONSE),
+      { ...DEFAULT_OPTIONS, isRetriableError: () => "BEST_EFFORT" },
+      NOT_FOUND_RESPONSE,
+      0,
+    ),
+  ).toEqual(42);
+
+  expect(
+    calcRetryDelay(
+      new FetchError("test", "some"),
+      {
+        ...DEFAULT_OPTIONS,
+        isRetriableError: (_: any, error: any) =>
+          error instanceof FetchError ? "NEVER_RETRY" : "RETRY",
+      },
+      NOT_FOUND_RESPONSE,
+      0,
+    ),
+  ).toEqual("no_retry");
+
+  expect(
+    calcRetryDelay(
+      new FetchError("test", "max-size"),
+      { ...DEFAULT_OPTIONS, isRetriableError: () => "BEST_EFFORT" },
+      NOT_FOUND_RESPONSE,
+      0,
+    ),
+  ).toEqual("no_retry");
+});
+
+test("when isRateLimitError=YES, isRetriableError=NO doesn't matter, and the request is always retriable", () => {
+  expect(() =>
+    throwIfErrorResponse(
+      {
+        ...DEFAULT_OPTIONS,
+        isRateLimitError: () => "RATE_LIMIT",
+        isRetriableError: () => "NEVER_RETRY",
+      },
+      SUCCESS_RESPONSE,
+    ),
+  ).toThrow(RestRateLimitError);
+
+  expect(
+    calcRetryDelay(
+      new RestRateLimitError("", 42, SUCCESS_RESPONSE),
+      {
+        ...DEFAULT_OPTIONS,
+        isRateLimitError: () => "RATE_LIMIT",
+        isRetriableError: () => "NEVER_RETRY",
+      },
+      SUCCESS_RESPONSE,
+      0,
+    ),
+  ).toEqual(42);
+});

package/src/__tests__/RestRequestCacheableLookup.test.ts

@@ -0,0 +1,88 @@
+import { Headers } from "node-fetch";
+import type { RequestInit } from "node-fetch";
+import { server } from "./helpers";
+
+let ORIGIN: string;
+
+beforeAll(async () => {
+  ORIGIN = await new Promise((resolve) => {
+    server.listen(0, () =>
+      resolve("http://127.0.0.1:" + (server.address() as { port: number }).port),
+    );
+  });
+});
+
+afterAll(async () => {
+  await new Promise((resolve) => server.close(resolve));
+});
+
+test("request works with family=0 and cacheable-lookup runtime loading", async () => {
+  const { RestClient } = require("..");
+  const client = new RestClient({ allowInternalIPs: true, family: 0 });
+  await expect(client.get(`${ORIGIN}/small`, undefined, "*/*").text()).resolves.toBe(
+    "ok",
+  );
+});
+
+test("lookupAsync call shape depends on family option", async () => {
+  type LookupAsync = (
+    hostname: string,
+    options?: { family: 4 | 6 },
+  ) => Promise<{ address: string; family: 4 | 6 }>;
+
+  const lookupAsyncMock: jest.MockedFunction<LookupAsync> = jest.fn(
+    async (_hostname: string, _options?: { family: 4 | 6 }) => ({
+      address: "93.184.216.34",
+      family: 4 as const,
+    }),
+  );
+
+  class MockCacheableLookup {
+    lookupAsync = lookupAsyncMock;
+  }
+
+  jest.resetModules();
+  jest.doMock("cacheable-lookup", () => ({
+    __esModule: true,
+    default: MockCacheableLookup,
+  }));
+
+  const { DEFAULT_OPTIONS } = require("..");
+  const { default: RestRequest } = require("../RestRequest");
+
+  lookupAsyncMock.mockClear();
+
+  const createFetchRequest = async (
+    request: unknown,
+  ): Promise<RequestInit & { url: string }> =>
+    (
+      request as {
+        _createFetchRequest(): Promise<RequestInit & { url: string }>;
+      }
+    )._createFetchRequest();
+
+  await createFetchRequest(
+    new RestRequest(
+      { ...DEFAULT_OPTIONS, allowInternalIPs: true, family: 0 },
+      "GET",
+      "http://example.com/test",
+      new Headers(),
+      "",
+    ),
+  );
+
+  await createFetchRequest(
+    new RestRequest(
+      { ...DEFAULT_OPTIONS, allowInternalIPs: true, family: 4 },
+      "GET",
+      "http://example.com/test",
+      new Headers(),
+      "",
+    ),
+  );
+
+  expect(lookupAsyncMock).toHaveBeenNthCalledWith(1, "example.com");
+  expect(lookupAsyncMock).toHaveBeenNthCalledWith(2, "example.com", {
+    family: 4,
+  });
+});

package/src/__tests__/RestStream.test.ts

@@ -0,0 +1,67 @@
+import Stream from "stream";
+import {
+  consumeReadable,
+  createRestStream,
+  server,
+  serverAssertConnectionsCount,
+} from "./helpers";
+
+let ORIGIN: string;
+
+beforeAll(async () => {
+  ORIGIN = await new Promise((resolve) => {
+    server.listen(0, () =>
+      resolve("http://127.0.0.1:" + (server.address() as any).port),
+    );
+  });
+});
+
+afterAll(async () => {
+  await new Promise((resolve) => server.close(resolve));
+});
+
+beforeEach(async () => {
+  await serverAssertConnectionsCount(0);
+});
+
+test("prefix", async () => {
+  const stream = await createRestStream(`${ORIGIN}/large`);
+  const prefix = await stream.consumeReturningPrefix(42);
+  expect(prefix.length).toEqual(42);
+  await serverAssertConnectionsCount(0);
+});
+
+test("stream_readable_proxy_closes_connection", async () => {
+  const stream = await createRestStream(`${ORIGIN}/large`);
+  const readable = Stream.Readable.from(stream);
+  expect((await consumeReadable(readable)).length).toBeGreaterThan(1024 * 10);
+  await serverAssertConnectionsCount(0);
+});
+
+test("read_cancelled_after_reading_preloaded_chunk_closes_connection", async () => {
+  const stream = await createRestStream(`${ORIGIN}/large`, 42);
+  await serverAssertConnectionsCount(1);
+
+  for await (const chunk of stream) {
+    expect(chunk).toEqual(stream.res.text);
+    await serverAssertConnectionsCount(1);
+    break;
+  }
+
+  await serverAssertConnectionsCount(0);
+});
+
+test("read_thrown_after_reading_preloaded_chunk_closes_connection", async () => {
+  const stream = await createRestStream(`${ORIGIN}/large`, 42);
+  await serverAssertConnectionsCount(1);
+
+  await expect(async () => {
+    for await (const chunk of stream) {
+      await serverAssertConnectionsCount(1);
+      expect(chunk).toEqual(stream.res.text);
+      throw Error("bailout");
+    }
+  }).rejects.toThrowError("bailout");
+
+  await serverAssertConnectionsCount(0);
+});

package/src/__tests__/helpers.ts

@@ -0,0 +1,173 @@
+import http from "http";
+import type { Readable } from "stream";
+import delay from "delay";
+import range from "lodash/range";
+import { Headers } from "node-fetch";
+import { DEFAULT_OPTIONS, RestRequest, RestResponse, RestStream } from "..";
+import type { RestFetchReaderOptions } from "../internal/RestFetchReader";
+import RestFetchReader from "../internal/RestFetchReader";
+
+// Two European Euro symbols in UTF-8.
+export const UTF8_BUF = Buffer.from([0xe2, 0x82, 0xac, 0xe2, 0x82, 0xac]);
+
+// Binary data, all possible bytes.
+export const BINARY_BUF = Buffer.from(range(256));
+
+// eslint-disable-next-line @typescript-eslint/no-misused-promises
+export const server = http.createServer(async (req, res) => {
+  res.setHeader("Content-Type", "text/plain; charset=utf-8");
+
+  // In node-fetch 2.6.13+, they stopped sending "Connection: close" by default:
+  // https://github.com/node-fetch/node-fetch/pull/1736 - which is a good thing:
+  // we can now send multiple requests through the same connection sequentially
+  // (it seems to be broken before). Since RestFetchReader calls
+  // controller.abort() when the client wants to interrupt the request, the
+  // connection is physically closed only if there is currently an active
+  // request (e.g. "/slow" below), and it's not closed when the request is
+  // successfully processed (which is, again, a good thing). So to be able to
+  // use serverAssertConnectionsCount() helper, we force the server to close the
+  // connection after each request, so serverAssertConnectionsCount() can work.
+  res.setHeader("Connection", "close");
+
+  // Returns a small response.
+  if (req.url === "/small") {
+    res.writeHead(200);
+    return res.end("ok");
+  }
+
+  // Returns a large, but quick response.
+  if (req.url === "/large") {
+    res.writeHead(200);
+    await write(res, 1024);
+    await delay(100);
+    await write(res, 1024 * 100);
+    return res.end("x".repeat(1024 * 200));
+  }
+
+  // Returns a large response, does it slowly.
+  if (req.url === "/slow") {
+    res.writeHead(200);
+    for (let timeStart = Date.now(); Date.now() - timeStart < 20000; ) {
+      const error = await write(res, 1024);
+      if (error) {
+        break;
+      } else {
+        await delay(10);
+      }
+    }
+
+    return res.end();
+  }
+
+  // Returns slow response as UTF-8 bytes, one after another.
+  if (req.url === "/small_slow_utf8") {
+    res.writeHead(200);
+    for (const byte of UTF8_BUF) {
+      await write(res, Buffer.from([byte]));
+      await delay(500);
+    }
+
+    return res.end();
+  }
+
+  // Returns a binary response.
+  if (req.url === "/binary") {
+    res.setHeader("Content-Type", "application/octet-stream");
+    res.writeHead(200);
+    for (const _ in range(10)) {
+      await write(res, BINARY_BUF);
+      await delay(50);
+    }
+
+    return res.end();
+  }
+
+  res.writeHead(404);
+  return res.end("not found");
+});
+
+export async function serverAssertConnectionsCount(expect: number) {
+  let got: number = 0;
+  for (let timeStart = Date.now(); Date.now() - timeStart < 10000; ) {
+    got = await new Promise((resolve) =>
+      server.getConnections((_, count) => resolve(count)),
+    );
+    if (got === expect) {
+      return;
+    }
+
+    await delay(50);
+  }
+
+  throw Error(`Expected ${expect} connections, but got ${got}`);
+}
+
+export class TimeoutError extends Error {}
+
+export function createFetchReader(
+  url: string,
+  options: RestFetchReaderOptions = {},
+) {
+  return new RestFetchReader(
+    url,
+    {},
+    {
+      onTimeout: () => {
+        throw new TimeoutError("timed out");
+      },
+      ...options,
+    },
+  );
+}
+
+export async function createRestStream(url: string, preloadBytes?: number) {
+  const reader = createFetchReader(url);
+  if (preloadBytes) {
+    await reader.preload(preloadBytes);
+  }
+
+  return new RestStream(
+    new RestResponse(
+      new RestRequest(DEFAULT_OPTIONS, "GET", "dummy", new Headers(), ""),
+      reader.agent,
+      reader.status,
+      reader.headers,
+      reader.textFetched.toString(),
+      reader.textIsPartial,
+    ),
+    reader[Symbol.asyncIterator](),
+  );
+}
+
+export async function consumeIterable(
+  iterable: AsyncIterable<string>,
+  delayMs?: number,
+) {
+  let rest = "";
+  for await (const chunk of iterable) {
+    rest += chunk;
+    if (delayMs) {
+      await delay(delayMs);
+    }
+  }
+
+  return rest;
+}
+
+export async function consumeReadable(stream: Readable) {
+  return new Promise<string>((resolve) => {
+    const chunks: string[] = [];
+    stream.on("data", (chunk) => {
+      chunks.push(chunk);
+    });
+    stream.on("end", () => {
+      resolve(chunks.join(""));
+    });
+  });
+}
+
+async function write(res: http.ServerResponse, bytes: number | Buffer) {
+  return new Promise((r) =>
+    res.write(typeof bytes === "number" ? "x".repeat(bytes) : bytes, r),
+  );
+}

package/src/index.ts

@@ -7,10 +7,7 @@ import RestRetriableError from "./errors/RestRetriableError";
 import RestTimeoutError from "./errors/RestTimeoutError";
 import RestTokenInvalidError from "./errors/RestTokenInvalidError";
 import depaginate from "./helpers/depaginate";
-import paceRequests from "./middlewares/paceRequests";
-import Pacer from "./pacers/Pacer";
-import PacerComposite from "./pacers/PacerComposite";
-import PacerQPS, { PacerQPSBackend } from "./pacers/PacerQPS";
+import paceRequests, { Pacer, PacerOutcome } from "./middlewares/paceRequests";
 import RestClient, { TokenGetter } from "./RestClient";
 import RestOptions, {
   RestLogEvent,
@@ -29,10 +26,8 @@ export {
   Headers,
   Middleware,
   Pacer,
-  PacerComposite,
   paceRequests,
-  PacerQPS,
-  PacerQPSBackend,
+  PacerOutcome,
   RestClient,
   RestContentSizeOverLimitError,
   RestError,

package/src/internal/RestFetchReader.ts

@@ -10,6 +10,7 @@ export interface RestFetchReaderOptions {
   heartbeat?: () => Promise<void>;
   onTimeout?: (reader: RestFetchReader, e: any) => void;
   onAfterRead?: (reader: RestFetchReader) => void;
+  responseEncoding?: NodeJS.BufferEncoding;
 }
 
 /**
@@ -160,7 +161,9 @@ export default class RestFetchReader {
       // how Node streams and setEncoding() handle decoding when the returned
       // chunks cross the boundaries of multi-byte characters (TL;DR: it works
       // fine, that's why we work with string and not Buffer here).
-      res.body.setEncoding(inferResBodyEncoding(res));
+      res.body.setEncoding(
+        this._options.responseEncoding ?? inferResBodyEncoding(res),
+      );
 
       await this._options.heartbeat?.();
       for await (const chunk of res.body) {

package/src/internal/ellipsis.ts

@@ -0,0 +1,16 @@
+const ELLIPSIS = "…";
+
+/**
+ * The fastest possible version of truncation. Lodash'es truncate() messes up
+ * with unicode a lot, so for e.g. logging purposes, it's super-slow.
+ */
+export default function ellipsis(string: any, length: number) {
+  string = ("" + string).trimEnd();
+  length = Math.max(length, ELLIPSIS.length);
+
+  if (string.length <= length) {
+    return string;
+  }
+
+  return string.substring(0, length - ELLIPSIS.length) + ELLIPSIS;
+}

package/src/internal/inspectPossibleJSON.ts

@@ -1,6 +1,6 @@
 import { inspect } from "util";
 import sortBy from "lodash/sortBy";
-import truncate from "lodash/truncate";
+import ellipsis from "./ellipsis";
 
 export default function inspectPossibleJSON(
   headers: { get(name: string): string | null },
@@ -65,7 +65,3 @@ function reorderObjectProps(
     Object.fromEntries(sortBy(entries, ([k, v]) => ranker(k, v))),
   );
 }
-
-function ellipsis(text: any, length: number) {
-  return truncate("" + text, { length }).trimEnd();
-}

package/src/internal/throwIfErrorResponse.ts

@@ -36,7 +36,7 @@ export default function throwIfErrorResponse(
         const retryAfterHeader = res.headers.get("Retry-After") || "0";
         throw new RestRateLimitError(
           `Rate limited by HTTP status ${STATUS_TOO_MANY_REQUESTS}`,
-          parseInt(retryAfterHeader) || 0,
+          1000 * parseInt(retryAfterHeader) || 0,
           res,
         );
       }