@continuedev/fetch 1.0.6 → 1.0.10

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@continuedev/fetch",
-  "version": "1.0.6",
+  "version": "1.0.10",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",
   "scripts": {
-    "test": "jest",
+    "test": "cross-env NODE_OPTIONS=--experimental-vm-modules jest",
     "build": "tsc"
   },
   "author": "Nate Sesti and Ty Dunn",
@@ -20,6 +20,10 @@
   },
   "devDependencies": {
     "@types/follow-redirects": "^1.14.4",
-    "jest": "^29.7.0"
+    "@types/jest": "^29.5.14",
+    "cross-env": "^7.0.3",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.3.4",
+    "ts-node": "^10.9.2"
   }
 }

package/src/fetch.test.ts

@@ -0,0 +1,172 @@
+import { globalAgent } from "https";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { getAgentOptions } from "./getAgentOptions.js";
+
+// Store original env
+const originalEnv = process.env;
+const originalGlobalAgentOptions = { ...globalAgent.options };
+
+// Temporary directory for test certificate files
+let tempDir: string;
+
+beforeEach(() => {
+  // Create a temporary directory for test files
+  tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "fetch-test-"));
+
+  process.env = { ...originalEnv };
+
+  // Reset globalAgent for each test
+  globalAgent.options = { ...originalGlobalAgentOptions };
+});
+
+afterEach(() => {
+  process.env = originalEnv;
+  globalAgent.options = originalGlobalAgentOptions;
+
+  // Clean up temporary directory
+  try {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  } catch (error) {
+    console.error(`Failed to remove temp directory: ${error}`);
+  }
+});
+
+// Helper function to create test certificate files
+function createTestCertFile(filename: string, content: string): string {
+  const filePath = path.join(tempDir, filename);
+  fs.writeFileSync(filePath, content, "utf8");
+  return filePath;
+}
+
+test("getAgentOptions returns basic configuration with default values", () => {
+  const options = getAgentOptions();
+
+  // Check default timeout (7200 seconds = 2 hours = 7,200,000 ms)
+  expect(options.timeout).toBe(7200000);
+  expect(options.sessionTimeout).toBe(7200000);
+  expect(options.keepAliveMsecs).toBe(7200000);
+  expect(options.keepAlive).toBe(true);
+
+  // Verify certificates array exists and contains items
+  expect(options.ca).toBeInstanceOf(Array);
+  expect(options.ca.length).toBeGreaterThan(0);
+
+  // Verify at least one of the real TLS root certificates is included
+  // This assumes there's at least one certificate with "CERTIFICATE" in it
+  expect(options.ca.some((cert: any) => cert.includes("CERTIFICATE"))).toBe(
+    true,
+  );
+});
+
+test("getAgentOptions respects custom timeout", () => {
+  const customTimeout = 300; // 5 minutes
+  const options = getAgentOptions({ timeout: customTimeout });
+
+  // Check timeout values (300 seconds = 300,000 ms)
+  expect(options.timeout).toBe(300000);
+  expect(options.sessionTimeout).toBe(300000);
+  expect(options.keepAliveMsecs).toBe(300000);
+});
+
+test("getAgentOptions uses verifySsl setting", () => {
+  // With verifySsl true
+  let options = getAgentOptions({ verifySsl: true });
+  expect(options.rejectUnauthorized).toBe(true);
+
+  // With verifySsl false
+  options = getAgentOptions({ verifySsl: false });
+  expect(options.rejectUnauthorized).toBe(false);
+});
+
+test("getAgentOptions incorporates custom CA bundle paths", () => {
+  // Create a test CA bundle file
+  const caBundleContent =
+    "-----BEGIN CERTIFICATE-----\nMIIDtTCCAp2gAwIBAgIJAMcuSp7chAYdMA==\n-----END CERTIFICATE-----";
+  const caBundlePath = createTestCertFile("ca-bundle.pem", caBundleContent);
+
+  // Single string path
+  let options = getAgentOptions({ caBundlePath });
+
+  // Verify that our test certificate is included in the CA list
+  expect(options.ca).toContain(caBundleContent);
+
+  // Create multiple test CA bundle files
+  const caContent1 =
+    "-----BEGIN CERTIFICATE-----\nABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\n-----END CERTIFICATE-----";
+  const caContent2 =
+    "-----BEGIN CERTIFICATE-----\n0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\n-----END CERTIFICATE-----";
+  const caPath1 = createTestCertFile("ca1.pem", caContent1);
+  const caPath2 = createTestCertFile("ca2.pem", caContent2);
+
+  // Array of paths
+  options = getAgentOptions({
+    caBundlePath: [caPath1, caPath2],
+  });
+
+  // Verify that both test certificates are included in the CA list
+  expect(options.ca).toContain(caContent1);
+  expect(options.ca).toContain(caContent2);
+});
+
+test("getAgentOptions includes global certs when running as binary", () => {
+  // Set up test certs in globalAgent
+  globalAgent.options.ca = ["global-cert-1", "global-cert-2"];
+
+  // Set IS_BINARY environment variable
+  process.env.IS_BINARY = "true";
+
+  const options = getAgentOptions();
+
+  // Test for global certs
+  expect(options.ca).toContain("global-cert-1");
+  expect(options.ca).toContain("global-cert-2");
+});
+
+test("getAgentOptions handles client certificate configuration", () => {
+  // Create test certificate files
+  const clientCertContent =
+    "-----BEGIN CERTIFICATE-----\nCLIENTCERT\n-----END CERTIFICATE-----";
+  const clientKeyContent =
+    "-----BEGIN PRIVATE KEY-----\nCLIENTKEY\n-----END PRIVATE KEY-----";
+  const certPath = createTestCertFile("client.cert", clientCertContent);
+  const keyPath = createTestCertFile("client.key", clientKeyContent);
+
+  const clientCertOptions = {
+    clientCertificate: {
+      cert: certPath,
+      key: keyPath,
+      passphrase: "secret-passphrase",
+    },
+  };
+
+  const options = getAgentOptions(clientCertOptions);
+
+  expect(options.cert).toBe(clientCertContent);
+  expect(options.key).toBe(clientKeyContent);
+  expect(options.passphrase).toBe("secret-passphrase");
+});
+
+test("getAgentOptions handles client certificate without passphrase", () => {
+  // Create test certificate files
+  const clientCertContent =
+    "-----BEGIN CERTIFICATE-----\nCLIENTCERT2\n-----END CERTIFICATE-----";
+  const clientKeyContent =
+    "-----BEGIN PRIVATE KEY-----\nCLIENTKEY2\n-----END PRIVATE KEY-----";
+  const certPath = createTestCertFile("client2.cert", clientCertContent);
+  const keyPath = createTestCertFile("client2.key", clientKeyContent);
+
+  const clientCertOptions = {
+    clientCertificate: {
+      cert: certPath,
+      key: keyPath,
+    },
+  };
+
+  const options = getAgentOptions(clientCertOptions);
+
+  expect(options.cert).toBe(clientCertContent);
+  expect(options.key).toBe(clientKeyContent);
+  expect(options.passphrase).toBeUndefined();
+});

package/src/fetch.ts

@@ -1,12 +1,10 @@
-import { globalAgent } from "https";
-import * as fs from "node:fs";
-import tls from "node:tls";
-
 import { RequestOptions } from "@continuedev/config-types";
 import * as followRedirects from "follow-redirects";
 import { HttpProxyAgent } from "http-proxy-agent";
 import { HttpsProxyAgent } from "https-proxy-agent";
 import fetch, { RequestInit, Response } from "node-fetch";
+import { getAgentOptions } from "./getAgentOptions.js";
+import { getProxyFromEnv, shouldBypassProxy } from "./util.js";
 
 const { http, https } = (followRedirects as any).default;
 
@@ -20,59 +18,23 @@ export async function fetchwithRequestOptions(
     url.host = "127.0.0.1";
   }
 
-  const TIMEOUT = 7200; // 7200 seconds = 2 hours
+  const agentOptions = getAgentOptions(requestOptions);
 
-  let globalCerts: string[] = [];
-  if (process.env.IS_BINARY) {
-    if (Array.isArray(globalAgent.options.ca)) {
-      globalCerts = [...globalAgent.options.ca.map((cert) => cert.toString())];
-    } else if (typeof globalAgent.options.ca !== "undefined") {
-      globalCerts.push(globalAgent.options.ca.toString());
-    }
-  }
-  const ca = Array.from(new Set([...tls.rootCertificates, ...globalCerts]));
-  const customCerts =
-    typeof requestOptions?.caBundlePath === "string"
-      ? [requestOptions?.caBundlePath]
-      : requestOptions?.caBundlePath;
-  if (customCerts) {
-    ca.push(
-      ...customCerts.map((customCert) => fs.readFileSync(customCert, "utf8")),
-    );
+  // Get proxy from options or environment variables
+  let proxy = requestOptions?.proxy;
+  if (!proxy) {
+    proxy = getProxyFromEnv(url.protocol);
   }
 
-  const timeout = (requestOptions?.timeout ?? TIMEOUT) * 1000; // measured in ms
-
-  const agentOptions: { [key: string]: any } = {
-    ca,
-    rejectUnauthorized: requestOptions?.verifySsl,
-    timeout,
-    sessionTimeout: timeout,
-    keepAlive: true,
-    keepAliveMsecs: timeout,
-  };
-
-  // Handle ClientCertificateOptions
-  if (requestOptions?.clientCertificate) {
-    agentOptions.cert = fs.readFileSync(
-      requestOptions.clientCertificate.cert,
-      "utf8",
-    );
-    agentOptions.key = fs.readFileSync(
-      requestOptions.clientCertificate.key,
-      "utf8",
-    );
-    if (requestOptions.clientCertificate.passphrase) {
-      agentOptions.passphrase = requestOptions.clientCertificate.passphrase;
-    }
-  }
-
-  const proxy = requestOptions?.proxy;
+  // Check if should bypass proxy based on requestOptions or NO_PROXY env var
+  const shouldBypass =
+    requestOptions?.noProxy?.includes(url.hostname) ||
+    shouldBypassProxy(url.hostname);
 
   // Create agent
   const protocol = url.protocol === "https:" ? https : http;
   const agent =
-    proxy && !requestOptions?.noProxy?.includes(url.hostname)
+    proxy && !shouldBypass
       ? protocol === https
         ? new HttpsProxyAgent(proxy, agentOptions)
         : new HttpProxyAgent(proxy, agentOptions)
@@ -107,19 +69,30 @@ export async function fetchwithRequestOptions(
   }
 
   // fetch the request with the provided options
-  const resp = await fetch(url, {
-    ...init,
-    body: updatedBody ?? init?.body,
-    headers: headers,
-    agent: agent,
-  });
+  try {
+    const resp = await fetch(url, {
+      ...init,
+      body: updatedBody ?? init?.body,
+      headers: headers,
+      agent: agent,
+    });
+
+    if (!resp.ok) {
+      const requestId = resp.headers.get("x-request-id");
+      if (requestId) {
+        console.log(`Request ID: ${requestId}, Status: ${resp.status}`);
+      }
+    }
 
-  if (!resp.ok) {
-    const requestId = resp.headers.get("x-request-id");
-    if (requestId) {
-      console.log(`Request ID: ${requestId}, Status: ${resp.status}`);
+    return resp;
+  } catch (error) {
+    if (error instanceof Error && error.name === "AbortError") {
+      // Return a Response object that streamResponse etc can handle
+      return new Response(null, {
+        status: 499, // Client Closed Request
+        statusText: "Client Closed Request",
+      });
     }
+    throw error;
   }
-
-  return resp;
 }

package/src/getAgentOptions.ts

@@ -0,0 +1,62 @@
+import { globalAgent } from "https";
+import * as fs from "node:fs";
+import tls from "node:tls";
+
+import { RequestOptions } from "@continuedev/config-types";
+
+/**
+ * Prepares agent options based on request options and certificates
+ */
+export function getAgentOptions(requestOptions?: RequestOptions): {
+  [key: string]: any;
+} {
+  const TIMEOUT = 7200; // 7200 seconds = 2 hours
+  const timeout = (requestOptions?.timeout ?? TIMEOUT) * 1000; // measured in ms
+
+  // Get root certificates
+  let globalCerts: string[] = [];
+  if (process.env.IS_BINARY) {
+    if (Array.isArray(globalAgent.options.ca)) {
+      globalCerts = [...globalAgent.options.ca.map((cert) => cert.toString())];
+    } else if (typeof globalAgent.options.ca !== "undefined") {
+      globalCerts.push(globalAgent.options.ca.toString());
+    }
+  }
+
+  const ca = Array.from(new Set([...tls.rootCertificates, ...globalCerts]));
+  const customCerts =
+    typeof requestOptions?.caBundlePath === "string"
+      ? [requestOptions?.caBundlePath]
+      : requestOptions?.caBundlePath;
+  if (customCerts) {
+    ca.push(
+      ...customCerts.map((customCert) => fs.readFileSync(customCert, "utf8")),
+    );
+  }
+
+  const agentOptions: { [key: string]: any } = {
+    ca,
+    rejectUnauthorized: requestOptions?.verifySsl,
+    timeout,
+    sessionTimeout: timeout,
+    keepAlive: true,
+    keepAliveMsecs: timeout,
+  };
+
+  // Handle ClientCertificateOptions
+  if (requestOptions?.clientCertificate) {
+    agentOptions.cert = fs.readFileSync(
+      requestOptions.clientCertificate.cert,
+      "utf8",
+    );
+    agentOptions.key = fs.readFileSync(
+      requestOptions.clientCertificate.key,
+      "utf8",
+    );
+    if (requestOptions.clientCertificate.passphrase) {
+      agentOptions.passphrase = requestOptions.clientCertificate.passphrase;
+    }
+  }
+
+  return agentOptions;
+}

package/src/stream.test.ts

@@ -0,0 +1,109 @@
+import { Readable } from "stream";
+import { parseDataLine, streamSse } from "./stream.js";
+
+function createMockResponse(sseLines: string[]): Response {
+  // Create a Readable stream that emits the SSE lines
+  const stream = new Readable({
+    read() {
+      for (const line of sseLines) {
+        this.push(line + "\n\n");
+      }
+      this.push(null); // End of stream
+    },
+  }) as any;
+
+  // Minimal Response mock
+  return {
+    status: 200,
+    body: stream,
+    text: async () => "",
+  } as unknown as Response;
+}
+
+describe("streamSse", () => {
+  it("yields parsed SSE data objects that ends with `data:[DONE]`", async () => {
+    const sseLines = [
+      'data: {"foo": "bar"}',
+      'data: {"baz": 42}',
+      "data:[DONE]",
+    ];
+    const response = createMockResponse(sseLines);
+
+    const results = [];
+    for await (const data of streamSse(response)) {
+      results.push(data);
+    }
+
+    expect(results).toEqual([{ foo: "bar" }, { baz: 42 }]);
+  });
+
+  it("yields parsed SSE data objects that ends with `data: [DONE]` (with a space before [DONE]", async () => {
+    const sseLines = [
+      'data: {"foo": "bar"}',
+      'data: {"baz": 42}',
+      "data: [DONE]",
+    ];
+    const response = createMockResponse(sseLines);
+
+    const results = [];
+    for await (const data of streamSse(response)) {
+      results.push(data);
+    }
+
+    expect(results).toEqual([{ foo: "bar" }, { baz: 42 }]);
+  });
+
+  it("throws on malformed JSON", async () => {
+    const sseLines = ['data: {"foo": "bar"', "data:[DONE]"];
+    const response = createMockResponse(sseLines);
+
+    const iterator = streamSse(response)[Symbol.asyncIterator]();
+    await expect(iterator.next()).rejects.toThrow(/Malformed JSON/);
+  });
+});
+
+describe("parseDataLine", () => {
+  test("parseDataLine should parse valid JSON data with 'data: ' prefix", () => {
+    const line = 'data: {"message":"hello","status":"ok"}';
+    const result = parseDataLine(line);
+    expect(result).toEqual({ message: "hello", status: "ok" });
+  });
+
+  test("parseDataLine should parse valid JSON data with 'data:' prefix (no space)", () => {
+    const line = 'data:{"message":"hello","status":"ok"}';
+    const result = parseDataLine(line);
+    expect(result).toEqual({ message: "hello", status: "ok" });
+  });
+
+  test("parseDataLine should throw error for malformed JSON", () => {
+    const line = "data: {invalid json}";
+    expect(() => parseDataLine(line)).toThrow(
+      "Malformed JSON sent from server",
+    );
+  });
+
+  test("parseDataLine should throw error when data contains error field", () => {
+    const line = 'data: {"error":"something went wrong"}';
+    expect(() => parseDataLine(line)).toThrow(
+      "Error streaming response: something went wrong",
+    );
+  });
+
+  test("parseDataLine should handle empty objects", () => {
+    const line = "data: {}";
+    const result = parseDataLine(line);
+    expect(result).toEqual({});
+  });
+
+  test("parseDataLine should handle arrays", () => {
+    const line = "data: [1,2,3]";
+    const result = parseDataLine(line);
+    expect(result).toEqual([1, 2, 3]);
+  });
+
+  test("parseDataLine should handle nested objects", () => {
+    const line = 'data: {"user":{"name":"John","age":30}}';
+    const result = parseDataLine(line);
+    expect(result).toEqual({ user: { name: "John", age: 30 } });
+  });
+});

package/src/stream.ts

@@ -2,7 +2,7 @@ export async function* toAsyncIterable(
   nodeReadable: NodeJS.ReadableStream,
 ): AsyncGenerator<Uint8Array> {
   for await (const chunk of nodeReadable) {
-    // @ts-expect-error
+    // @ts-ignore
     yield chunk as Uint8Array;
   }
 }
@@ -10,6 +10,10 @@ export async function* toAsyncIterable(
 export async function* streamResponse(
   response: Response,
 ): AsyncGenerator<string> {
+  if (response.status === 499) {
+    return; // In case of client-side cancellation, just return
+  }
+
   if (response.status !== 200) {
     throw new Error(await response.text());
   }
@@ -21,26 +25,34 @@ export async function* streamResponse(
   // Get the major version of Node.js
   const nodeMajorVersion = parseInt(process.versions.node.split(".")[0], 10);
 
-  if (nodeMajorVersion >= 20) {
-    // Use the new API for Node 20 and above
-    const stream = (ReadableStream as any).from(response.body);
-    for await (const chunk of stream.pipeThrough(
-      new TextDecoderStream("utf-8"),
-    )) {
-      yield chunk;
+  try {
+    if (nodeMajorVersion >= 20) {
+      // Use the new API for Node 20 and above
+      const stream = (ReadableStream as any).from(response.body);
+      for await (const chunk of stream.pipeThrough(
+        new TextDecoderStream("utf-8"),
+      )) {
+        yield chunk;
+      }
+    } else {
+      // Fallback for Node versions below 20
+      // Streaming with this method doesn't work as version 20+ does
+      const decoder = new TextDecoder("utf-8");
+      const nodeStream = response.body as unknown as NodeJS.ReadableStream;
+      for await (const chunk of toAsyncIterable(nodeStream)) {
+        yield decoder.decode(chunk, { stream: true });
+      }
     }
-  } else {
-    // Fallback for Node versions below 20
-    // Streaming with this method doesn't work as version 20+ does
-    const decoder = new TextDecoder("utf-8");
-    const nodeStream = response.body as unknown as NodeJS.ReadableStream;
-    for await (const chunk of toAsyncIterable(nodeStream)) {
-      yield decoder.decode(chunk, { stream: true });
+  } catch (e) {
+    if (e instanceof Error && e.name.startsWith("AbortError")) {
+      return; // In case of client-side cancellation, just return
     }
+    throw e;
   }
 }
 
-function parseDataLine(line: string): any {
+// Export for testing purposes
+export function parseDataLine(line: string): any {
   const json = line.startsWith("data: ")
     ? line.slice("data: ".length)
     : line.slice("data:".length);
@@ -53,12 +65,20 @@ function parseDataLine(line: string): any {
 
     return data;
   } catch (e) {
+    // If the error was thrown by our error check, rethrow it
+    if (
+      e instanceof Error &&
+      e.message.startsWith("Error streaming response:")
+    ) {
+      throw e;
+    }
+    // Otherwise it's a JSON parsing error
     throw new Error(`Malformed JSON sent from server: ${json}`);
   }
 }
 
 function parseSseLine(line: string): { done: boolean; data: any } {
-  if (line.startsWith("data: [DONE]")) {
+  if (line.startsWith("data:[DONE]") || line.startsWith("data: [DONE]")) {
     return { done: true, data: undefined };
   }
   if (line.startsWith("data:")) {