@continuedev/fetch 1.0.6 → 1.0.9

package/src/fetch.test.ts

@@ -0,0 +1,172 @@
+import { globalAgent } from "https";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { getAgentOptions } from "./getAgentOptions.js";
+
+// Store original env
+const originalEnv = process.env;
+const originalGlobalAgentOptions = { ...globalAgent.options };
+
+// Temporary directory for test certificate files
+let tempDir: string;
+
+beforeEach(() => {
+  // Create a temporary directory for test files
+  tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "fetch-test-"));
+
+  process.env = { ...originalEnv };
+
+  // Reset globalAgent for each test
+  globalAgent.options = { ...originalGlobalAgentOptions };
+});
+
+afterEach(() => {
+  process.env = originalEnv;
+  globalAgent.options = originalGlobalAgentOptions;
+
+  // Clean up temporary directory
+  try {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  } catch (error) {
+    console.error(`Failed to remove temp directory: ${error}`);
+  }
+});
+
+// Helper function to create test certificate files
+function createTestCertFile(filename: string, content: string): string {
+  const filePath = path.join(tempDir, filename);
+  fs.writeFileSync(filePath, content, "utf8");
+  return filePath;
+}
+
+test("getAgentOptions returns basic configuration with default values", () => {
+  const options = getAgentOptions();
+
+  // Check default timeout (7200 seconds = 2 hours = 7,200,000 ms)
+  expect(options.timeout).toBe(7200000);
+  expect(options.sessionTimeout).toBe(7200000);
+  expect(options.keepAliveMsecs).toBe(7200000);
+  expect(options.keepAlive).toBe(true);
+
+  // Verify certificates array exists and contains items
+  expect(options.ca).toBeInstanceOf(Array);
+  expect(options.ca.length).toBeGreaterThan(0);
+
+  // Verify at least one of the real TLS root certificates is included
+  // This assumes there's at least one certificate with "CERTIFICATE" in it
+  expect(options.ca.some((cert: any) => cert.includes("CERTIFICATE"))).toBe(
+    true,
+  );
+});
+
+test("getAgentOptions respects custom timeout", () => {
+  const customTimeout = 300; // 5 minutes
+  const options = getAgentOptions({ timeout: customTimeout });
+
+  // Check timeout values (300 seconds = 300,000 ms)
+  expect(options.timeout).toBe(300000);
+  expect(options.sessionTimeout).toBe(300000);
+  expect(options.keepAliveMsecs).toBe(300000);
+});
+
+test("getAgentOptions uses verifySsl setting", () => {
+  // With verifySsl true
+  let options = getAgentOptions({ verifySsl: true });
+  expect(options.rejectUnauthorized).toBe(true);
+
+  // With verifySsl false
+  options = getAgentOptions({ verifySsl: false });
+  expect(options.rejectUnauthorized).toBe(false);
+});
+
+test("getAgentOptions incorporates custom CA bundle paths", () => {
+  // Create a test CA bundle file
+  const caBundleContent =
+    "-----BEGIN CERTIFICATE-----\nMIIDtTCCAp2gAwIBAgIJAMcuSp7chAYdMA==\n-----END CERTIFICATE-----";
+  const caBundlePath = createTestCertFile("ca-bundle.pem", caBundleContent);
+
+  // Single string path
+  let options = getAgentOptions({ caBundlePath });
+
+  // Verify that our test certificate is included in the CA list
+  expect(options.ca).toContain(caBundleContent);
+
+  // Create multiple test CA bundle files
+  const caContent1 =
+    "-----BEGIN CERTIFICATE-----\nABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\n-----END CERTIFICATE-----";
+  const caContent2 =
+    "-----BEGIN CERTIFICATE-----\n0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\n-----END CERTIFICATE-----";
+  const caPath1 = createTestCertFile("ca1.pem", caContent1);
+  const caPath2 = createTestCertFile("ca2.pem", caContent2);
+
+  // Array of paths
+  options = getAgentOptions({
+    caBundlePath: [caPath1, caPath2],
+  });
+
+  // Verify that both test certificates are included in the CA list
+  expect(options.ca).toContain(caContent1);
+  expect(options.ca).toContain(caContent2);
+});
+
+test("getAgentOptions includes global certs when running as binary", () => {
+  // Set up test certs in globalAgent
+  globalAgent.options.ca = ["global-cert-1", "global-cert-2"];
+
+  // Set IS_BINARY environment variable
+  process.env.IS_BINARY = "true";
+
+  const options = getAgentOptions();
+
+  // Test for global certs
+  expect(options.ca).toContain("global-cert-1");
+  expect(options.ca).toContain("global-cert-2");
+});
+
+test("getAgentOptions handles client certificate configuration", () => {
+  // Create test certificate files
+  const clientCertContent =
+    "-----BEGIN CERTIFICATE-----\nCLIENTCERT\n-----END CERTIFICATE-----";
+  const clientKeyContent =
+    "-----BEGIN PRIVATE KEY-----\nCLIENTKEY\n-----END PRIVATE KEY-----";
+  const certPath = createTestCertFile("client.cert", clientCertContent);
+  const keyPath = createTestCertFile("client.key", clientKeyContent);
+
+  const clientCertOptions = {
+    clientCertificate: {
+      cert: certPath,
+      key: keyPath,
+      passphrase: "secret-passphrase",
+    },
+  };
+
+  const options = getAgentOptions(clientCertOptions);
+
+  expect(options.cert).toBe(clientCertContent);
+  expect(options.key).toBe(clientKeyContent);
+  expect(options.passphrase).toBe("secret-passphrase");
+});
+
+test("getAgentOptions handles client certificate without passphrase", () => {
+  // Create test certificate files
+  const clientCertContent =
+    "-----BEGIN CERTIFICATE-----\nCLIENTCERT2\n-----END CERTIFICATE-----";
+  const clientKeyContent =
+    "-----BEGIN PRIVATE KEY-----\nCLIENTKEY2\n-----END PRIVATE KEY-----";
+  const certPath = createTestCertFile("client2.cert", clientCertContent);
+  const keyPath = createTestCertFile("client2.key", clientKeyContent);
+
+  const clientCertOptions = {
+    clientCertificate: {
+      cert: certPath,
+      key: keyPath,
+    },
+  };
+
+  const options = getAgentOptions(clientCertOptions);
+
+  expect(options.cert).toBe(clientCertContent);
+  expect(options.key).toBe(clientKeyContent);
+  expect(options.passphrase).toBeUndefined();
+});

package/src/fetch.ts

@@ -1,12 +1,10 @@
-import { globalAgent } from "https";
-import * as fs from "node:fs";
-import tls from "node:tls";
-
 import { RequestOptions } from "@continuedev/config-types";
 import * as followRedirects from "follow-redirects";
 import { HttpProxyAgent } from "http-proxy-agent";
 import { HttpsProxyAgent } from "https-proxy-agent";
 import fetch, { RequestInit, Response } from "node-fetch";
+import { getAgentOptions } from "./getAgentOptions.js";
+import { getProxyFromEnv, shouldBypassProxy } from "./util.js";
 
 const { http, https } = (followRedirects as any).default;
 
@@ -20,59 +18,23 @@ export async function fetchwithRequestOptions(
     url.host = "127.0.0.1";
   }
 
-  const TIMEOUT = 7200; // 7200 seconds = 2 hours
+  const agentOptions = getAgentOptions(requestOptions);
 
-  let globalCerts: string[] = [];
-  if (process.env.IS_BINARY) {
-    if (Array.isArray(globalAgent.options.ca)) {
-      globalCerts = [...globalAgent.options.ca.map((cert) => cert.toString())];
-    } else if (typeof globalAgent.options.ca !== "undefined") {
-      globalCerts.push(globalAgent.options.ca.toString());
-    }
-  }
-  const ca = Array.from(new Set([...tls.rootCertificates, ...globalCerts]));
-  const customCerts =
-    typeof requestOptions?.caBundlePath === "string"
-      ? [requestOptions?.caBundlePath]
-      : requestOptions?.caBundlePath;
-  if (customCerts) {
-    ca.push(
-      ...customCerts.map((customCert) => fs.readFileSync(customCert, "utf8")),
-    );
+  // Get proxy from options or environment variables
+  let proxy = requestOptions?.proxy;
+  if (!proxy) {
+    proxy = getProxyFromEnv(url.protocol);
   }
 
-  const timeout = (requestOptions?.timeout ?? TIMEOUT) * 1000; // measured in ms
-
-  const agentOptions: { [key: string]: any } = {
-    ca,
-    rejectUnauthorized: requestOptions?.verifySsl,
-    timeout,
-    sessionTimeout: timeout,
-    keepAlive: true,
-    keepAliveMsecs: timeout,
-  };
-
-  // Handle ClientCertificateOptions
-  if (requestOptions?.clientCertificate) {
-    agentOptions.cert = fs.readFileSync(
-      requestOptions.clientCertificate.cert,
-      "utf8",
-    );
-    agentOptions.key = fs.readFileSync(
-      requestOptions.clientCertificate.key,
-      "utf8",
-    );
-    if (requestOptions.clientCertificate.passphrase) {
-      agentOptions.passphrase = requestOptions.clientCertificate.passphrase;
-    }
-  }
-
-  const proxy = requestOptions?.proxy;
+  // Check if should bypass proxy based on requestOptions or NO_PROXY env var
+  const shouldBypass =
+    requestOptions?.noProxy?.includes(url.hostname) ||
+    shouldBypassProxy(url.hostname);
 
   // Create agent
   const protocol = url.protocol === "https:" ? https : http;
   const agent =
-    proxy && !requestOptions?.noProxy?.includes(url.hostname)
+    proxy && !shouldBypass
       ? protocol === https
         ? new HttpsProxyAgent(proxy, agentOptions)
         : new HttpProxyAgent(proxy, agentOptions)
@@ -107,19 +69,30 @@ export async function fetchwithRequestOptions(
   }
 
   // fetch the request with the provided options
-  const resp = await fetch(url, {
-    ...init,
-    body: updatedBody ?? init?.body,
-    headers: headers,
-    agent: agent,
-  });
+  try {
+    const resp = await fetch(url, {
+      ...init,
+      body: updatedBody ?? init?.body,
+      headers: headers,
+      agent: agent,
+    });
+
+    if (!resp.ok) {
+      const requestId = resp.headers.get("x-request-id");
+      if (requestId) {
+        console.log(`Request ID: ${requestId}, Status: ${resp.status}`);
+      }
+    }
 
-  if (!resp.ok) {
-    const requestId = resp.headers.get("x-request-id");
-    if (requestId) {
-      console.log(`Request ID: ${requestId}, Status: ${resp.status}`);
+    return resp;
+  } catch (error) {
+    if (error instanceof Error && error.name === "AbortError") {
+      // Return a Response object that streamResponse etc can handle
+      return new Response(null, {
+        status: 499, // Client Closed Request
+        statusText: "Client Closed Request",
+      });
     }
+    throw error;
   }
-
-  return resp;
 }

package/src/getAgentOptions.ts

@@ -0,0 +1,62 @@
+import { globalAgent } from "https";
+import * as fs from "node:fs";
+import tls from "node:tls";
+
+import { RequestOptions } from "@continuedev/config-types";
+
+/**
+ * Prepares agent options based on request options and certificates
+ */
+export function getAgentOptions(requestOptions?: RequestOptions): {
+  [key: string]: any;
+} {
+  const TIMEOUT = 7200; // 7200 seconds = 2 hours
+  const timeout = (requestOptions?.timeout ?? TIMEOUT) * 1000; // measured in ms
+
+  // Get root certificates
+  let globalCerts: string[] = [];
+  if (process.env.IS_BINARY) {
+    if (Array.isArray(globalAgent.options.ca)) {
+      globalCerts = [...globalAgent.options.ca.map((cert) => cert.toString())];
+    } else if (typeof globalAgent.options.ca !== "undefined") {
+      globalCerts.push(globalAgent.options.ca.toString());
+    }
+  }
+
+  const ca = Array.from(new Set([...tls.rootCertificates, ...globalCerts]));
+  const customCerts =
+    typeof requestOptions?.caBundlePath === "string"
+      ? [requestOptions?.caBundlePath]
+      : requestOptions?.caBundlePath;
+  if (customCerts) {
+    ca.push(
+      ...customCerts.map((customCert) => fs.readFileSync(customCert, "utf8")),
+    );
+  }
+
+  const agentOptions: { [key: string]: any } = {
+    ca,
+    rejectUnauthorized: requestOptions?.verifySsl,
+    timeout,
+    sessionTimeout: timeout,
+    keepAlive: true,
+    keepAliveMsecs: timeout,
+  };
+
+  // Handle ClientCertificateOptions
+  if (requestOptions?.clientCertificate) {
+    agentOptions.cert = fs.readFileSync(
+      requestOptions.clientCertificate.cert,
+      "utf8",
+    );
+    agentOptions.key = fs.readFileSync(
+      requestOptions.clientCertificate.key,
+      "utf8",
+    );
+    if (requestOptions.clientCertificate.passphrase) {
+      agentOptions.passphrase = requestOptions.clientCertificate.passphrase;
+    }
+  }
+
+  return agentOptions;
+}

package/src/stream.test.ts

@@ -0,0 +1,49 @@
+// The parseDataLine function is not exported, so we need to import it
+// indirectly by re-exporting it for testing purposes
+import { parseDataLine } from "./stream.js";
+
+describe("parseDataLine", () => {
+  test("parseDataLine should parse valid JSON data with 'data: ' prefix", () => {
+    const line = 'data: {"message":"hello","status":"ok"}';
+    const result = parseDataLine(line);
+    expect(result).toEqual({ message: "hello", status: "ok" });
+  });
+
+  test("parseDataLine should parse valid JSON data with 'data:' prefix (no space)", () => {
+    const line = 'data:{"message":"hello","status":"ok"}';
+    const result = parseDataLine(line);
+    expect(result).toEqual({ message: "hello", status: "ok" });
+  });
+
+  test("parseDataLine should throw error for malformed JSON", () => {
+    const line = "data: {invalid json}";
+    expect(() => parseDataLine(line)).toThrow(
+      "Malformed JSON sent from server",
+    );
+  });
+
+  test("parseDataLine should throw error when data contains error field", () => {
+    const line = 'data: {"error":"something went wrong"}';
+    expect(() => parseDataLine(line)).toThrow(
+      "Error streaming response: something went wrong",
+    );
+  });
+
+  test("parseDataLine should handle empty objects", () => {
+    const line = "data: {}";
+    const result = parseDataLine(line);
+    expect(result).toEqual({});
+  });
+
+  test("parseDataLine should handle arrays", () => {
+    const line = "data: [1,2,3]";
+    const result = parseDataLine(line);
+    expect(result).toEqual([1, 2, 3]);
+  });
+
+  test("parseDataLine should handle nested objects", () => {
+    const line = 'data: {"user":{"name":"John","age":30}}';
+    const result = parseDataLine(line);
+    expect(result).toEqual({ user: { name: "John", age: 30 } });
+  });
+});

package/src/stream.ts

@@ -10,6 +10,9 @@ export async function* toAsyncIterable(
 export async function* streamResponse(
   response: Response,
 ): AsyncGenerator<string> {
+  if (response.status === 499) {
+    return; // In case of client-side cancellation, just return
+  }
   if (response.status !== 200) {
     throw new Error(await response.text());
   }
@@ -21,26 +24,34 @@ export async function* streamResponse(
   // Get the major version of Node.js
   const nodeMajorVersion = parseInt(process.versions.node.split(".")[0], 10);
 
-  if (nodeMajorVersion >= 20) {
-    // Use the new API for Node 20 and above
-    const stream = (ReadableStream as any).from(response.body);
-    for await (const chunk of stream.pipeThrough(
-      new TextDecoderStream("utf-8"),
-    )) {
-      yield chunk;
+  try {
+    if (nodeMajorVersion >= 20) {
+      // Use the new API for Node 20 and above
+      const stream = (ReadableStream as any).from(response.body);
+      for await (const chunk of stream.pipeThrough(
+        new TextDecoderStream("utf-8"),
+      )) {
+        yield chunk;
+      }
+    } else {
+      // Fallback for Node versions below 20
+      // Streaming with this method doesn't work as version 20+ does
+      const decoder = new TextDecoder("utf-8");
+      const nodeStream = response.body as unknown as NodeJS.ReadableStream;
+      for await (const chunk of toAsyncIterable(nodeStream)) {
+        yield decoder.decode(chunk, { stream: true });
+      }
     }
-  } else {
-    // Fallback for Node versions below 20
-    // Streaming with this method doesn't work as version 20+ does
-    const decoder = new TextDecoder("utf-8");
-    const nodeStream = response.body as unknown as NodeJS.ReadableStream;
-    for await (const chunk of toAsyncIterable(nodeStream)) {
-      yield decoder.decode(chunk, { stream: true });
+  } catch (e) {
+    if (e instanceof Error && e.name.startsWith("AbortError")) {
+      return; // In case of client-side cancellation, just return
     }
+    throw e;
   }
 }
 
-function parseDataLine(line: string): any {
+// Export for testing purposes
+export function parseDataLine(line: string): any {
   const json = line.startsWith("data: ")
     ? line.slice("data: ".length)
     : line.slice("data:".length);
@@ -53,6 +64,14 @@ function parseDataLine(line: string): any {
 
     return data;
   } catch (e) {
+    // If the error was thrown by our error check, rethrow it
+    if (
+      e instanceof Error &&
+      e.message.startsWith("Error streaming response:")
+    ) {
+      throw e;
+    }
+    // Otherwise it's a JSON parsing error
     throw new Error(`Malformed JSON sent from server: ${json}`);
   }
 }

package/src/util.test.ts

@@ -0,0 +1,102 @@
+import { jest } from "@jest/globals";
+import { getProxyFromEnv, shouldBypassProxy } from "./util.js";
+
+// Reset environment variables after each test
+afterEach(() => {
+  jest.resetModules();
+  process.env = {};
+});
+
+// Tests for getProxyFromEnv
+test("getProxyFromEnv returns undefined when no proxy is set", () => {
+  expect(getProxyFromEnv("http:")).toBeUndefined();
+  expect(getProxyFromEnv("https:")).toBeUndefined();
+});
+
+test("getProxyFromEnv returns HTTP_PROXY for http protocol", () => {
+  process.env.HTTP_PROXY = "http://proxy.example.com";
+  expect(getProxyFromEnv("http:")).toBe("http://proxy.example.com");
+});
+
+test("getProxyFromEnv returns lowercase http_proxy for http protocol", () => {
+  process.env.http_proxy = "http://proxy.example.com";
+  expect(getProxyFromEnv("http:")).toBe("http://proxy.example.com");
+});
+
+test("getProxyFromEnv prefers HTTP_PROXY over http_proxy for http protocol", () => {
+  process.env.HTTP_PROXY = "http://upper.example.com";
+  process.env.http_proxy = "http://lower.example.com";
+  expect(getProxyFromEnv("http:")).toBe("http://upper.example.com");
+});
+
+test("getProxyFromEnv returns HTTPS_PROXY for https protocol", () => {
+  process.env.HTTPS_PROXY = "https://secure.example.com";
+  expect(getProxyFromEnv("https:")).toBe("https://secure.example.com");
+});
+
+test("getProxyFromEnv returns lowercase https_proxy for https protocol", () => {
+  process.env.https_proxy = "https://secure.example.com";
+  expect(getProxyFromEnv("https:")).toBe("https://secure.example.com");
+});
+
+test("getProxyFromEnv falls back to HTTP_PROXY for https protocol when HTTPS_PROXY is not set", () => {
+  process.env.HTTP_PROXY = "http://fallback.example.com";
+  expect(getProxyFromEnv("https:")).toBe("http://fallback.example.com");
+});
+
+test("getProxyFromEnv prefers HTTPS_PROXY over other env vars for https protocol", () => {
+  process.env.HTTPS_PROXY = "https://preferred.example.com";
+  process.env.https_proxy = "https://notused1.example.com";
+  process.env.HTTP_PROXY = "http://notused2.example.com";
+  process.env.http_proxy = "http://notused3.example.com";
+  expect(getProxyFromEnv("https:")).toBe("https://preferred.example.com");
+});
+
+// Tests for shouldBypassProxy
+test("shouldBypassProxy returns false when NO_PROXY is not set", () => {
+  expect(shouldBypassProxy("example.com")).toBe(false);
+});
+
+test("shouldBypassProxy returns true for exact hostname match", () => {
+  process.env.NO_PROXY = "example.com,another.com";
+  expect(shouldBypassProxy("example.com")).toBe(true);
+});
+
+test("shouldBypassProxy returns false when hostname doesn't match any NO_PROXY entry", () => {
+  process.env.NO_PROXY = "example.com,another.com";
+  expect(shouldBypassProxy("different.com")).toBe(false);
+});
+
+test("shouldBypassProxy handles lowercase no_proxy", () => {
+  process.env.no_proxy = "example.com";
+  expect(shouldBypassProxy("example.com")).toBe(true);
+});
+
+test("shouldBypassProxy works with wildcard domains", () => {
+  process.env.NO_PROXY = "*.example.com";
+  expect(shouldBypassProxy("sub.example.com")).toBe(true);
+  expect(shouldBypassProxy("example.com")).toBe(false);
+  expect(shouldBypassProxy("different.com")).toBe(false);
+});
+
+test("shouldBypassProxy works with domain suffix", () => {
+  process.env.NO_PROXY = ".example.com";
+  expect(shouldBypassProxy("sub.example.com")).toBe(true);
+  expect(shouldBypassProxy("example.com")).toBe(true);
+  expect(shouldBypassProxy("different.com")).toBe(false);
+});
+
+test("shouldBypassProxy handles multiple entries with different patterns", () => {
+  process.env.NO_PROXY = "internal.local,*.example.com,.test.com";
+  expect(shouldBypassProxy("internal.local")).toBe(true);
+  expect(shouldBypassProxy("sub.example.com")).toBe(true);
+  expect(shouldBypassProxy("sub.test.com")).toBe(true);
+  expect(shouldBypassProxy("test.com")).toBe(true);
+  expect(shouldBypassProxy("example.org")).toBe(false);
+});
+
+test("shouldBypassProxy ignores whitespace in NO_PROXY", () => {
+  process.env.NO_PROXY = " example.com , *.test.org ";
+  expect(shouldBypassProxy("example.com")).toBe(true);
+  expect(shouldBypassProxy("subdomain.test.org")).toBe(true);
+});

package/src/util.ts

@@ -0,0 +1,43 @@
+/**
+ * Gets the proxy settings from environment variables
+ * @param protocol The URL protocol (http: or https:)
+ * @returns The proxy URL if available, otherwise undefined
+ */
+export function getProxyFromEnv(protocol: string): string | undefined {
+  if (protocol === "https:") {
+    return (
+      process.env.HTTPS_PROXY ||
+      process.env.https_proxy ||
+      process.env.HTTP_PROXY ||
+      process.env.http_proxy
+    );
+  } else {
+    return process.env.HTTP_PROXY || process.env.http_proxy;
+  }
+}
+
+/**
+ * Checks if a hostname should bypass proxy based on NO_PROXY environment variable
+ * @param hostname The hostname to check
+ * @returns True if the hostname should bypass proxy
+ */
+export function shouldBypassProxy(hostname: string): boolean {
+  const noProxy = process.env.NO_PROXY || process.env.no_proxy;
+  if (!noProxy) return false;
+
+  const noProxyItems = noProxy.split(",").map((item) => item.trim());
+
+  return noProxyItems.some((item) => {
+    // Exact match
+    if (item === hostname) return true;
+
+    // Wildcard domain match (*.example.com)
+    if (item.startsWith("*.") && hostname.endsWith(item.substring(1)))
+      return true;
+
+    // Domain suffix match (.example.com)
+    if (item.startsWith(".") && hostname.endsWith(item.slice(1))) return true;
+
+    return false;
+  });
+}

package/tsconfig.json

@@ -14,7 +14,7 @@
     "resolveJsonModule": true,
     "isolatedModules": true,
     "noEmitOnError": false,
-    "types": ["node"],
+    "types": ["node", "jest"],
     "outDir": "dist",
     "declaration": true
     // "sourceMap": true

package/jest.config.d.ts

@@ -1,7 +0,0 @@
-/**
- * For a detailed explanation regarding each configuration property, visit:
- * https://jestjs.io/docs/configuration
- */
-import type { Config } from 'jest';
-declare const config: Config;
-export default config;