@continuedev/fetch 1.0.12 → 1.0.14

package/src/getAgentOptions.ts

@@ -1,38 +1,19 @@
-import { globalAgent } from "https";
-import * as fs from "node:fs";
-import tls from "node:tls";
-
 import { RequestOptions } from "@continuedev/config-types";
+import { CertsCache, getCertificateContent } from "./certs.js";
 
 /**
  * Prepares agent options based on request options and certificates
  */
-export function getAgentOptions(requestOptions?: RequestOptions): {
+export async function getAgentOptions(
+  requestOptions?: RequestOptions,
+): Promise<{
   [key: string]: any;
-} {
+}> {
   const TIMEOUT = 7200; // 7200 seconds = 2 hours
   const timeout = (requestOptions?.timeout ?? TIMEOUT) * 1000; // measured in ms
 
-  // Get root certificates
-  let globalCerts: string[] = [];
-  if (process.env.IS_BINARY) {
-    if (Array.isArray(globalAgent.options.ca)) {
-      globalCerts = [...globalAgent.options.ca.map((cert) => cert.toString())];
-    } else if (typeof globalAgent.options.ca !== "undefined") {
-      globalCerts.push(globalAgent.options.ca.toString());
-    }
-  }
-
-  const ca = Array.from(new Set([...tls.rootCertificates, ...globalCerts]));
-  const customCerts =
-    typeof requestOptions?.caBundlePath === "string"
-      ? [requestOptions?.caBundlePath]
-      : requestOptions?.caBundlePath;
-  if (customCerts) {
-    ca.push(
-      ...customCerts.map((customCert) => fs.readFileSync(customCert, "utf8")),
-    );
-  }
+  const certsCache = CertsCache.getInstance();
+  const ca = await certsCache.getCa(requestOptions?.caBundlePath);
 
   const agentOptions: { [key: string]: any } = {
     ca,
@@ -45,18 +26,31 @@ export function getAgentOptions(requestOptions?: RequestOptions): {
 
   // Handle ClientCertificateOptions
   if (requestOptions?.clientCertificate) {
-    agentOptions.cert = fs.readFileSync(
-      requestOptions.clientCertificate.cert,
-      "utf8",
-    );
-    agentOptions.key = fs.readFileSync(
-      requestOptions.clientCertificate.key,
-      "utf8",
-    );
+    const { cert, key, passphrase } = requestOptions.clientCertificate;
+
+    agentOptions.cert = getCertificateContent(cert);
+    agentOptions.key = getCertificateContent(key);
+
     if (requestOptions.clientCertificate.passphrase) {
-      agentOptions.passphrase = requestOptions.clientCertificate.passphrase;
+      agentOptions.passphrase = passphrase;
     }
   }
 
+  if (process.env.VERBOSE_FETCH) {
+    console.log(`Fetch agent options:`);
+    console.log(
+      `\tTimeout (sessionTimeout/keepAliveMsecs): ${agentOptions.timeout}`,
+    );
+    console.log(`\tTotal CA certs: ${ca.length}`);
+    console.log(`\tGlobal/Root CA certs: ${certsCache.fixedCa.length}`);
+    console.log(`\tCustom CA certs: ${ca.length - certsCache.fixedCa.length}`);
+    console.log(
+      `\tClient certificate: ${requestOptions?.clientCertificate ? "Yes" : "No"}`,
+    );
+    console.log(
+      `\trejectUnauthorized/verifySsl: ${agentOptions.rejectUnauthorized ?? "not set (defaults to true)"}`,
+    );
+  }
+
   return agentOptions;
 }

package/src/node-fetch-patch.js

@@ -0,0 +1,518 @@
+/*
+This is a patched version of index.js from node-fetch 3.3.2 (see packages/fetch/node_modules/node-fetch/src/index.js)
+This is to fix "Premature close" errors caused by chunked encoding assumptions - see https://github.com/continuedev/continue/issues/5502
+The patch adjusts boundary chunk detection logic (see "PATCH" comment next to patch)
+And is based on https://github.com/node-fetch/node-fetch/issues/1576
+*/
+
+/**
+ * Index.js
+ *
+ * a request API compatible with window.fetch
+ *
+ * All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/.
+ */
+
+import { Buffer } from "node:buffer";
+import http from "node:http";
+import https from "node:https";
+import Stream, { PassThrough, pipeline as pump } from "node:stream";
+import zlib from "node:zlib";
+
+import dataUriToBuffer from "data-uri-to-buffer";
+
+import {
+  Blob,
+  blobFrom,
+  blobFromSync,
+  File,
+  fileFrom,
+  fileFromSync,
+} from "fetch-blob/from.js";
+import { FormData } from "formdata-polyfill/esm.min.js";
+import { clone, writeToStream } from "node-fetch/src/body.js";
+import { AbortError } from "node-fetch/src/errors/abort-error.js";
+import { FetchError } from "node-fetch/src/errors/fetch-error.js";
+import Headers, { fromRawHeaders } from "node-fetch/src/headers.js";
+import Request, { getNodeRequestOptions } from "node-fetch/src/request.js";
+import Response from "node-fetch/src/response.js";
+import { isRedirect } from "node-fetch/src/utils/is-redirect.js";
+import {
+  isDomainOrSubdomain,
+  isSameProtocol,
+} from "node-fetch/src/utils/is.js";
+import { parseReferrerPolicyFromHeader } from "node-fetch/src/utils/referrer.js";
+
+export {
+  AbortError,
+  Blob,
+  blobFrom,
+  blobFromSync,
+  FetchError,
+  File,
+  fileFrom,
+  fileFromSync,
+  FormData,
+  Headers,
+  isRedirect,
+  Request,
+  Response,
+};
+
+const supportedSchemas = new Set(["data:", "http:", "https:"]);
+
+/**
+ * Fetch function
+ *
+ * @param   {string | URL | import('./request').default} url - Absolute url or Request instance
+ * @param   {*} [options_] - Fetch options
+ * @return  {Promise<import('./response').default>}
+ */
+export default async function fetch(url, options_) {
+  return new Promise((resolve, reject) => {
+    // Build request object
+    const request = new Request(url, options_);
+    const { parsedURL, options } = getNodeRequestOptions(request);
+    if (!supportedSchemas.has(parsedURL.protocol)) {
+      throw new TypeError(
+        `node-fetch cannot load ${url}. URL scheme "${parsedURL.protocol.replace(/:$/, "")}" is not supported.`,
+      );
+    }
+
+    if (parsedURL.protocol === "data:") {
+      const data = dataUriToBuffer(request.url);
+      const response = new Response(data, {
+        headers: { "Content-Type": data.typeFull },
+      });
+      resolve(response);
+      return;
+    }
+
+    // Wrap http.request into fetch
+    const send = (parsedURL.protocol === "https:" ? https : http).request;
+    const { signal } = request;
+    let response = null;
+
+    const abort = () => {
+      const error = new AbortError("The operation was aborted.");
+      reject(error);
+      if (request.body && request.body instanceof Stream.Readable) {
+        request.body.destroy(error);
+      }
+
+      if (!response || !response.body) {
+        return;
+      }
+
+      response.body.emit("error", error);
+    };
+
+    if (signal && signal.aborted) {
+      abort();
+      return;
+    }
+
+    const abortAndFinalize = () => {
+      abort();
+      finalize();
+    };
+
+    // Send request
+    const request_ = send(parsedURL.toString(), options);
+
+    if (signal) {
+      signal.addEventListener("abort", abortAndFinalize);
+    }
+
+    const finalize = () => {
+      request_.abort();
+      if (signal) {
+        signal.removeEventListener("abort", abortAndFinalize);
+      }
+    };
+
+    request_.on("error", (error) => {
+      reject(
+        new FetchError(
+          `request to ${request.url} failed, reason: ${error.message}`,
+          "system",
+          error,
+        ),
+      );
+      finalize();
+    });
+
+    fixResponseChunkedTransferBadEnding(request_, (error) => {
+      if (response && response.body) {
+        response.body.destroy(error);
+      }
+    });
+
+    /* c8 ignore next 18 */
+    if (process.version < "v14") {
+      // Before Node.js 14, pipeline() does not fully support async iterators and does not always
+      // properly handle when the socket close/end events are out of order.
+      request_.on("socket", (s) => {
+        let endedWithEventsCount;
+        s.prependListener("end", () => {
+          endedWithEventsCount = s._eventsCount;
+        });
+        s.prependListener("close", (hadError) => {
+          // if end happened before close but the socket didn't emit an error, do it now
+          if (response && endedWithEventsCount < s._eventsCount && !hadError) {
+            const error = new Error("Premature close");
+            error.code = "ERR_STREAM_PREMATURE_CLOSE";
+            response.body.emit("error", error);
+          }
+        });
+      });
+    }
+
+    request_.on("response", (response_) => {
+      request_.setTimeout(0);
+      const headers = fromRawHeaders(response_.rawHeaders);
+
+      // HTTP fetch step 5
+      if (isRedirect(response_.statusCode)) {
+        // HTTP fetch step 5.2
+        const location = headers.get("Location");
+
+        // HTTP fetch step 5.3
+        let locationURL = null;
+        try {
+          locationURL =
+            location === null ? null : new URL(location, request.url);
+        } catch {
+          // error here can only be invalid URL in Location: header
+          // do not throw when options.redirect == manual
+          // let the user extract the errorneous redirect URL
+          if (request.redirect !== "manual") {
+            reject(
+              new FetchError(
+                `uri requested responds with an invalid redirect URL: ${location}`,
+                "invalid-redirect",
+              ),
+            );
+            finalize();
+            return;
+          }
+        }
+
+        // HTTP fetch step 5.5
+        switch (request.redirect) {
+          case "error":
+            reject(
+              new FetchError(
+                `uri requested responds with a redirect, redirect mode is set to error: ${request.url}`,
+                "no-redirect",
+              ),
+            );
+            finalize();
+            return;
+          case "manual":
+            // Nothing to do
+            break;
+          case "follow": {
+            // HTTP-redirect fetch step 2
+            if (locationURL === null) {
+              break;
+            }
+
+            // HTTP-redirect fetch step 5
+            if (request.counter >= request.follow) {
+              reject(
+                new FetchError(
+                  `maximum redirect reached at: ${request.url}`,
+                  "max-redirect",
+                ),
+              );
+              finalize();
+              return;
+            }
+
+            // HTTP-redirect fetch step 6 (counter increment)
+            // Create a new Request object.
+            const requestOptions = {
+              headers: new Headers(request.headers),
+              follow: request.follow,
+              counter: request.counter + 1,
+              agent: request.agent,
+              compress: request.compress,
+              method: request.method,
+              body: clone(request),
+              signal: request.signal,
+              size: request.size,
+              referrer: request.referrer,
+              referrerPolicy: request.referrerPolicy,
+            };
+
+            // when forwarding sensitive headers like "Authorization",
+            // "WWW-Authenticate", and "Cookie" to untrusted targets,
+            // headers will be ignored when following a redirect to a domain
+            // that is not a subdomain match or exact match of the initial domain.
+            // For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com"
+            // will forward the sensitive headers, but a redirect to "bar.com" will not.
+            // headers will also be ignored when following a redirect to a domain using
+            // a different protocol. For example, a redirect from "https://foo.com" to "http://foo.com"
+            // will not forward the sensitive headers
+            if (
+              !isDomainOrSubdomain(request.url, locationURL) ||
+              !isSameProtocol(request.url, locationURL)
+            ) {
+              for (const name of [
+                "authorization",
+                "www-authenticate",
+                "cookie",
+                "cookie2",
+              ]) {
+                requestOptions.headers.delete(name);
+              }
+            }
+
+            // HTTP-redirect fetch step 9
+            if (
+              response_.statusCode !== 303 &&
+              request.body &&
+              options_.body instanceof Stream.Readable
+            ) {
+              reject(
+                new FetchError(
+                  "Cannot follow redirect with body being a readable stream",
+                  "unsupported-redirect",
+                ),
+              );
+              finalize();
+              return;
+            }
+
+            // HTTP-redirect fetch step 11
+            if (
+              response_.statusCode === 303 ||
+              ((response_.statusCode === 301 || response_.statusCode === 302) &&
+                request.method === "POST")
+            ) {
+              requestOptions.method = "GET";
+              requestOptions.body = undefined;
+              requestOptions.headers.delete("content-length");
+            }
+
+            // HTTP-redirect fetch step 14
+            const responseReferrerPolicy =
+              parseReferrerPolicyFromHeader(headers);
+            if (responseReferrerPolicy) {
+              requestOptions.referrerPolicy = responseReferrerPolicy;
+            }
+
+            // HTTP-redirect fetch step 15
+            resolve(fetch(new Request(locationURL, requestOptions)));
+            finalize();
+            return;
+          }
+
+          default:
+            return reject(
+              new TypeError(
+                `Redirect option '${request.redirect}' is not a valid value of RequestRedirect`,
+              ),
+            );
+        }
+      }
+
+      // Prepare response
+      if (signal) {
+        response_.once("end", () => {
+          signal.removeEventListener("abort", abortAndFinalize);
+        });
+      }
+
+      let body = pump(response_, new PassThrough(), (error) => {
+        if (error) {
+          reject(error);
+        }
+      });
+      // see https://github.com/nodejs/node/pull/29376
+      /* c8 ignore next 3 */
+      if (process.version < "v12.10") {
+        response_.on("aborted", abortAndFinalize);
+      }
+
+      const responseOptions = {
+        url: request.url,
+        status: response_.statusCode,
+        statusText: response_.statusMessage,
+        headers,
+        size: request.size,
+        counter: request.counter,
+        highWaterMark: request.highWaterMark,
+      };
+
+      // HTTP-network fetch step 12.1.1.3
+      const codings = headers.get("Content-Encoding");
+
+      // HTTP-network fetch step 12.1.1.4: handle content codings
+
+      // in following scenarios we ignore compression support
+      // 1. compression support is disabled
+      // 2. HEAD request
+      // 3. no Content-Encoding header
+      // 4. no content response (204)
+      // 5. content not modified response (304)
+      if (
+        !request.compress ||
+        request.method === "HEAD" ||
+        codings === null ||
+        response_.statusCode === 204 ||
+        response_.statusCode === 304
+      ) {
+        response = new Response(body, responseOptions);
+        resolve(response);
+        return;
+      }
+
+      // For Node v6+
+      // Be less strict when decoding compressed responses, since sometimes
+      // servers send slightly invalid responses that are still accepted
+      // by common browsers.
+      // Always using Z_SYNC_FLUSH is what cURL does.
+      const zlibOptions = {
+        flush: zlib.Z_SYNC_FLUSH,
+        finishFlush: zlib.Z_SYNC_FLUSH,
+      };
+
+      // For gzip
+      if (codings === "gzip" || codings === "x-gzip") {
+        body = pump(body, zlib.createGunzip(zlibOptions), (error) => {
+          if (error) {
+            reject(error);
+          }
+        });
+        response = new Response(body, responseOptions);
+        resolve(response);
+        return;
+      }
+
+      // For deflate
+      if (codings === "deflate" || codings === "x-deflate") {
+        // Handle the infamous raw deflate response from old servers
+        // a hack for old IIS and Apache servers
+        const raw = pump(response_, new PassThrough(), (error) => {
+          if (error) {
+            reject(error);
+          }
+        });
+        raw.once("data", (chunk) => {
+          // See http://stackoverflow.com/questions/37519828
+          if ((chunk[0] & 0x0f) === 0x08) {
+            body = pump(body, zlib.createInflate(), (error) => {
+              if (error) {
+                reject(error);
+              }
+            });
+          } else {
+            body = pump(body, zlib.createInflateRaw(), (error) => {
+              if (error) {
+                reject(error);
+              }
+            });
+          }
+
+          response = new Response(body, responseOptions);
+          resolve(response);
+        });
+        raw.once("end", () => {
+          // Some old IIS servers return zero-length OK deflate responses, so
+          // 'data' is never emitted. See https://github.com/node-fetch/node-fetch/pull/903
+          if (!response) {
+            response = new Response(body, responseOptions);
+            resolve(response);
+          }
+        });
+        return;
+      }
+
+      // For br
+      if (codings === "br") {
+        body = pump(body, zlib.createBrotliDecompress(), (error) => {
+          if (error) {
+            reject(error);
+          }
+        });
+        response = new Response(body, responseOptions);
+        resolve(response);
+        return;
+      }
+
+      // Otherwise, use response as-is
+      response = new Response(body, responseOptions);
+      resolve(response);
+    });
+
+    // eslint-disable-next-line promise/prefer-await-to-then
+    writeToStream(request_, request).catch(reject);
+  });
+}
+
+function fixResponseChunkedTransferBadEnding(request, errorCallback) {
+  const LAST_CHUNK = Buffer.from("0\r\n\r\n");
+
+  let isChunkedTransfer = false;
+  let properLastChunkReceived = false;
+  let previousChunk;
+
+  request.on("response", (response) => {
+    const { headers } = response;
+    isChunkedTransfer =
+      headers["transfer-encoding"] === "chunked" && !headers["content-length"];
+  });
+
+  request.on("socket", (socket) => {
+    const onSocketClose = () => {
+      if (isChunkedTransfer && !properLastChunkReceived) {
+        const error = new Error("Premature close");
+        error.code = "ERR_STREAM_PREMATURE_CLOSE";
+        errorCallback(error);
+      }
+    };
+
+    // const onData = buf => {
+    // 	properLastChunkReceived = Buffer.compare(buf.slice(-5), LAST_CHUNK) === 0;
+
+    // 	// Sometimes final 0-length chunk and end of message code are in separate packets
+    // 	if (!properLastChunkReceived && previousChunk) {
+    // 		properLastChunkReceived = (
+    // 			Buffer.compare(previousChunk.slice(-3), LAST_CHUNK.slice(0, 3)) === 0 &&
+    // 			Buffer.compare(buf.slice(-2), LAST_CHUNK.slice(3)) === 0
+    // 		);
+    // 	}
+
+    // 	previousChunk = buf;
+    // };
+
+    // PATCH - THIS IS WHERE THE PATCH IS - FIXES BOUNDARY CHUNKING ISSUE
+    // See https://github.com/node-fetch/node-fetch/issues/1576
+    const onData = (buf) => {
+      properLastChunkReceived = Buffer.compare(buf.slice(-5), LAST_CHUNK) === 0;
+
+      // Sometimes final 0-length chunk and end of message code are in separate packets
+      if (!properLastChunkReceived && previousChunk) {
+        if (buf.length < 5) {
+          properLastChunkReceived =
+            Buffer.compare(
+              Buffer.from([...previousChunk.slice(-5), ...buf]).slice(-5),
+              LAST_CHUNK,
+            ) === 0;
+        }
+      }
+
+      previousChunk = buf;
+    };
+
+    socket.prependListener("close", onSocketClose);
+    socket.on("data", onData);
+
+    request.on("close", () => {
+      socket.removeListener("close", onSocketClose);
+      socket.removeListener("data", onData);
+    });
+  });
+}

package/src/node-fetch-patch.test.js

@@ -0,0 +1,67 @@
+import { Buffer } from "node:buffer";
+import { describe, expect, it } from "vitest";
+
+// Tests the boundary chunking patch to node fetch made in node-fetch-patch.js (see that file for details).
+describe("Chunked Transfer Encoding Patch Logic", () => {
+  // The LAST_CHUNK marker "0\r\n\r\n"
+  const LAST_CHUNK = Buffer.from("0\r\n\r\n");
+
+  it("should detect complete marker at the end (normal case)", () => {
+    const normalCase = Buffer.from("data data data 0\r\n\r\n");
+    const result = Buffer.compare(normalCase.slice(-5), LAST_CHUNK) === 0;
+    expect(result).toBe(true);
+  });
+
+  it("should handle chunks split across original pattern (0\\r\\n | \\r\\n)", () => {
+    const previousChunk = Buffer.from("data data 0\r\n");
+    const lastChunk = Buffer.from("\r\n");
+
+    const result =
+      Buffer.compare(previousChunk.slice(-3), LAST_CHUNK.slice(0, 3)) === 0 &&
+      Buffer.compare(lastChunk.slice(-2), LAST_CHUNK.slice(3)) === 0;
+
+    expect(result).toBe(true);
+  });
+
+  it("should handle problematic case from issue #1576 with new patch", () => {
+    // The problematic case: data ends with "0\r\n0\r" and the next chunk is "\n\r\n"
+    const previousChunk = Buffer.from([
+      ...Array(10).fill(42),
+      48,
+      13,
+      10,
+      48,
+      13,
+    ]); // some data + "0\r\n0\r"
+    const lastChunk = Buffer.from([10, 13, 10]); // "\n\r\n"
+
+    // Check 1: Direct check for complete marker (should fail)
+    const check1 = Buffer.compare(lastChunk.slice(-5), LAST_CHUNK) === 0;
+    expect(check1).toBe(false);
+
+    // Check 2: Original split pattern check (should fail)
+    const check2 =
+      Buffer.compare(previousChunk.slice(-3), LAST_CHUNK.slice(0, 3)) === 0 &&
+      Buffer.compare(lastChunk.slice(-2), LAST_CHUNK.slice(3)) === 0;
+    expect(check2).toBe(false);
+
+    // Check 3: New patch check (should pass)
+    const lastChunkLength = lastChunk.length;
+    const newPatchCheck =
+      lastChunkLength < 5 &&
+      Buffer.compare(
+        Buffer.from([...previousChunk.slice(-5), ...lastChunk]).slice(-5),
+        LAST_CHUNK,
+      ) === 0;
+    expect(newPatchCheck).toBe(true);
+
+    // Verification of the combined buffer
+    const combinedBuffer = Buffer.from([
+      ...previousChunk.slice(-5),
+      ...lastChunk,
+    ]);
+    const combinedResult =
+      Buffer.compare(combinedBuffer.slice(-5), LAST_CHUNK) === 0;
+    expect(combinedResult).toBe(true);
+  });
+});

package/src/node_modules/.vite/vitest/d41d8cd98f00b204e9800998ecf8427e/results.json

@@ -0,0 +1 @@
+{"version":"3.2.0","results":[[":node-fetch-bug.test.ts",{"duration":10.538165999999933,"failed":true}],[":boundary-assumption.test.ts",{"duration":392.1355840000324,"failed":false}],[":chunk-transformer.test.ts",{"duration":0,"failed":true}],[":fetch.test.ts",{"duration":115.30825000000186,"failed":false}],[":getAgentOptions.test.ts",{"duration":6.5474579999936395,"failed":false}],[":stream.test.ts",{"duration":68.50500000000466,"failed":false}],[":premature-close.test.ts",{"duration":5329.249708,"failed":true}],[":util.test.ts",{"duration":3.016583999999966,"failed":false}]]}

package/src/node_modules/.vite/vitest/da39a3ee5e6b4b0d3255bfef95601890afd80709/results.json

@@ -0,0 +1 @@
+{"version":"3.2.1","results":[[":node-fetch-patch.test.ts",{"duration":125.8410000000149,"failed":true}],[":remove/node-fetch-patch.test.ts",{"duration":214.92558300006203,"failed":true}],[":patch-test.test.js",{"duration":1.4479579999999999,"failed":false}],[":premature-close.test.ts",{"duration":5396.810541999992,"failed":true}],[":util.test.ts",{"duration":4.095832999795675,"failed":false}]]}

package/src/stream.test.ts

@@ -1,4 +1,5 @@
 import { Readable } from "stream";
+import { describe, expect, it, test } from "vitest";
 import { parseDataLine, streamSse } from "./stream.js";
 
 function createMockResponse(sseLines: string[]): Response {