@continuedev/fetch 1.0.12 → 1.0.14

package/dist/node-fetch-patch.js

@@ -0,0 +1,395 @@
+/*
+This is a patched version of index.js from node-fetch 3.3.2 (see packages/fetch/node_modules/node-fetch/src/index.js)
+This is to fix "Premature close" errors caused by chunked encoding assumptions - see https://github.com/continuedev/continue/issues/5502
+The patch adjusts boundary chunk detection logic (see "PATCH" comment next to patch)
+And is based on https://github.com/node-fetch/node-fetch/issues/1576
+*/
+/**
+ * Index.js
+ *
+ * a request API compatible with window.fetch
+ *
+ * All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/.
+ */
+import { Buffer } from "node:buffer";
+import http from "node:http";
+import https from "node:https";
+import Stream, { PassThrough, pipeline as pump } from "node:stream";
+import zlib from "node:zlib";
+import dataUriToBuffer from "data-uri-to-buffer";
+import { Blob, blobFrom, blobFromSync, File, fileFrom, fileFromSync, } from "fetch-blob/from.js";
+import { FormData } from "formdata-polyfill/esm.min.js";
+import { clone, writeToStream } from "node-fetch/src/body.js";
+import { AbortError } from "node-fetch/src/errors/abort-error.js";
+import { FetchError } from "node-fetch/src/errors/fetch-error.js";
+import Headers, { fromRawHeaders } from "node-fetch/src/headers.js";
+import Request, { getNodeRequestOptions } from "node-fetch/src/request.js";
+import Response from "node-fetch/src/response.js";
+import { isRedirect } from "node-fetch/src/utils/is-redirect.js";
+import { isDomainOrSubdomain, isSameProtocol, } from "node-fetch/src/utils/is.js";
+import { parseReferrerPolicyFromHeader } from "node-fetch/src/utils/referrer.js";
+export { AbortError, Blob, blobFrom, blobFromSync, FetchError, File, fileFrom, fileFromSync, FormData, Headers, isRedirect, Request, Response, };
+const supportedSchemas = new Set(["data:", "http:", "https:"]);
+/**
+ * Fetch function
+ *
+ * @param   {string | URL | import('./request').default} url - Absolute url or Request instance
+ * @param   {*} [options_] - Fetch options
+ * @return  {Promise<import('./response').default>}
+ */
+export default async function fetch(url, options_) {
+    return new Promise((resolve, reject) => {
+        // Build request object
+        const request = new Request(url, options_);
+        const { parsedURL, options } = getNodeRequestOptions(request);
+        if (!supportedSchemas.has(parsedURL.protocol)) {
+            throw new TypeError(`node-fetch cannot load ${url}. URL scheme "${parsedURL.protocol.replace(/:$/, "")}" is not supported.`);
+        }
+        if (parsedURL.protocol === "data:") {
+            const data = dataUriToBuffer(request.url);
+            const response = new Response(data, {
+                headers: { "Content-Type": data.typeFull },
+            });
+            resolve(response);
+            return;
+        }
+        // Wrap http.request into fetch
+        const send = (parsedURL.protocol === "https:" ? https : http).request;
+        const { signal } = request;
+        let response = null;
+        const abort = () => {
+            const error = new AbortError("The operation was aborted.");
+            reject(error);
+            if (request.body && request.body instanceof Stream.Readable) {
+                request.body.destroy(error);
+            }
+            if (!response || !response.body) {
+                return;
+            }
+            response.body.emit("error", error);
+        };
+        if (signal && signal.aborted) {
+            abort();
+            return;
+        }
+        const abortAndFinalize = () => {
+            abort();
+            finalize();
+        };
+        // Send request
+        const request_ = send(parsedURL.toString(), options);
+        if (signal) {
+            signal.addEventListener("abort", abortAndFinalize);
+        }
+        const finalize = () => {
+            request_.abort();
+            if (signal) {
+                signal.removeEventListener("abort", abortAndFinalize);
+            }
+        };
+        request_.on("error", (error) => {
+            reject(new FetchError(`request to ${request.url} failed, reason: ${error.message}`, "system", error));
+            finalize();
+        });
+        fixResponseChunkedTransferBadEnding(request_, (error) => {
+            if (response && response.body) {
+                response.body.destroy(error);
+            }
+        });
+        /* c8 ignore next 18 */
+        if (process.version < "v14") {
+            // Before Node.js 14, pipeline() does not fully support async iterators and does not always
+            // properly handle when the socket close/end events are out of order.
+            request_.on("socket", (s) => {
+                let endedWithEventsCount;
+                s.prependListener("end", () => {
+                    endedWithEventsCount = s._eventsCount;
+                });
+                s.prependListener("close", (hadError) => {
+                    // if end happened before close but the socket didn't emit an error, do it now
+                    if (response && endedWithEventsCount < s._eventsCount && !hadError) {
+                        const error = new Error("Premature close");
+                        error.code = "ERR_STREAM_PREMATURE_CLOSE";
+                        response.body.emit("error", error);
+                    }
+                });
+            });
+        }
+        request_.on("response", (response_) => {
+            request_.setTimeout(0);
+            const headers = fromRawHeaders(response_.rawHeaders);
+            // HTTP fetch step 5
+            if (isRedirect(response_.statusCode)) {
+                // HTTP fetch step 5.2
+                const location = headers.get("Location");
+                // HTTP fetch step 5.3
+                let locationURL = null;
+                try {
+                    locationURL =
+                        location === null ? null : new URL(location, request.url);
+                }
+                catch {
+                    // error here can only be invalid URL in Location: header
+                    // do not throw when options.redirect == manual
+                    // let the user extract the errorneous redirect URL
+                    if (request.redirect !== "manual") {
+                        reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, "invalid-redirect"));
+                        finalize();
+                        return;
+                    }
+                }
+                // HTTP fetch step 5.5
+                switch (request.redirect) {
+                    case "error":
+                        reject(new FetchError(`uri requested responds with a redirect, redirect mode is set to error: ${request.url}`, "no-redirect"));
+                        finalize();
+                        return;
+                    case "manual":
+                        // Nothing to do
+                        break;
+                    case "follow": {
+                        // HTTP-redirect fetch step 2
+                        if (locationURL === null) {
+                            break;
+                        }
+                        // HTTP-redirect fetch step 5
+                        if (request.counter >= request.follow) {
+                            reject(new FetchError(`maximum redirect reached at: ${request.url}`, "max-redirect"));
+                            finalize();
+                            return;
+                        }
+                        // HTTP-redirect fetch step 6 (counter increment)
+                        // Create a new Request object.
+                        const requestOptions = {
+                            headers: new Headers(request.headers),
+                            follow: request.follow,
+                            counter: request.counter + 1,
+                            agent: request.agent,
+                            compress: request.compress,
+                            method: request.method,
+                            body: clone(request),
+                            signal: request.signal,
+                            size: request.size,
+                            referrer: request.referrer,
+                            referrerPolicy: request.referrerPolicy,
+                        };
+                        // when forwarding sensitive headers like "Authorization",
+                        // "WWW-Authenticate", and "Cookie" to untrusted targets,
+                        // headers will be ignored when following a redirect to a domain
+                        // that is not a subdomain match or exact match of the initial domain.
+                        // For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com"
+                        // will forward the sensitive headers, but a redirect to "bar.com" will not.
+                        // headers will also be ignored when following a redirect to a domain using
+                        // a different protocol. For example, a redirect from "https://foo.com" to "http://foo.com"
+                        // will not forward the sensitive headers
+                        if (!isDomainOrSubdomain(request.url, locationURL) ||
+                            !isSameProtocol(request.url, locationURL)) {
+                            for (const name of [
+                                "authorization",
+                                "www-authenticate",
+                                "cookie",
+                                "cookie2",
+                            ]) {
+                                requestOptions.headers.delete(name);
+                            }
+                        }
+                        // HTTP-redirect fetch step 9
+                        if (response_.statusCode !== 303 &&
+                            request.body &&
+                            options_.body instanceof Stream.Readable) {
+                            reject(new FetchError("Cannot follow redirect with body being a readable stream", "unsupported-redirect"));
+                            finalize();
+                            return;
+                        }
+                        // HTTP-redirect fetch step 11
+                        if (response_.statusCode === 303 ||
+                            ((response_.statusCode === 301 || response_.statusCode === 302) &&
+                                request.method === "POST")) {
+                            requestOptions.method = "GET";
+                            requestOptions.body = undefined;
+                            requestOptions.headers.delete("content-length");
+                        }
+                        // HTTP-redirect fetch step 14
+                        const responseReferrerPolicy = parseReferrerPolicyFromHeader(headers);
+                        if (responseReferrerPolicy) {
+                            requestOptions.referrerPolicy = responseReferrerPolicy;
+                        }
+                        // HTTP-redirect fetch step 15
+                        resolve(fetch(new Request(locationURL, requestOptions)));
+                        finalize();
+                        return;
+                    }
+                    default:
+                        return reject(new TypeError(`Redirect option '${request.redirect}' is not a valid value of RequestRedirect`));
+                }
+            }
+            // Prepare response
+            if (signal) {
+                response_.once("end", () => {
+                    signal.removeEventListener("abort", abortAndFinalize);
+                });
+            }
+            let body = pump(response_, new PassThrough(), (error) => {
+                if (error) {
+                    reject(error);
+                }
+            });
+            // see https://github.com/nodejs/node/pull/29376
+            /* c8 ignore next 3 */
+            if (process.version < "v12.10") {
+                response_.on("aborted", abortAndFinalize);
+            }
+            const responseOptions = {
+                url: request.url,
+                status: response_.statusCode,
+                statusText: response_.statusMessage,
+                headers,
+                size: request.size,
+                counter: request.counter,
+                highWaterMark: request.highWaterMark,
+            };
+            // HTTP-network fetch step 12.1.1.3
+            const codings = headers.get("Content-Encoding");
+            // HTTP-network fetch step 12.1.1.4: handle content codings
+            // in following scenarios we ignore compression support
+            // 1. compression support is disabled
+            // 2. HEAD request
+            // 3. no Content-Encoding header
+            // 4. no content response (204)
+            // 5. content not modified response (304)
+            if (!request.compress ||
+                request.method === "HEAD" ||
+                codings === null ||
+                response_.statusCode === 204 ||
+                response_.statusCode === 304) {
+                response = new Response(body, responseOptions);
+                resolve(response);
+                return;
+            }
+            // For Node v6+
+            // Be less strict when decoding compressed responses, since sometimes
+            // servers send slightly invalid responses that are still accepted
+            // by common browsers.
+            // Always using Z_SYNC_FLUSH is what cURL does.
+            const zlibOptions = {
+                flush: zlib.Z_SYNC_FLUSH,
+                finishFlush: zlib.Z_SYNC_FLUSH,
+            };
+            // For gzip
+            if (codings === "gzip" || codings === "x-gzip") {
+                body = pump(body, zlib.createGunzip(zlibOptions), (error) => {
+                    if (error) {
+                        reject(error);
+                    }
+                });
+                response = new Response(body, responseOptions);
+                resolve(response);
+                return;
+            }
+            // For deflate
+            if (codings === "deflate" || codings === "x-deflate") {
+                // Handle the infamous raw deflate response from old servers
+                // a hack for old IIS and Apache servers
+                const raw = pump(response_, new PassThrough(), (error) => {
+                    if (error) {
+                        reject(error);
+                    }
+                });
+                raw.once("data", (chunk) => {
+                    // See http://stackoverflow.com/questions/37519828
+                    if ((chunk[0] & 0x0f) === 0x08) {
+                        body = pump(body, zlib.createInflate(), (error) => {
+                            if (error) {
+                                reject(error);
+                            }
+                        });
+                    }
+                    else {
+                        body = pump(body, zlib.createInflateRaw(), (error) => {
+                            if (error) {
+                                reject(error);
+                            }
+                        });
+                    }
+                    response = new Response(body, responseOptions);
+                    resolve(response);
+                });
+                raw.once("end", () => {
+                    // Some old IIS servers return zero-length OK deflate responses, so
+                    // 'data' is never emitted. See https://github.com/node-fetch/node-fetch/pull/903
+                    if (!response) {
+                        response = new Response(body, responseOptions);
+                        resolve(response);
+                    }
+                });
+                return;
+            }
+            // For br
+            if (codings === "br") {
+                body = pump(body, zlib.createBrotliDecompress(), (error) => {
+                    if (error) {
+                        reject(error);
+                    }
+                });
+                response = new Response(body, responseOptions);
+                resolve(response);
+                return;
+            }
+            // Otherwise, use response as-is
+            response = new Response(body, responseOptions);
+            resolve(response);
+        });
+        // eslint-disable-next-line promise/prefer-await-to-then
+        writeToStream(request_, request).catch(reject);
+    });
+}
+function fixResponseChunkedTransferBadEnding(request, errorCallback) {
+    const LAST_CHUNK = Buffer.from("0\r\n\r\n");
+    let isChunkedTransfer = false;
+    let properLastChunkReceived = false;
+    let previousChunk;
+    request.on("response", (response) => {
+        const { headers } = response;
+        isChunkedTransfer =
+            headers["transfer-encoding"] === "chunked" && !headers["content-length"];
+    });
+    request.on("socket", (socket) => {
+        const onSocketClose = () => {
+            if (isChunkedTransfer && !properLastChunkReceived) {
+                const error = new Error("Premature close");
+                error.code = "ERR_STREAM_PREMATURE_CLOSE";
+                errorCallback(error);
+            }
+        };
+        // const onData = buf => {
+        // 	properLastChunkReceived = Buffer.compare(buf.slice(-5), LAST_CHUNK) === 0;
+        // 	// Sometimes final 0-length chunk and end of message code are in separate packets
+        // 	if (!properLastChunkReceived && previousChunk) {
+        // 		properLastChunkReceived = (
+        // 			Buffer.compare(previousChunk.slice(-3), LAST_CHUNK.slice(0, 3)) === 0 &&
+        // 			Buffer.compare(buf.slice(-2), LAST_CHUNK.slice(3)) === 0
+        // 		);
+        // 	}
+        // 	previousChunk = buf;
+        // };
+        // PATCH - THIS IS WHERE THE PATCH IS - FIXES BOUNDARY CHUNKING ISSUE
+        // See https://github.com/node-fetch/node-fetch/issues/1576
+        const onData = (buf) => {
+            properLastChunkReceived = Buffer.compare(buf.slice(-5), LAST_CHUNK) === 0;
+            // Sometimes final 0-length chunk and end of message code are in separate packets
+            if (!properLastChunkReceived && previousChunk) {
+                if (buf.length < 5) {
+                    properLastChunkReceived =
+                        Buffer.compare(Buffer.from([...previousChunk.slice(-5), ...buf]).slice(-5), LAST_CHUNK) === 0;
+                }
+            }
+            previousChunk = buf;
+        };
+        socket.prependListener("close", onSocketClose);
+        socket.on("data", onData);
+        request.on("close", () => {
+            socket.removeListener("close", onSocketClose);
+            socket.removeListener("data", onData);
+        });
+    });
+}

package/dist/node-fetch-patch.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/node-fetch-patch.test.js

@@ -0,0 +1,50 @@
+import { Buffer } from "node:buffer";
+import { describe, expect, it } from "vitest";
+// Tests the boundary chunking patch to node fetch made in node-fetch-patch.js (see that file for details).
+describe("Chunked Transfer Encoding Patch Logic", () => {
+    // The LAST_CHUNK marker "0\r\n\r\n"
+    const LAST_CHUNK = Buffer.from("0\r\n\r\n");
+    it("should detect complete marker at the end (normal case)", () => {
+        const normalCase = Buffer.from("data data data 0\r\n\r\n");
+        const result = Buffer.compare(normalCase.slice(-5), LAST_CHUNK) === 0;
+        expect(result).toBe(true);
+    });
+    it("should handle chunks split across original pattern (0\\r\\n | \\r\\n)", () => {
+        const previousChunk = Buffer.from("data data 0\r\n");
+        const lastChunk = Buffer.from("\r\n");
+        const result = Buffer.compare(previousChunk.slice(-3), LAST_CHUNK.slice(0, 3)) === 0 &&
+            Buffer.compare(lastChunk.slice(-2), LAST_CHUNK.slice(3)) === 0;
+        expect(result).toBe(true);
+    });
+    it("should handle problematic case from issue #1576 with new patch", () => {
+        // The problematic case: data ends with "0\r\n0\r" and the next chunk is "\n\r\n"
+        const previousChunk = Buffer.from([
+            ...Array(10).fill(42),
+            48,
+            13,
+            10,
+            48,
+            13,
+        ]); // some data + "0\r\n0\r"
+        const lastChunk = Buffer.from([10, 13, 10]); // "\n\r\n"
+        // Check 1: Direct check for complete marker (should fail)
+        const check1 = Buffer.compare(lastChunk.slice(-5), LAST_CHUNK) === 0;
+        expect(check1).toBe(false);
+        // Check 2: Original split pattern check (should fail)
+        const check2 = Buffer.compare(previousChunk.slice(-3), LAST_CHUNK.slice(0, 3)) === 0 &&
+            Buffer.compare(lastChunk.slice(-2), LAST_CHUNK.slice(3)) === 0;
+        expect(check2).toBe(false);
+        // Check 3: New patch check (should pass)
+        const lastChunkLength = lastChunk.length;
+        const newPatchCheck = lastChunkLength < 5 &&
+            Buffer.compare(Buffer.from([...previousChunk.slice(-5), ...lastChunk]).slice(-5), LAST_CHUNK) === 0;
+        expect(newPatchCheck).toBe(true);
+        // Verification of the combined buffer
+        const combinedBuffer = Buffer.from([
+            ...previousChunk.slice(-5),
+            ...lastChunk,
+        ]);
+        const combinedResult = Buffer.compare(combinedBuffer.slice(-5), LAST_CHUNK) === 0;
+        expect(combinedResult).toBe(true);
+    });
+});

package/dist/stream.js

@@ -16,12 +16,14 @@ export async function* streamResponse(response) {
     }
     // Get the major version of Node.js
     const nodeMajorVersion = parseInt(process.versions.node.split(".")[0], 10);
+    let chunks = 0;
     try {
         if (nodeMajorVersion >= 20) {
             // Use the new API for Node 20 and above
             const stream = ReadableStream.from(response.body);
             for await (const chunk of stream.pipeThrough(new TextDecoderStream("utf-8"))) {
                 yield chunk;
+                chunks++;
             }
         }
         else {
@@ -31,12 +33,28 @@ export async function* streamResponse(response) {
             const nodeStream = response.body;
             for await (const chunk of toAsyncIterable(nodeStream)) {
                 yield decoder.decode(chunk, { stream: true });
+                chunks++;
             }
         }
     }
     catch (e) {
-        if (e instanceof Error && e.name.startsWith("AbortError")) {
-            return; // In case of client-side cancellation, just return
+        if (e instanceof Error) {
+            if (e.name.startsWith("AbortError")) {
+                return; // In case of client-side cancellation, just return
+            }
+            if (e.message.toLowerCase().includes("premature close")) {
+                // Premature close can happen for various reasons, including:
+                // - Malformed chunks of data received from the server
+                // - The server closed the connection before sending the complete response
+                // - Long delays from the server during streaming
+                // - 'Keep alive' header being used in combination with an http agent and a set, low number of maxSockets
+                if (chunks === 0) {
+                    throw new Error("Stream was closed before any data was received. Try again. (Premature Close)");
+                }
+                else {
+                    throw new Error("The response was cancelled mid-stream. Try again. (Premature Close).");
+                }
+            }
         }
         throw e;
     }
@@ -106,8 +124,13 @@ export async function* streamJSON(response) {
         let position;
         while ((position = buffer.indexOf("\n")) >= 0) {
             const line = buffer.slice(0, position);
-            const data = JSON.parse(line);
-            yield data;
+            try {
+                const data = JSON.parse(line);
+                yield data;
+            }
+            catch (e) {
+                throw new Error(`Malformed JSON sent from server: ${line}`);
+            }
             buffer = buffer.slice(position + 1);
         }
     }

package/dist/stream.test.js

@@ -1,4 +1,5 @@
 import { Readable } from "stream";
+import { describe, expect, it, test } from "vitest";
 import { parseDataLine, streamSse } from "./stream.js";
 function createMockResponse(sseLines) {
     // Create a Readable stream that emits the SSE lines

package/dist/util.d.ts

@@ -1,12 +1,17 @@
+import { RequestOptions } from "@continuedev/config-types";
 /**
  * Gets the proxy settings from environment variables
  * @param protocol The URL protocol (http: or https:)
  * @returns The proxy URL if available, otherwise undefined
  */
 export declare function getProxyFromEnv(protocol: string): string | undefined;
+export declare function getProxy(protocol: string, requestOptions?: RequestOptions): string | undefined;
+export declare function getEnvNoProxyPatterns(): string[];
+export declare function getReqOptionsNoProxyPatterns(options: RequestOptions | undefined): string[];
+export declare function patternMatchesHostname(hostname: string, pattern: string): boolean;
 /**
  * Checks if a hostname should bypass proxy based on NO_PROXY environment variable
  * @param hostname The hostname to check
  * @returns True if the hostname should bypass proxy
  */
-export declare function shouldBypassProxy(hostname: string): boolean;
+export declare function shouldBypassProxy(hostname: string, requestOptions: RequestOptions | undefined): boolean;

package/dist/util.js

@@ -14,26 +14,66 @@ export function getProxyFromEnv(protocol) {
         return process.env.HTTP_PROXY || process.env.http_proxy;
     }
 }
+// Note that request options proxy (per model) takes precedence over environment variables
+export function getProxy(protocol, requestOptions) {
+    if (requestOptions?.proxy) {
+        return requestOptions.proxy;
+    }
+    return getProxyFromEnv(protocol);
+}
+export function getEnvNoProxyPatterns() {
+    const envValue = process.env.NO_PROXY || process.env.no_proxy;
+    if (envValue) {
+        return envValue
+            .split(",")
+            .map((item) => item.trim().toLowerCase())
+            .filter((i) => !!i);
+    }
+    else {
+        return [];
+    }
+}
+export function getReqOptionsNoProxyPatterns(options) {
+    return (options?.noProxy?.map((i) => i.trim().toLowerCase()).filter((i) => !!i) ??
+        []);
+}
+export function patternMatchesHostname(hostname, pattern) {
+    // Split hostname and pattern to separate hostname and port
+    const [hostnameWithoutPort, hostnamePort] = hostname.toLowerCase().split(":");
+    const [patternWithoutPort, patternPort] = pattern.toLowerCase().split(":");
+    // If pattern specifies a port but hostname doesn't match it, no match
+    if (patternPort && (!hostnamePort || hostnamePort !== patternPort)) {
+        return false;
+    }
+    // Now compare just the hostname parts
+    // exact match
+    if (patternWithoutPort === hostnameWithoutPort) {
+        return true;
+    }
+    // wildcard domain match (*.example.com)
+    if (patternWithoutPort.startsWith("*.") &&
+        hostnameWithoutPort.endsWith(patternWithoutPort.substring(1))) {
+        return true;
+    }
+    // Domain suffix match (.example.com)
+    if (patternWithoutPort.startsWith(".") &&
+        hostnameWithoutPort.endsWith(patternWithoutPort.slice(1))) {
+        return true;
+    }
+    // TODO IP address ranges
+    // TODO CIDR notation
+    return false;
+}
 /**
  * Checks if a hostname should bypass proxy based on NO_PROXY environment variable
  * @param hostname The hostname to check
  * @returns True if the hostname should bypass proxy
  */
-export function shouldBypassProxy(hostname) {
-    const noProxy = process.env.NO_PROXY || process.env.no_proxy;
-    if (!noProxy)
-        return false;
-    const noProxyItems = noProxy.split(",").map((item) => item.trim());
-    return noProxyItems.some((item) => {
-        // Exact match
-        if (item === hostname)
-            return true;
-        // Wildcard domain match (*.example.com)
-        if (item.startsWith("*.") && hostname.endsWith(item.substring(1)))
-            return true;
-        // Domain suffix match (.example.com)
-        if (item.startsWith(".") && hostname.endsWith(item.slice(1)))
-            return true;
-        return false;
-    });
+export function shouldBypassProxy(hostname, requestOptions) {
+    const ignores = [
+        ...getEnvNoProxyPatterns(),
+        ...getReqOptionsNoProxyPatterns(requestOptions),
+    ];
+    const hostLowerCase = hostname.toLowerCase();
+    return ignores.some((ignore) => patternMatchesHostname(hostLowerCase, ignore));
 }