@continuedev/fetch 1.0.9 → 1.0.11

package/dist/fetch.js

@@ -5,6 +5,61 @@ import fetch, { Response } from "node-fetch";
 import { getAgentOptions } from "./getAgentOptions.js";
 import { getProxyFromEnv, shouldBypassProxy } from "./util.js";
 const { http, https } = followRedirects.default;
+function logRequest(method, url, headers, body, proxy, shouldBypass) {
+    console.log("=== FETCH REQUEST ===");
+    console.log(`Method: ${method}`);
+    console.log(`URL: ${url.toString()}`);
+    // Log headers in curl format
+    console.log("Headers:");
+    for (const [key, value] of Object.entries(headers)) {
+        console.log(`  -H '${key}: ${value}'`);
+    }
+    // Log proxy information
+    if (proxy && !shouldBypass) {
+        console.log(`Proxy: ${proxy}`);
+    }
+    // Log body
+    if (body) {
+        console.log(`Body: ${body}`);
+    }
+    // Generate equivalent curl command
+    let curlCommand = `curl -X ${method}`;
+    for (const [key, value] of Object.entries(headers)) {
+        curlCommand += ` -H '${key}: ${value}'`;
+    }
+    if (body) {
+        curlCommand += ` -d '${body}'`;
+    }
+    if (proxy && !shouldBypass) {
+        curlCommand += ` --proxy '${proxy}'`;
+    }
+    curlCommand += ` '${url.toString()}'`;
+    console.log(`Equivalent curl: ${curlCommand}`);
+    console.log("=====================");
+}
+async function logResponse(resp) {
+    console.log("=== FETCH RESPONSE ===");
+    console.log(`Status: ${resp.status} ${resp.statusText}`);
+    console.log("Response Headers:");
+    resp.headers.forEach((value, key) => {
+        console.log(`  ${key}: ${value}`);
+    });
+    // Clone response to read body without consuming it
+    const respClone = resp.clone();
+    try {
+        const responseText = await respClone.text();
+        console.log(`Response Body: ${responseText}`);
+    }
+    catch (e) {
+        console.log("Could not read response body:", e);
+    }
+    console.log("======================");
+}
+function logError(error) {
+    console.log("=== FETCH ERROR ===");
+    console.log(`Error: ${error}`);
+    console.log("===================");
+}
 export async function fetchwithRequestOptions(url_, init, requestOptions) {
     const url = typeof url_ === "string" ? new URL(url_) : url_;
     if (url.host === "localhost") {
@@ -52,14 +107,24 @@ export async function fetchwithRequestOptions(url_, init, requestOptions) {
     catch (e) {
         console.log("Unable to parse HTTP request body: ", e);
     }
+    const finalBody = updatedBody ?? init?.body;
+    const method = init?.method || "GET";
+    // Verbose logging for debugging - log request details
+    if (process.env.VERBOSE_FETCH) {
+        logRequest(method, url, headers, finalBody, proxy, shouldBypass);
+    }
     // fetch the request with the provided options
     try {
         const resp = await fetch(url, {
             ...init,
-            body: updatedBody ?? init?.body,
+            body: finalBody,
             headers: headers,
             agent: agent,
         });
+        // Verbose logging for debugging - log response details
+        if (process.env.VERBOSE_FETCH) {
+            await logResponse(resp);
+        }
         if (!resp.ok) {
             const requestId = resp.headers.get("x-request-id");
             if (requestId) {
@@ -69,6 +134,10 @@ export async function fetchwithRequestOptions(url_, init, requestOptions) {
         return resp;
     }
     catch (error) {
+        // Verbose logging for errors
+        if (process.env.VERBOSE_FETCH) {
+            logError(error);
+        }
         if (error instanceof Error && error.name === "AbortError") {
             // Return a Response object that streamResponse etc can handle
             return new Response(null, {

package/dist/stream.js

@@ -1,6 +1,6 @@
 export async function* toAsyncIterable(nodeReadable) {
     for await (const chunk of nodeReadable) {
-        // @ts-expect-error
+        // @ts-ignore
         yield chunk;
     }
 }
@@ -64,7 +64,7 @@ export function parseDataLine(line) {
     }
 }
 function parseSseLine(line) {
-    if (line.startsWith("data: [DONE]")) {
+    if (line.startsWith("data:[DONE]") || line.startsWith("data: [DONE]")) {
         return { done: true, data: undefined };
     }
     if (line.startsWith("data:")) {

package/dist/stream.test.js

@@ -1,6 +1,56 @@
-// The parseDataLine function is not exported, so we need to import it
-// indirectly by re-exporting it for testing purposes
-import { parseDataLine } from "./stream.js";
+import { Readable } from "stream";
+import { parseDataLine, streamSse } from "./stream.js";
+function createMockResponse(sseLines) {
+    // Create a Readable stream that emits the SSE lines
+    const stream = new Readable({
+        read() {
+            for (const line of sseLines) {
+                this.push(line + "\n\n");
+            }
+            this.push(null); // End of stream
+        },
+    });
+    // Minimal Response mock
+    return {
+        status: 200,
+        body: stream,
+        text: async () => "",
+    };
+}
+describe("streamSse", () => {
+    it("yields parsed SSE data objects that ends with `data:[DONE]`", async () => {
+        const sseLines = [
+            'data: {"foo": "bar"}',
+            'data: {"baz": 42}',
+            "data:[DONE]",
+        ];
+        const response = createMockResponse(sseLines);
+        const results = [];
+        for await (const data of streamSse(response)) {
+            results.push(data);
+        }
+        expect(results).toEqual([{ foo: "bar" }, { baz: 42 }]);
+    });
+    it("yields parsed SSE data objects that ends with `data: [DONE]` (with a space before [DONE]", async () => {
+        const sseLines = [
+            'data: {"foo": "bar"}',
+            'data: {"baz": 42}',
+            "data: [DONE]",
+        ];
+        const response = createMockResponse(sseLines);
+        const results = [];
+        for await (const data of streamSse(response)) {
+            results.push(data);
+        }
+        expect(results).toEqual([{ foo: "bar" }, { baz: 42 }]);
+    });
+    it("throws on malformed JSON", async () => {
+        const sseLines = ['data: {"foo": "bar"', "data:[DONE]"];
+        const response = createMockResponse(sseLines);
+        const iterator = streamSse(response)[Symbol.asyncIterator]();
+        await expect(iterator.next()).rejects.toThrow(/Malformed JSON/);
+    });
+});
 describe("parseDataLine", () => {
     test("parseDataLine should parse valid JSON data with 'data: ' prefix", () => {
         const line = 'data: {"message":"hello","status":"ok"}';

package/jest.globals.d.ts

@@ -0,0 +1,14 @@
+// This file declares the Jest globals for TypeScript
+import '@jest/globals';
+
+declare global {
+  const describe: typeof import('@jest/globals').describe;
+  const expect: typeof import('@jest/globals').expect;
+  const it: typeof import('@jest/globals').it;
+  const test: typeof import('@jest/globals').test;
+  const beforeAll: typeof import('@jest/globals').beforeAll;
+  const afterAll: typeof import('@jest/globals').afterAll;
+  const beforeEach: typeof import('@jest/globals').beforeEach;
+  const afterEach: typeof import('@jest/globals').afterEach;
+  const jest: typeof import('@jest/globals').jest;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@continuedev/fetch",
-  "version": "1.0.9",
+  "version": "1.0.11",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/fetch.ts

@@ -2,12 +2,81 @@ import { RequestOptions } from "@continuedev/config-types";
 import * as followRedirects from "follow-redirects";
 import { HttpProxyAgent } from "http-proxy-agent";
 import { HttpsProxyAgent } from "https-proxy-agent";
-import fetch, { RequestInit, Response } from "node-fetch";
+import fetch, { BodyInit, RequestInit, Response } from "node-fetch";
 import { getAgentOptions } from "./getAgentOptions.js";
 import { getProxyFromEnv, shouldBypassProxy } from "./util.js";
 
 const { http, https } = (followRedirects as any).default;
 
+function logRequest(
+  method: string,
+  url: URL,
+  headers: { [key: string]: string },
+  body: BodyInit | null | undefined,
+  proxy?: string,
+  shouldBypass?: boolean,
+) {
+  console.log("=== FETCH REQUEST ===");
+  console.log(`Method: ${method}`);
+  console.log(`URL: ${url.toString()}`);
+
+  // Log headers in curl format
+  console.log("Headers:");
+  for (const [key, value] of Object.entries(headers)) {
+    console.log(`  -H '${key}: ${value}'`);
+  }
+
+  // Log proxy information
+  if (proxy && !shouldBypass) {
+    console.log(`Proxy: ${proxy}`);
+  }
+
+  // Log body
+  if (body) {
+    console.log(`Body: ${body}`);
+  }
+
+  // Generate equivalent curl command
+  let curlCommand = `curl -X ${method}`;
+  for (const [key, value] of Object.entries(headers)) {
+    curlCommand += ` -H '${key}: ${value}'`;
+  }
+  if (body) {
+    curlCommand += ` -d '${body}'`;
+  }
+  if (proxy && !shouldBypass) {
+    curlCommand += ` --proxy '${proxy}'`;
+  }
+  curlCommand += ` '${url.toString()}'`;
+  console.log(`Equivalent curl: ${curlCommand}`);
+  console.log("=====================");
+}
+
+async function logResponse(resp: Response) {
+  console.log("=== FETCH RESPONSE ===");
+  console.log(`Status: ${resp.status} ${resp.statusText}`);
+  console.log("Response Headers:");
+  resp.headers.forEach((value, key) => {
+    console.log(`  ${key}: ${value}`);
+  });
+
+  // Clone response to read body without consuming it
+  const respClone = resp.clone();
+  try {
+    const responseText = await respClone.text();
+    console.log(`Response Body: ${responseText}`);
+  } catch (e) {
+    console.log("Could not read response body:", e);
+  }
+  console.log("======================");
+}
+
+function logError(error: unknown) {
+  console.log("=== FETCH ERROR ===");
+  console.log(`Error: ${error}`);
+  console.log("===================");
+}
+
 export async function fetchwithRequestOptions(
   url_: URL | string,
   init?: RequestInit,
@@ -68,15 +137,28 @@ export async function fetchwithRequestOptions(
     console.log("Unable to parse HTTP request body: ", e);
   }
 
+  const finalBody = updatedBody ?? init?.body;
+  const method = init?.method || "GET";
+
+  // Verbose logging for debugging - log request details
+  if (process.env.VERBOSE_FETCH) {
+    logRequest(method, url, headers, finalBody, proxy, shouldBypass);
+  }
+
   // fetch the request with the provided options
   try {
     const resp = await fetch(url, {
       ...init,
-      body: updatedBody ?? init?.body,
+      body: finalBody,
       headers: headers,
       agent: agent,
     });
 
+    // Verbose logging for debugging - log response details
+    if (process.env.VERBOSE_FETCH) {
+      await logResponse(resp);
+    }
+
     if (!resp.ok) {
       const requestId = resp.headers.get("x-request-id");
       if (requestId) {
@@ -86,6 +168,11 @@ export async function fetchwithRequestOptions(
 
     return resp;
   } catch (error) {
+    // Verbose logging for errors
+    if (process.env.VERBOSE_FETCH) {
+      logError(error);
+    }
+
     if (error instanceof Error && error.name === "AbortError") {
       // Return a Response object that streamResponse etc can handle
       return new Response(null, {

package/src/stream.test.ts

@@ -1,6 +1,66 @@
-// The parseDataLine function is not exported, so we need to import it
-// indirectly by re-exporting it for testing purposes
-import { parseDataLine } from "./stream.js";
+import { Readable } from "stream";
+import { parseDataLine, streamSse } from "./stream.js";
+
+function createMockResponse(sseLines: string[]): Response {
+  // Create a Readable stream that emits the SSE lines
+  const stream = new Readable({
+    read() {
+      for (const line of sseLines) {
+        this.push(line + "\n\n");
+      }
+      this.push(null); // End of stream
+    },
+  }) as any;
+
+  // Minimal Response mock
+  return {
+    status: 200,
+    body: stream,
+    text: async () => "",
+  } as unknown as Response;
+}
+
+describe("streamSse", () => {
+  it("yields parsed SSE data objects that ends with `data:[DONE]`", async () => {
+    const sseLines = [
+      'data: {"foo": "bar"}',
+      'data: {"baz": 42}',
+      "data:[DONE]",
+    ];
+    const response = createMockResponse(sseLines);
+
+    const results = [];
+    for await (const data of streamSse(response)) {
+      results.push(data);
+    }
+
+    expect(results).toEqual([{ foo: "bar" }, { baz: 42 }]);
+  });
+
+  it("yields parsed SSE data objects that ends with `data: [DONE]` (with a space before [DONE]", async () => {
+    const sseLines = [
+      'data: {"foo": "bar"}',
+      'data: {"baz": 42}',
+      "data: [DONE]",
+    ];
+    const response = createMockResponse(sseLines);
+
+    const results = [];
+    for await (const data of streamSse(response)) {
+      results.push(data);
+    }
+
+    expect(results).toEqual([{ foo: "bar" }, { baz: 42 }]);
+  });
+
+  it("throws on malformed JSON", async () => {
+    const sseLines = ['data: {"foo": "bar"', "data:[DONE]"];
+    const response = createMockResponse(sseLines);
+
+    const iterator = streamSse(response)[Symbol.asyncIterator]();
+    await expect(iterator.next()).rejects.toThrow(/Malformed JSON/);
+  });
+});
 
 describe("parseDataLine", () => {
   test("parseDataLine should parse valid JSON data with 'data: ' prefix", () => {

package/src/stream.ts

@@ -2,7 +2,7 @@ export async function* toAsyncIterable(
   nodeReadable: NodeJS.ReadableStream,
 ): AsyncGenerator<Uint8Array> {
   for await (const chunk of nodeReadable) {
-    // @ts-expect-error
+    // @ts-ignore
     yield chunk as Uint8Array;
   }
 }
@@ -13,6 +13,7 @@ export async function* streamResponse(
   if (response.status === 499) {
     return; // In case of client-side cancellation, just return
   }
+
   if (response.status !== 200) {
     throw new Error(await response.text());
   }
@@ -77,7 +78,7 @@ export function parseDataLine(line: string): any {
 }
 
 function parseSseLine(line: string): { done: boolean; data: any } {
-  if (line.startsWith("data: [DONE]")) {
+  if (line.startsWith("data:[DONE]") || line.startsWith("data: [DONE]")) {
     return { done: true, data: undefined };
   }
   if (line.startsWith("data:")) {

package/tsconfig.json

@@ -19,5 +19,5 @@
     "declaration": true
     // "sourceMap": true
   },
-  "include": ["src/**/*"]
+  "include": ["src/**/*", "jest.globals.d.ts"]
 }