@continuedev/fetch 1.0.0

package/jest.config.ts

@@ -0,0 +1,199 @@
+/**
+ * For a detailed explanation regarding each configuration property, visit:
+ * https://jestjs.io/docs/configuration
+ */
+
+import type {Config} from 'jest';
+
+const config: Config = {
+  // All imported modules in your tests should be mocked automatically
+  // automock: false,
+
+  // Stop running tests after `n` failures
+  // bail: 0,
+
+  // The directory where Jest should store its cached dependency information
+  // cacheDirectory: "/private/var/folders/n_/c3r8w0s95xl271j98y2lzhz80000gn/T/jest_dx",
+
+  // Automatically clear mock calls, instances, contexts and results before every test
+  clearMocks: true,
+
+  // Indicates whether the coverage information should be collected while executing the test
+  collectCoverage: true,
+
+  // An array of glob patterns indicating a set of files for which coverage information should be collected
+  // collectCoverageFrom: undefined,
+
+  // The directory where Jest should output its coverage files
+  coverageDirectory: "coverage",
+
+  // An array of regexp pattern strings used to skip coverage collection
+  // coveragePathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // Indicates which provider should be used to instrument code for coverage
+  coverageProvider: "v8",
+
+  // A list of reporter names that Jest uses when writing coverage reports
+  // coverageReporters: [
+  //   "json",
+  //   "text",
+  //   "lcov",
+  //   "clover"
+  // ],
+
+  // An object that configures minimum threshold enforcement for coverage results
+  // coverageThreshold: undefined,
+
+  // A path to a custom dependency extractor
+  // dependencyExtractor: undefined,
+
+  // Make calling deprecated APIs throw helpful error messages
+  // errorOnDeprecated: false,
+
+  // The default configuration for fake timers
+  // fakeTimers: {
+  //   "enableGlobally": false
+  // },
+
+  // Force coverage collection from ignored files using an array of glob patterns
+  // forceCoverageMatch: [],
+
+  // A path to a module which exports an async function that is triggered once before all test suites
+  // globalSetup: undefined,
+
+  // A path to a module which exports an async function that is triggered once after all test suites
+  // globalTeardown: undefined,
+
+  // A set of global variables that need to be available in all test environments
+  // globals: {},
+
+  // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
+  // maxWorkers: "50%",
+
+  // An array of directory names to be searched recursively up from the requiring module's location
+  // moduleDirectories: [
+  //   "node_modules"
+  // ],
+
+  // An array of file extensions your modules use
+  // moduleFileExtensions: [
+  //   "js",
+  //   "mjs",
+  //   "cjs",
+  //   "jsx",
+  //   "ts",
+  //   "tsx",
+  //   "json",
+  //   "node"
+  // ],
+
+  // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
+  // moduleNameMapper: {},
+
+  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
+  // modulePathIgnorePatterns: [],
+
+  // Activates notifications for test results
+  // notify: false,
+
+  // An enum that specifies notification mode. Requires { notify: true }
+  // notifyMode: "failure-change",
+
+  // A preset that is used as a base for Jest's configuration
+  // preset: undefined,
+
+  // Run tests from one or more projects
+  // projects: undefined,
+
+  // Use this configuration option to add custom reporters to Jest
+  // reporters: undefined,
+
+  // Automatically reset mock state before every test
+  // resetMocks: false,
+
+  // Reset the module registry before running each individual test
+  // resetModules: false,
+
+  // A path to a custom resolver
+  // resolver: undefined,
+
+  // Automatically restore mock state and implementation before every test
+  // restoreMocks: false,
+
+  // The root directory that Jest should scan for tests and modules within
+  // rootDir: undefined,
+
+  // A list of paths to directories that Jest should use to search for files in
+  // roots: [
+  //   "<rootDir>"
+  // ],
+
+  // Allows you to use a custom runner instead of Jest's default test runner
+  // runner: "jest-runner",
+
+  // The paths to modules that run some code to configure or set up the testing environment before each test
+  // setupFiles: [],
+
+  // A list of paths to modules that run some code to configure or set up the testing framework before each test
+  // setupFilesAfterEnv: [],
+
+  // The number of seconds after which a test is considered as slow and reported as such in the results.
+  // slowTestThreshold: 5,
+
+  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
+  // snapshotSerializers: [],
+
+  // The test environment that will be used for testing
+  // testEnvironment: "jest-environment-node",
+
+  // Options that will be passed to the testEnvironment
+  // testEnvironmentOptions: {},
+
+  // Adds a location field to test results
+  // testLocationInResults: false,
+
+  // The glob patterns Jest uses to detect test files
+  // testMatch: [
+  //   "**/__tests__/**/*.[jt]s?(x)",
+  //   "**/?(*.)+(spec|test).[tj]s?(x)"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+  // testPathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // The regexp pattern or array of patterns that Jest uses to detect test files
+  // testRegex: [],
+
+  // This option allows the use of a custom results processor
+  // testResultsProcessor: undefined,
+
+  // This option allows use of a custom test runner
+  // testRunner: "jest-circus/runner",
+
+  // A map from regular expressions to paths to transformers
+  // transform: undefined,
+
+  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+  // transformIgnorePatterns: [
+  //   "/node_modules/",
+  //   "\\.pnp\\.[^\\/]+$"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+  // unmockedModulePathPatterns: undefined,
+
+  // Indicates whether each individual test should be reported during the run
+  // verbose: undefined,
+
+  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+  // watchPathIgnorePatterns: [],
+
+  // Whether to use watchman for file crawling
+  // watchman: true,
+};
+
+export default config;

package/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@continuedev/fetch",
+  "version": "1.0.0",
+  "description": "",
+  "main": "src/index.ts",
+  "scripts": {
+    "test": "jest"
+  },
+  "author": "Nate Sesti and Ty Dunn",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@continuedev/config-types": "^1.0.4",
+    "follow-redirects": "^1.15.6",
+    "http-proxy-agent": "^7.0.2",
+    "https-proxy-agent": "^7.0.5",
+    "node-fetch": "^3.3.2"
+  },
+  "devDependencies": {
+    "@types/follow-redirects": "^1.14.4",
+    "jest": "^29.7.0"
+  }
+}

package/src/fetch.ts

@@ -0,0 +1,117 @@
+import { RequestOptions } from "@continuedev/config-types/src/index";
+import * as followRedirects from "follow-redirects";
+import { HttpProxyAgent } from "http-proxy-agent";
+import { globalAgent } from "https";
+import { HttpsProxyAgent } from "https-proxy-agent";
+import fetch, { RequestInit, Response } from "node-fetch";
+import * as fs from "node:fs";
+import tls from "node:tls";
+
+const { http, https } = (followRedirects as any).default;
+
+export function fetchwithRequestOptions(
+  url_: URL | string,
+  init?: RequestInit,
+  requestOptions?: RequestOptions,
+): Promise<Response> {
+  let url = url_;
+  if (typeof url === "string") {
+    url = new URL(url);
+  }
+
+  const TIMEOUT = 7200; // 7200 seconds = 2 hours
+
+  let globalCerts: string[] = [];
+  if (process.env.IS_BINARY) {
+    if (Array.isArray(globalAgent.options.ca)) {
+      globalCerts = [...globalAgent.options.ca.map((cert) => cert.toString())];
+    } else if (typeof globalAgent.options.ca !== "undefined") {
+      globalCerts.push(globalAgent.options.ca.toString());
+    }
+  }
+  const ca = Array.from(new Set([...tls.rootCertificates, ...globalCerts]));
+  const customCerts =
+    typeof requestOptions?.caBundlePath === "string"
+      ? [requestOptions?.caBundlePath]
+      : requestOptions?.caBundlePath;
+  if (customCerts) {
+    ca.push(
+      ...customCerts.map((customCert) => fs.readFileSync(customCert, "utf8")),
+    );
+  }
+
+  const timeout = (requestOptions?.timeout ?? TIMEOUT) * 1000; // measured in ms
+
+  const agentOptions: { [key: string]: any } = {
+    ca,
+    rejectUnauthorized: requestOptions?.verifySsl,
+    timeout,
+    sessionTimeout: timeout,
+    keepAlive: true,
+    keepAliveMsecs: timeout,
+  };
+
+  // Handle ClientCertificateOptions
+  if (requestOptions?.clientCertificate) {
+    agentOptions.cert = fs.readFileSync(
+      requestOptions.clientCertificate.cert,
+      "utf8",
+    );
+    agentOptions.key = fs.readFileSync(
+      requestOptions.clientCertificate.key,
+      "utf8",
+    );
+    if (requestOptions.clientCertificate.passphrase) {
+      agentOptions.passphrase = requestOptions.clientCertificate.passphrase;
+    }
+  }
+
+  const proxy = requestOptions?.proxy;
+
+  // Create agent
+  const protocol = url.protocol === "https:" ? https : http;
+  const agent =
+    proxy && !requestOptions?.noProxy?.includes(url.hostname)
+      ? protocol === https
+        ? new HttpsProxyAgent(proxy, agentOptions)
+        : new HttpProxyAgent(proxy, agentOptions)
+      : new protocol.Agent(agentOptions);
+
+  let headers: { [key: string]: string } = {};
+  for (const [key, value] of Object.entries(init?.headers || {})) {
+    headers[key] = value as string;
+  }
+  headers = {
+    ...headers,
+    ...requestOptions?.headers,
+  };
+
+  // Replace localhost with 127.0.0.1
+  if (url.hostname === "localhost") {
+    url.hostname = "127.0.0.1";
+  }
+
+  // add extra body properties if provided
+  let updatedBody: string | undefined = undefined;
+  try {
+    if (requestOptions?.extraBodyProperties && typeof init?.body === "string") {
+      const parsedBody = JSON.parse(init.body);
+      updatedBody = JSON.stringify({
+        ...parsedBody,
+        ...requestOptions.extraBodyProperties,
+      });
+    }
+  } catch (e) {
+    console.log("Unable to parse HTTP request body: ", e);
+  }
+
+  // fetch the request with the provided options
+  const resp = fetch(url, {
+    ...init,
+    body: updatedBody ?? init?.body,
+    headers: headers,
+    agent: agent,
+  });
+
+  return resp;
+}

package/src/index.ts

@@ -0,0 +1,8 @@
+export {
+  streamJSON,
+  streamResponse,
+  streamSse,
+  toAsyncIterable,
+} from "./stream";
+
+export { fetchwithRequestOptions } from "./fetch";

package/src/stream.ts

@@ -0,0 +1,113 @@
+export async function* toAsyncIterable(
+  nodeReadable: NodeJS.ReadableStream,
+): AsyncGenerator<Uint8Array> {
+  for await (const chunk of nodeReadable) {
+    yield chunk as Uint8Array;
+  }
+}
+
+export async function* streamResponse(
+  response: Response,
+): AsyncGenerator<string> {
+  if (response.status !== 200) {
+    throw new Error(await response.text());
+  }
+
+  if (!response.body) {
+    throw new Error("No response body returned.");
+  }
+
+  // Get the major version of Node.js
+  const nodeMajorVersion = parseInt(process.versions.node.split(".")[0], 10);
+
+  if (nodeMajorVersion >= 20) {
+    // Use the new API for Node 20 and above
+    const stream = (ReadableStream as any).from(response.body);
+    for await (const chunk of stream.pipeThrough(
+      new TextDecoderStream("utf-8"),
+    )) {
+      yield chunk;
+    }
+  } else {
+    // Fallback for Node versions below 20
+    // Streaming with this method doesn't work as version 20+ does
+    const decoder = new TextDecoder("utf-8");
+    const nodeStream = response.body as unknown as NodeJS.ReadableStream;
+    for await (const chunk of toAsyncIterable(nodeStream)) {
+      yield decoder.decode(chunk, { stream: true });
+    }
+  }
+}
+
+function parseDataLine(line: string): any {
+  const json = line.startsWith("data: ")
+    ? line.slice("data: ".length)
+    : line.slice("data:".length);
+
+  try {
+    const data = JSON.parse(json);
+    if (data.error) {
+      throw new Error(`Error streaming response: ${data.error}`);
+    }
+
+    return data;
+  } catch (e) {
+    throw new Error(`Malformed JSON sent from server: ${json}`);
+  }
+}
+
+function parseSseLine(line: string): { done: boolean; data: any } {
+  if (line.startsWith("data: [DONE]")) {
+    return { done: true, data: undefined };
+  }
+  if (line.startsWith("data:")) {
+    return { done: false, data: parseDataLine(line) };
+  }
+  if (line.startsWith(": ping")) {
+    return { done: true, data: undefined };
+  }
+  return { done: false, data: undefined };
+}
+
+export async function* streamSse(response: Response): AsyncGenerator<any> {
+  let buffer = "";
+  for await (const value of streamResponse(response)) {
+    buffer += value;
+
+    let position: number;
+    while ((position = buffer.indexOf("\n")) >= 0) {
+      const line = buffer.slice(0, position);
+      buffer = buffer.slice(position + 1);
+
+      const { done, data } = parseSseLine(line);
+      if (done) {
+        break;
+      }
+      if (data) {
+        yield data;
+      }
+    }
+  }
+
+  if (buffer.length > 0) {
+    const { done, data } = parseSseLine(buffer);
+    if (!done && data) {
+      yield data;
+    }
+  }
+}
+
+export async function* streamJSON(response: Response): AsyncGenerator<any> {
+  let buffer = "";
+  for await (const value of streamResponse(response)) {
+    buffer += value;
+
+    let position;
+    while ((position = buffer.indexOf("\n")) >= 0) {
+      const line = buffer.slice(0, position);
+      const data = JSON.parse(line);
+      yield data;
+      buffer = buffer.slice(position + 1);
+    }
+  }
+}

package/tsconfig.json

@@ -0,0 +1,108 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig to read more about this file */
+
+    /* Projects */
+    // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
+    // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
+    // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
+    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
+    // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
+    // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
+
+    /* Language and Environment */
+    "target": "es2016",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
+    // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
+    // "jsx": "preserve",                                /* Specify what JSX code is generated. */
+    // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
+    // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
+    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
+    // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
+    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
+    // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
+    // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
+    // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
+    // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
+
+    /* Modules */
+    "module": "commonjs",                                /* Specify what module code is generated. */
+    // "rootDir": "./",                                  /* Specify the root folder within your source files. */
+    // "moduleResolution": "node10",                     /* Specify how TypeScript looks up a file from a given module specifier. */
+    // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
+    // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
+    // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
+    // "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */
+    // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
+    // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
+    // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
+    // "allowImportingTsExtensions": true,               /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
+    // "resolvePackageJsonExports": true,                /* Use the package.json 'exports' field when resolving package imports. */
+    // "resolvePackageJsonImports": true,                /* Use the package.json 'imports' field when resolving imports. */
+    // "customConditions": [],                           /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
+    // "resolveJsonModule": true,                        /* Enable importing .json files. */
+    // "allowArbitraryExtensions": true,                 /* Enable importing files with any extension, provided a declaration file is present. */
+    // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
+
+    /* JavaScript Support */
+    // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
+    // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
+    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
+
+    /* Emit */
+    // "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
+    // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
+    // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
+    // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
+    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
+    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
+    // "outDir": "./",                                   /* Specify an output folder for all emitted files. */
+    // "removeComments": true,                           /* Disable emitting comments. */
+    // "noEmit": true,                                   /* Disable emitting files from a compilation. */
+    // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
+    // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
+    // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
+    // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
+    // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
+    // "newLine": "crlf",                                /* Set the newline character for emitting files. */
+    // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
+    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
+    // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
+    // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
+    // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
+
+    /* Interop Constraints */
+    // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
+    // "verbatimModuleSyntax": true,                     /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
+    // "isolatedDeclarations": true,                     /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */
+    // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
+    "esModuleInterop": true,                             /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
+    // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
+    "forceConsistentCasingInFileNames": true,            /* Ensure that casing is correct in imports. */
+
+    /* Type Checking */
+    "strict": true,                                      /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
+    // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
+    // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
+    // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
+    // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
+    // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
+    // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
+    // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
+    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
+    // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
+    // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
+    // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
+    // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
+    // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
+    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
+    // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
+    // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
+
+    /* Completeness */
+    // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
+    "skipLibCheck": true                                 /* Skip type checking all .d.ts files. */
+  }
+}