getraw 0.1.0

package/tests/unit/networking/client.test.ts

@@ -0,0 +1,272 @@
+import { describe, expect, test, mock, beforeEach, afterEach } from "bun:test";
+import { HttpClient } from "../../../src/networking/client";
+import { CookieJar } from "../../../src/networking/cookies";
+import { USER_AGENTS } from "../../../src/networking/user-agents";
+
+function mockFetchResponse(
+  body: string,
+  status = 200,
+  headers: Record<string, string> = {},
+): Response {
+  return new Response(body, {
+    status,
+    headers: { "content-type": "text/plain", ...headers },
+  });
+}
+
+describe("HttpClient - User-Agent rotation", () => {
+  test("uses configured user agent by default", async () => {
+    const capturedHeaders: Record<string, string>[] = [];
+
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = mock(async (_url: string | URL | Request, init?: RequestInit) => {
+      const hdrs = init?.headers as Record<string, string> | undefined;
+      if (hdrs) capturedHeaders.push(hdrs);
+      return mockFetchResponse("ok");
+    });
+
+    try {
+      const client = new HttpClient({ userAgent: "TestAgent/1.0", timeout: 5000 });
+      await client.get("http://example.com/");
+      expect(capturedHeaders[0]?.["User-Agent"]).toBe("TestAgent/1.0");
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+
+  test("rotates user agents when rotateUserAgent is true", async () => {
+    const capturedUAs: string[] = [];
+
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = mock(async (_url: string | URL | Request, init?: RequestInit) => {
+      const hdrs = init?.headers as Record<string, string> | undefined;
+      if (hdrs?.["User-Agent"]) capturedUAs.push(hdrs["User-Agent"]);
+      return mockFetchResponse("ok");
+    });
+
+    try {
+      const client = new HttpClient({ rotateUserAgent: true, timeout: 5000 });
+      for (let i = 0; i < 5; i++) {
+        await client.get(`http://example.com/${i}`);
+      }
+      const known = USER_AGENTS.map((e) => e.ua);
+      expect(capturedUAs.every((ua) => known.includes(ua))).toBe(true);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+});
+
+describe("HttpClient - Retry logic", () => {
+  test("retries on 503 status", async () => {
+    let callCount = 0;
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = mock(async () => {
+      callCount++;
+      if (callCount < 3) return mockFetchResponse("error", 503);
+      return mockFetchResponse("ok", 200);
+    });
+
+    try {
+      const client = new HttpClient({ maxRetries: 3, retryDelay: 1, timeout: 5000 });
+      const res = await client.get("http://example.com/");
+      expect(res.ok).toBe(true);
+      expect(callCount).toBe(3);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+
+  test("retries on 429 status", async () => {
+    let callCount = 0;
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = mock(async () => {
+      callCount++;
+      if (callCount < 2) return mockFetchResponse("rate limited", 429);
+      return mockFetchResponse("ok", 200);
+    });
+
+    try {
+      const client = new HttpClient({ maxRetries: 3, retryDelay: 1, timeout: 5000 });
+      const res = await client.get("http://example.com/");
+      expect(res.ok).toBe(true);
+      expect(callCount).toBe(2);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+
+  test("returns last error response after exhausting retries", async () => {
+    let callCount = 0;
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = mock(async () => {
+      callCount++;
+      return mockFetchResponse("error", 503);
+    });
+
+    try {
+      const client = new HttpClient({ maxRetries: 2, retryDelay: 1, timeout: 5000 });
+      const res = await client.get("http://example.com/");
+      expect(res.status).toBe(503);
+      expect(res.ok).toBe(false);
+      expect(callCount).toBe(3);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+
+  test("does not retry on 404", async () => {
+    let callCount = 0;
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = mock(async () => {
+      callCount++;
+      return mockFetchResponse("not found", 404);
+    });
+
+    try {
+      const client = new HttpClient({ maxRetries: 3, retryDelay: 1, timeout: 5000 });
+      const res = await client.get("http://example.com/");
+      expect(res.status).toBe(404);
+      expect(callCount).toBe(1);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+});
+
+describe("HttpClient - Rate limiting", () => {
+  test("enforces delay between requests", async () => {
+    const timestamps: number[] = [];
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = mock(async () => {
+      timestamps.push(Date.now());
+      return mockFetchResponse("ok");
+    });
+
+    try {
+      const client = new HttpClient({ requestsPerSecond: 5, timeout: 5000 });
+      await client.get("http://example.com/1");
+      await client.get("http://example.com/2");
+      await client.get("http://example.com/3");
+
+      const minInterval = 1000 / 5;
+      for (let i = 1; i < timestamps.length; i++) {
+        const gap = timestamps[i] - timestamps[i - 1];
+        expect(gap).toBeGreaterThanOrEqual(minInterval - 5);
+      }
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+
+  test("no delay when rate limiting is disabled", async () => {
+    const timestamps: number[] = [];
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = mock(async () => {
+      timestamps.push(Date.now());
+      return mockFetchResponse("ok");
+    });
+
+    try {
+      const client = new HttpClient({ requestsPerSecond: 0, timeout: 5000 });
+      await client.get("http://example.com/1");
+      await client.get("http://example.com/2");
+      const gap = timestamps[1] - timestamps[0];
+      expect(gap).toBeLessThan(100);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+});
+
+describe("HttpClient - Cookie jar", () => {
+  test("sends Cookie header when jar has matching cookies", async () => {
+    const capturedHeaders: Record<string, string>[] = [];
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = mock(async (_url: string | URL | Request, init?: RequestInit) => {
+      const hdrs = init?.headers as Record<string, string> | undefined;
+      if (hdrs) capturedHeaders.push(hdrs);
+      return mockFetchResponse("ok");
+    });
+
+    try {
+      const jar = new CookieJar();
+      jar.set({
+        domain: "example.com",
+        includeSubdomains: false,
+        path: "/",
+        secure: false,
+        expires: Math.floor(Date.now() / 1000) + 86400,
+        name: "auth",
+        value: "token123",
+      });
+      const client = new HttpClient({ cookieJar: jar, timeout: 5000 });
+      await client.get("http://example.com/");
+      expect(capturedHeaders[0]?.["Cookie"]).toContain("auth=token123");
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+});
+
+describe("HttpClient - Caching", () => {
+  test("caches GET responses when cacheTtl > 0", async () => {
+    let callCount = 0;
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = mock(async () => {
+      callCount++;
+      return mockFetchResponse("cached body");
+    });
+
+    try {
+      const client = new HttpClient({ cacheTtl: 60, timeout: 5000 });
+      const r1 = await client.get("http://example.com/cached");
+      const r2 = await client.get("http://example.com/cached");
+      expect(callCount).toBe(1);
+      expect(r1.body).toBe(r2.body);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+
+  test("bypasses cache with noCache option", async () => {
+    let callCount = 0;
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = mock(async () => {
+      callCount++;
+      return mockFetchResponse("body");
+    });
+
+    try {
+      const client = new HttpClient({ cacheTtl: 60, timeout: 5000 });
+      await client.get("http://example.com/nocache", { noCache: true });
+      await client.get("http://example.com/nocache", { noCache: true });
+      expect(callCount).toBe(2);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+});
+
+describe("HttpClient - Referer", () => {
+  test("sets Referer header from options", async () => {
+    const capturedHeaders: Record<string, string>[] = [];
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = mock(async (_url: string | URL | Request, init?: RequestInit) => {
+      const hdrs = init?.headers as Record<string, string> | undefined;
+      if (hdrs) capturedHeaders.push(hdrs);
+      return mockFetchResponse("ok");
+    });
+
+    try {
+      const client = new HttpClient({
+        referer: "https://referrer.example.com/",
+        timeout: 5000,
+      });
+      await client.get("http://example.com/");
+      expect(capturedHeaders[0]?.["Referer"]).toBe("https://referrer.example.com/");
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+});

package/tests/unit/networking/cookies.test.ts

@@ -0,0 +1,256 @@
+import { describe, expect, test, beforeEach } from "bun:test";
+import {
+  parseNetscapeCookieFile,
+  serializeNetscapeCookieFile,
+  CookieJar,
+  type Cookie,
+} from "../../../src/networking/cookies";
+
+const NETSCAPE_FILE = `# Netscape HTTP Cookie File
+# This is a comment
+.example.com	TRUE	/	FALSE	1893456000	session	abc123
+.youtube.com	TRUE	/	TRUE	1893456000	VISITOR_INFO1_LIVE	xyz789
+.example.com	TRUE	/path	FALSE	0	tracker	val1
+`;
+
+describe("parseNetscapeCookieFile", () => {
+  test("parses valid Netscape cookie file", () => {
+    const cookies = parseNetscapeCookieFile(NETSCAPE_FILE);
+    expect(cookies).toHaveLength(3);
+  });
+
+  test("parses domain correctly", () => {
+    const cookies = parseNetscapeCookieFile(NETSCAPE_FILE);
+    expect(cookies[0].domain).toBe(".example.com");
+    expect(cookies[1].domain).toBe(".youtube.com");
+  });
+
+  test("parses includeSubdomains flag", () => {
+    const cookies = parseNetscapeCookieFile(NETSCAPE_FILE);
+    expect(cookies[0].includeSubdomains).toBe(true);
+    expect(cookies[1].includeSubdomains).toBe(true);
+  });
+
+  test("parses secure flag", () => {
+    const cookies = parseNetscapeCookieFile(NETSCAPE_FILE);
+    expect(cookies[0].secure).toBe(false);
+    expect(cookies[1].secure).toBe(true);
+  });
+
+  test("parses expiry", () => {
+    const cookies = parseNetscapeCookieFile(NETSCAPE_FILE);
+    expect(cookies[0].expires).toBe(1893456000);
+    expect(cookies[2].expires).toBe(0);
+  });
+
+  test("parses name and value", () => {
+    const cookies = parseNetscapeCookieFile(NETSCAPE_FILE);
+    expect(cookies[0].name).toBe("session");
+    expect(cookies[0].value).toBe("abc123");
+    expect(cookies[1].name).toBe("VISITOR_INFO1_LIVE");
+    expect(cookies[1].value).toBe("xyz789");
+  });
+
+  test("skips comment lines", () => {
+    const cookies = parseNetscapeCookieFile(NETSCAPE_FILE);
+    expect(cookies.every((c) => !c.name.startsWith("#"))).toBe(true);
+  });
+
+  test("skips empty lines", () => {
+    const cookies = parseNetscapeCookieFile("\n\n\n");
+    expect(cookies).toHaveLength(0);
+  });
+
+  test("skips malformed lines", () => {
+    const bad = "not\tenough\tfields\n";
+    const cookies = parseNetscapeCookieFile(bad);
+    expect(cookies).toHaveLength(0);
+  });
+});
+
+describe("serializeNetscapeCookieFile", () => {
+  test("round-trips cookies", () => {
+    const original = parseNetscapeCookieFile(NETSCAPE_FILE);
+    const serialized = serializeNetscapeCookieFile(original);
+    const reparsed = parseNetscapeCookieFile(serialized);
+    expect(reparsed).toHaveLength(original.length);
+    for (let i = 0; i < original.length; i++) {
+      expect(reparsed[i].name).toBe(original[i].name);
+      expect(reparsed[i].value).toBe(original[i].value);
+      expect(reparsed[i].domain).toBe(original[i].domain);
+    }
+  });
+
+  test("includes header comment", () => {
+    const serialized = serializeNetscapeCookieFile([]);
+    expect(serialized).toContain("# Netscape HTTP Cookie File");
+  });
+});
+
+describe("CookieJar", () => {
+  let jar: CookieJar;
+  const futureExpiry = Math.floor(Date.now() / 1000) + 86400;
+  const pastExpiry = Math.floor(Date.now() / 1000) - 1;
+
+  beforeEach(() => {
+    jar = new CookieJar();
+  });
+
+  test("load parses netscape file", () => {
+    jar.load(NETSCAPE_FILE);
+    expect(jar.size()).toBe(3);
+  });
+
+  test("set and get cookie", () => {
+    const cookie: Cookie = {
+      domain: "example.com",
+      includeSubdomains: false,
+      path: "/",
+      secure: false,
+      expires: futureExpiry,
+      name: "test",
+      value: "hello",
+    };
+    jar.set(cookie);
+    const retrieved = jar.get("example.com", "/", "test");
+    expect(retrieved).toBeDefined();
+    expect(retrieved?.value).toBe("hello");
+  });
+
+  test("get returns undefined for expired cookies", () => {
+    const cookie: Cookie = {
+      domain: "example.com",
+      includeSubdomains: false,
+      path: "/",
+      secure: false,
+      expires: pastExpiry,
+      name: "expired",
+      value: "yes",
+    };
+    jar.set(cookie);
+    const retrieved = jar.get("example.com", "/", "expired");
+    expect(retrieved).toBeUndefined();
+  });
+
+  test("set overwrites existing cookie with same key", () => {
+    const cookie: Cookie = {
+      domain: "example.com",
+      includeSubdomains: false,
+      path: "/",
+      secure: false,
+      expires: futureExpiry,
+      name: "mykey",
+      value: "v1",
+    };
+    jar.set(cookie);
+    jar.set({ ...cookie, value: "v2" });
+    expect(jar.size()).toBe(1);
+    expect(jar.get("example.com", "/", "mykey")?.value).toBe("v2");
+  });
+
+  test("getForUrl matches by host", () => {
+    jar.load(NETSCAPE_FILE);
+    const cookies = jar.getForUrl("https://www.example.com/");
+    expect(cookies.some((c) => c.name === "session")).toBe(true);
+  });
+
+  test("getForUrl excludes secure cookies on http", () => {
+    jar.set({
+      domain: "example.com",
+      includeSubdomains: false,
+      path: "/",
+      secure: true,
+      expires: futureExpiry,
+      name: "secureOnly",
+      value: "s",
+    });
+    const httpCookies = jar.getForUrl("http://example.com/");
+    expect(httpCookies.some((c) => c.name === "secureOnly")).toBe(false);
+    const httpsCookies = jar.getForUrl("https://example.com/");
+    expect(httpsCookies.some((c) => c.name === "secureOnly")).toBe(true);
+  });
+
+  test("getForUrl filters by path", () => {
+    jar.set({
+      domain: "example.com",
+      includeSubdomains: false,
+      path: "/api",
+      secure: false,
+      expires: futureExpiry,
+      name: "apiOnly",
+      value: "a",
+    });
+    const root = jar.getForUrl("http://example.com/");
+    expect(root.some((c) => c.name === "apiOnly")).toBe(false);
+    const api = jar.getForUrl("http://example.com/api/v1");
+    expect(api.some((c) => c.name === "apiOnly")).toBe(true);
+  });
+
+  test("getCookieHeader returns key=value pairs", () => {
+    jar.set({
+      domain: "example.com",
+      includeSubdomains: false,
+      path: "/",
+      secure: false,
+      expires: futureExpiry,
+      name: "a",
+      value: "1",
+    });
+    jar.set({
+      domain: "example.com",
+      includeSubdomains: false,
+      path: "/",
+      secure: false,
+      expires: futureExpiry,
+      name: "b",
+      value: "2",
+    });
+    const header = jar.getCookieHeader("http://example.com/");
+    expect(header).toContain("a=1");
+    expect(header).toContain("b=2");
+  });
+
+  test("removeExpired clears expired cookies", () => {
+    jar.set({
+      domain: "example.com",
+      includeSubdomains: false,
+      path: "/",
+      secure: false,
+      expires: pastExpiry,
+      name: "dead",
+      value: "x",
+    });
+    jar.set({
+      domain: "example.com",
+      includeSubdomains: false,
+      path: "/",
+      secure: false,
+      expires: futureExpiry,
+      name: "alive",
+      value: "y",
+    });
+    expect(jar.size()).toBe(2);
+    jar.removeExpired();
+    expect(jar.size()).toBe(1);
+  });
+
+  test("clear empties the jar", () => {
+    jar.load(NETSCAPE_FILE);
+    jar.clear();
+    expect(jar.size()).toBe(0);
+  });
+
+  test("session cookies (expires=0) never expire", () => {
+    jar.set({
+      domain: "example.com",
+      includeSubdomains: false,
+      path: "/",
+      secure: false,
+      expires: 0,
+      name: "session",
+      value: "s",
+    });
+    const retrieved = jar.get("example.com", "/", "session");
+    expect(retrieved).toBeDefined();
+  });
+});

package/tests/unit/networking/proxy.test.ts

@@ -0,0 +1,137 @@
+import { describe, expect, test } from "bun:test";
+import {
+  parseProxyUrl,
+  buildProxyAuthHeader,
+  createProxyAgent,
+  buildConnectRequest,
+  ProxyParseError,
+} from "../../../src/networking/proxy";
+
+describe("parseProxyUrl", () => {
+  test("parses http proxy", () => {
+    const proxy = parseProxyUrl("http://proxy.example.com:8080");
+    expect(proxy.protocol).toBe("http");
+    expect(proxy.host).toBe("proxy.example.com");
+    expect(proxy.port).toBe(8080);
+    expect(proxy.username).toBeUndefined();
+    expect(proxy.password).toBeUndefined();
+  });
+
+  test("parses https proxy", () => {
+    const proxy = parseProxyUrl("https://proxy.example.com:443");
+    expect(proxy.protocol).toBe("https");
+    expect(proxy.port).toBe(443);
+  });
+
+  test("parses socks5 proxy", () => {
+    const proxy = parseProxyUrl("socks5://127.0.0.1:1080");
+    expect(proxy.protocol).toBe("socks5");
+    expect(proxy.host).toBe("127.0.0.1");
+    expect(proxy.port).toBe(1080);
+  });
+
+  test("parses proxy with username and password", () => {
+    const proxy = parseProxyUrl("http://user:pass@proxy.example.com:8080");
+    expect(proxy.username).toBe("user");
+    expect(proxy.password).toBe("pass");
+  });
+
+  test("parses proxy with url-encoded credentials", () => {
+    const proxy = parseProxyUrl("http://my%40user:p%40ss@proxy.example.com:8080");
+    expect(proxy.username).toBe("my@user");
+    expect(proxy.password).toBe("p@ss");
+  });
+
+  test("defaults port for socks5 when omitted", () => {
+    const proxy = parseProxyUrl("socks5://127.0.0.1");
+    expect(proxy.port).toBe(1080);
+  });
+
+  test("throws ProxyParseError for invalid URL", () => {
+    expect(() => parseProxyUrl("not a url")).toThrow(ProxyParseError);
+  });
+
+  test("throws ProxyParseError for unsupported protocol", () => {
+    expect(() => parseProxyUrl("ftp://proxy.example.com:21")).toThrow(ProxyParseError);
+  });
+});
+
+describe("buildProxyAuthHeader", () => {
+  test("returns Basic auth header when credentials present", () => {
+    const proxy = parseProxyUrl("http://user:pass@proxy.example.com:8080");
+    const header = buildProxyAuthHeader(proxy);
+    expect(header).toBe(`Basic ${btoa("user:pass")}`);
+  });
+
+  test("returns undefined when no credentials", () => {
+    const proxy = parseProxyUrl("http://proxy.example.com:8080");
+    const header = buildProxyAuthHeader(proxy);
+    expect(header).toBeUndefined();
+  });
+
+  test("handles password-less username", () => {
+    const proxy = parseProxyUrl("http://user@proxy.example.com:8080");
+    const header = buildProxyAuthHeader(proxy);
+    expect(header).toBe(`Basic ${btoa("user:")}`);
+  });
+});
+
+describe("createProxyAgent", () => {
+  test("returns agent with parsed proxy", () => {
+    const agent = createProxyAgent("http://proxy.example.com:8080");
+    expect(agent.proxy.host).toBe("proxy.example.com");
+  });
+
+  test("getProxyUrl reconstructs proxy URL without credentials", () => {
+    const agent = createProxyAgent("http://proxy.example.com:8080");
+    expect(agent.getProxyUrl()).toBe("http://proxy.example.com:8080");
+  });
+
+  test("getProxyUrl includes credentials when present", () => {
+    const agent = createProxyAgent("http://user:pass@proxy.example.com:8080");
+    expect(agent.getProxyUrl()).toContain("user");
+  });
+
+  test("getAuthHeader returns header when credentials present", () => {
+    const agent = createProxyAgent("http://user:pass@proxy.example.com:8080");
+    expect(agent.getAuthHeader()).toBeDefined();
+  });
+
+  test("getAuthHeader returns undefined when no credentials", () => {
+    const agent = createProxyAgent("http://proxy.example.com:8080");
+    expect(agent.getAuthHeader()).toBeUndefined();
+  });
+});
+
+describe("buildConnectRequest", () => {
+  test("builds valid CONNECT request", () => {
+    const proxy = parseProxyUrl("http://proxy.example.com:8080");
+    const req = buildConnectRequest({
+      targetHost: "api.example.com",
+      targetPort: 443,
+      proxy,
+    });
+    expect(req).toContain("CONNECT api.example.com:443 HTTP/1.1");
+    expect(req).toContain("Host: api.example.com:443");
+  });
+
+  test("includes Proxy-Authorization when credentials present", () => {
+    const proxy = parseProxyUrl("http://user:pass@proxy.example.com:8080");
+    const req = buildConnectRequest({
+      targetHost: "api.example.com",
+      targetPort: 443,
+      proxy,
+    });
+    expect(req).toContain("Proxy-Authorization: Basic");
+  });
+
+  test("omits Proxy-Authorization when no credentials", () => {
+    const proxy = parseProxyUrl("http://proxy.example.com:8080");
+    const req = buildConnectRequest({
+      targetHost: "api.example.com",
+      targetPort: 443,
+      proxy,
+    });
+    expect(req).not.toContain("Proxy-Authorization");
+  });
+});