@caido-utils/backend 1.6.1 → 1.7.0

package/dist/http/client.d.ts

@@ -0,0 +1,8 @@
+import type { HttpClientOptions, HttpClientRequest, HttpClientResponse, Result } from "./types";
+export declare class HttpClient {
+    private readonly options;
+    private readonly semaphore;
+    constructor(options: HttpClientOptions);
+    send(request: HttpClientRequest): Promise<Result<HttpClientResponse>>;
+    private execute;
+}

package/dist/http/client.js

@@ -0,0 +1,119 @@
+import { RequestSpec } from "caido:utils";
+import { checkSignal, waitForUnpause } from "./signal.js";
+function createSemaphore(limit) {
+  let active = 0;
+  const waiting = [];
+  function acquire() {
+    if (active < limit) {
+      active++;
+      return Promise.resolve();
+    }
+    return new Promise((resolve) => {
+      waiting.push(resolve);
+    });
+  }
+  function release() {
+    const next = waiting.shift();
+    if (next !== void 0) {
+      next();
+    } else {
+      active--;
+    }
+  }
+  return { acquire, release };
+}
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+export class HttpClient {
+  options;
+  semaphore;
+  constructor(options) {
+    this.options = options;
+    this.semaphore = createSemaphore(options.concurrency);
+  }
+  async send(request) {
+    if (this.options.signal !== void 0 && checkSignal(this.options.signal) === "stopped") {
+      return { kind: "Error", error: "Stopped" };
+    }
+    if (this.options.signal !== void 0) {
+      await waitForUnpause(this.options.signal);
+    }
+    if (this.options.signal !== void 0 && checkSignal(this.options.signal) === "stopped") {
+      return { kind: "Error", error: "Stopped" };
+    }
+    await this.semaphore.acquire();
+    try {
+      if (this.options.delayMs > 0) {
+        await delay(this.options.delayMs);
+      }
+      return await this.execute(request);
+    } finally {
+      this.semaphore.release();
+    }
+  }
+  async execute(request, attempt = 0) {
+    try {
+      const spec = new RequestSpec(request.url);
+      spec.setMethod(request.method);
+      const headers = { ...request.headers };
+      if (this.options.userAgents !== void 0 && this.options.userAgents.length > 0 && headers["User-Agent"] === void 0) {
+        const index = Math.floor(
+          Math.random() * this.options.userAgents.length
+        );
+        headers["User-Agent"] = this.options.userAgents[index];
+      }
+      for (const [name, value] of Object.entries(headers)) {
+        spec.setHeader(name, value);
+      }
+      if (request.body !== void 0) {
+        spec.setBody(request.body);
+      }
+      const result = await this.options.sdk.requests.send(spec, {
+        save: true,
+        timeouts: {
+          global: this.options.timeoutMs
+        }
+      });
+      const response = result.response;
+      const status = response.getCode();
+      const caidoId = result.request.getId() !== void 0 ? String(result.request.getId()) : void 0;
+      const contentTypeHeader = response.getHeader("content-type");
+      const responseHeaders = {};
+      if (contentTypeHeader !== void 0 && contentTypeHeader[0] !== void 0) {
+        responseHeaders["content-type"] = contentTypeHeader[0];
+      }
+      const locationHeader = response.getHeader("location");
+      if (locationHeader !== void 0 && locationHeader[0] !== void 0) {
+        responseHeaders["location"] = locationHeader[0];
+      }
+      const bodyBuffer = response.getBody();
+      const body = bodyBuffer !== void 0 ? bodyBuffer.toText() : "";
+      const maxRetries = this.options.retries ?? 0;
+      if (attempt < maxRetries && (status === 429 || status >= 500)) {
+        const retryDelay = this.options.retryDelayMs ?? 1e3;
+        await delay(retryDelay * (attempt + 1));
+        return this.execute(request, attempt + 1);
+      }
+      return {
+        kind: "Ok",
+        value: {
+          status,
+          headers: responseHeaders,
+          body,
+          url: result.request.getUrl() ?? request.url,
+          caidoId
+        }
+      };
+    } catch (error) {
+      const maxRetries = this.options.retries ?? 0;
+      if (attempt < maxRetries) {
+        const retryDelay = this.options.retryDelayMs ?? 1e3;
+        await delay(retryDelay * (attempt + 1));
+        return this.execute(request, attempt + 1);
+      }
+      const message = error instanceof Error ? error.message : "Unknown error";
+      return { kind: "Error", error: message };
+    }
+  }
+}

package/dist/http/index.d.ts

@@ -1 +1,6 @@
+export { HttpClient } from "./client";
 export { HttpRequest, type HttpRequestOptions } from "./request";
+export { checkSignal, waitForUnpause } from "./signal";
+export { getStatusText } from "./status";
+export type { HttpClientOptions, HttpClientRequest, HttpClientResponse, HttpMethod, Result, Signal, } from "./types";
+export { parseUrl } from "./url";

package/dist/http/index.js

@@ -1 +1,5 @@
+export { HttpClient } from "./client.js";
 export { HttpRequest } from "./request.js";
+export { checkSignal, waitForUnpause } from "./signal.js";
+export { getStatusText } from "./status.js";
+export { parseUrl } from "./url.js";

package/dist/http/signal.d.ts

@@ -0,0 +1,3 @@
+import type { Signal } from "./types";
+export declare function checkSignal(signal: Signal): "continue" | "stopped";
+export declare function waitForUnpause(signal: Signal): Promise<void>;

package/dist/http/signal.js

@@ -0,0 +1,20 @@
+const PAUSE_POLL_MS = 200;
+export function checkSignal(signal) {
+  if (signal.stopped) return "stopped";
+  return "continue";
+}
+export function waitForUnpause(signal) {
+  if (!signal.paused || signal.stopped) {
+    return Promise.resolve();
+  }
+  return new Promise((resolve) => {
+    const poll = () => {
+      if (!signal.paused || signal.stopped) {
+        resolve();
+        return;
+      }
+      setTimeout(poll, PAUSE_POLL_MS);
+    };
+    setTimeout(poll, PAUSE_POLL_MS);
+  });
+}

package/dist/http/status.d.ts

@@ -0,0 +1 @@
+export declare function getStatusText(code: string): string;

package/dist/http/status.js

@@ -0,0 +1,22 @@
+const STATUS_TEXTS = {
+  "200": "OK",
+  "201": "Created",
+  "204": "No Content",
+  "301": "Moved Permanently",
+  "302": "Found",
+  "304": "Not Modified",
+  "307": "Temporary Redirect",
+  "308": "Permanent Redirect",
+  "400": "Bad Request",
+  "401": "Unauthorized",
+  "403": "Forbidden",
+  "404": "Not Found",
+  "405": "Method Not Allowed",
+  "500": "Internal Server Error",
+  "501": "Not Implemented",
+  "502": "Bad Gateway",
+  "503": "Service Unavailable"
+};
+export function getStatusText(code) {
+  return STATUS_TEXTS[code] ?? "Unknown";
+}

package/dist/http/types.d.ts

@@ -0,0 +1,36 @@
+import type { SDK } from "caido:plugin";
+export type Signal = {
+    stopped: boolean;
+    paused: boolean;
+};
+export type Result<T> = {
+    kind: "Ok";
+    value: T;
+} | {
+    kind: "Error";
+    error: string;
+};
+export type HttpMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS";
+export type HttpClientOptions = {
+    sdk: SDK;
+    concurrency: number;
+    delayMs: number;
+    timeoutMs: number;
+    signal?: Signal;
+    userAgents?: string[];
+    retries?: number;
+    retryDelayMs?: number;
+};
+export type HttpClientRequest = {
+    url: string;
+    method: HttpMethod;
+    headers?: Record<string, string>;
+    body?: string;
+};
+export type HttpClientResponse = {
+    status: number;
+    headers: Record<string, string>;
+    body: string;
+    url: string;
+    caidoId: string | undefined;
+};

package/dist/http/url.d.ts

@@ -0,0 +1,5 @@
+export declare function parseUrl(url: string): {
+    hostname: string;
+    pathname: string;
+    query: string;
+};

package/dist/http/url.js

@@ -0,0 +1,9 @@
+import parse from "url-parse";
+export function parseUrl(url) {
+  const parsed = parse(url);
+  return {
+    hostname: parsed.hostname,
+    pathname: parsed.pathname,
+    query: parsed.query
+  };
+}

package/dist/index.d.ts

@@ -1,7 +1,7 @@
 export { spawnCommand } from "./process";
 export { sha256 } from "./crypto";
 export { resolveFilterQuery } from "./filter";
-export { HttpRequest, type HttpRequestOptions } from "./http";
+export { HttpClient, HttpRequest, checkSignal, getStatusText, parseUrl, waitForUnpause, type HttpClientOptions, type HttpClientRequest, type HttpClientResponse, type HttpMethod, type HttpRequestOptions, type Result, type Signal, } from "./http";
 export { Queue } from "./pool";
 export type { PoolCallbacks, PoolConfig } from "./pool";
 export { getProjectId } from "./project";

package/dist/index.js

@@ -1,7 +1,14 @@
 export { spawnCommand } from "./process/index.js";
 export { sha256 } from "./crypto/index.js";
 export { resolveFilterQuery } from "./filter/index.js";
-export { HttpRequest } from "./http/index.js";
+export {
+  HttpClient,
+  HttpRequest,
+  checkSignal,
+  getStatusText,
+  parseUrl,
+  waitForUnpause
+} from "./http/index.js";
 export { Queue } from "./pool/index.js";
 export { getProjectId } from "./project/index.js";
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@caido-utils/backend",
-  "version": "1.6.1",
+  "version": "1.7.0",
   "type": "module",
   "files": [
     "dist"