@caido-utils/backend 1.2.0 → 1.5.0

package/README.md

@@ -0,0 +1,225 @@
+# @caido-utils/backend
+
+Backend utilities for Caido plugins.
+
+## Installation
+
+```bash
+pnpm add @caido-utils/backend
+```
+
+**Peer dependency:** `@caido/sdk-backend`
+
+```ts
+import { queryRequests, resolveFilterQuery, Queue } from "@caido-utils/backend";
+```
+
+---
+
+## Requests
+
+### queryRequests
+
+Paginated query over all requests with optional filtering, deduplication, and scope checking.
+
+```ts
+const items = await queryRequests(sdk, {
+  filter: 'req.host.cont:"example.com"',
+  excludeStaticAssets: true,
+  deduplicate: true,
+  inScope: true,
+  onPage: (pageItems) => console.log(`Got ${pageItems.length} items`),
+});
+```
+
+#### Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `filter` | `string` | - | HTTPQL filter string |
+| `pageSize` | `number` | `1000` | Items per page |
+| `excludeStaticAssets` | `boolean` | `false` | Filter out images, fonts, JS, CSS, media |
+| `deduplicate` | `boolean` | `false` | Keep last occurrence per `method\|host\|path` |
+| `inScope` | `boolean` | `false` | Post-filter using `sdk.requests.inScope()` |
+| `onPage` | `(items[]) => void` | - | Callback after each page |
+
+Always applies base filter `resp.roundtrip.gt:0` (only requests with responses).
+
+---
+
+### Fetch
+
+Concurrent HTTP request sender with configurable parallelism and delay.
+
+```ts
+const fetcher = new Fetch(sdk, { threads: 5, delay: 100 });
+const response = await fetcher.send(requestSpec);
+```
+
+#### Constructor Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `threads` | `number` | `10` | Max concurrent requests |
+| `delay` | `number` | `0` | Delay in ms after each request completes |
+
+#### Methods
+
+| Method | Signature | Description |
+|--------|-----------|-------------|
+| `send` | `(request: RequestSpec \| RequestSpecRaw) => Promise<RequestResponse>` | Queue and send a request |
+
+---
+
+## Filter
+
+### resolveFilterQuery
+
+Resolves custom `filter:` tokens into real HTTPQL.
+
+```ts
+const resolved = resolveFilterQuery('filter:1hr AND req.host.cont:"example.com"');
+// => 'req.created_at.gt:"2026-02-16 15:00:00" AND req.host.cont:"example.com"'
+```
+
+#### Token Mapping
+
+| Token | Expands To |
+|-------|------------|
+| `filter:inscope` | `preset:inscope` |
+| `filter:recent` | `req.created_at.gt:"<15 minutes ago>"` |
+| `filter:1hr` | `req.created_at.gt:"<1 hour ago>"` |
+| `filter:6hr` | `req.created_at.gt:"<6 hours ago>"` |
+| `filter:12hr` | `req.created_at.gt:"<12 hours ago>"` |
+| `filter:24hr` | `req.created_at.gt:"<24 hours ago>"` |
+| `filter:<unknown>` | Removed |
+
+Timestamps use local time in `YYYY-MM-DD HH:MM:SS` format.
+
+---
+
+## Pool
+
+### Queue
+
+Concurrent async task queue with pause/resume/stop.
+
+```ts
+const queue = new Queue<string>(
+  { concurrency: 5, delayMs: 100 },
+  async (url) => {
+    await processUrl(url);
+  },
+  {
+    onError: (url, error) => console.error(`Failed: ${url}`, error),
+  },
+);
+
+queue.enqueue(["https://a.com", "https://b.com"]);
+queue.pause();
+queue.resume();
+queue.stop();
+```
+
+#### Config
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `concurrency` | `number` | required | Max parallel workers |
+| `delayMs` | `number` | `0` | Delay between items per worker |
+
+#### Callbacks
+
+| Callback | Signature | Description |
+|----------|-----------|-------------|
+| `onError` | `(item: T, error: string) => void` | Called when a task throws |
+
+#### Methods
+
+| Method | Signature | Description |
+|--------|-----------|-------------|
+| `enqueue` | `(items: T \| T[]) => void` | Add items to the queue |
+| `pause` | `() => void` | Pause processing |
+| `resume` | `() => void` | Resume processing |
+| `stop` | `() => void` | Stop and clear queue |
+| `isRunning` | `() => boolean` | Check if queue is active |
+| `isPaused` | `() => boolean` | Check if paused |
+| `setConcurrency` | `(n: number) => void` | Change max workers at runtime |
+| `pending` | `number` | Items waiting |
+| `completed` | `number` | Items processed |
+
+---
+
+## Crypto
+
+### sha256
+
+```ts
+const hash = sha256("content");
+// => "ed7002b439e9ac845f22357d822bac..."
+```
+
+Returns hex-encoded SHA-256 hash.
+
+---
+
+## Process
+
+### spawnCommand
+
+Run a binary bundled in the plugin's assets directory.
+
+```ts
+const output = await spawnCommand(sdk, "my-tool", ["--flag", "value"], stdinData);
+```
+
+| Arg | Type | Description |
+|-----|------|-------------|
+| `sdk` | `SDK` | Backend SDK instance |
+| `binary` | `string` | Filename in `sdk.meta.assetsPath()` |
+| `args` | `string[]` | Command-line arguments |
+| `stdin` | `string` | Optional stdin input |
+
+Returns stdout as a string. Throws on non-zero exit code. Automatically sets executable permissions if needed.
+
+---
+
+## Project
+
+### getProjectId
+
+```ts
+const projectId = await getProjectId(sdk);
+```
+
+Returns the current Caido project ID. Throws `"Project not found"` if no project is active.
+
+---
+
+## Storage
+
+File I/O within the plugin's data directory.
+
+### storeFile
+
+```ts
+const filePath = await storeFile(sdk, {
+  data: responseBody,
+  contentType: "application/json", // determines file extension
+});
+```
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `data` | `string` | File content |
+| `contentType` | `string` | Optional — determines file extension |
+
+Returns the absolute file path. Supported extensions: `.js`, `.html`, `.json`, `.xml`, `.css`, `.txt`, `.ts`.
+
+### readFile / deleteFile / fileExists
+
+```ts
+const content = await readFile(filePath);
+await deleteFile(filePath);
+const exists = await fileExists(filePath);
+```

package/dist/filter/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Resolves custom `filter:` tokens in a query into real HTTPQL.
+ * e.g. `filter:1hr` → `req.created_at.gt:"2026-02-17 14:00:00"`
+ *      `filter:inscope` → `preset:inscope`
+ */
+export declare function resolveFilterQuery(query: string): string;

package/dist/filter/index.js

@@ -0,0 +1,29 @@
+const TIME_FILTERS = {
+  recent: 15 * 60 * 1e3,
+  "1hr": 60 * 60 * 1e3,
+  "6hr": 6 * 60 * 60 * 1e3,
+  "12hr": 12 * 60 * 60 * 1e3,
+  "24hr": 24 * 60 * 60 * 1e3
+};
+function formatDateTime(date) {
+  const y = date.getFullYear();
+  const mo = String(date.getMonth() + 1).padStart(2, "0");
+  const d = String(date.getDate()).padStart(2, "0");
+  const h = String(date.getHours()).padStart(2, "0");
+  const mi = String(date.getMinutes()).padStart(2, "0");
+  const s = String(date.getSeconds()).padStart(2, "0");
+  return `${y}-${mo}-${d} ${h}:${mi}:${s}`;
+}
+export function resolveFilterQuery(query) {
+  return query.replace(/filter:(\w+)/g, (_, value) => {
+    const ms = TIME_FILTERS[value];
+    if (ms !== void 0) {
+      const cutoff = new Date(Date.now() - ms);
+      return `req.created_at.gt:"${formatDateTime(cutoff)}"`;
+    }
+    if (value === "inscope") {
+      return "preset:inscope";
+    }
+    return "";
+  }).replace(/\s{2,}/g, " ").trim();
+}

package/dist/http/index.d.ts

@@ -0,0 +1 @@
+export { HttpRequest, type HttpRequestOptions } from "./request";

package/dist/http/index.js

@@ -0,0 +1 @@
+export { HttpRequest } from "./request.js";

package/dist/http/request.d.ts

@@ -0,0 +1,16 @@
+import type { SDK } from "caido:plugin";
+export type HttpRequestOptions = {
+    url: string;
+    method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH" | "HEAD" | "OPTIONS";
+    headers?: Record<string, string> | Array<{
+        name: string;
+        value: string;
+    }>;
+    body?: string;
+};
+export declare class HttpRequest {
+    private sdk;
+    private maxRedirects;
+    constructor(sdk: SDK, maxRedirects?: number);
+    send(options: HttpRequestOptions): Promise<any>;
+}

package/dist/http/request.js

@@ -0,0 +1,77 @@
+import { RequestSpec } from "caido:utils";
+import parse from "url-parse";
+const DEFAULT_HEADERS = {
+  "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
+  "Sec-Fetch-Site": "same-origin",
+  "Sec-Fetch-Mode": "navigate",
+  "Sec-Fetch-User": "?1",
+  "Sec-Fetch-Dest": "document"
+};
+function normalizeHeaders(headers) {
+  if (headers === void 0) {
+    return {};
+  }
+  if (Array.isArray(headers)) {
+    return headers.reduce(
+      (acc, { name, value }) => {
+        acc[name] = value;
+        return acc;
+      },
+      {}
+    );
+  }
+  return headers;
+}
+function resolveRedirectUrl(currentUrl, location) {
+  if (location.startsWith("http://") || location.startsWith("https://")) {
+    return location;
+  }
+  const parsed = parse(currentUrl);
+  if (location.startsWith("/")) {
+    return `${parsed.protocol}//${parsed.host}${location}`;
+  }
+  const basePath = parsed.pathname.substring(
+    0,
+    parsed.pathname.lastIndexOf("/") + 1
+  );
+  return `${parsed.protocol}//${parsed.host}${basePath}${location}`;
+}
+export class HttpRequest {
+  sdk;
+  maxRedirects;
+  constructor(sdk, maxRedirects = 5) {
+    this.sdk = sdk;
+    this.maxRedirects = maxRedirects;
+  }
+  async send(options) {
+    let currentUrl = options.url;
+    let redirectCount = 0;
+    while (redirectCount <= this.maxRedirects) {
+      const spec = new RequestSpec(currentUrl);
+      if (options.method !== void 0) {
+        spec.setMethod(options.method);
+      }
+      const normalizedHeaders = normalizeHeaders(options.headers);
+      const headers = { ...DEFAULT_HEADERS, ...normalizedHeaders };
+      for (const [name, value] of Object.entries(headers)) {
+        spec.setHeader(name, value);
+      }
+      if (options.body !== void 0) {
+        spec.setBody(options.body);
+      }
+      const result = await this.sdk.requests.send(spec);
+      const statusCode = result.response.getCode();
+      if ([301, 302, 303, 307, 308].includes(statusCode)) {
+        const locationHeader = result.response.getHeader("Location");
+        if (!locationHeader) {
+          return result;
+        }
+        currentUrl = resolveRedirectUrl(currentUrl, locationHeader.toString());
+        redirectCount++;
+        continue;
+      }
+      return result;
+    }
+    throw new Error(`Too many redirects (max: ${this.maxRedirects})`);
+  }
+}

package/dist/index.d.ts

@@ -1,7 +1,9 @@
+export { spawnCommand } from "@caido-utils/shared";
 export { sha256 } from "./crypto";
+export { resolveFilterQuery } from "./filter";
+export { HttpRequest, type HttpRequestOptions } from "./http";
 export { Queue } from "./pool";
 export type { PoolCallbacks, PoolConfig } from "./pool";
-export { spawnCommand } from "./process";
 export { getProjectId } from "./project";
-export { Fetch, queryRequests, type FetchOptions, type QueryRequestsOptions, } from "./requests";
+export { Fetch, filterAuthHeaders, queryRequests, type FetchOptions, type QueryRequestsOptions, } from "./requests";
 export { deleteFile, fileExists, readFile, storeFile } from "./storage";

package/dist/index.js

@@ -1,9 +1,12 @@
+export { spawnCommand } from "@caido-utils/shared";
 export { sha256 } from "./crypto/index.js";
+export { resolveFilterQuery } from "./filter/index.js";
+export { HttpRequest } from "./http/index.js";
 export { Queue } from "./pool/index.js";
-export { spawnCommand } from "./process/index.js";
 export { getProjectId } from "./project/index.js";
 export {
   Fetch,
+  filterAuthHeaders,
   queryRequests
 } from "./requests/index.js";
 export { deleteFile, fileExists, readFile, storeFile } from "./storage/index.js";

package/dist/requests/headers.d.ts

@@ -0,0 +1,6 @@
+type Header = {
+    name: string;
+    value: string;
+};
+export declare function filterAuthHeaders(headers: Header[]): string[];
+export {};

package/dist/requests/headers.js

@@ -0,0 +1,17 @@
+const AUTH_HEADERS = [
+  "authorization",
+  "x-api-key",
+  "x-auth-token",
+  "x-access-token",
+  "x-csrf-token",
+  "x-xsrf-token"
+];
+export function filterAuthHeaders(headers) {
+  const result = [];
+  for (const { name, value } of headers) {
+    if (AUTH_HEADERS.includes(name.toLowerCase())) {
+      result.push(`${name}: ${value}`);
+    }
+  }
+  return result;
+}

package/dist/requests/index.d.ts

@@ -1,2 +1,3 @@
 export { Fetch, type FetchOptions } from "./fetcher";
+export { filterAuthHeaders } from "./headers";
 export { queryRequests, type QueryRequestsOptions } from "./query";

package/dist/requests/index.js

@@ -1,2 +1,3 @@
 export { Fetch } from "./fetcher.js";
+export { filterAuthHeaders } from "./headers.js";
 export { queryRequests } from "./query.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@caido-utils/backend",
-  "version": "1.2.0",
+  "version": "1.5.0",
   "type": "module",
   "files": [
     "dist"
@@ -14,6 +14,9 @@
   "scripts": {
     "build": "unbuild"
   },
+  "dependencies": {
+    "@caido-utils/shared": "workspace:*"
+  },
   "peerDependencies": {
     "@caido/sdk-backend": ">=0.46.0"
   },

package/dist/process/index.d.ts

@@ -1 +0,0 @@
-export { spawnCommand } from "./spawn";

package/dist/process/index.js

@@ -1 +0,0 @@
-export { spawnCommand } from "./spawn.js";

package/dist/process/spawn.d.ts

@@ -1,2 +0,0 @@
-import type { SDK } from "caido:plugin";
-export declare function spawnCommand(sdk: SDK, binary: string, args: string[], stdin?: string): Promise<string>;

package/dist/process/spawn.js

@@ -1,58 +0,0 @@
-import {
-  spawn as nodeSpawn
-} from "child_process";
-import { access, constants } from "fs/promises";
-import * as path from "path";
-async function ensureBinaryExecutable(binaryPath, sdk) {
-  try {
-    await access(binaryPath, constants.F_OK | constants.X_OK);
-  } catch {
-    try {
-      await makeExecutable(binaryPath);
-    } catch (chmodError) {
-      sdk.console.error(
-        `Failed to make ${binaryPath} executable: ${chmodError}`
-      );
-      throw new Error(`Cannot make binary executable: ${chmodError}`);
-    }
-  }
-}
-async function makeExecutable(binaryPath) {
-  const child = nodeSpawn("chmod", ["+x", binaryPath], {
-    shell: true
-  });
-  await driveChild(child);
-}
-export async function spawnCommand(sdk, binary, args, stdin) {
-  try {
-    const binaryPath = path.join(sdk.meta.assetsPath(), binary);
-    await ensureBinaryExecutable(binaryPath, sdk);
-    const child = nodeSpawn(binaryPath, args);
-    if (stdin !== void 0) {
-      child.stdin.write(stdin);
-      child.stdin.end();
-    }
-    const output = await driveChild(child);
-    return output;
-  } catch (error) {
-    sdk.console.error(`Spawn command error (${binary}): ${error}`);
-    throw error;
-  }
-}
-async function driveChild(child, allowExitCode1 = false) {
-  let output = "";
-  child.stdout.on("data", (data) => {
-    output += data.toString();
-  });
-  let error = "";
-  child.stderr.on("data", (data) => {
-    error += data.toString();
-  });
-  const exitCode = await new Promise((resolve) => {
-    child.on("close", resolve);
-  });
-  if (exitCode !== void 0 && exitCode !== 0 && !(allowExitCode1 && exitCode === 1)) {
-    throw new Error(`subprocess error exit ${exitCode}, ${error}`);
-  }
-  return output;
-}