@pensar/apex 2.5.0 → 2.6.1-canary.0cc77dcc

package/src/core/agents/specialized/attackSurface/grpcSchema.ts

@@ -0,0 +1,142 @@
+import { z } from "zod";
+
+// Zod-only leaf module: keeps this shareable between the live tool and
+// `schemas.ts` without a circular import through `whiteboxAttackSurface`.
+
+export const EndpointTransportEnum = z.enum([
+  "http",
+  "grpc",
+  "grpc_web",
+  "connect",
+]);
+
+export const GrpcStreamingTypeEnum = z.enum([
+  "unary",
+  "server_stream",
+  "client_stream",
+  "bidi",
+]);
+
+const GrpcSchemaSourceEnum = z.enum([
+  "proto",
+  "reflection",
+  "protoset",
+  "js_bundle",
+  "gateway_derived",
+  "unknown",
+]);
+
+const GrpcGatewayTypeEnum = z.enum(["rest", "graphql", "connect"]);
+
+export const GrpcEndpointMetadataSchema = z.object({
+  serviceFqn: z
+    .string()
+    .describe("Fully-qualified service name, e.g. 'account.v1.AccountService'"),
+  method: z.string().describe("RPC method name, e.g. 'GetAccount'"),
+  streamingType: GrpcStreamingTypeEnum.default("unary"),
+  reflectionAvailable: z.boolean().optional(),
+  schemaSource: GrpcSchemaSourceEnum.optional(),
+  host: z
+    .string()
+    .nullish()
+    .describe("Authority override; null falls back to the app domain"),
+  frontingGatewayOperation: z
+    .object({
+      gatewayType: GrpcGatewayTypeEnum,
+      operation: z.string(),
+    })
+    .nullish(),
+});
+
+export type EndpointTransport = z.infer<typeof EndpointTransportEnum>;
+export type GrpcEndpointMetadata = z.infer<typeof GrpcEndpointMetadataSchema>;
+
+// ---------------------------------------------------------------------------
+// gRPC pentest context — the slice of gRPC metadata a pentest agent needs to
+// run the gRPC test battery instead of treating the method as an HTTP path.
+// Lives here (leaf module) so it can be shared by the pentest agent, the
+// swarm target shape, tool contexts, and orchestration tools without cycles.
+// ---------------------------------------------------------------------------
+
+export const GrpcPentestContextSchema = z.object({
+  transport: z.enum(["grpc", "grpc_web", "connect"]),
+  serviceFqn: z.string(),
+  method: z.string(),
+  streamingType: GrpcStreamingTypeEnum.optional(),
+  reflectionAvailable: z.boolean().optional(),
+  schemaSource: GrpcSchemaSourceEnum.optional(),
+  host: z
+    .string()
+    .nullish()
+    .describe("Authority override; null falls back to the app domain"),
+  frontingGatewayOperation: z
+    .object({
+      gatewayType: GrpcGatewayTypeEnum,
+      operation: z.string(),
+    })
+    .nullish(),
+});
+
+export type GrpcPentestContext = z.infer<typeof GrpcPentestContextSchema>;
+
+/**
+ * Derive minimal gRPC metadata (`serviceFqn` / `method`) from a wire path of
+ * the form `/package.Service/Method`. Used when an endpoint is marked with a
+ * gRPC transport but carries no `grpc` object, so it still runs the gRPC
+ * battery instead of silently degrading to HTTP.
+ */
+function deriveGrpcMetadataFromPath(
+  path: string | undefined,
+):
+  | (Partial<GrpcEndpointMetadata> &
+      Pick<GrpcEndpointMetadata, "serviceFqn" | "method">)
+  | undefined {
+  if (!path) return undefined;
+  const wire = path.replace(/^\//, "").split("?")[0];
+  const slash = wire.lastIndexOf("/");
+  if (slash <= 0 || slash === wire.length - 1) return undefined;
+  return { serviceFqn: wire.slice(0, slash), method: wire.slice(slash + 1) };
+}
+
+/**
+ * Build a {@link GrpcPentestContext} from an endpoint's `transport` + `grpc`
+ * metadata (as produced by whitebox discovery / `document_endpoint`). Returns
+ * `undefined` for plain HTTP endpoints, so callers can attach the result to a
+ * target unconditionally. When `transport` is a gRPC variant but the `grpc`
+ * object is missing, `serviceFqn` / `method` are derived from the wire `path`
+ * so the target is still tested as gRPC.
+ */
+export function toGrpcPentestContext(
+  transport: EndpointTransport | undefined,
+  grpc: GrpcEndpointMetadata | undefined,
+  path?: string,
+): GrpcPentestContext | undefined {
+  if (!transport || transport === "http") return undefined;
+  const derived = grpc ?? deriveGrpcMetadataFromPath(path);
+  if (!derived) return undefined;
+  return {
+    transport,
+    serviceFqn: derived.serviceFqn,
+    method: derived.method,
+    streamingType: derived.streamingType,
+    reflectionAvailable: derived.reflectionAvailable,
+    schemaSource: derived.schemaSource,
+    host: derived.host,
+    frontingGatewayOperation: derived.frontingGatewayOperation,
+  };
+}
+
+/**
+ * When a gRPC method's authority (`grpc.host`) differs from the app domain,
+ * build the `grpc://<authority>/<wire-path>` target so the method's real
+ * authority reaches the pentest agent. Returns `undefined` when there is no
+ * host override, so callers fall back to their default target construction.
+ */
+export function grpcAuthorityTarget(
+  grpc: GrpcPentestContext | undefined,
+  path: string,
+): string | undefined {
+  if (!grpc?.host) return undefined;
+  const wirePath = path.startsWith("/") ? path : `/${path}`;
+  return `grpc://${grpc.host}${wirePath}`;
+}

package/src/core/agents/specialized/attackSurface/jsExtraction.test.ts

@@ -0,0 +1,237 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { HttpResponse } from "../../../tools/backends/types";
+import type { ToolContext } from "../../offSecAgent/tools/types";
+import {
+  extractJavascriptEndpoints,
+  extractJavascriptEndpointsFromHtml,
+} from "./jsExtraction";
+
+const PAGE_URL = "https://example.com/app";
+const FETCH_PATTERN = "fetch\\s*\\(\\s*['\"]([^'\"]+)['\"]...";
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+});
+
+describe("extractJavascriptEndpointsFromHtml", () => {
+  it("extracts endpoints across patterns in first-seen order with exact metadata", () => {
+    const html = [
+      "<html><head>",
+      '<script src="/static/app.js"></script>',
+      "<script>",
+      "fetch('/api/users');",
+      'url = "/api/users";',
+      "fetch('/api/users');",
+      "fetch(`/api/orders/123`);",
+      "axios.get('/api/items/45');",
+      'xhr.open("GET", "/api/session");',
+      'action = "/api/submit";',
+      "</script>",
+      "</head><body></body></html>",
+    ].join("\n");
+
+    const result = extractJavascriptEndpointsFromHtml(html, PAGE_URL);
+
+    // Scan order is per script tag, then per pattern family. The axios call
+    // is matched by its first capture group (the method name), which the
+    // root-relative check then filters out — current supported behavior.
+    expect(result.totalAjaxCalls).toBe(6);
+    expect(result.endpoints).toEqual([
+      {
+        endpoint: "/api/users",
+        pattern: FETCH_PATTERN,
+        source: "inline-script",
+      },
+      {
+        endpoint: "/api/orders/123",
+        pattern: "fetch\\s*\\(\\s*`([^`]+)`...",
+        source: "inline-script",
+      },
+      {
+        endpoint: "/api/session",
+        pattern: "\\.open\\s*\\(\\s*['\"](?:GET|POST|...",
+        source: "inline-script",
+      },
+      {
+        endpoint: "/api/submit",
+        pattern: "action\\s*[:=]\\s*['\"]([^'\"]+)['...",
+        source: "inline-script",
+      },
+    ]);
+    expect(result.externalJSFiles).toEqual(["/static/app.js"]);
+    expect(result.filesAnalyzed).toBe(2);
+    expect(result.success).toBe(true);
+    expect(result.url).toBe(PAGE_URL);
+    expect(result.message).toBe(
+      "Found 4 unique endpoints in JavaScript (6 total calls).",
+    );
+  });
+
+  it("keeps the first record's metadata and position for duplicates that differ", () => {
+    const html = `<script>
+      fetch('/api/first');
+      fetch('/api/second');
+      url = "/api/first";
+      href = "/api/second";
+    </script>`;
+
+    const result = extractJavascriptEndpointsFromHtml(html, PAGE_URL);
+
+    expect(result.totalAjaxCalls).toBe(4);
+    expect(result.endpoints).toEqual([
+      {
+        endpoint: "/api/first",
+        pattern: FETCH_PATTERN,
+        source: "inline-script",
+      },
+      {
+        endpoint: "/api/second",
+        pattern: FETCH_PATTERN,
+        source: "inline-script",
+      },
+    ]);
+  });
+
+  it("parameterizes numeric segments and dedupes the parameterized patterns", () => {
+    const html = `<script>
+      fetch('/api/items/123');
+      fetch('/api/items/456');
+      fetch('/api/items/123');
+    </script>`;
+
+    const result = extractJavascriptEndpointsFromHtml(html, PAGE_URL);
+
+    expect(result.endpoints?.map((e) => e.endpoint)).toEqual([
+      "/api/items/123",
+      "/api/items/456",
+    ]);
+    expect(result.parameterizedPatterns).toEqual(["/api/items/{id}"]);
+    expect(result.totalAjaxCalls).toBe(3);
+  });
+
+  it("collects external script sources only when requested", () => {
+    const html = `<html><head>
+      <script src="/static/app.js"></script>
+      <script>fetch('/api/inline');</script>
+    </head></html>`;
+
+    const withExternal = extractJavascriptEndpointsFromHtml(html, PAGE_URL);
+    const withoutExternal = extractJavascriptEndpointsFromHtml(
+      html,
+      PAGE_URL,
+      false,
+    );
+
+    expect(withExternal.externalJSFiles).toEqual(["/static/app.js"]);
+    expect(withExternal.filesAnalyzed).toBe(2);
+    expect(withoutExternal.externalJSFiles).toEqual([]);
+    expect(withoutExternal.filesAnalyzed).toBe(1);
+    expect(withoutExternal.endpoints).toEqual(withExternal.endpoints);
+  });
+
+  it("ignores endpoints that are not root-relative paths", () => {
+    const html = `<script>
+      fetch('https://api.example.com/absolute');
+      fetch('relative/path');
+      fetch('/api/root-relative');
+    </script>`;
+
+    const result = extractJavascriptEndpointsFromHtml(html, PAGE_URL);
+
+    expect(result.endpoints?.map((e) => e.endpoint)).toEqual([
+      "/api/root-relative",
+    ]);
+  });
+
+  it("parses typed and JSON script blocks like plain inline scripts", () => {
+    const html = `<html><head>
+      <script type="module">fetch('/api/module');</script>
+      <script type="application/json">{url: "/api/data-block"}</script>
+    </head></html>`;
+
+    const result = extractJavascriptEndpointsFromHtml(html, PAGE_URL);
+
+    expect(result.endpoints?.map((e) => e.endpoint)).toEqual([
+      "/api/module",
+      "/api/data-block",
+    ]);
+  });
+
+  it("returns an empty success for pages with no script content", () => {
+    const result = extractJavascriptEndpointsFromHtml(
+      "<html><body><p>no scripts</p></body></html>",
+      PAGE_URL,
+    );
+
+    expect(result.success).toBe(true);
+    expect(result.endpoints).toEqual([]);
+    expect(result.totalAjaxCalls).toBe(0);
+    expect(result.message).toBe(
+      "Found 0 unique endpoints in JavaScript (0 total calls).",
+    );
+  });
+});
+
+describe("extractJavascriptEndpoints (fetching helper)", () => {
+  const html = `<script>fetch('/api/via-fetch');</script>`;
+
+  // The page fetch routes through the tool http backend, never
+  // a bare global fetch — so the test injects a fake backend on the context.
+  function ctxWith(
+    request: (...args: unknown[]) => Promise<HttpResponse>,
+  ): ToolContext {
+    return { backends: { http: { request } } } as unknown as ToolContext;
+  }
+
+  it("sends the session cookie as a Cookie header and matches the pure parser output", async () => {
+    const request = vi.fn(
+      async () => ({ success: true, body: html }) as HttpResponse,
+    );
+
+    const result = await extractJavascriptEndpoints({
+      url: PAGE_URL,
+      sessionCookie: "sid=abc",
+      ctx: ctxWith(request),
+    });
+
+    expect(request).toHaveBeenCalledWith({
+      url: PAGE_URL,
+      method: "GET",
+      followRedirects: true,
+      headers: { Cookie: "sid=abc" },
+    });
+    expect(result).toEqual(extractJavascriptEndpointsFromHtml(html, PAGE_URL));
+  });
+
+  it("omits headers when no session cookie is given", async () => {
+    const request = vi.fn(
+      async () => ({ success: true, body: html }) as HttpResponse,
+    );
+
+    await extractJavascriptEndpoints({ url: PAGE_URL, ctx: ctxWith(request) });
+
+    expect(request).toHaveBeenCalledWith({
+      url: PAGE_URL,
+      method: "GET",
+      followRedirects: true,
+      headers: undefined,
+    });
+  });
+
+  it("returns a failure result when the page fetch fails", async () => {
+    const request = vi.fn(
+      async () =>
+        ({ success: false, error: "connection refused" }) as HttpResponse,
+    );
+
+    const result = await extractJavascriptEndpoints({
+      url: PAGE_URL,
+      ctx: ctxWith(request),
+    });
+
+    expect(result.success).toBe(false);
+    expect(result.message).toBe(
+      "JavaScript extraction error: connection refused",
+    );
+  });
+});

package/src/core/agents/specialized/attackSurface/jsExtraction.ts

@@ -0,0 +1,174 @@
+/**
+ * JavaScript endpoint extraction utilities
+ */
+
+import { resolveBackends } from "../../../tools/backends";
+import type { ToolContext } from "../../offSecAgent/tools/types";
+
+export interface ExtractJavascriptEndpointsParams {
+  url: string;
+  sessionCookie?: string;
+  includeExternalJS?: boolean;
+  /** Routes the page fetch through the caller's tool backend — no bare `fetch`. */
+  ctx: ToolContext;
+}
+
+export interface EndpointInfo {
+  endpoint: string;
+  pattern: string;
+  source: string;
+}
+
+export interface ExtractJavascriptEndpointsResult {
+  success: boolean;
+  url?: string;
+  endpoints?: EndpointInfo[];
+  parameterizedPatterns?: string[];
+  totalAjaxCalls?: number;
+  externalJSFiles?: string[];
+  filesAnalyzed?: number;
+  message: string;
+}
+
+/**
+ * Extract endpoint URLs from JavaScript code in a page using pattern matching.
+ *
+ * Uses regex patterns to find:
+ * - AJAX calls ($.ajax, $.get, $.post)
+ * - Fetch API calls
+ * - Axios requests
+ * - XMLHttpRequest calls
+ * - URL assignments
+ */
+export async function extractJavascriptEndpoints(
+  params: ExtractJavascriptEndpointsParams,
+): Promise<ExtractJavascriptEndpointsResult> {
+  try {
+    const { url, sessionCookie, includeExternalJS = true, ctx } = params;
+
+    // Fetch the page
+    const pageResult = await resolveBackends(ctx).http.request({
+      url,
+      method: "GET",
+      followRedirects: true,
+      headers: sessionCookie ? { Cookie: sessionCookie } : undefined,
+    });
+    if (!pageResult.success) {
+      return {
+        success: false,
+        message: `JavaScript extraction error: ${pageResult.error ?? "request failed"}`,
+      };
+    }
+    const html = pageResult.body;
+
+    return extractJavascriptEndpointsFromHtml(html, url, includeExternalJS);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      success: false,
+      message: `JavaScript extraction error: ${message}`,
+    };
+  }
+}
+
+/**
+ * Pure HTML variant of {@link extractJavascriptEndpoints} for callers that
+ * already hold the page body — no second fetch of the page.
+ */
+export function extractJavascriptEndpointsFromHtml(
+  html: string,
+  url: string,
+  includeExternalJS = true,
+): ExtractJavascriptEndpointsResult {
+  // Regex patterns to extract endpoints
+  const endpointPatterns = [
+    // jQuery AJAX
+    /\$\.ajax\s*\(\s*\{\s*url\s*:\s*['"]([^'"]+)['"]/g,
+    /\$\.get\s*\(\s*['"]([^'"]+)['"]/g,
+    /\$\.post\s*\(\s*['"]([^'"]+)['"]/g,
+    /\$\.getJSON\s*\(\s*['"]([^'"]+)['"]/g,
+    // Fetch API
+    /fetch\s*\(\s*['"]([^'"]+)['"]/g,
+    /fetch\s*\(\s*`([^`]+)`/g,
+    // Axios
+    /axios\.(get|post|put|delete|patch)\s*\(\s*['"]([^'"]+)['"]/g,
+    // XMLHttpRequest
+    /\.open\s*\(\s*['"](?:GET|POST|PUT|DELETE|PATCH)['"]\s*,\s*['"]([^'"]+)['"]/gi,
+    // URL construction
+    /url\s*[:=]\s*['"]([^'"]+)['"]/gi,
+    /href\s*[:=]\s*['"]([^'"]+)['"]/gi,
+    /action\s*[:=]\s*['"]([^'"]+)['"]/gi,
+  ];
+
+  const endpoints: EndpointInfo[] = [];
+  const jsFiles: string[] = [];
+
+  // Extract inline script content
+  const scriptTagRegex = /<script[^>]*>([\s\S]*?)<\/script>/gi;
+  for (
+    let scriptMatch = scriptTagRegex.exec(html);
+    scriptMatch !== null;
+    scriptMatch = scriptTagRegex.exec(html)
+  ) {
+    const scriptContent = scriptMatch[1];
+
+    // Apply all patterns to script content
+    for (const pattern of endpointPatterns) {
+      const patternCopy = new RegExp(pattern.source, pattern.flags);
+      for (
+        let match = patternCopy.exec(scriptContent);
+        match !== null;
+        match = patternCopy.exec(scriptContent)
+      ) {
+        const endpoint = match[1] || match[2];
+        if (endpoint?.startsWith("/")) {
+          endpoints.push({
+            endpoint,
+            pattern: `${pattern.source.substring(0, 30)}...`,
+            source: "inline-script",
+          });
+        }
+      }
+    }
+  }
+
+  // Extract external JS file URLs
+  if (includeExternalJS) {
+    const scriptSrcRegex = /<script[^>]+src=['"]([^'"]+)['"]/gi;
+    for (
+      let srcMatch = scriptSrcRegex.exec(html);
+      srcMatch !== null;
+      srcMatch = scriptSrcRegex.exec(html)
+    ) {
+      jsFiles.push(srcMatch[1]);
+    }
+  }
+
+  // Deduplicate, keeping the first record and first-seen order (O(n))
+  const byEndpoint = new Map<string, EndpointInfo>();
+  for (const record of endpoints) {
+    if (!byEndpoint.has(record.endpoint)) {
+      byEndpoint.set(record.endpoint, record);
+    }
+  }
+  const uniqueEndpoints = Array.from(byEndpoint.values());
+
+  // Parameterize endpoints (replace numeric IDs with {id})
+  const parameterizedEndpoints = uniqueEndpoints.map((e) => ({
+    ...e,
+    pattern: e.endpoint.replace(/\/\d+/g, "/{id}"),
+  }));
+
+  return {
+    success: true,
+    url,
+    endpoints: uniqueEndpoints,
+    parameterizedPatterns: Array.from(
+      new Set(parameterizedEndpoints.map((e) => e.pattern)),
+    ),
+    totalAjaxCalls: endpoints.length,
+    externalJSFiles: jsFiles,
+    filesAnalyzed: 1 + jsFiles.length,
+    message: `Found ${uniqueEndpoints.length} unique endpoints in JavaScript (${endpoints.length} total calls).`,
+  };
+}