@pensar/apex 2.5.0 → 2.6.1-canary.0cc77dcc

package/src/core/agents/specialized/findingJudge/index.ts

@@ -0,0 +1,166 @@
+/**
+ * Finding Judge
+ *
+ * Agentic validation for submitted findings. The judge runs after the POC
+ * exits successfully and before persistence to catch hallucinated findings,
+ * hardcoded evidence, and mismatched claims.
+ */
+
+import type { AIModel } from "../../../ai";
+import { createLogger } from "../../../logger/structured";
+import { scopedLogger } from "../../../util/lazyLogger";
+import type { ToolContext } from "../../offSecAgent/tools";
+import type {
+  FindingJudgeAgentOutput,
+  FindingJudgeInput,
+  FindingJudgeResult,
+} from "./types";
+
+const log = scopedLogger(() => createLogger("FindingJudge"));
+
+export type {
+  FindingJudgeAgentOutput,
+  FindingJudgeInput,
+  FindingJudgeResult,
+  FindingType,
+} from "./types";
+
+export type FindingJudgeRuntimeContext = Pick<
+  ToolContext,
+  | "session"
+  | "authConfig"
+  | "abortSignal"
+  | "eventBus"
+  | "sandbox"
+  | "backends"
+  | "target"
+  | "enableThinking"
+  | "thinkingEffort"
+  | "openAIReasoningEffort"
+  | "languageModelMiddleware"
+  | "usageRecorder"
+  | "streamIdFactory"
+> & {
+  model: AIModel;
+  /**
+   * Subagent id used to tag the judge's own stream events. Callers that
+   * spawn the judge as a nested subagent must pass the same id they
+   * emitted in the `subagent-spawn` lifecycle event so the live UI and
+   * persisted logs line up. Defaults to `"finding-judge"`.
+   */
+  subagentId?: string;
+  /** Human-readable label for readable OTel span names. Defaults to "Finding Judge". */
+  subagentName?: string;
+};
+
+/**
+ * Validate a vulnerability finding by running a bounded verifier agent.
+ *
+ * Infrastructure failures reject the finding as unverified rather than
+ * preserving it as a vulnerability.
+ */
+export async function judgeFinding(
+  input: FindingJudgeInput,
+  ctx: FindingJudgeRuntimeContext,
+): Promise<FindingJudgeResult> {
+  try {
+    // Lazy import avoids the tool-registry cycle:
+    // offSec tools -> documentFinding -> findingJudge -> OffensiveSecurityAgent.
+    const { FindingJudgeAgent } = await import("./agent");
+    const agent = new FindingJudgeAgent({
+      finding: input,
+      model: ctx.model,
+      session: ctx.session,
+      authConfig: ctx.authConfig,
+      abortSignal: ctx.abortSignal,
+      eventBus: ctx.eventBus,
+      subagentId: ctx.subagentId,
+      subagentName: ctx.subagentName,
+      sandbox: ctx.sandbox,
+      backends: ctx.backends,
+      target: input.target ?? ctx.target ?? ctx.session.targets[0],
+      enableThinking: ctx.enableThinking,
+      thinkingEffort: ctx.thinkingEffort,
+      openAIReasoningEffort: ctx.openAIReasoningEffort,
+      languageModelMiddleware: ctx.languageModelMiddleware,
+      usageRecorder: ctx.usageRecorder,
+      streamIdFactory: ctx.streamIdFactory,
+    });
+
+    const result = await agent.consume();
+    if (!result) {
+      throw new Error("Finding judge agent finished without a response.");
+    }
+
+    return normalizeJudgeResult(result);
+  } catch (err: unknown) {
+    const fallback = createJudgeFailureResult(err, ctx.model);
+    log.warn("Agentic validation failed", {
+      model: fallback.error?.model,
+      type: fallback.error?.type,
+      message: fallback.error?.message,
+    });
+    return fallback;
+  }
+}
+
+function normalizeJudgeResult(
+  result: FindingJudgeAgentOutput,
+): FindingJudgeResult {
+  return {
+    valid: result.valid,
+    findingType: result.findingType,
+    confidence: result.confidence,
+    reasoning: result.reasoning,
+    concerns: result.concerns,
+    verificationSteps: result.verificationSteps,
+    toolEvidence: result.toolEvidence,
+    reproducedPoc: result.reproducedPoc,
+    webResearchUsed: result.webResearchUsed,
+    limitations: result.limitations,
+  };
+}
+
+export function createJudgeFailureResult(
+  err: unknown,
+  model: AIModel,
+): FindingJudgeResult {
+  const errObj =
+    typeof err === "object" && err !== null
+      ? (err as Record<string, unknown>)
+      : {};
+  const message = err instanceof Error ? err.message : String(err);
+  const type = err instanceof Error ? err.constructor.name : typeof err;
+  const stack = err instanceof Error ? err.stack : undefined;
+  const statusCode = errObj.status ?? errObj.statusCode ?? errObj.code;
+  const statusStr = statusCode != null ? ` [status=${statusCode}]` : "";
+  const isAbort =
+    err instanceof Error &&
+    (err.name === "AbortError" || err.constructor.name === "DOMException");
+  const confidence = isAbort ? 0.3 : 0.4;
+  const diagnostic = `${type}${statusStr}: ${message.substring(0, 200)}`;
+
+  return {
+    valid: false,
+    findingType: "informational",
+    confidence,
+    reasoning: `Agentic finding judge could not complete (${diagnostic}). Rejecting the finding as unverified so unavailable judge infrastructure does not preserve false positives as vulnerabilities.`,
+    concerns: [
+      "Agentic judge infrastructure failed before producing a completed verification judgment.",
+      "The submitted PoC may have executed successfully, but the judge did not independently verify that it proves a material vulnerability.",
+    ],
+    verificationSteps: [],
+    toolEvidence: [],
+    reproducedPoc: false,
+    webResearchUsed: false,
+    limitations: [
+      "No independent judge verification was completed due to infrastructure failure.",
+    ],
+    error: {
+      message: `${message}${statusStr}`,
+      type,
+      model: String(model ?? "undefined"),
+      stack: stack?.substring(0, 500),
+    },
+  };
+}

package/src/core/agents/specialized/findingJudge/prompts.ts

@@ -0,0 +1,140 @@
+import type { FindingJudgeInput } from "./types";
+
+const MAX_POC_SCRIPT_CHARS = 12_000;
+const MAX_OUTPUT_CHARS = 16_000;
+const MAX_EVIDENCE_CHARS = 8_000;
+const MAX_DESCRIPTION_CHARS = 4_000;
+
+function truncate(value: string, limit: number): string {
+  if (value.length <= limit) return value;
+  return `${value.substring(0, limit)}\n... [truncated]`;
+}
+
+export const FINDING_JUDGE_SYSTEM_PROMPT = `You are a security finding validation specialist running as an agentic verifier inside an automated penetration testing pipeline.
+
+You are the last line of defense against false positives, hallucinated findings, fabricated evidence, and overstated impact. Your job is to independently verify whether a submitted proof-of-concept and its output support the claimed finding.
+
+## Operating Rules
+
+- Use tools before accepting a finding. At minimum, rerun or inspect the PoC with execute_command, compare behavior with http_request, read saved artifacts, or research the relevant technology with web_search/get_page.
+- Stay in scope. Only interact with the target endpoint/host and artifacts from this session.
+- Do not create, edit, delete, or document findings. You only validate and call the response tool.
+- Do not run destructive payloads or broaden exploitation. Verification should be enough to confirm or refute the claim.
+- Distinguish "the PoC exited 0" from "the vulnerability is proven." A successful exit code is only supporting evidence.
+- If your investigation completes and the evidence does not support the claim, reject it with actionable concerns.
+- If tools fail or the environment blocks verification, report the limitation honestly in your response. Infrastructure failure fallback is handled by the caller, not by you.
+- Cite concrete observations from tool output, HTTP responses, or web research in reasoning, concerns, and toolEvidence.
+
+## Evaluation Criteria
+
+### Materiality
+- A finding is a vulnerability only when the observed behavior is exploitable now in a way that creates a concrete security risk for the assessed target and threat model.
+- Do not reward theoretical "missing control" findings unless the PoC shows a realistic abuse path, affected security boundary, and material consequence.
+- Evaluate behavior in context: application purpose, deployment model, data sensitivity, exposed trust boundaries, and whether the observed behavior is intended for that environment. Expected, documented, demo, sandbox, or training behavior is usually expected-behavior or informational unless the claim demonstrates impact outside that intended use.
+- Reject CVSS 0.0 findings rather than downgrading them. A non-zero LOW severity can still be valid when the PoC demonstrates real exploitability and security risk.
+- If a PoC only proves that input is accepted, an error is returned, a public endpoint is reachable, or a control is absent, reject when it does not show an exploitable security risk.
+
+### Correctness
+- SQL injection: output should show inaccessible data, query errors, or behavioral differences indicating injection success.
+- XSS: output should show payload reflection/execution in a browser-executable context.
+- Command injection / RCE: output should show execution on the target system, not only local command execution.
+- SSRF: output should show responses from internal or otherwise protected services.
+- Path traversal: output should show file contents outside the intended path.
+- IDOR: output should show access to another user's/account's resource while authentication exists.
+- Authorization / access-control boundary bypass (RSC/prefetch/redirect-body leaks, missing per-route checks): the evidence must establish the access-model delta — which identity/role was used, and proof that the returned data is genuinely out of scope for that identity (cross-tenant, another user's, or admin-only data a member should not see). A 200 response, a leaked RSC payload, or a skipped redirect alone does NOT prove sensitive exposure. If the bypass is real but the returned data is non-sensitive or plausibly same-tenant/expected-visible, classify as a valid lower-impact finding and require the claim's impact/severity to match — do NOT accept it as high-confidence sensitive/cross-tenant exposure.
+- Authentication bypass: output should show protected access without valid credentials.
+- Email/HTML injection: ideally verify delivery and rendering. If only server acceptance is shown, confidence must stay below 0.7 and limitations must state rendering was not independently verified.
+- Missing rate limiting / anti-automation: a modest burst (~20 requests) with no observed throttling, delays, or 429s is sufficient to demonstrate the gap. For endpoints with real-world side effects (sending email/SMS/push, creating tickets/notifications, contact/support/feedback forms, payments), as few as 2–3 attempts plus the observed response is sufficient — do NOT require volume-testing that would spam real recipients. Evidence must state exact request count, timing/distribution, and observed statuses.
+
+### Common Low-Signal Patterns
+Reject these unless the submitted evidence demonstrates concrete harm:
+- Missing rate limiting on non-sensitive, low-value, intentionally public, or low-abuse-potential endpoints.
+- Verbose errors, stack traces, framework versions, or internal paths without secrets, protected data, or a working exploit chain.
+- Missing input validation that only permits nulls, empty strings, negative values, long text, type coercion, or malformed JSON with no security boundary crossed.
+- Missing authentication on public, non-sensitive, catalog, status, listener, metadata, demo, or intentionally unauthenticated endpoints.
+- Client-side validation bypasses where direct API calls produce no server-side security impact.
+- User enumeration, timing differences, weak pseudonymization, and email masking weaknesses without credible account takeover, sensitive data exposure, or targeting impact.
+- Unhandled type errors, method confusion, and 500 responses that only prove fragile error handling.
+
+### Hallucination and Hardcoding
+Reject or lower confidence when:
+- The script prints "evidence" with echo/print/console.log that was never received from the target.
+- The script constructs fake HTTP responses or error messages.
+- The success condition is hardcoded or independent of target response content.
+- The script always exits 0.
+- Evidence text does not match live responses or the claimed endpoint.
+
+### Severity and Classification
+- Accept as vulnerability only when real target interaction demonstrates unintended, exploitable behavior that creates security risk.
+- Classify as informational when the observation is real but not currently exploitable or not security-impacting.
+- Classify as expected-behavior when the behavior is clearly by design.
+- Reject when the PoC fabricates evidence, does not demonstrate the claim, or the demonstrated impact fundamentally mismatches the finding.
+- Guard against overstated confidentiality impact: when the demonstrated result is weaker than the claimed impact (e.g. a boundary was bypassed but no sensitive/cross-tenant data was shown, or only an identifier's existence — not its contents — was proven), keep confidence below 0.7 and raise a concern that the impact/severity should be reduced to match the evidence.
+
+## Response Requirements
+
+Call the response tool exactly once. Include:
+- valid: true only when the finding should be persisted as an exploitable vulnerability. Use valid: false for informational, expected-behavior, CVSS 0.0, non-exploitable, or merely best-practice observations.
+- findingType: vulnerability, informational, or expected-behavior.
+- confidence: calibrated 0.0 to 1.0. Below 0.7 requires concerns.
+- reasoning: concise explanation grounded in concrete observations.
+- concerns: actionable concerns; empty only for valid high-confidence results.
+- verificationSteps: actions you performed.
+- toolEvidence: concrete observations from provided artifacts or tool outputs.
+- reproducedPoc: whether you reran or independently reproduced the PoC.
+- webResearchUsed: whether web research materially informed the judgment.
+- limitations: remaining gaps.`;
+
+export function buildFindingJudgePrompt(input: FindingJudgeInput): string {
+  const { pocScript, pocType, pocOutput, claim } = input;
+
+  return `# Finding Validation Request
+
+## Target
+
+**Target:** ${input.target || "Unknown"}
+**Claimed Endpoint:** ${claim.endpoint}
+**POC Path:** ${input.pocPath || "Not provided"}
+
+## Claimed Vulnerability
+
+**Title:** ${claim.title}
+**Vulnerability Class:** ${claim.vulnerabilityClass || "Unknown"}
+
+### Description
+${truncate(claim.description, MAX_DESCRIPTION_CHARS)}
+
+### Claimed Impact
+${truncate(claim.impact, MAX_DESCRIPTION_CHARS)}
+
+### Evidence Provided by Agent
+\`\`\`
+${truncate(claim.evidence, MAX_EVIDENCE_CHARS)}
+\`\`\`
+
+## Previously Executed POC (${pocType})
+
+\`\`\`${pocType === "javascript" ? "js" : pocType === "python" ? "python" : "bash"}
+${truncate(pocScript, MAX_POC_SCRIPT_CHARS)}
+\`\`\`
+
+## Previous POC Execution Output
+
+**Exit Code:** ${pocOutput.exitCode}
+
+### stdout
+\`\`\`
+${truncate(pocOutput.stdout || "(empty)", MAX_OUTPUT_CHARS)}
+\`\`\`
+
+### stderr
+\`\`\`
+${truncate(pocOutput.stderr || "(empty)", MAX_OUTPUT_CHARS / 2)}
+\`\`\`
+
+## Task
+
+Independently validate whether the submitted POC and observed behavior support the claimed finding. Use the available minimal tools to rerun or inspect the POC, compare live target behavior, read saved artifacts, or research public documentation/CVEs as needed.
+
+Do not document or mutate anything. When done, call response with your structured judgment.`;
+}

package/src/core/agents/specialized/findingJudge/types.ts

@@ -0,0 +1,115 @@
+import { z } from "zod";
+
+export const FINDING_TYPES = [
+  "vulnerability",
+  "informational",
+  "expected-behavior",
+] as const;
+
+export type FindingType = (typeof FINDING_TYPES)[number];
+
+export interface FindingJudgeInput {
+  /** The full POC script content */
+  pocScript: string;
+  /** Script language */
+  pocType: "bash" | "python" | "javascript";
+  /** Relative path to the persisted POC file, when available */
+  pocPath?: string;
+  /** Target URL/host being assessed */
+  target?: string;
+  /** POC execution output */
+  pocOutput: {
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+  };
+  /** The vulnerability claim to validate against */
+  claim: {
+    title: string;
+    description: string;
+    impact: string;
+    evidence: string;
+    endpoint: string;
+    vulnerabilityClass?: string;
+  };
+}
+
+interface FindingJudgeVerificationDetails {
+  /** Concrete actions the judge took before deciding */
+  verificationSteps?: string[];
+  /** Specific observations from tools or provided artifacts */
+  toolEvidence?: string[];
+  /** Whether the judge independently reran or otherwise reproduced the POC */
+  reproducedPoc?: boolean;
+  /** Whether public web research influenced the decision */
+  webResearchUsed?: boolean;
+  /** Known gaps in verification */
+  limitations?: string[];
+}
+
+export interface FindingJudgeResult extends FindingJudgeVerificationDetails {
+  /** Whether the finding should be persisted as an exploitable vulnerability */
+  valid: boolean;
+  /** Classification of the finding */
+  findingType: FindingType;
+  /** Confidence in the judgment (0.0-1.0) */
+  confidence: number;
+  /** Explanation of the judgment */
+  reasoning: string;
+  /** Specific concerns identified (empty if valid) */
+  concerns: string[];
+  /** Present when the judge fell back due to an error — contains diagnostics */
+  error?: {
+    message: string;
+    type: string;
+    model: string;
+    stack?: string;
+  };
+}
+
+export const FindingJudgeOutputSchema = z.object({
+  valid: z
+    .boolean()
+    .describe(
+      "Whether the POC demonstrates a currently exploitable vulnerability that should be persisted",
+    ),
+  findingType: z
+    .enum(FINDING_TYPES)
+    .describe(
+      "Classification: 'vulnerability' for genuine security issues, 'informational' for low-risk observations worth noting, 'expected-behavior' for findings that describe intentional design decisions",
+    ),
+  confidence: z
+    .number()
+    .min(0)
+    .max(1)
+    .describe(
+      "Confidence in the judgment, a decimal between 0.0 (no confidence) and 1.0 (certain)",
+    ),
+  reasoning: z
+    .string()
+    .describe(
+      "Brief explanation of the key tool observations and analysis that led to this judgment",
+    ),
+  concerns: z
+    .array(z.string())
+    .describe(
+      "Specific concerns identified. Empty array only when the finding is a valid high-confidence vulnerability.",
+    ),
+  verificationSteps: z
+    .array(z.string())
+    .describe("Concrete verification actions performed before deciding."),
+  toolEvidence: z
+    .array(z.string())
+    .describe("Specific observations from tool output or provided artifacts."),
+  reproducedPoc: z
+    .boolean()
+    .describe("Whether the judge reran or independently reproduced the POC."),
+  webResearchUsed: z
+    .boolean()
+    .describe("Whether web_search/get_page materially informed the judgment."),
+  limitations: z
+    .array(z.string())
+    .describe("Remaining verification gaps or environmental limitations."),
+});
+
+export type FindingJudgeAgentOutput = z.infer<typeof FindingJudgeOutputSchema>;

package/src/core/agents/specialized/googleSignInPrompt.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, it } from "vitest";
+import {
+  browserEngineForGoogleSignIn,
+  GOOGLE_SIGNIN_PROMPT_GUIDANCE,
+} from "./googleSignInPrompt";
+
+describe("GOOGLE_SIGNIN_PROMPT_GUIDANCE", () => {
+  it("teaches Google SSO rather than the target native form", () => {
+    expect(GOOGLE_SIGNIN_PROMPT_GUIDANCE).toContain(
+      "Do NOT fill the target's native username or password form",
+    );
+    expect(GOOGLE_SIGNIN_PROMPT_GUIDANCE).toContain("accounts.google.com");
+    expect(GOOGLE_SIGNIN_PROMPT_GUIDANCE).toContain(
+      'credentialField="password"',
+    );
+    expect(GOOGLE_SIGNIN_PROMPT_GUIDANCE).toContain("Google employee ID");
+    expect(GOOGLE_SIGNIN_PROMPT_GUIDANCE).toContain(
+      "this browser or app may not be secure",
+    );
+    expect(GOOGLE_SIGNIN_PROMPT_GUIDANCE).toContain("complete_authentication");
+  });
+});
+
+describe("browserEngineForGoogleSignIn", () => {
+  it("uses Chrome for Google Sign-In and Camoufox otherwise", () => {
+    expect(browserEngineForGoogleSignIn(true)).toBe("chrome");
+    expect(browserEngineForGoogleSignIn(false)).toBe("camoufox");
+  });
+});

package/src/core/agents/specialized/googleSignInPrompt.ts

@@ -0,0 +1,18 @@
+import type { BrowserEngine } from "../offSecAgent/tools/playwrightMcp";
+
+export type { BrowserEngine };
+
+export const GOOGLE_SIGNIN_PROMPT_GUIDANCE = `## Sign in with Google
+
+When a credential is marked Sign-in: Google:
+1. Open the credential's Login URL on the target. Click the target's Google / Sign in with Google button. Do NOT fill the target's native username or password form.
+2. On accounts.google.com, snapshot, then \`browser_fill\` the identifier (email) with \`credentialId\` + \`credentialField="username"\`. Continue, then fill the password with \`credentialId\` + \`credentialField="password"\`. Never type a password or employee ID into the prompt or into \`execute_command\`.
+3. If Google asks to verify the account, fill **Google employee ID** with \`credentialId\` + \`credentialField="Google employee ID"\` (the name is in Additional secret fields).
+4. Fail closed: if Google shows "this browser or app may not be secure", a CAPTCHA, or a blocked-app interstitial, call \`complete_authentication\` with success=false and name the barrier. Do not try account recovery, password reset, or a different browser profile.
+5. After Google returns to the target, snapshot to confirm the session, then \`complete_authentication\` as usual.`;
+
+export function browserEngineForGoogleSignIn(
+  hasGoogleSignIn: boolean,
+): BrowserEngine {
+  return hasGoogleSignIn ? "chrome" : "camoufox";
+}

package/src/core/agents/specialized/mobileOtpPrompt.ts

@@ -0,0 +1,9 @@
+export const MOBILE_OTP_PROMPT_GUIDANCE = `## Mobile OTP credentials
+
+Read each credential's \`Authentication method\` before starting:
+- \`sms-passwordless\`: \`phoneNumber\` is the login identifier, not MFA after a password. Reserve immediately before filling that field, then request the code.
+- \`sms-mfa\`: fill username and password first. If submitting the password triggers the SMS code, reserve immediately before submitting. If it reveals a separate send-code action, reserve immediately before that action. Fill a destination phone field only if the target asks, using \`credentialId\` + \`credentialField="phoneNumber"\`.
+
+Never ask for, write, log, or pass a public phone number. Reserve with \`sms_list_messages\` and \`reserve=true\`; the Console chooses the shared number. A 429 means that number is busy: do not wait or poll inside the tool. Make at most two separate reservation attempts (with an \`execute_command\` sleep between them), then report the authentication block.
+
+At the action that sends the code, record \`Date.now()\` as \`sinceMs\`. Run \`execute_command\` \`sleep 5\`, then call \`sms_list_messages\` with \`sinceMs\` and \`claim=true\`. If no message arrives, make one delayed list retry; otherwise report the result rather than waiting indefinitely. Fill the claimed OTP and continue. Do not report \`phone_verification\` as a barrier for either Mobile OTP flow. TOTP via an environment variable is unchanged.`;