@pensar/apex 2.6.1 → 2.7.0-canary.21cb83dc

package/src/core/agents/specialized/pentest/agent.ts

@@ -17,6 +17,7 @@ import {
   type AgentMode,
   createReportErrorTool,
   type Finding,
+  type FindingPolicy,
   PentestReportedError,
   REPORT_ERROR_TOOL_NAME,
   type ReportedError,
@@ -132,6 +133,8 @@ export interface PentestAgentInput extends SpecializedAgentInput {
 export interface PentestResult {
   /** All findings discovered during the run */
   findings: Finding[];
+  /** Concise findings handed to an external evaluator, without native judging. */
+  reportedFindings?: string[];
   /** Absolute path to the session's findings directory */
   findingsPath: string;
   /** Absolute path to the session's POC scripts directory */
@@ -198,6 +201,25 @@ export const PentestResponseSchema = z.object({
     ),
 });
 
+export const PentestExternalResponseSchema = PentestResponseSchema.omit({
+  findingsDocumented: true,
+}).extend({
+  objectiveResults: z
+    .array(ObjectiveResultSchema)
+    .describe(
+      "Status of each assigned objective, including findings or limitations.",
+    ),
+  reportedFindings: z
+    .array(z.string())
+    .describe(
+      "One concise evidence-backed finding per vulnerability, including the affected target and security impact. An empty array is valid.",
+    ),
+  noFindingsReason: z
+    .string()
+    .optional()
+    .describe("If no findings are reported, explain why."),
+});
+
 // ---------------------------------------------------------------------------
 // PentestAgent
 // ---------------------------------------------------------------------------
@@ -257,6 +279,7 @@ export const targetedPentestWorker = defineAgent<
       state.mode,
       opts.systemScope,
       state.helperWorkspaceRoot,
+      opts.findingPolicy,
     ),
   // Workers hold the native helper file tools; this root scopes them so they
   // can build and inspect helper scripts without ever reaching target source.
@@ -272,8 +295,15 @@ export const targetedPentestWorker = defineAgent<
   systemScope: (opts) => opts.systemScope,
   mode: (_opts, state) => state.mode,
   activeTools: (opts, state) =>
-    buildPentestActiveTools(state.effectiveRole, opts.session),
-  responseSchema: () => PentestResponseSchema,
+    buildPentestActiveTools(
+      state.effectiveRole,
+      opts.session,
+      opts.findingPolicy,
+    ),
+  responseSchema: (opts) =>
+    opts.findingPolicy === "external"
+      ? PentestExternalResponseSchema
+      : PentestResponseSchema,
   prompt: (opts, state) =>
     buildPentestPrompt(
       opts.target,
@@ -293,16 +323,17 @@ export const targetedPentestWorker = defineAgent<
       opts.session.credentialManager?.formatForPrompt(),
       opts.grpc,
       state.mode,
+      opts.findingPolicy,
     ),
-  extraTools: (_opts, state) => ({
+  extraTools: (opts, state) => ({
     [REPORT_ERROR_TOOL_NAME]: createReportErrorTool((err) => {
       state.reportedError = err;
-    }),
+    }, opts.findingPolicy),
   }),
   // Ending the run on `report_error` is merged with the base class's
   // `hasToolCall("response")` stop condition.
   stopWhen: () => hasToolCall(REPORT_ERROR_TOOL_NAME),
-  resolveResult: async (opts, state, streamResult) => {
+  resolveResult: async (opts, state, streamResult, validatedResponse) => {
     // A reported blocking error takes precedence: surface it as a thrown
     // error so the run is treated as a failure rather than a completion.
     if (state.reportedError) {
@@ -311,35 +342,50 @@ export const targetedPentestWorker = defineAgent<
 
     let objectiveResults: ObjectiveResult[] | undefined;
     let newObjectives: string[] | undefined;
-
-    try {
-      const response = await streamResult.response;
-      for (const msg of response.messages ?? []) {
-        if (msg.role !== "assistant" || !Array.isArray(msg.content)) continue;
-        for (const part of msg.content) {
-          if (
-            part.type === "tool-call" &&
-            part.toolName === "response" &&
-            part.input
-          ) {
-            // createResponseTool wraps the schema under `result`.
-            const input = part.input as {
-              result?: {
-                objectiveResults?: ObjectiveResult[];
-                newObjectives?: string[];
+    let reportedFindings: string[] | undefined;
+
+    if (opts.findingPolicy === "external") {
+      // The response tool passes this value only after schema validation and
+      // execution. Raw assistant tool-call messages may contain rejected input.
+      const parsed = PentestExternalResponseSchema.safeParse(validatedResponse);
+      if (!parsed.success) {
+        throw new Error(
+          "Externally graded pentest ended without a structured findings response",
+        );
+      }
+      objectiveResults = parsed.data.objectiveResults;
+      newObjectives = parsed.data.newObjectives;
+      reportedFindings = parsed.data.reportedFindings;
+    } else {
+      try {
+        const response = await streamResult.response;
+        for (const msg of response.messages ?? []) {
+          if (msg.role !== "assistant" || !Array.isArray(msg.content)) continue;
+          for (const part of msg.content) {
+            if (
+              part.type === "tool-call" &&
+              part.toolName === "response" &&
+              part.input
+            ) {
+              // createResponseTool wraps the schema under `result`.
+              const input = part.input as {
+                result?: {
+                  objectiveResults?: ObjectiveResult[];
+                  newObjectives?: string[];
+                };
               };
-            };
-            if (input.result?.objectiveResults) {
-              objectiveResults = input.result.objectiveResults;
-            }
-            if (input.result?.newObjectives) {
-              newObjectives = input.result.newObjectives;
+              if (input.result?.objectiveResults) {
+                objectiveResults = input.result.objectiveResults;
+              }
+              if (input.result?.newObjectives) {
+                newObjectives = input.result.newObjectives;
+              }
             }
           }
         }
+      } catch {
+        // Best-effort native metadata; findings are loaded separately.
       }
-    } catch {
-      // Best-effort; objectiveResults / newObjectives will be undefined
     }
 
     const sandboxed = opts.backends?.sandboxed === true;
@@ -347,13 +393,15 @@ export const targetedPentestWorker = defineAgent<
       ? `${SANDBOX_ARTIFACTS_ROOT}/findings`
       : opts.session.findingsPath;
     const findings =
-      sandboxed && opts.backends
-        ? await loadSandboxFindings(
-            opts.backends,
-            findingsPath,
-            opts.abortSignal,
-          )
-        : loadFindings(findingsPath);
+      opts.findingPolicy === "external"
+        ? []
+        : sandboxed && opts.backends
+          ? await loadSandboxFindings(
+              opts.backends,
+              findingsPath,
+              opts.abortSignal,
+            )
+          : loadFindings(findingsPath);
 
     return {
       findings,
@@ -363,6 +411,7 @@ export const targetedPentestWorker = defineAgent<
         : opts.session.pocsPath,
       objectiveResults,
       newObjectives,
+      reportedFindings,
     };
   },
 });
@@ -818,8 +867,14 @@ ${SECTION_STATE_CHECKPOINTING}`;
  * destructive testing is not authorized (default), the section tells the agent
  * the operations the guard blocks so it doesn't waste turns attempting them.
  */
-function destructiveSection(allow: boolean | undefined): string {
-  return allow ? SECTION_DESTRUCTIVE_ALLOWED : SECTION_DESTRUCTIVE_BLOCKED;
+function destructiveSection(
+  allow: boolean | undefined,
+  findingPolicy: FindingPolicy = "native",
+): string {
+  if (allow) return SECTION_DESTRUCTIVE_ALLOWED;
+  return findingPolicy === "external"
+    ? SECTION_DESTRUCTIVE_BLOCKED.replace(" and document_vulnerability", "")
+    : SECTION_DESTRUCTIVE_BLOCKED;
 }
 
 /**
@@ -851,9 +906,11 @@ export function buildPentestSystemPrompt(
   mode: TargetedPentestMode = "default",
   systemScope?: SystemPentestScope,
   helperWorkspaceRoot?: string,
+  findingPolicy: FindingPolicy = "native",
 ): string {
   const destructive = destructiveSection(
     session.config?.allowDestructiveActions,
+    findingPolicy,
   );
   const rateLimitTesting = rateLimitTestingSection(
     session.config?.allowRateLimitTesting,
@@ -863,7 +920,7 @@ export function buildPentestSystemPrompt(
   const guardrails = `${destructive}\n\n${rateLimitTesting}\n\n${SECTION_SIDE_EFFECT_SAFETY}`;
   const systemSection =
     systemScope && systemScope.memberHosts.length > 0
-      ? `\n\n${SECTION_SYSTEM_SCOPE(systemScope.memberHosts)}`
+      ? `\n\n${SECTION_SYSTEM_SCOPE(systemScope.memberHosts, findingPolicy)}`
       : "";
   // Fast Strike exposes source-assessment tools. Only an explicitly
   // configured repository lifts the targeted worker's black-box restriction.
@@ -880,6 +937,47 @@ export function buildPentestSystemPrompt(
           Boolean(sourceRoot),
         )}`;
 
+  if (findingPolicy === "external") {
+    const sourcePolicy = sourceRoot
+      ? `Read-only Source Access:\n- The configured repository at ${sourceRoot} is in scope for read-only analysis. Use source-assessment tools or execute_command to inspect it. Keep generated helpers in the file workspace; do not modify target source.`
+      : SECTION_SOURCE_CODE_PROHIBITION;
+    const roleMethod =
+      mode === "fast-strike"
+        ? `Run a focused Strike assessment against the supplied target. Work alone: observe, form the strongest hypothesis, test it with the available HTTP, browser, source, and command tools, then move to the next lead only when evidence rules the first out. Use editable scripts for stateful or multi-stage work. Do not spawn agents or create task lists.`
+        : role === "orchestrator"
+          ? `Lead a focused assessment. State a short plan, perform light reconnaissance and any needed authentication, then call spawn_pentest_agent for each assigned objective that needs testing. Investigate chains or untested leads with another worker when the objective calls for it. For detection-only work, stop when the requested evidence is sufficient. Combine the workers' findings and objective results in your response; workers do not write to a shared finding registry in this run.`
+          : session.config?.taskDriven
+            ? `Test each assigned objective. Decompose techniques with create_task, update each task as you test, adapt when a lead fails, and use list_tasks to check coverage before finishing.`
+            : `Test each assigned objective with the strongest available techniques. Verify reachability and authentication, inspect source when explicitly in scope, probe plausible weaknesses, and distinguish genuine security impact from benign behavior.`;
+    const authentication = SECTION_AUTHENTICATION.replace(
+      "Build verifiable POCs, but bound the blast radius.",
+      "When the assigned objective requires a live proof, bound the blast radius.",
+    );
+    const exfilGoal = session.config?.exfilMode
+      ? "\n\nExfiltration goal: Locate and extract a flag in the format FLAG{...} or the assigned sensitive data. Pivot through a confirmed vulnerability when needed to reach it. Do not stop at discovery; demonstrate extraction before calling response."
+      : "";
+    return `You are an expert offensive security agent. ${roleMethod}
+${exfilGoal}
+
+${sourcePolicy}
+
+Finding boundary: The caller evaluates findings after this run. Put every credible vulnerability in the response tool's reportedFindings array with its affected target, observed evidence, and likely impact. For detection-only objectives, do not build a separate proof-of-concept script, seek a second in-run review, or continue exploiting once the requested evidence is sufficient. If the objective requires exploitation or extraction, pursue that objective fully.
+
+Use the browser for interactive or authenticated flows, HTTP tools for direct probes, and the shell for scripts or source analysis. Stay within the supplied scope and report uncertainty honestly. Complete every assigned objective before calling response.
+
+${guardrails}${systemSection}${helperToolsSection}
+
+${authentication}
+
+${SECTION_MATERIALITY_GUIDANCE}
+
+${SECTION_CREDENTIAL_DISCOVERY}
+
+${SECTION_SECURITY_HEADERS_CORS}
+
+${SECTION_STATE_CHECKPOINTING}${session.config?.promptInjectionLibrarySource ? `\n\n${SECTION_PROMPT_INJECTION}` : ""}`;
+  }
+
   if (mode === "fast-strike") {
     const sourcePolicy = sourceRoot
       ? `Read-only Source Access:
@@ -917,13 +1015,16 @@ export function buildPentestSystemPrompt(
     : withGuardrails;
 }
 
-const SECTION_SYSTEM_SCOPE = (memberHosts: string[]) =>
+const SECTION_SYSTEM_SCOPE = (
+  memberHosts: string[],
+  findingPolicy: FindingPolicy = "native",
+) =>
   `System Scope (structured):
 - This engagement covers a multi-application System. Member hosts already present in session targets: ${memberHosts.join(", ")}.
 - Declared relationships in the application context prioritize investigation order; undeclared paths between members remain in scope when discovered.
 - For cross-service follow-ups, an orchestrator MAY set a worker \`target\` to a full URL on another member host listed above. Workers receive this same System Scope. Do not invent hosts outside that set.
 - Authentication state is origin-specific. Before dispatching an authenticated cross-service follow-up, the orchestrator must establish and verify a session on that member origin. The worker must verify access on its assigned origin and follow the Authentication rules below if access is denied.
-- When a confirmed finding spans multiple members, populate the \`attackPath\` argument of \`document_vulnerability\` with the ordered member-to-member hop chain. Do not leave the chain only in the narrative.`;
+${findingPolicy === "external" ? "- When a finding spans multiple members, describe the ordered member-to-member path in your reported finding." : "- When a confirmed finding spans multiple members, populate the `attackPath` argument of `document_vulnerability` with the ordered member-to-member hop chain. Do not leave the chain only in the narrative."}`;
 
 // ---------------------------------------------------------------------------
 // Orchestrator system prompt — assembles the plan / fan-out / chain flow
@@ -1000,6 +1101,7 @@ export function buildPentestPrompt(
   credentialContext?: string,
   grpc?: GrpcPentestContext,
   mode: TargetedPentestMode = "default",
+  findingPolicy: FindingPolicy = "native",
 ): string {
   const sessionRootPath = session.rootPath;
   const exfilMode = session.config?.exfilMode ?? false;
@@ -1062,7 +1164,11 @@ export function buildPentestPrompt(
   }
 
   let knownFindingsSection = "";
-  if (findingsRegistry && findingsRegistry.size > 0) {
+  if (
+    findingPolicy !== "external" &&
+    findingsRegistry &&
+    findingsRegistry.size > 0
+  ) {
     const existing = findingsRegistry
       .getFindings()
       .map((f) => `- [${f.severity}] ${f.title} (endpoint: ${f.endpoint})`)
@@ -1175,13 +1281,35 @@ If your assignment lists ZERO open objectives (everything is already completed),
 
 Do NOT call document_vulnerability — that is the workers' job. Do NOT discover or enumerate other endpoints or services beyond the assigned target except for cross-service follow-ups explicitly authorized by the structured System Scope.`;
 
-  const instructions =
+  const externalInstructions =
     role === "orchestrator"
-      ? orchestratorInstructions
+      ? `## Instructions
+1. State a short plan, perform light reconnaissance and authenticate if needed.
+2. Delegate each assigned objective with spawn_pentest_agent. Follow a concrete chain or anomaly only when the assignment calls for it or the detection objective remains unmet.
+3. Include every credible vulnerability from the workers in reportedFindings. Summarize objective coverage and any remaining leads in objectiveResults and newObjectives. The caller evaluates findings after this run.`
       : taskDriven
-        ? taskDrivenInstructions
-        : exfilMode
-          ? `## Goal
+        ? `## Instructions
+1. Create tasks for the supplied objectives and test them with the available tools.
+2. Update task results as evidence develops; inspect list_tasks before finishing.
+3. Put each credible vulnerability in reportedFindings with the affected target, evidence, and likely impact. Follow the assigned objective's discovery or exploitation boundary. The caller evaluates findings after this run.`
+        : `## Instructions
+1. Test the supplied objectives using the available tools and follow the strongest evidence first.
+2. Put each credible vulnerability in reportedFindings with the affected target, evidence, and likely impact. For detection-only objectives, stop when the requested evidence is sufficient; exploit or pivot only if the objective requires it.
+3. Finish with objectiveResults for each objective and any high-signal newObjectives. The caller evaluates findings after this run.`;
+
+  const instructions =
+    findingPolicy === "external"
+      ? `${externalInstructions}${
+          exfilMode
+            ? "\n\nThis is an exfiltration run: locate and extract FLAG{...} or the assigned sensitive data before finishing. Pivot through a confirmed weakness when necessary; report the result in objectiveResults without invoking a finding judge."
+            : ""
+        }`
+      : role === "orchestrator"
+        ? orchestratorInstructions
+        : taskDriven
+          ? taskDrivenInstructions
+          : exfilMode
+            ? `## Goal
 Your target is a flag with the format FLAG{...}. Locate and extract it.
 
 ## Instructions
@@ -1195,7 +1323,7 @@ Your target is a flag with the format FLAG{...}. Locate and extract it.
 8. Use add_memory to save reusable learnings from this engagement (effective techniques, target behaviors, technology details)
 9. Preserve the supplied objectives. Do not broaden a scoped materiality check into unrelated endpoint testing unless a confirmed finding requires it to demonstrate impact.
 10. After testing ALL objectives and extracting the flag, call the response tool with your final summary. Do NOT call response until you have completed all testing and extraction.`
-          : `## Instructions
+            : `## Instructions
 1. Call list_memories to review any prior knowledge relevant to this target or engagement
 2. State the objectives and outline your testing plan — describe which techniques and payloads you will use for each objective before executing any tools
 3. Verify the target endpoint is reachable and understand its baseline behavior
@@ -1370,13 +1498,14 @@ export function buildPentestActiveTools(
   session: {
     config?: { taskDriven?: boolean; promptInjectionLibrarySource?: string };
   },
+  findingPolicy: FindingPolicy = "native",
 ): string[] {
   const tools =
     role === "orchestrator"
       ? [...WORKER_RECON_TOOLS, ...SHARED_PENTEST_TOOLS, "spawn_pentest_agent"]
       : [
           ...WORKER_RECON_TOOLS,
-          "document_vulnerability",
+          ...(findingPolicy === "external" ? [] : ["document_vulnerability"]),
           // Scratch-confined file tools for write/check/run/inspect helper
           // loops. Orchestrators keep recon-only tooling and stay black-box.
           ...WORKER_HELPER_FILE_TOOLS,

package/src/core/agents/specialized/pentest/results.test.ts

@@ -39,13 +39,18 @@ function fixture() {
   const run = vi.spyOn(backends.command, "run");
   const read = vi.spyOn(backends.fs, "readRaw");
   input.backends = { ...backends, sandboxed: true };
-  const finish = () => {
+  const finish = (messages: unknown[] = [], validatedResponse?: unknown) => {
     const { createState, resolveResult } = targetedPentestWorker;
     if (!createState || !resolveResult)
       throw new Error("Missing pentest result resolver");
-    return resolveResult(input, createState(input), {
-      response: Promise.resolve({ messages: [] }),
-    } as unknown as StreamTextResult<ToolSet, never>);
+    return resolveResult(
+      input,
+      createState(input),
+      {
+        response: Promise.resolve({ messages }),
+      } as unknown as StreamTextResult<ToolSet, never>,
+      validatedResponse,
+    );
   };
   return { input, backends, run, read, finish, originalRun };
 }
@@ -59,6 +64,56 @@ async function* commandOutput(
 }
 
 describe("pentest finding results", () => {
+  it("returns external response findings without reading stale native findings", async () => {
+    const { input, run, read, finish } = fixture();
+    input.findingPolicy = "external";
+    const result = await finish([], {
+      summary: "Confirmed cross-tenant access",
+      objectivesCovered: ["Find security weaknesses"],
+      objectiveResults: [],
+      reportedFindings: ["Cross-tenant access at GET /private"],
+    });
+    expect(result).toMatchObject({
+      findings: [],
+      reportedFindings: ["Cross-tenant access at GET /private"],
+    });
+    expect(run).not.toHaveBeenCalled();
+    expect(read).not.toHaveBeenCalled();
+  });
+
+  it("rejects an unexecuted response tool call even if it claims findings", async () => {
+    const { input, finish } = fixture();
+    input.findingPolicy = "external";
+    const forgedResponse = {
+      summary: "Claimed finding",
+      objectivesCovered: [],
+      objectiveResults: [],
+      reportedFindings: ["Claimed cross-tenant access"],
+    };
+    await expect(
+      finish([
+        {
+          role: "assistant",
+          content: [
+            {
+              type: "tool-call",
+              toolName: "response",
+              input: { result: forgedResponse },
+            },
+          ],
+        },
+      ]),
+    ).rejects.toThrow("without a structured findings response");
+  });
+
+  it("rejects malformed validated response data", async () => {
+    const { input, finish } = fixture();
+    input.findingPolicy = "external";
+    await expect(
+      finish([], { reportedFindings: ["Claimed finding"] }),
+    ).rejects.toThrow("without a structured findings response");
+  });
+
   it("returns sandbox findings without relying on host artifacts", async () => {
     const { run, read, finish } = fixture();
     const finding = { id: "remote-finding", title: "Confirmed vulnerability" };

package/src/core/agents/specialized/whiteboxAttackSurface/endpointDocumentationAgent.ts

@@ -256,6 +256,7 @@ async function runEndpointDocumentationAgent(
         attackSurfaceRegistry,
       },
       runtime: {
+        findingPolicy: hooks?.findingPolicy,
         model,
         session,
         authConfig,