@pensar/apex 2.5.0 → 2.6.1-canary.4c944004

package/src/core/tools/backends/resolve.ts

@@ -0,0 +1,152 @@
+import { appendLocalWorkspaceFile } from "../../agents/offSecAgent/tools/fileWorkspace";
+import type { ToolContext } from "../../agents/offSecAgent/tools/types";
+import { LocalBackends, runLocalProgram } from "./local";
+import type { RunOpts, ToolBackends, WriteResult } from "./types";
+import { windowsProgramInvocation } from "./windowsProgram";
+
+export const SANDBOX_ARTIFACTS_ROOT = "/workspace/repo/.pensar";
+
+const localByContext = new WeakMap<ToolContext, ToolBackends>();
+
+/** The backends a tool should execute against: injected, else local (memoised per context). */
+export function resolveBackends(ctx: ToolContext): ToolBackends {
+  if (ctx.backends) return ctx.backends;
+  let local = localByContext.get(ctx);
+  if (!local) {
+    local = LocalBackends(ctx);
+    localByContext.set(ctx, local);
+  }
+  return local;
+}
+
+const artifactsByContext = new WeakMap<ToolContext, ToolBackends["fs"]>();
+
+/**
+ * Classic findings persist host session artifacts even when execution is
+ * remote. The default instance allows unlimited retained text for artifact
+ * writers; pass `maxTextFileBytes` for a fresh, session-scoped instance whose
+ * reads are byte-capped (not memoised alongside the unlimited default).
+ */
+export function resolveArtifactFs(
+  ctx: ToolContext,
+  options: { maxTextFileBytes?: number } = {},
+): ToolBackends["fs"] {
+  if (ctx.backends) return ctx.backends.fs;
+  if (options.maxTextFileBytes !== undefined) {
+    return LocalBackends(
+      {
+        ...ctx,
+        sandbox: undefined,
+        agentCwd: ctx.session.rootPath,
+        fileWorkspaceRoot: ctx.session.rootPath,
+      },
+      undefined,
+      { maxTextFileBytes: options.maxTextFileBytes },
+    ).fs;
+  }
+  let fs = artifactsByContext.get(ctx);
+  if (!fs) {
+    fs = LocalBackends(
+      {
+        ...ctx,
+        sandbox: undefined,
+        agentCwd: ctx.session.rootPath,
+        fileWorkspaceRoot: ctx.session.rootPath,
+      },
+      undefined,
+      { maxTextFileBytes: Infinity },
+    ).fs;
+    artifactsByContext.set(ctx, fs);
+  }
+  return fs;
+}
+
+/** Keep the injected read/write journal unchanged; classic artifacts use native append. */
+export async function appendArtifactSummary(
+  ctx: ToolContext,
+  path: string,
+  header: string,
+  entry: string,
+): Promise<WriteResult> {
+  if (ctx.backends) {
+    const fs = resolveArtifactFs(ctx);
+    const existing = await fs.readRaw(path);
+    return fs.write(
+      path,
+      (existing.success ? existing.content : header) + entry,
+      { mode: "overwrite" },
+    );
+  }
+  await appendLocalWorkspaceFile(
+    {
+      ...ctx,
+      sandbox: undefined,
+      agentCwd: ctx.session.rootPath,
+      fileWorkspaceRoot: ctx.session.rootPath,
+    },
+    path,
+    header,
+    entry,
+  );
+  return { success: true, error: "", path };
+}
+
+function posixQuote(value: string): string {
+  return `'${value.replace(/'/g, `'\\''`)}'`;
+}
+
+// Safe literal option arguments (flags) pass through unquoted so journalled
+// command bytes stay readable; anything else is single-quoted.
+function posixFlagArg(value: string): string {
+  return /^[-A-Za-z0-9_@+=:,./]+$/.test(value) ? value : posixQuote(value);
+}
+
+/**
+ * Transport for one program invocation: runner, option args, and a final
+ * target path. The final argument is always single-quoted (remote transports
+ * journal these exact command bytes); earlier args pass through unquoted
+ * when they are safe literals.
+ */
+export function resolveProgramRunner(ctx: ToolContext) {
+  const command =
+    ctx.backends || ctx.sandbox ? resolveBackends(ctx).command : undefined;
+  // Injected command backends see only RunOpts, so the agent's configured
+  // environment is merged here with per-call envVars winning. The local and
+  // classic sandbox transports already merge it themselves. Options pass
+  // through untouched when there is nothing to merge.
+  const withConfiguredEnv = (options: RunOpts | undefined) =>
+    ctx.environmentVariables || options?.envVars
+      ? {
+          ...options,
+          envVars: { ...ctx.environmentVariables, ...options?.envVars },
+        }
+      : options;
+  return (runner: string, args: string[], options?: RunOpts) => {
+    if (command?.platform === "windows") {
+      const invocation = windowsProgramInvocation(runner, args);
+      return command.run(invocation.command, {
+        ...options,
+        envVars: {
+          ...ctx.environmentVariables,
+          ...options?.envVars,
+          ...invocation.envVars,
+        },
+      });
+    }
+    const commandText = [
+      runner,
+      ...args.slice(0, -1).map(posixFlagArg),
+      posixQuote(args[args.length - 1] ?? ""),
+    ].join(" ");
+    return command
+      ? command.run(commandText, withConfiguredEnv(options))
+      : runLocalProgram(ctx, commandText, runner, args, options);
+  };
+}
+
+/** Native scripts use argv; remote scripts retain their journalled command bytes. */
+export function resolveScriptRunner(ctx: ToolContext) {
+  const runProgram = resolveProgramRunner(ctx);
+  return (runner: string, scriptPath: string, options?: RunOpts) =>
+    runProgram(runner, [scriptPath], options);
+}

package/src/core/tools/backends/types.ts

@@ -0,0 +1,279 @@
+/** Host- and engine-neutral tool backend contracts. */
+
+import type { CreateFileResult } from "../../agents/offSecAgent/tools/createFile";
+import type { EmailAdapterResolver } from "../../agents/offSecAgent/tools/email/adapters";
+import type {
+  BrowserClickResult,
+  BrowserConsoleResult,
+  BrowserEvaluateResult,
+  BrowserFillResult,
+  BrowserNavigateResult,
+  BrowserScreenshotResult,
+} from "../../agents/offSecAgent/tools/playwrightMcp";
+import type { SmsInbox } from "../../agents/offSecAgent/tools/smsInbox";
+
+export type {
+  ApplyPatchResult,
+  FilePatchResult,
+} from "../../agents/offSecAgent/tools/applyPatch";
+export type { GlobResult } from "../../agents/offSecAgent/tools/glob";
+export type { GrepResult } from "../../agents/offSecAgent/tools/grep";
+export type { ListFilesResult } from "../../agents/offSecAgent/tools/listFiles";
+export type {
+  BrowserClickResult,
+  BrowserConsoleResult,
+  BrowserEvaluateResult,
+  BrowserFillResult,
+  BrowserNavigateResult,
+  BrowserScreenshotResult,
+} from "../../agents/offSecAgent/tools/playwrightMcp";
+export type { ReadFileResult } from "../../agents/offSecAgent/tools/readFile";
+
+import type { ApplyPatchResult } from "../../agents/offSecAgent/tools/applyPatch";
+import type { GlobResult } from "../../agents/offSecAgent/tools/glob";
+import type { GrepResult } from "../../agents/offSecAgent/tools/grep";
+import type { ListFilesResult } from "../../agents/offSecAgent/tools/listFiles";
+import type { ReadFileResult } from "../../agents/offSecAgent/tools/readFile";
+
+/**
+ * Whole-file write result. `create_file` returns this directly; `update_file`
+ * reads, replaces, then writes and layers its own `replacements` count on top.
+ */
+export type WriteResult = CreateFileResult;
+
+/** Arguments for {@link FsBackend.git}. Only `status`/`diff` are used locally. */
+export interface GitArgs {
+  /** Limit a `diff` to one path. */
+  path?: string;
+  /** `diff` the index instead of the working tree. */
+  staged?: boolean;
+  /** Sandbox-only source-materialisation ops (clone/snapshot/restore). */
+  repoUrl?: string;
+  ref?: string;
+  message?: string;
+}
+
+/** Raw git output, matching the shared `runGit` seam plus the working dir. */
+export interface GitResult {
+  stdoutTruncated?: boolean;
+  success: boolean;
+  stdout: string;
+  stderr: string;
+  cwd: string;
+}
+
+export interface ReadOpts {
+  startLine?: number;
+  endLine?: number;
+  /**
+   * Byte-window mode (canary #1098): 0-based UTF-8 codepoint-aligned offset,
+   * paired with {@link byteCount}. Mutually exclusive with startLine/endLine.
+   * The result carries `stoppedAtByte` as the continuation cursor.
+   */
+  byteOffset?: number;
+  /** Bytes to read from {@link byteOffset} (capped at the read's max window). */
+  byteCount?: number;
+}
+
+export interface ListOpts {
+  recursive?: boolean;
+}
+
+export interface GrepQuery {
+  pattern: string;
+  directory?: string;
+  flags?: string;
+}
+
+export interface GlobOpts {
+  path?: string;
+}
+
+export type WriteMode = "create" | "overwrite";
+
+/**
+ * Write options (canary #1095 scoped-workspace mutations). `mode` keeps the
+ * create-vs-overwrite intent; `expected` layers optimistic concurrency on top:
+ * `null` = exclusive create (one winner via `link`), a string = the content the
+ * caller last read (rejected if the file changed since), `undefined` = no check.
+ */
+export interface WriteOpts {
+  permissions?: number;
+  timeoutSeconds?: number;
+  mode: WriteMode;
+  expected?: string | null;
+}
+
+/**
+ * Delete options (canary #1095). `confineToCwd` scopes resolution to the agent
+ * cwd when no file-workspace root is set; `expected` (a hash/content) rejects a
+ * delete if the file changed since it was read.
+ */
+export interface DeleteOpts {
+  confineToCwd?: boolean;
+  expected?: string;
+}
+
+/**
+ * Whole-file, unformatted read (no line numbers, no truncation cap). `update_file`
+ * needs the exact literal bytes for its search-and-replace match, which
+ * {@link FsBackend.read}'s numbered/capped output can't provide.
+ */
+export interface RawReadResult {
+  success: boolean;
+  error: string;
+  content: string;
+  path: string;
+}
+
+export interface FsBackend {
+  read(path: string, o?: ReadOpts): Promise<ReadFileResult>;
+  /** Uncapped, unformatted read for callers that need exact file bytes (`update_file`). */
+  readRaw(path: string): Promise<RawReadResult>;
+  list(dir: string, o?: ListOpts): Promise<ListFilesResult>;
+  grep(q: GrepQuery): Promise<GrepResult>;
+  glob(pattern: string, o?: GlobOpts): Promise<GlobResult>;
+  write(path: string, content: string, o: WriteOpts): Promise<WriteResult>;
+  delete(path: string, o?: DeleteOpts): Promise<void>;
+  applyPatch(diff: string): Promise<ApplyPatchResult>;
+  git(
+    op: "status" | "diff" | "clone" | "snapshot" | "restore",
+    args?: GitArgs,
+  ): Promise<GitResult>;
+}
+
+/**
+ * One streamed command event. `stdout`/`stderr` carry a monotonic `seq` so a
+ * surviving executor can reattach by offset; `end` carries the
+ * final exit code and whether a timeout fired.
+ */
+export type CommandEvent =
+  | { type: "start"; pid?: number }
+  | { type: "stdout"; seq: number; bytes: string }
+  | { type: "stderr"; seq: number; bytes: string }
+  | {
+      type: "end";
+      exitCode: number;
+      timedOut: boolean;
+      stdoutTruncated?: boolean;
+      stderrTruncated?: boolean;
+    };
+
+export interface RunOpts {
+  /** Seconds; no timeout when omitted. */
+  timeoutSeconds?: number;
+  /** Extra env for this command only (isolated via `env … bash -lc`). */
+  envVars?: Record<string, string>;
+  abortSignal?: AbortSignal;
+}
+
+export interface CommandBackend {
+  /** Target command interpreter; custom transports default to POSIX, regardless of host OS. */
+  platform?: "posix" | "windows";
+  run(cmd: string, o?: RunOpts): AsyncIterable<CommandEvent>;
+}
+
+export type HttpMethod =
+  | "GET"
+  | "POST"
+  | "PUT"
+  | "DELETE"
+  | "PATCH"
+  | "OPTIONS"
+  | "HEAD";
+
+export interface HttpRequest {
+  url: string;
+  method?: HttpMethod;
+  headers?: Record<string, string>;
+  body?: string;
+  followRedirects?: boolean;
+  /** `get_page` folds into `http_request` here. */
+  extract?: "readability";
+}
+
+export interface HttpOpts {
+  /** Milliseconds; no timeout when omitted. */
+  timeoutMs?: number;
+  abortSignal?: AbortSignal;
+}
+
+/** Matches `http_request`'s result, plus the readability page title. */
+export interface HttpResponse {
+  success: boolean;
+  status: number;
+  statusText: string;
+  headers: Record<string, string>;
+  body: string;
+  url: string;
+  redirected: boolean;
+  redirectChain?: string[];
+  error?: string;
+  method?: string;
+  title?: string;
+  capture?: import("../../agents/offSecAgent/tools/httpRequest").HttpRequestResult["capture"];
+  contentTruncated?: boolean;
+  stopReason?: import("../../agents/offSecAgent/tools/getPage").GetPageResponse["stopReason"];
+}
+
+export interface HttpBackend {
+  request(req: HttpRequest, o?: HttpOpts): Promise<HttpResponse>;
+}
+
+export interface BrowserSnapshotResult {
+  success: boolean;
+  snapshot?: string;
+  error?: string;
+}
+
+export interface BrowserCookiesResult {
+  success: boolean;
+  cookies?: Array<{
+    name: string;
+    value: string;
+    domain: string;
+    path: string;
+    httpOnly: boolean;
+    secure: boolean;
+  }>;
+  cookieHeader?: string;
+  error?: string;
+}
+
+/** Today's `sandboxPlaywright` surface. */
+export interface BrowserBackend {
+  navigate(url: string): Promise<BrowserNavigateResult>;
+  snapshot(): Promise<BrowserSnapshotResult>;
+  screenshot(o: { filename: string }): Promise<BrowserScreenshotResult>;
+  click(o: { element: string; ref?: string }): Promise<BrowserClickResult>;
+  fill(o: {
+    element: string;
+    ref?: string;
+    value: string;
+  }): Promise<BrowserFillResult>;
+  evaluate(o: { script: string }): Promise<BrowserEvaluateResult>;
+  console(): Promise<BrowserConsoleResult>;
+  getCookies(o?: { urls?: string[] }): Promise<BrowserCookiesResult>;
+}
+
+/** The #1051 inbox seams, relocated. */
+export interface InboxBackend {
+  email: EmailAdapterResolver;
+  sms: SmsInbox;
+}
+
+export interface ToolBackends {
+  fs: FsBackend;
+  command: CommandBackend;
+  http: HttpBackend;
+  browser: BrowserBackend;
+  inbox: InboxBackend;
+  /**
+   * True when these backends route through a real sandbox whose filesystem is
+   * the contained workspace root (not the host). Tools that write artifacts
+   * (e.g. `document_vulnerability`'s PoC/finding files) must target the sandbox
+   * workspace, not `session.rootPath`, when this is set. `LocalBackends` leaves
+   * it undefined (host paths); console's sandbox-routed backends set it true.
+   */
+  sandboxed?: boolean;
+}

package/src/core/tools/backends/whitebox.test.ts

@@ -0,0 +1,275 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, it, vi } from "vitest";
+import type { ToolContext } from "../../agents/offSecAgent/tools/types";
+import { profileCodebase } from "../../whitebox/repoProfile";
+import type { CommandBackend } from "./types";
+import { resolveWhiteboxBackend } from "./whitebox";
+
+function response(command: string) {
+  if (command.includes("'test' '-d'")) return { stdout: "", exitCode: 0 };
+  if (command.includes("find ."))
+    return { stdout: "src/server.ts\npackage.json\n", exitCode: 0 };
+  if (command.includes("'cat'"))
+    return { stdout: '{"scripts":{"test":"vitest"}}', exitCode: 0 };
+  if (command.includes("'rev-parse'"))
+    return { stdout: "deadbeef", exitCode: 0 };
+  return { stdout: "", exitCode: 1 };
+}
+
+describe("resolved whitebox execution", () => {
+  it("keeps injected profile command identities and order unchanged", async () => {
+    const calls: Array<[string, unknown]> = [];
+    const command: CommandBackend = {
+      async *run(cmd, options) {
+        calls.push([cmd, JSON.parse(JSON.stringify(options))]);
+        const result = response(cmd);
+        yield { type: "stdout", seq: 0, bytes: result.stdout };
+        yield { type: "end", exitCode: result.exitCode, timedOut: false };
+      },
+    };
+    const root = "/nonexistent-remote-repository";
+    // Fixed command journal from before the transport consolidation.
+    const baseline = [
+      [
+        "cd '/nonexistent-remote-repository' && 'test' '-d' '.'",
+        {
+          timeoutSeconds: 5,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'bash' '-c' 'find . -mindepth 1 \\( -type d \\( -name '\\''.git'\\'' -o -name '\\''node_modules'\\'' -o -name '\\''vendor'\\'' -o -name '\\''third_party'\\'' -o -name '\\''dist'\\'' -o -name '\\''build'\\'' -o -name '\\''coverage'\\'' -o -name '\\''.next'\\'' -o -name '\\''target'\\'' -o -name '\\''__pycache__'\\'' \\) -prune \\) -o -type f -print | LC_ALL=C sort'",
+        {
+          timeoutSeconds: 30,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'cat' 'package.json'",
+        {
+          timeoutSeconds: 5,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'git' 'rev-parse' 'HEAD'",
+        {
+          timeoutSeconds: 5,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'git' 'submodule' 'status'",
+        {
+          timeoutSeconds: 5,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'tokei'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'rg'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'ast-grep'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'comby'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'semgrep'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'codeql'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'bandit'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'gosec'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'cargo-geiger'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'cargo-audit'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'gitleaks'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'trufflehog'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'noseyparker'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'osv-scanner'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'trivy'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'grype'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'npm'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'pip-audit'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'govulncheck'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'brakeman'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'spotbugs'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+      [
+        "cd '/nonexistent-remote-repository' && 'which' 'jazzer'",
+        {
+          timeoutSeconds: 2,
+        },
+      ],
+    ];
+    const ctx = { backends: { command } } as ToolContext;
+    const actual = await profileCodebase(
+      root,
+      resolveWhiteboxBackend(ctx).profile,
+    );
+    expect(actual.languages).toEqual(["typescript"]);
+    expect(actual.testCommands).toEqual(["npm run test"]);
+    expect(calls).toEqual(baseline);
+    expect(calls).toHaveLength(27);
+  });
+
+  it("profiles an owned sandbox without reading its path on the host", async () => {
+    const sessionRoot = await mkdtemp(join(tmpdir(), "whitebox-remote-"));
+    try {
+      const execute = vi.fn(async (cmd: string) => {
+        const result = response(cmd);
+        return { ...result, success: result.exitCode === 0, stderr: "" };
+      });
+      const local = vi.fn();
+      const ctx = {
+        agentCwd: "/nonexistent-remote-repository",
+        session: { rootPath: sessionRoot },
+        sandbox: { type: "linux", execute },
+        commandShell: { execute: local },
+      } as unknown as ToolContext;
+      const profile = await profileCodebase(
+        ctx.agentCwd,
+        resolveWhiteboxBackend(ctx).profile,
+      );
+      expect(profile.languages).toEqual(["typescript"]);
+      expect(profile.testCommands).toEqual(["npm run test"]);
+      expect(execute).toHaveBeenCalledTimes(27);
+      expect(local).not.toHaveBeenCalled();
+    } finally {
+      await rm(sessionRoot, { recursive: true, force: true });
+    }
+  });
+
+  it("preserves native argv quoting and capture limits for local analysis", async () => {
+    const ctx = { agentCwd: process.cwd() } as ToolContext;
+    const literal = "spaces ' quotes $HOME ; literal";
+    const result = await resolveWhiteboxBackend(ctx).run(
+      [
+        process.execPath,
+        "-e",
+        "process.stdout.write(process.argv[1])",
+        literal,
+      ],
+      { cwd: process.cwd(), timeoutSeconds: 5, maxTotalBytes: 12 },
+    );
+    expect(result.stdout).toBe(literal.slice(0, 12));
+    expect(result.outputTruncated).toBe(true);
+    expect(result.exitCode).toBe(0);
+  });
+
+  it("does not fall back after an injected analyzer fails", async () => {
+    const failure = new Error("analyzer backend unavailable");
+    const execute = vi.fn();
+    const ctx = {
+      backends: {
+        command: {
+          async *run() {
+            yield await Promise.reject<never>(failure);
+          },
+        },
+      },
+      sandbox: { execute },
+    } as unknown as ToolContext;
+    await expect(
+      resolveWhiteboxBackend(ctx).run(["scanner"], {
+        cwd: "/remote",
+        timeoutSeconds: 5,
+        maxTotalBytes: 1024,
+      }),
+    ).rejects.toBe(failure);
+    expect(execute).not.toHaveBeenCalled();
+  });
+});