@pensar/apex 2.5.0 → 2.6.1-canary.0cc77dcc

package/src/core/whitebox/profiles.ts

@@ -0,0 +1,294 @@
+import type { CatalogRecord } from "./types";
+
+export const WHITEBOX_CATALOG: CatalogRecord[] = [
+  {
+    id: "repo.day-one-recon",
+    kind: "review-pass",
+    title: "Day-one repository recon",
+    tags: ["repo", "recon", "orientation"],
+    tools: ["tokei", "git", "tree", "rg"],
+    guidance:
+      "Profile languages, manifests, current commit, submodules, generated/vendor exclusions, build/test/run commands, recent security-sensitive changes, and local runtime hints before deep analysis.",
+  },
+  {
+    id: "entry.http-routes",
+    kind: "entry-point",
+    title: "HTTP route definitions",
+    languages: [
+      "typescript",
+      "javascript",
+      "python",
+      "go",
+      "rust",
+      "java",
+      "kotlin",
+      "ruby",
+    ],
+    tags: ["entrypoint", "http", "routes"],
+    patterns: [
+      "@(app|router|blueprint)\\.(get|post|put|patch|delete)",
+      "app\\.(get|post|put|patch|delete)\\(",
+      "http\\.HandleFunc|mux\\.Handle|chi\\.Route",
+      "Router::new\\(\\)\\.route|actix_web::web::resource",
+      "@(Get|Post|Put|Delete|Patch|Request)Mapping",
+      "resources :|namespace .* do",
+    ],
+    guidance:
+      "Map route -> handler -> auth middleware -> downstream sink. Consolidate methods per route and keep file/line/handler evidence.",
+  },
+  {
+    id: "entry.async-handlers",
+    kind: "entry-point",
+    title: "Async, webhook, serverless, and realtime entry points",
+    tags: ["entrypoint", "queues", "webhooks", "serverless", "websocket"],
+    patterns: [
+      "SQS|Kafka|Celery|Bull|Sidekiq|RabbitMQ",
+      "webhook|signature|X-Hub-Signature|stripe-signature",
+      "exports\\.handler|lambda\\.Handler|APIGatewayProxy",
+      "WebSocket|SSE|EventSource|socket\\.io",
+      "cron|schedule|watch\\(",
+    ],
+    guidance:
+      "Treat queue consumers, webhooks, Lambda handlers, WebSockets, CLIs, cron jobs, file watchers, and signal handlers as attacker-controlled entry points when inputs cross trust boundaries.",
+  },
+  {
+    id: "boundary.trust-map",
+    kind: "trust-boundary",
+    title: "Trust-boundary map",
+    tags: ["trust-boundary", "tenant", "internal-services"],
+    guidance:
+      "Identify user-to-app, frontend-to-backend, app-to-DB, app-to-internal-service, app-to-third-party, tenant-to-tenant, background job tenant context, and internal/admin surfaces.",
+  },
+  {
+    id: "sink.command-exec",
+    kind: "sink",
+    title: "Command execution",
+    languages: [
+      "typescript",
+      "javascript",
+      "python",
+      "go",
+      "java",
+      "kotlin",
+      "rust",
+    ],
+    tags: ["rce", "command-injection", "sink"],
+    patterns: [
+      "child_process\\.exec\\(",
+      "subprocess\\..*shell\\s*=\\s*True",
+      "\\b(exec|eval|system|popen)\\(",
+      "os/exec|Runtime\\.exec|ProcessBuilder",
+      "Command::new\\([^)]*sh|sh\\s+-c",
+    ],
+    guidance:
+      "Trace command arguments backward to route, queue, webhook, config, archive, or tenant-controlled input. Prefer execFile/argument arrays and allowlists.",
+  },
+  {
+    id: "sink.injection",
+    kind: "sink",
+    title: "SQL and NoSQL injection",
+    tags: ["sqli", "nosql", "sink"],
+    patterns: [
+      "\\.raw\\(|\\.extra\\(|RawSQL|text\\(f",
+      "\\$queryRawUnsafe|knex\\.raw|fmt\\.Sprintf.*SELECT",
+      "db\\.Query\\(fmt\\.Sprintf|execute\\([^,]+\\+",
+      "\\$where|req\\.body.*\\$",
+    ],
+    guidance:
+      "Find raw query APIs, dynamic identifiers, template literals/format strings, raw Mongo filters, and second-order injection paths.",
+  },
+  {
+    id: "sink.deserialization",
+    kind: "sink",
+    title: "Unsafe deserialization",
+    languages: [
+      "python",
+      "java",
+      "kotlin",
+      "typescript",
+      "javascript",
+      "php",
+      "rust",
+    ],
+    tags: ["deserialization", "rce", "sink"],
+    patterns: [
+      "pickle\\.loads|cPickle|dill|shelve|marshal",
+      "yaml\\.load\\(|yaml\\.full_load",
+      "ObjectInputStream|readObject",
+      "SnakeYAML|@JsonTypeInfo|enableDefaultTyping",
+      "node-serialize|unserialize\\(",
+      "bincode|rmp_serde",
+    ],
+    guidance:
+      "Assume attacker-controlled object formats are high-risk. Verify reachable parser paths, gadget availability, and resource-exhaustion vectors.",
+  },
+  {
+    id: "sink.ssrf-path-files",
+    kind: "sink",
+    title: "SSRF and filesystem access",
+    tags: [
+      "ssrf",
+      "node",
+      "javascript",
+      "typescript",
+      "path-traversal",
+      "file-read",
+      "file-write",
+      "zip-slip",
+      "sink",
+    ],
+    patterns: [
+      "requests\\.(get|post)|urllib|http\\.Get|fetch\\(|axios\\.|got\\(",
+      "open\\(|fs\\.readFile|fs\\.writeFile|path\\.join|os\\.path\\.join",
+      "ZipFile|tarfile|archive|extractall|unzip",
+    ],
+    guidance:
+      "Trace URL/file/archive inputs to network clients, redirects, metadata endpoints, path joins, upload destinations, and archive extraction.",
+  },
+  {
+    id: "sink.template-xxe-xss",
+    kind: "sink",
+    title: "Template, XML, and client-side injection",
+    tags: ["ssti", "xxe", "xss", "frontend", "sink"],
+    patterns: [
+      "render_template_string|Template\\(|compile\\(",
+      "resolve_entities|XmlDocument|DocumentBuilderFactory|lxml",
+      "dangerouslySetInnerHTML|innerHTML|outerHTML|document\\.write|v-html|\\[innerHTML\\]",
+      "postMessage|Access-Control-Allow-Origin|Content-Security-Policy",
+    ],
+    guidance:
+      "Check output context, parser configuration, origin checks, CORS/CSP, stored payload flows, SVG/Markdown/CSV renderers, and production source maps.",
+  },
+  {
+    id: "sink.crypto-auth",
+    kind: "sink",
+    title: "Crypto, token, and auth bypass primitives",
+    tags: ["crypto", "auth", "jwt", "secrets", "sink"],
+    patterns: [
+      "MD5|SHA1|ECB|Math\\.random|random\\.random|rand\\(",
+      "verify\\s*=\\s*False|InsecureSkipVerify|rejectUnauthorized\\s*:\\s*false",
+      "algorithm.*none|jwt\\.decode|kid|jku|x5u",
+      "compare\\(|==.*token|==.*secret",
+    ],
+    guidance:
+      "Review custom crypto, password hashing, token entropy, constant-time comparison, key storage, TLS verification, JWT algorithm confusion, and claim-use-before-signature verification.",
+  },
+  {
+    id: "review.auth-session",
+    kind: "review-pass",
+    title: "Authentication and session review",
+    tags: ["auth", "session", "oauth", "saml", "mfa"],
+    guidance:
+      "Read auth code end-to-end. Cover session cookies, JWTs, API keys, OAuth, SAML, mTLS, HMAC, password reset, registration, MFA, logout, and impersonation flows before hunting isolated bugs.",
+  },
+  {
+    id: "review.authorization-business-logic",
+    kind: "review-pass",
+    title: "Authorization, tenancy, and business logic",
+    tags: ["authz", "idor", "tenant", "business-logic", "graphql"],
+    guidance:
+      "Look for resource-level authz gaps, tenant context from request data, GraphQL field/mutation auth, mass assignment, state-machine skips, quota/payment/coupon races, webhook replay, and weak rate limits.",
+  },
+  {
+    id: "review.parsers-uploads",
+    kind: "review-pass",
+    title: "Parsers, uploads, and output encoding",
+    tags: ["parser", "upload", "encoding", "redos"],
+    guidance:
+      "Map JSON, XML, YAML, multipart, protobuf, MessagePack, CSV, Markdown, SVG, archive, image parser, Unicode normalization, double encoding, and ReDoS surfaces.",
+  },
+  {
+    id: "review.storage-data",
+    kind: "review-pass",
+    title: "Storage, ORMs, and data layer",
+    tags: ["orm", "storage", "cache", "search", "export"],
+    guidance:
+      "Review ORM raw APIs, dynamic identifiers, second-order injection, sensitive data at rest, CSV formula injection, cache poisoning/key collisions, and Elasticsearch/Lucene query injection.",
+  },
+  {
+    id: "review.infra-supply-chain",
+    kind: "review-pass",
+    title: "Infrastructure, runtime, and supply chain",
+    tags: ["iac", "docker", "kubernetes", "ci", "supply-chain"],
+    guidance:
+      "Review Docker root/secrets/latest tags, Kubernetes privileged/host/RBAC/network policy/service accounts, IaC public resources/IAM wildcards/encryption, GitHub Actions pull_request_target/unpinned actions/OIDC, reverse proxy normalization, debug/log leaks, dependency confusion, install scripts, vendored binaries, and image provenance.",
+  },
+  {
+    id: "scanner.static",
+    kind: "scanner",
+    title: "Static analysis tools",
+    tags: ["scanner", "sast"],
+    tools: [
+      "semgrep",
+      "codeql",
+      "bandit",
+      "gosec",
+      "cargo-geiger",
+      "clippy",
+      "brakeman",
+      "spotbugs",
+      "find-sec-bugs",
+    ],
+    guidance:
+      "Run installed SAST tools opportunistically. Normalize results into candidates; verify exploitability before documenting findings.",
+  },
+  {
+    id: "scanner.secrets-deps",
+    kind: "scanner",
+    title: "Secrets, dependency, and supply-chain scanners",
+    tags: ["scanner", "secrets", "dependencies", "supply-chain"],
+    tools: [
+      "gitleaks",
+      "trufflehog",
+      "noseyparker",
+      "osv-scanner",
+      "trivy",
+      "grype",
+      "pip-audit",
+      "npm",
+      "govulncheck",
+      "cargo-audit",
+    ],
+    guidance:
+      "Run secrets and dependency scans with bounded output. Prefer direct dependencies and attack-path relevance over raw vulnerability counts.",
+  },
+  {
+    id: "scanner.repo-intelligence",
+    kind: "scanner",
+    title: "Repo intelligence and history",
+    tags: ["scanner", "git", "history", "code-query"],
+    tools: ["rg", "ast-grep", "comby", "git"],
+    guidance:
+      "Use ripgrep, ast-grep, comby, git pickaxe, and git history on auth/crypto/session/admin/permission/tenant files to locate risky changes and incomplete fixes.",
+  },
+  {
+    id: "fuzzer.runtime",
+    kind: "fuzzer",
+    title: "Runtime fuzzing and crash triage",
+    languages: ["python", "go", "rust", "java", "kotlin", "c", "cpp"],
+    tags: ["fuzzing", "rce", "memory-corruption", "parser"],
+    tools: [
+      "atheris",
+      "cargo-fuzz",
+      "go test -fuzz",
+      "jazzer",
+      "asan",
+      "ubsan",
+    ],
+    guidance:
+      "Generate harnesses in session scratch space, bound runtime, keep logs/crashes as artifacts, and minimize reproducers before promoting candidates.",
+  },
+];
+
+export const DEFAULT_WHITEBOX_EXCLUDED_DIRS = [
+  ".git",
+  "node_modules",
+  "vendor",
+  "third_party",
+  "dist",
+  "build",
+  "coverage",
+  ".next",
+  "target",
+  "__pycache__",
+];

package/src/core/whitebox/repoProfile.test.ts

@@ -0,0 +1,217 @@
+import { execFileSync } from "node:child_process";
+import { mkdir, mkdtemp, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { PerCommandShell } from "../agents/offSecAgent/tools/perCommandShell";
+import type { ToolContext } from "../agents/offSecAgent/tools/types";
+import type { SessionInfo } from "../session";
+import { LocalBackends } from "../tools/backends";
+import type { CommandBackend, CommandEvent } from "../tools/backends/types";
+import { DEFAULT_WHITEBOX_EXCLUDED_DIRS } from "./profiles";
+import { profileCodebase } from "./repoProfile";
+
+async function tempDir(prefix: string): Promise<string> {
+  return mkdtemp(join(tmpdir(), prefix));
+}
+
+function mockSession(rootPath: string): SessionInfo {
+  return {
+    id: "ses_test",
+    version: "1.0.0",
+    targets: [],
+    time: { created: Date.now(), updated: Date.now() },
+    rootPath,
+    logsPath: join(rootPath, "logs"),
+    findingsPath: join(rootPath, "findings"),
+    scratchpadPath: join(rootPath, "scratchpad"),
+    pocsPath: join(rootPath, "pocs"),
+    config: {},
+  };
+}
+
+describe("profileCodebase via LocalBackends", () => {
+  let shell: PerCommandShell | undefined;
+
+  afterEach(() => {
+    shell?.dispose();
+    shell = undefined;
+  });
+
+  it("matches today's host walk for a fixture tree: languages, manifests, entry hints, IaC/CI, git info, and pruning", async () => {
+    const root = await tempDir("apex-repoprofile-fixture-");
+    const outside = await tempDir("apex-repoprofile-outside-");
+
+    await writeFile(
+      join(root, "package.json"),
+      JSON.stringify({
+        scripts: {
+          build: "tsc",
+          test: "vitest",
+          dev: "tsx watch src/index.ts",
+        },
+      }),
+    );
+    await writeFile(join(root, "package-lock.json"), "{}");
+    await mkdir(join(root, "src"), { recursive: true });
+    await writeFile(join(root, "src", "routes.ts"), "app.get('/x', handler);");
+    await mkdir(join(root, "node_modules", "left-pad"), { recursive: true });
+    await writeFile(
+      join(root, "node_modules", "left-pad", "index.js"),
+      "module.exports = () => {};",
+    );
+    await mkdir(join(root, ".github", "workflows"), { recursive: true });
+    await writeFile(join(root, ".github", "workflows", "ci.yml"), "on: push\n");
+    await mkdir(join(root, "infra"), { recursive: true });
+    await writeFile(join(root, "infra", "main.tf"), 'resource "x" {}\n');
+    await writeFile(join(outside, "secret.txt"), "outside the root");
+    await symlink(outside, join(root, "escaped-link"));
+
+    const gitEnv = {
+      ...process.env,
+      GIT_AUTHOR_NAME: "apex-test",
+      GIT_AUTHOR_EMAIL: "apex-test@example.com",
+      GIT_COMMITTER_NAME: "apex-test",
+      GIT_COMMITTER_EMAIL: "apex-test@example.com",
+    };
+    execFileSync("git", ["init", "-q"], { cwd: root, env: gitEnv });
+    execFileSync("git", ["add", "-A"], { cwd: root, env: gitEnv });
+    execFileSync("git", ["commit", "-q", "-m", "init"], {
+      cwd: root,
+      env: gitEnv,
+    });
+    const expectedCommit = execFileSync("git", ["rev-parse", "HEAD"], {
+      cwd: root,
+      env: gitEnv,
+    })
+      .toString()
+      .trim();
+
+    shell = new PerCommandShell({ cwd: root });
+    const ctx = {
+      agentCwd: root,
+      session: mockSession(root),
+      commandShell: shell,
+    } as ToolContext;
+    const { command } = LocalBackends(ctx);
+
+    const profile = await profileCodebase(root, command);
+    expect(profile).toEqual(await profileCodebase(root));
+
+    expect(profile.rootPath).toBe(root);
+    expect(profile.currentCommit).toBe(expectedCommit);
+    // Only "typescript" — if node_modules/left-pad's .js leaked through the
+    // prune, "javascript" would show up here too.
+    expect(profile.languages).toEqual(["typescript"]);
+    expect(profile.packageManagers).toEqual(["npm"]);
+    expect([...profile.manifestFiles].sort()).toEqual([
+      "package-lock.json",
+      "package.json",
+    ]);
+    expect(profile.lockfiles).toEqual(["package-lock.json"]);
+    expect(profile.buildCommands).toEqual(["npm run build"]);
+    expect(profile.testCommands).toEqual(["npm run test"]);
+    expect(profile.runCommands).toEqual(["npm run dev"]);
+    expect(profile.entryPointHints).toEqual(["src/routes.ts"]);
+    expect(profile.iacFiles).toEqual(["infra/main.tf"]);
+    expect(profile.ciFiles).toEqual([".github/workflows/ci.yml"]);
+    expect(profile.nativeCode).toBe(false);
+    expect(profile.submodules).toEqual([]);
+    expect(profile.excludedDirs).toEqual(DEFAULT_WHITEBOX_EXCLUDED_DIRS);
+    expect(profile.toolAvailability).toHaveLength(22);
+    for (const tool of profile.toolAvailability) {
+      expect(typeof tool.name).toBe("string");
+      expect(typeof tool.available).toBe("boolean");
+      if (tool.available) expect(typeof tool.path).toBe("string");
+    }
+
+    // The symlinked escape hatch is walked into by neither `-type d` nor
+    // `-type f`, matching the old Dirent-based skip.
+    const serialized = JSON.stringify(profile);
+    expect(serialized).not.toContain("escaped-link");
+    expect(serialized).not.toContain("secret.txt");
+  });
+});
+
+describe("profileCodebase via a synthetic command backend", () => {
+  it("propagates an injected backend failure without profiling the host", async () => {
+    const root = await tempDir("apex-repoprofile-no-fallback-");
+    const failure = new Error("remote command unavailable");
+    const command: CommandBackend = {
+      async *run() {
+        yield await Promise.reject<CommandEvent>(failure);
+      },
+    };
+    await expect(profileCodebase(root, command)).rejects.toBe(failure);
+  });
+
+  it("never touches the host filesystem — profiles purely from the backend's responses", async () => {
+    const calls: string[] = [];
+    const packageJson = JSON.stringify({
+      scripts: { build: "tsc", test: "vitest" },
+      packageManager: "bun@1.2.3",
+    });
+    const files = ["src/handler.py", "requirements.txt", "Dockerfile"].join(
+      "\n",
+    );
+
+    function respond(cmd: string): { stdout: string; exitCode: number } {
+      if (cmd.includes("'-d'")) return { stdout: "", exitCode: 0 };
+      if (cmd.includes("find .")) return { stdout: files, exitCode: 0 };
+      if (cmd.includes("package.json"))
+        return { stdout: packageJson, exitCode: 0 };
+      if (cmd.includes("rev-parse"))
+        return { stdout: "deadbeefcafe\n", exitCode: 0 };
+      if (cmd.includes("submodule")) return { stdout: "", exitCode: 0 };
+      if (cmd.includes("'which' 'semgrep'"))
+        return { stdout: "/usr/bin/semgrep\n", exitCode: 0 };
+      if (cmd.includes("'which'")) return { stdout: "", exitCode: 1 };
+      return { stdout: "", exitCode: 1 };
+    }
+
+    const spyCommand: CommandBackend = {
+      async *run(cmd: string): AsyncIterable<CommandEvent> {
+        calls.push(cmd);
+        const { stdout, exitCode } = respond(cmd);
+        yield { type: "start" };
+        if (stdout) yield { type: "stdout", seq: 0, bytes: stdout };
+        yield { type: "end", exitCode, timedOut: false };
+      },
+    };
+
+    const root = "/apex-nonexistent-profile-root-xyz";
+    const profile = await profileCodebase(root, spyCommand);
+
+    expect(profile.rootPath).toBe(root);
+    expect(profile.currentCommit).toBe("deadbeefcafe");
+    expect(profile.languages).toEqual(["python"]);
+    expect(profile.packageManagers).toEqual(["pip"]);
+    expect(profile.manifestFiles).toEqual(["requirements.txt", "Dockerfile"]);
+    expect(profile.lockfiles).toEqual([]);
+    expect(profile.buildCommands).toEqual(["bun run build"]);
+    expect(profile.testCommands).toEqual(["bun run test"]);
+    expect(profile.runCommands).toEqual([]);
+    expect(profile.entryPointHints).toEqual(["src/handler.py"]);
+    expect(profile.nativeCode).toBe(false);
+    expect(profile.submodules).toEqual([]);
+
+    const semgrep = profile.toolAvailability.find((t) => t.name === "semgrep");
+    expect(semgrep).toEqual({
+      name: "semgrep",
+      available: true,
+      path: "/usr/bin/semgrep",
+    });
+    expect(
+      profile.toolAvailability.filter((t) => t.name !== "semgrep"),
+    ).toSatisfy((tools: typeof profile.toolAvailability) =>
+      tools.every((t) => t.available === false),
+    );
+
+    // Every read went through the backend — none of it hit a real path.
+    expect(calls.some((c) => c.includes("find ."))).toBe(true);
+    expect(calls.some((c) => c.includes("package.json"))).toBe(true);
+    expect(calls.some((c) => c.includes("rev-parse"))).toBe(true);
+    expect(calls.some((c) => c.includes("submodule"))).toBe(true);
+    expect(calls.filter((c) => c.includes("'which'"))).toHaveLength(22);
+  });
+});