@pensar/apex 2.5.0 → 2.6.1-canary.025b1ecf

package/src/core/eventBus.ts

@@ -0,0 +1,410 @@
+import { EventEmitter } from "node:events";
+import type { TextStreamPart, ToolSet } from "ai";
+
+/**
+ * Typed event map for the agent event bus.
+ *
+ * Events mirror the AI SDK's `TextStreamPart` types, plus native identity
+ * (`sessionId`/`messageId`/`partId`) for routing without inference.
+ * `subagentId` is kept as a backward-compatible alias of `sessionId`.
+ * Lifecycle events (`subagent-spawn`, `subagent-complete`) and side-channel
+ * events (`command-output`, `error`) round out the map.
+ */
+export type AgentEventMap = {
+  "text-delta": {
+    text: string;
+    subagentId?: string;
+    sessionId?: string;
+    messageId?: string;
+    partId?: string;
+  };
+  /**
+   * Extended-thinking / reasoning output streamed incrementally. Mirrors
+   * `text-delta` but carries the model's reasoning tokens (Anthropic
+   * extended thinking, OpenAI reasoning) so consumers can surface a
+   * "thinking" indicator instead of appearing to hang between tool calls.
+   */
+  "reasoning-delta": {
+    text: string;
+    subagentId?: string;
+    sessionId?: string;
+    messageId?: string;
+  };
+  /** A reasoning block opened. Lets consumers time the thinking window. */
+  "reasoning-start": {
+    subagentId?: string;
+    sessionId?: string;
+    messageId?: string;
+  };
+  /** A reasoning block closed. Paired with `reasoning-start` to derive duration. */
+  "reasoning-end": {
+    subagentId?: string;
+    sessionId?: string;
+    messageId?: string;
+  };
+  "tool-call-start": {
+    toolCallId: string;
+    toolName: string;
+    subagentId?: string;
+    sessionId?: string;
+    messageId?: string;
+    partId?: string;
+  };
+  "tool-call-delta": {
+    toolCallId: string;
+    argsTextDelta: string;
+    subagentId?: string;
+    sessionId?: string;
+    messageId?: string;
+    partId?: string;
+  };
+  "tool-call-complete": {
+    toolCallId: string;
+    toolName: string;
+    args: unknown;
+    subagentId?: string;
+    sessionId?: string;
+    messageId?: string;
+    partId?: string;
+  };
+  "tool-result": {
+    toolCallId: string;
+    toolName: string;
+    result: unknown;
+    subagentId?: string;
+    sessionId?: string;
+    messageId?: string;
+    partId?: string;
+  };
+  "subagent-spawn": {
+    subagentId: string;
+    /** The spawned child's own session id (`ses_…`). */
+    sessionId?: string;
+    name?: string;
+    input: unknown;
+    /** Omitted means top-level. Auto-populated by {@link AgentEventBus.attachChild}. */
+    parentSubagentId?: string;
+    /** Parent agent's session id (`ses_…`). Auto-populated by {@link AgentEventBus.attachChild}. */
+    parentSessionId?: string;
+  };
+  "subagent-complete": {
+    subagentId: string;
+    /** The completed child's own session id (`ses_…`). */
+    sessionId?: string;
+    status: "completed" | "failed";
+    /** Omitted means top-level. Auto-populated by {@link AgentEventBus.attachChild}. */
+    parentSubagentId?: string;
+    /** Parent agent's session id (`ses_…`). Auto-populated by {@link AgentEventBus.attachChild}. */
+    parentSessionId?: string;
+  };
+  "workflow-phase-start": {
+    phase: "discovery" | "pentesting" | "reporting";
+    label: string;
+    metadata?: Record<string, unknown>;
+  };
+  "workflow-phase-complete": {
+    phase: "discovery" | "pentesting" | "reporting";
+    summary: Record<string, unknown>;
+  };
+  "app-analysis-progress": {
+    totalApps: number;
+    completedApps: number;
+    appName?: string;
+  };
+  "step-finish": {
+    messages: unknown[];
+    subagentId?: string;
+    sessionId?: string;
+    /** The assistant message (`msg_…`) that just closed this step. */
+    messageId?: string;
+  };
+  "command-output": { data: string; subagentId?: string };
+  error: { error: unknown; subagentId?: string };
+  "trace-record": {
+    record: import("./agents/offSecAgent/trace").TraceRecord;
+    subagentId?: string;
+  };
+};
+
+/**
+ * Per-event decision used by {@link AgentEventBus.attachChild}.
+ *
+ * - `"forward"` — the event may originate inside a subagent; bubble it to the
+ *   parent bus so parent-side consumers (W&B uploader, dashboard buffers,
+ *   future persistence/metrics layers) receive it.
+ * - `"parent-only"` — the event is emitted only by orchestrator-level workflow
+ *   code on the parent bus; if it ever appears on a child bus, drop it on the
+ *   floor rather than re-emit on the parent (which would either be dead code
+ *   or, worse, a double-fire).
+ *
+ * The map type below is `{ readonly [K in keyof AgentEventMap]: ... }`, so
+ * adding a new event to {@link AgentEventMap} without an explicit policy
+ * entry is a TypeScript compile error. This invariant is the long-term
+ * safety net: hand-curated allowlists silently omit events; an exhaustive
+ * mapped type cannot.
+ */
+type ChildForwardPolicy = "forward" | "parent-only";
+
+const CHILD_BUS_FORWARD_POLICY: {
+  readonly [K in keyof AgentEventMap]: ChildForwardPolicy;
+} = {
+  "text-delta": "forward",
+  "reasoning-delta": "forward",
+  "reasoning-start": "forward",
+  "reasoning-end": "forward",
+  "tool-call-start": "forward",
+  "tool-call-delta": "forward",
+  "tool-call-complete": "forward",
+  "tool-result": "forward",
+  "subagent-spawn": "forward",
+  "subagent-complete": "forward",
+  "step-finish": "forward",
+  "command-output": "forward",
+  error: "forward",
+  "trace-record": "forward",
+  // Emitted only by orchestrator workflow code (pentest.ts,
+  // whiteboxAttackSurface.ts) on the parent bus directly. If a future change
+  // makes any of these subagent-emitted, flip the value here deliberately.
+  "workflow-phase-start": "parent-only",
+  "workflow-phase-complete": "parent-only",
+  "app-analysis-progress": "parent-only",
+};
+
+const CHILD_BUS_FORWARDED_EVENTS = (
+  Object.keys(CHILD_BUS_FORWARD_POLICY) as (keyof AgentEventMap)[]
+).filter((key) => CHILD_BUS_FORWARD_POLICY[key] === "forward");
+
+/**
+ * Centralized, typed event bus for agent streaming output.
+ *
+ * Replaces the callback-based `ConsumeCallbacks` / `SubagentConsumeCallbacks`
+ * pattern with a publish-subscribe model.  Multiple consumers (TUI rendering,
+ * DB persistence, metrics, logging) subscribe independently.
+ *
+ * Usage:
+ * ```ts
+ * const bus = new AgentEventBus();
+ * bus.on("text-delta", (e) => process.stdout.write(e.text));
+ * bus.on("tool-call-complete", (e) => console.log(`→ ${e.toolName}`));
+ * await agent.consume();
+ * ```
+ */
+export class AgentEventBus {
+  private readonly emitter = new EventEmitter();
+
+  constructor() {
+    this.emitter.setMaxListeners(50);
+  }
+
+  on<K extends keyof AgentEventMap>(
+    event: K,
+    handler: (payload: AgentEventMap[K]) => void,
+  ): this {
+    this.emitter.on(event, handler);
+    return this;
+  }
+
+  once<K extends keyof AgentEventMap>(
+    event: K,
+    handler: (payload: AgentEventMap[K]) => void,
+  ): this {
+    this.emitter.once(event, handler);
+    return this;
+  }
+
+  off<K extends keyof AgentEventMap>(
+    event: K,
+    handler: (payload: AgentEventMap[K]) => void,
+  ): this {
+    this.emitter.off(event, handler);
+    return this;
+  }
+
+  emit<K extends keyof AgentEventMap>(
+    event: K,
+    payload: AgentEventMap[K],
+  ): boolean {
+    return this.emitter.emit(event, payload);
+  }
+
+  removeAllListeners(event?: keyof AgentEventMap): this {
+    if (event) {
+      this.emitter.removeAllListeners(event);
+    } else {
+      this.emitter.removeAllListeners();
+    }
+    return this;
+  }
+
+  /**
+   * Forward selected events from a child bus to a parent bus, stamping
+   * `sessionId`/`subagentId` (and `parentSessionId`/`parentSubagentId`
+   * for lifecycle events) so the parent's routing works without inference.
+   */
+  static attachChild(
+    child: AgentEventBus,
+    parent: AgentEventBus | undefined,
+    childSessionId: string,
+  ): void {
+    if (!parent) return;
+    for (const key of CHILD_BUS_FORWARDED_EVENTS) {
+      // Re-bind so the handler's payload type tracks `key` as a single K —
+      // without this, the union of payload shapes is too wide for `emit`.
+      const forwardKey = key as keyof AgentEventMap;
+      child.on(forwardKey, (payload: AgentEventMap[typeof forwardKey]) => {
+        const p = payload as Record<string, unknown>;
+
+        const isLifecycle =
+          forwardKey === "subagent-spawn" || forwardKey === "subagent-complete";
+
+        if (isLifecycle) {
+          // Nested subagent case: childSessionId becomes the grandchild's parent id.
+          const next: Record<string, unknown> = { ...p };
+          if (!p.parentSubagentId) next.parentSubagentId = childSessionId;
+          if (!p.parentSessionId) next.parentSessionId = childSessionId;
+          parent.emit(forwardKey, next as never);
+          return;
+        }
+
+        const next: Record<string, unknown> = { ...p };
+        if (!p.subagentId) next.subagentId = childSessionId;
+        if (!p.sessionId) next.sessionId = childSessionId;
+        parent.emit(forwardKey, next as never);
+      });
+    }
+  }
+
+  /**
+   * Emit the appropriate bus event for an AI SDK `fullStream` chunk.
+   *
+   * Maps Vercel AI SDK `TextStreamPart` types to `AgentEventMap` keys:
+   *   text-delta        → text-delta
+   *   reasoning-start   → reasoning-start
+   *   reasoning-delta   → reasoning-delta
+   *   reasoning-end     → reasoning-end
+   *   tool-input-start  → tool-call-start
+   *   tool-input-delta  → tool-call-delta
+   *   tool-call         → tool-call-complete
+   *   tool-result       → tool-result
+   *   error             → error
+   *
+   * `ids` accepts a bare `subagentId` string for backward compatibility, or
+   * a {@link StreamIdContext} to also stamp `sessionId`/`messageId`/`partId`.
+   */
+  emitStreamPart(
+    chunk: TextStreamPart<ToolSet>,
+    ids?: string | StreamIdContext,
+  ): void {
+    const ctx: StreamIdContext =
+      typeof ids === "string" ? { subagentId: ids } : (ids ?? {});
+    // No sessionId fallback: subagentId stays undefined for the orchestrator.
+    const subagentId = ctx.subagentId;
+    const sessionId = ctx.sessionId;
+    const messageId = ctx.messageId;
+
+    switch (chunk.type) {
+      case "text-delta":
+        this.emit("text-delta", {
+          text: chunk.text,
+          subagentId,
+          sessionId,
+          messageId,
+          partId: ctx.textPartId,
+        });
+        break;
+      case "reasoning-start":
+        this.emit("reasoning-start", { subagentId, sessionId, messageId });
+        break;
+      case "reasoning-delta":
+        this.emit("reasoning-delta", {
+          text: chunk.text,
+          subagentId,
+          sessionId,
+          messageId,
+        });
+        break;
+      case "reasoning-end":
+        this.emit("reasoning-end", { subagentId, sessionId, messageId });
+        break;
+      case "tool-input-start":
+        this.emit("tool-call-start", {
+          toolCallId: chunk.id,
+          toolName: chunk.toolName,
+          subagentId,
+          sessionId,
+          messageId,
+          partId: ctx.toolPartId?.(chunk.id),
+        });
+        break;
+      case "tool-input-delta":
+        this.emit("tool-call-delta", {
+          toolCallId: chunk.id,
+          argsTextDelta: chunk.delta,
+          subagentId,
+          sessionId,
+          messageId,
+          partId: ctx.toolPartId?.(chunk.id),
+        });
+        break;
+      case "tool-call": {
+        const tc = chunk as {
+          toolCallId: string;
+          toolName: string;
+          input?: unknown;
+          args?: unknown;
+        };
+        this.emit("tool-call-complete", {
+          toolCallId: tc.toolCallId,
+          toolName: tc.toolName,
+          args: tc.args ?? tc.input,
+          subagentId,
+          sessionId,
+          messageId,
+          partId: ctx.toolPartId?.(tc.toolCallId),
+        });
+        break;
+      }
+      case "tool-result": {
+        const tr = chunk as {
+          toolCallId: string;
+          toolName: string;
+          result?: unknown;
+          output?: unknown;
+        };
+        this.emit("tool-result", {
+          toolCallId: tr.toolCallId,
+          toolName: tr.toolName,
+          result: tr.result ?? tr.output,
+          subagentId,
+          sessionId,
+          messageId,
+          partId: ctx.toolPartId?.(tr.toolCallId),
+        });
+        break;
+      }
+      case "error":
+        this.emit("error", {
+          error: (chunk as { type: "error"; error: unknown }).error,
+          subagentId,
+        });
+        break;
+    }
+  }
+}
+
+/**
+ * Identity context the emitting agent threads through
+ * {@link AgentEventBus.emitStreamPart}.
+ */
+export interface StreamIdContext {
+  /** Legacy multi-agent delineator; equals {@link sessionId} when set. */
+  subagentId?: string;
+  /** Emitting agent's session id (`ses_…`). */
+  sessionId?: string;
+  /** Current open assistant message id (`msg_…`) for this step. */
+  messageId?: string;
+  /** Current text part id (`prt_…`); minted per text run by the agent. */
+  textPartId?: string;
+  /** Resolves a stable part id (`prt_…`) for a `toolCallId`, minting on first use. */
+  toolPartId?: (toolCallId: string) => string;
+}

package/src/core/findings/attackSurfaceRegistry.ts

@@ -0,0 +1,294 @@
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+
+// ---------------------------------------------------------------------------
+// Fingerprinting helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Normalise a URL for comparison: lowercase, strip trailing slash,
+ * strip query string, fragment, and protocol.
+ */
+function normalizeAssetUrl(url: string): string {
+  let u = url.trim().toLowerCase();
+
+  const hashIdx = u.indexOf("#");
+  if (hashIdx !== -1) u = u.substring(0, hashIdx);
+
+  const qIdx = u.indexOf("?");
+  if (qIdx !== -1) u = u.substring(0, qIdx);
+
+  u = u.replace(/^https?:\/\//, "");
+  u = u.replace(/^www\./, "");
+
+  if (u.length > 1 && u.endsWith("/")) {
+    u = u.replace(/\/+$/, "");
+  }
+
+  return u;
+}
+
+/**
+ * Normalise an asset name for comparison: lowercase, collapse
+ * non-alphanumeric runs into a single dash, trim leading/trailing dashes.
+ */
+function normalizeAssetName(name: string): string {
+  return name
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-|-$/g, "");
+}
+
+/**
+ * Build fingerprint keys for an asset.
+ *
+ * - `urlKey`:  `[<app>||]<normalized_url>||<asset_type>` — precise per-URL dedup
+ *              (only set when the asset has a URL)
+ * - `nameKey`: `[<app>||]<normalized_name>||<asset_type>` — catches the same logical
+ *              asset rediscovered with different metadata
+ *
+ * When `appName` is present, keys are scoped to that app so that
+ * `/api/users` in App A and `/api/users` in App B are treated as
+ * distinct assets.
+ */
+function generateAssetFingerprint(asset: AssetRecord): {
+  urlKey: string | null;
+  nameKey: string;
+} {
+  const normalizedName = normalizeAssetName(asset.assetName);
+  const assetType = asset.assetType.toLowerCase();
+  const url = asset.details?.url;
+  const appPrefix = asset.appName
+    ? `${normalizeAssetName(asset.appName)}||`
+    : "";
+
+  return {
+    urlKey: url ? `${appPrefix}${normalizeAssetUrl(url)}||${assetType}` : null,
+    nameKey: `${appPrefix}${normalizedName}||${assetType}`,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+/**
+ * Minimal asset shape the registry needs for dedup.
+ * Compatible with `DocumentedEndpointRecord` (persisted record)
+ * and the inline records passed by `document_endpoint`.
+ */
+export interface AssetRecord {
+  appName?: string;
+  assetName: string;
+  assetType: string;
+  description: string;
+  details?: {
+    url?: string;
+    [key: string]: unknown;
+  };
+  riskLevel?: string;
+}
+
+export interface AssetDuplicateCheckResult {
+  duplicate: boolean;
+  matchedAsset?: AssetRecord;
+  matchType?: "exact-url" | "exact-name";
+}
+
+// ---------------------------------------------------------------------------
+// AttackSurfaceRegistry
+// ---------------------------------------------------------------------------
+
+/**
+ * Public surface {@link AttackSurfaceRegistry} exposes to workflows.
+ * Extracted so a DB-backed registry (atomic upsert-by-fingerprint) can stand
+ * in for the in-memory implementation behind {@link WorkflowSeams}
+ * `registries` — see `../workflows/seams.ts`.
+ */
+export interface AttackSurfaceRegistryOps {
+  readonly size: number;
+  getAssets(): readonly AssetRecord[];
+  isDuplicate(asset: AssetRecord): AssetDuplicateCheckResult;
+  register(asset: AssetRecord): Promise<AssetDuplicateCheckResult>;
+  unregister(asset: AssetRecord): Promise<void>;
+}
+
+/**
+ * Thread-safe, in-memory registry of known attack surface assets.
+ *
+ * Supports two dedup tiers:
+ *   1. **Exact URL** — same normalised URL + same asset type
+ *   2. **Exact name** — same normalised name + same asset type
+ *
+ * The registry is designed to be created once per session and shared
+ * across concurrent attack surface agents via `ToolContext`.
+ */
+export class AttackSurfaceRegistry implements AttackSurfaceRegistryOps {
+  private urlKeys = new Map<string, AssetRecord>();
+  private nameKeys = new Map<string, AssetRecord>();
+  private assets: AssetRecord[] = [];
+
+  private mutex: Promise<void> = Promise.resolve();
+
+  /** How many assets are tracked. */
+  get size(): number {
+    return this.assets.length;
+  }
+
+  /** Return a snapshot of all tracked assets. */
+  getAssets(): readonly AssetRecord[] {
+    return [...this.assets];
+  }
+
+  // -----------------------------------------------------------------------
+  // Factories
+  // -----------------------------------------------------------------------
+
+  /**
+   * Create a registry pre-loaded with assets from a directory.
+   * Reads every `.json` file, parses each as an asset record, and indexes it.
+   * Malformed files are silently skipped.
+   */
+  static fromDirectory(assetsPath: string): AttackSurfaceRegistry {
+    const registry = new AttackSurfaceRegistry();
+
+    if (!existsSync(assetsPath)) return registry;
+
+    const files = readdirSync(assetsPath).filter((f) => f.endsWith(".json"));
+
+    for (const file of files) {
+      try {
+        const raw = readFileSync(join(assetsPath, file), "utf-8");
+        const asset = JSON.parse(raw) as AssetRecord;
+        if (
+          asset &&
+          typeof asset.assetName === "string" &&
+          typeof asset.assetType === "string"
+        ) {
+          registry.indexAsset(asset);
+        }
+      } catch {
+        // skip malformed files
+      }
+    }
+
+    return registry;
+  }
+
+  /**
+   * Create a registry pre-loaded with an array of assets (e.g. from Console DB).
+   */
+  static fromAssets(assets: AssetRecord[]): AttackSurfaceRegistry {
+    const registry = new AttackSurfaceRegistry();
+    for (const a of assets) {
+      registry.indexAsset(a);
+    }
+    return registry;
+  }
+
+  // -----------------------------------------------------------------------
+  // Core API
+  // -----------------------------------------------------------------------
+
+  /**
+   * Check whether an asset is a duplicate of something already registered.
+   * Synchronous.
+   *
+   * Tier 1: exact URL + asset type
+   * Tier 2: exact name + asset type
+   */
+  isDuplicate(asset: AssetRecord): AssetDuplicateCheckResult {
+    const { urlKey, nameKey } = generateAssetFingerprint(asset);
+
+    if (urlKey) {
+      const urlMatch = this.urlKeys.get(urlKey);
+      if (urlMatch) {
+        return {
+          duplicate: true,
+          matchedAsset: urlMatch,
+          matchType: "exact-url",
+        };
+      }
+    }
+
+    const nameMatch = this.nameKeys.get(nameKey);
+    if (nameMatch) {
+      return {
+        duplicate: true,
+        matchedAsset: nameMatch,
+        matchType: "exact-name",
+      };
+    }
+
+    return { duplicate: false };
+  }
+
+  /**
+   * Register a new asset in the registry.
+   *
+   * Thread-safe via an async mutex. Returns the dedup check result.
+   * If the asset is a duplicate it is **not** added to the registry.
+   */
+  async register(asset: AssetRecord): Promise<AssetDuplicateCheckResult> {
+    return new Promise<AssetDuplicateCheckResult>((resolve) => {
+      this.mutex = this.mutex.then(() => {
+        const check = this.isDuplicate(asset);
+        if (check.duplicate) {
+          resolve(check);
+        } else {
+          this.indexAsset(asset);
+          resolve({ duplicate: false });
+        }
+      });
+    });
+  }
+
+  /**
+   * Remove a previously-registered asset from all indexes.
+   *
+   * Rollback counterpart to {@link register}: call it when an asset was
+   * accepted by the registry but the caller failed to persist it.
+   */
+  async unregister(asset: AssetRecord): Promise<void> {
+    return new Promise<void>((resolve) => {
+      this.mutex = this.mutex.then(() => {
+        this.removeAsset(asset);
+        resolve();
+      });
+    });
+  }
+
+  // -----------------------------------------------------------------------
+  // Internal
+  // -----------------------------------------------------------------------
+
+  private removeAsset(asset: AssetRecord): void {
+    const { urlKey, nameKey } = generateAssetFingerprint(asset);
+
+    if (urlKey && this.urlKeys.get(urlKey) === asset) {
+      this.urlKeys.delete(urlKey);
+    }
+    if (this.nameKeys.get(nameKey) === asset) {
+      this.nameKeys.delete(nameKey);
+    }
+
+    const idx = this.assets.indexOf(asset);
+    if (idx !== -1) {
+      this.assets.splice(idx, 1);
+    }
+  }
+
+  private indexAsset(asset: AssetRecord): void {
+    const { urlKey, nameKey } = generateAssetFingerprint(asset);
+
+    if (urlKey && !this.urlKeys.has(urlKey)) {
+      this.urlKeys.set(urlKey, asset);
+    }
+    if (!this.nameKeys.has(nameKey)) {
+      this.nameKeys.set(nameKey, asset);
+    }
+
+    this.assets.push(asset);
+  }
+}