@hoanghainh/komagu 2.2.1 → 2.4.0

package/operate/cli/commands/inventory.py

@@ -0,0 +1,187 @@
+"""Inventory subcommand: compare code routes, OpenAPI spec, and gateway traffic to find Shadow/Zombie APIs."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from operate.cli.ui import color, print_summary_table, step, step_fail, step_info, step_ok, step_warn
+from operate.inventory.api_inventory import (
+    diff_api_inventory,
+    load_code_routes,
+    load_openapi_spec,
+    load_traffic_log,
+)
+
+
+def register_subcommand(subparsers: argparse._SubParsersAction) -> None:
+    """Register the inventory subcommand parser."""
+    parser = subparsers.add_parser(
+        "inventory",
+        help="Reconcile code routes, OpenAPI spec, and gateway traffic to detect Shadow and Zombie APIs",
+        description="Detects Shadow APIs (undocumented code/traffic) and Zombie APIs (deprecated routes still active).",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Reconcile LSA code routes against the published OpenAPI spec
+  komagu inventory --routes OUT/routes.json --openapi openapi.json
+
+  # Include Kong Gateway traffic log to spot Zombie APIs and active Shadow endpoints
+  komagu inventory --routes OUT/routes.json --openapi openapi.json --traffic-log /var/log/kong/access.log
+
+  # CI gate: fail if any undocumented Shadow API is detected
+  komagu inventory --routes OUT/routes.json --openapi openapi.json --fail-on-shadow
+
+  # Output machine-readable JSON diff to a custom path
+  komagu inventory --routes OUT/routes.json --openapi openapi.json --out OUT/api-diff.json
+""",
+    )
+    parser.add_argument(
+        "--routes",
+        "-r",
+        type=Path,
+        required=True,
+        help="Path to code routes JSON (e.g. OUT/routes.json or routes.candidates.json from LSA)",
+    )
+    parser.add_argument(
+        "--openapi",
+        "-s",
+        type=Path,
+        required=True,
+        help="Path to OpenAPI / Swagger JSON or YAML specification",
+    )
+    parser.add_argument(
+        "--traffic-log",
+        "--kong-log",
+        "-t",
+        type=Path,
+        default=None,
+        help="Path to Kong HTTP-Log (JSON lines) or NGINX Combined access log",
+    )
+    parser.add_argument(
+        "--out",
+        "-o",
+        type=Path,
+        default=None,
+        help="Path to write the api-inventory-diff.json output (defaults to OUT/api-inventory-diff.json)",
+    )
+    parser.add_argument(
+        "--fail-on-shadow",
+        action="store_true",
+        help="Exit with non-zero code if any shadow API is detected",
+    )
+    parser.add_argument(
+        "--fail-on-zombie",
+        action="store_true",
+        help="Exit with non-zero code if any zombie API is detected",
+    )
+    parser.set_defaults(func=run_inventory)
+
+
+def run_inventory(args: argparse.Namespace) -> int:
+    """Execute API inventory reconciliation."""
+    routes_file = Path(args.routes).resolve()
+    openapi_file = Path(args.openapi).resolve()
+    traffic_file = Path(args.traffic_log).resolve() if args.traffic_log else None
+
+    if not routes_file.is_file():
+        step_fail(f"Routes file not found: {routes_file}")
+        return 2
+
+    if not openapi_file.is_file():
+        step_fail(f"OpenAPI spec file not found: {openapi_file}")
+        return 2
+
+    step("Parsing routes extracted from source code")
+    code_routes = load_code_routes(routes_file)
+    step_ok(f"Loaded {len(code_routes)} endpoints from code routes")
+
+    step("Parsing OpenAPI specification")
+    spec_endpoints = load_openapi_spec(openapi_file)
+    step_ok(f"Loaded {len(spec_endpoints)} endpoints from OpenAPI specification")
+
+    traffic_data = {}
+    if traffic_file:
+        if not traffic_file.is_file():
+            step_warn(f"Traffic log file not found: {traffic_file} (skipping traffic analysis)")
+        else:
+            step("Analyzing gateway traffic log (Kong / NGINX)")
+            traffic_data = load_traffic_log(traffic_file)
+            total_hits = sum(t["traffic_hits"] for t in traffic_data.values())
+            step_ok(f"Parsed {len(traffic_data)} active endpoints ({total_hits} total requests) from traffic log")
+
+    step("Reconciling API inventories across 3 surfaces")
+    diff_doc = diff_api_inventory(
+        code_routes=code_routes,
+        spec_endpoints=spec_endpoints,
+        traffic_data=traffic_data,
+        routes_file_name=routes_file.name,
+        openapi_file_name=openapi_file.name,
+        traffic_file_name=traffic_file.name if traffic_file else None,
+    )
+
+    s = diff_doc["summary"]
+    shadow_count = s["shadow_count"]
+    zombie_count = s["zombie_count"]
+    missing_count = s["missing_code_count"]
+    dormant_count = s["dormant_count"]
+
+    # Determine output file path
+    out_file = args.out
+    if not out_file:
+        out_dir = routes_file.parent
+        out_file = out_dir / "api-inventory-diff.json"
+
+    out_file.parent.mkdir(parents=True, exist_ok=True)
+    with open(out_file, "w", encoding="utf-8") as fh:
+        json.dump(diff_doc, fh, ensure_ascii=False, indent=2)
+        fh.write("\n")
+    step_ok(f"Saved inventory diff to {out_file}")
+
+    # Display summary card
+    c_yellow_bold = "\033[33m\033[1m"
+    c_red_bold = "\033[31m\033[1m"
+    c_green = "\033[32m"
+    c_bold = "\033[1m"
+
+    summary_rows = [
+        ("Code Routes Discovered", str(s["total_code_routes"])),
+        ("OpenAPI Endpoints Documented", str(s["total_spec_endpoints"])),
+        ("Gateway Traffic Endpoints", str(s["total_traffic_endpoints"])),
+        ("Shadow APIs (Undocumented)", color(str(shadow_count), c_yellow_bold if shadow_count > 0 else c_green)),
+        ("Zombie APIs (Deprecated Active)", color(str(zombie_count), c_red_bold if zombie_count > 0 else c_green)),
+        ("Documented Missing in Code", str(missing_count)),
+        ("Dormant Code Endpoints", str(dormant_count)),
+    ]
+    print_summary_table(
+        "API Inventory Reconciliation Summary",
+        summary_rows,
+        footer="Shadow APIs expose undocumented attack surface; Zombie APIs must be retired."
+    )
+
+    if shadow_count > 0:
+        print("  " + color("⚠ Detected Shadow APIs (Top 5):", "\033[33m", c_bold))
+        for sh in diff_doc["shadow_apis"][:5]:
+            hits = f"({sh['traffic_hits']} traffic hits)" if sh["traffic_hits"] > 0 else ""
+            norm_str = color(sh["normalized"], c_bold)
+            disc_str = sh["discovered_in"]
+            print(f"    • {norm_str} [{disc_str}] {hits}")
+
+    if zombie_count > 0:
+        print("  " + color("✖ Detected Zombie APIs (Top 5):", "\033[31m", c_bold))
+        for z in diff_doc["zombie_apis"][:5]:
+            norm_str = color(z["normalized"], c_bold)
+            hits_str = f"{z['traffic_hits']} active traffic hits!"
+            print(f"    • {norm_str} ({hits_str})")
+
+    exit_code = 0
+    if args.fail_on_shadow and shadow_count > 0:
+        step_fail(f"Failing gate: {shadow_count} shadow APIs detected")
+        exit_code = 1
+    if args.fail_on_zombie and zombie_count > 0:
+        step_fail(f"Failing gate: {zombie_count} zombie APIs detected")
+        exit_code = 1
+
+    return exit_code

package/operate/cli/commands/simulate.py

@@ -1,4 +1,4 @@
-"""Command: komagu simulate - Empirical Breach & Attack Simulation (BAS)."""
+"""Command: komagu simulate - Empirical Breach & Attack Simulation (BAS) (#317)."""
 
 from __future__ import annotations
 
@@ -7,7 +7,7 @@ import json
 import sys
 import tempfile
 from pathlib import Path
-from typing import Any, List
+from typing import Any, Dict, List, Optional
 
 from operate.cli.ui import (
     BOLD,
@@ -23,12 +23,18 @@ from operate.cli.ui import (
     step_ok,
     step_warn,
 )
+from operate.simulate.engine import (
+    DeclarativeScenario,
+    generate_execution_evidence,
+)
+from operate.simulate.loader import (
+    discover_scenarios,
+    load_scenario_file,
+    validate_scenario,
+)
 from operate.simulate.scenarios import (
     ExecutiveReportGenerator,
-    IDCFrontierRansomwareScenario,
-    MetabaseZeroDayScenario,
     ScenarioResult,
-    TimesCarBreachScenario,
 )
 
 
@@ -41,28 +47,71 @@ def register_subcommand(subparsers: argparse._SubParsersAction) -> None:
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog="""
 Examples:
+  # List all discovered attack scenarios
+  komagu simulate --list
+
+  # Validate a custom declarative scenario JSON against bas-scenario.schema.json
+  komagu simulate --validate ./custom-scenario.json
+
   # Run all simulated attack scenarios against a target staging API
   komagu simulate --target http://localhost:5001
 
   # Simulate the Times Car mass IDOR harvesting attack specifically
   komagu simulate --scenario times-car-breach --target http://staging-api:8000
 
-  # Simulate Metabase unauthenticated SQL injection probe
-  komagu simulate --scenario metabase-cve-2026-72898 --target http://localhost:5001
-
-  # Simulate IDC Frontier ransomware immutable storage attack
-  komagu simulate --scenario idc-frontier-ransomware --target /tmp/backup-dir
+  # Run a custom scenario file with parameter override
+  komagu simulate --scenario-file ./my-scenario.json --set endpoint_pattern=/api/v2/items/{id}
 
-  # Export Executive C-Level Report for Board of Directors
-  komagu simulate --target http://localhost:5001 --output ./executive-bas-report.md --format markdown
+  # Export Executive C-Level Report and METI SCS audit evidence
+  komagu simulate --target http://localhost:5001 --output ./bas-report.md --evidence ./bas-evidence.json
 """,
     )
+    parser.add_argument(
+        "--list",
+        action="store_true",
+        help="List all available built-in and discovered scenarios",
+    )
+    parser.add_argument(
+        "--validate",
+        metavar="PATH",
+        help="Validate a scenario JSON file against schema and exit",
+    )
     parser.add_argument(
         "--scenario",
         "-s",
         default="all",
-        choices=["all", "times-car-breach", "metabase-cve-2026-72898", "idc-frontier-ransomware"],
-        help="Attack scenario to execute (default: all)",
+        help="Attack scenario ID to execute, or 'all' (default: all)",
+    )
+    parser.add_argument(
+        "--scenario-file",
+        action="append",
+        dest="scenario_files",
+        metavar="PATH",
+        help="Path to custom scenario file (repeatable)",
+    )
+    parser.add_argument(
+        "--scenario-dir",
+        action="append",
+        dest="scenario_dirs",
+        metavar="DIR",
+        help="Path to directory containing scenario files (repeatable)",
+    )
+    parser.add_argument(
+        "--set",
+        action="append",
+        dest="param_sets",
+        metavar="KEY=VALUE",
+        help="Override scenario parameter (repeatable, e.g. --set token=secret)",
+    )
+    parser.add_argument(
+        "--allow-draft",
+        action="store_true",
+        help="Allow execution of scenarios with status != 'approved'",
+    )
+    parser.add_argument(
+        "--allow-destructive",
+        action="store_true",
+        help="Allow execution of steps marked destructive: true",
     )
     parser.add_argument(
         "--target",
@@ -89,6 +138,10 @@ Examples:
         "-o",
         help="Output file path for detailed executive report",
     )
+    parser.add_argument(
+        "--evidence",
+        help="Output file path for METI SCS / ISO 27001 audit evidence JSON (#271)",
+    )
     parser.add_argument(
         "--format",
         default="table",
@@ -100,59 +153,118 @@ Examples:
 
 def execute_simulate(args: argparse.Namespace) -> int:
     """Execute Breach & Attack Simulation scenarios."""
+    # Mode 1: Validate scenario file
+    if args.validate:
+        v_path = Path(args.validate).resolve()
+        if not v_path.is_file():
+            step_fail(f"Scenario file not found: {v_path}")
+            return 2
+        try:
+            doc = json.loads(v_path.read_text(encoding="utf-8"))
+        except Exception as ex:
+            step_fail(f"Invalid JSON in {v_path}: {ex}")
+            return 2
+
+        errs = validate_scenario(doc)
+        if errs:
+            step_fail(f"Scenario {v_path.name} failed schema validation:")
+            for err in errs:
+                print(f"  • {err}")
+            return 2
+        step_ok(f"Scenario {color(doc.get('id', v_path.name), BOLD)} is valid ({doc.get('status', 'draft')})")
+        return 0
+
+    # Discover all scenarios
+    scenarios_map = discover_scenarios(
+        scenario_files=args.scenario_files,
+        scenario_dirs=args.scenario_dirs,
+        include_builtins=True,
+    )
+
+    # Mode 2: List scenarios
+    if args.list:
+        step("Discovered Breach & Attack Simulation (BAS) Scenarios:")
+        rows = []
+        for sid, sc in sorted(scenarios_map.items()):
+            status = sc.get("status", "draft")
+            status_color = GREEN if status == "approved" else (YELLOW if status == "draft" else RED)
+            date_str = sc.get("incident", {}).get("date", "-")
+            title = sc.get("title", sid)
+            rows.append((f"{color(sid, BOLD)} (v{sc.get('version', '1.0.0')})", f"[{color(status, status_color)}] {title} ({date_str})"))
+
+        print_summary_table(
+            title="Available BAS Scenarios",
+            rows=rows,
+            footer="Execute with: komagu simulate --scenario <id> --target <url>"
+        )
+        return 0
+
+    # Mode 3: Run simulation
     step(f"Initializing Komagu Breach & Attack Simulation against {color(args.target, BOLD)}")
 
+    # Parse --set parameter overrides
+    overrides: Dict[str, Any] = {}
+    if args.param_sets:
+        for p_set in args.param_sets:
+            if "=" in p_set:
+                k, v = p_set.split("=", 1)
+                overrides[k.strip()] = v.strip()
+
+    # Legacy CLI flag bridges
+    if args.endpoint:
+        overrides["endpoint_pattern"] = args.endpoint
+        overrides["query_path"] = args.endpoint
+    if args.param:
+        overrides["target_param"] = args.param
+    if args.token:
+        overrides["attacker_token"] = args.token
+
+    if args.scenario == "all":
+        scenarios_to_run = list(scenarios_map.keys())
+    else:
+        if args.scenario not in scenarios_map:
+            step_fail(f"Unknown scenario ID: '{args.scenario}'. Use --list to view available scenarios.")
+            return 2
+        scenarios_to_run = [args.scenario]
+
     results: List[ScenarioResult] = []
-    scenarios_to_run = [args.scenario] if args.scenario != "all" else [
-        "times-car-breach",
-        "metabase-cve-2026-72898",
-        "idc-frontier-ransomware",
-    ]
-
-    for sc in scenarios_to_run:
-        if sc == "times-car-breach":
-            ep = args.endpoint or "/api/legacy-orders/{id}"
-            step_info(f"Launching Scenario: Times Car Mass BOLA/IDOR Harvesting ({ep})")
-            runner = TimesCarBreachScenario(
-                target=args.target,
-                endpoint_pattern=ep,
-                attacker_token=args.token,
-            )
-            res = runner.run()
-            results.append(res)
-            if res.passed:
-                step_ok(f"Times Car Simulation: {res.resilience_score}% resilient (0 breaches)")
-            else:
-                step_fail(f"Times Car Simulation: {res.breaches_leaked} data leaks detected ({res.resilience_score}% resilient)")
-
-        elif sc == "metabase-cve-2026-72898":
-            ep = args.endpoint or "/api/products"
-            step_info(f"Launching Scenario: Metabase Zero-Day SQLi Simulation ({ep}?{args.param}=...)")
-            runner = MetabaseZeroDayScenario(
-                target=args.target,
-                query_path=ep,
-                target_param=args.param,
-            )
-            res = runner.run()
-            results.append(res)
-            if res.passed:
-                step_ok(f"Metabase Zero-Day Simulation: {res.resilience_score}% resilient (All SQLi probes blocked)")
-            else:
-                step_fail(f"Metabase Zero-Day Simulation: SQLi probes unhandled ({res.resilience_score}% resilient)")
+    runners: List[DeclarativeScenario] = []
+
+    for sid in scenarios_to_run:
+        sc_data = scenarios_map.get(sid)
+        if not sc_data:
+            continue
 
-        elif sc == "idc-frontier-ransomware":
-            # A fresh private dir, not a fixed /tmp path another local user could pre-create or symlink
-            storage_path = (
-                args.target if not args.target.startswith("http") else tempfile.mkdtemp(prefix="komagu-backup-sim-")
+        step_info(f"Launching Scenario: {sc_data.get('title', sid)} ({sid})")
+
+        # Handle local dir target if needed for idc-frontier-ransomware
+        eff_target = args.target
+        if sid == "idc-frontier-ransomware" and args.target.startswith("http"):
+            eff_target = tempfile.mkdtemp(prefix="komagu-backup-sim-")
+            overrides["target_dir"] = eff_target
+
+        try:
+            runner = DeclarativeScenario(
+                scenario_data=sc_data,
+                target=eff_target,
+                param_overrides=overrides,
+                allow_draft=args.allow_draft,
+                allow_destructive=args.allow_destructive,
             )
-            step_info(f"Launching Scenario: IDC Frontier Cloud Storage Ransomware Simulation ({storage_path})")
-            runner = IDCFrontierRansomwareScenario(target_dir=storage_path)
+            runners.append(runner)
             res = runner.run()
             results.append(res)
+
             if res.passed:
-                step_ok(f"IDC Frontier Simulation: {res.resilience_score}% resilient (WORM lock enforced)")
+                step_ok(f"{sc_data.get('title', sid)}: {res.resilience_score}% resilient (0 breaches)")
             else:
-                step_warn(f"IDC Frontier Simulation: Writable storage ({res.resilience_score}% resilient)")
+                step_fail(f"{sc_data.get('title', sid)}: {res.breaches_leaked} data leaks detected ({res.resilience_score}% resilient)")
+        except PermissionError as p_err:
+            step_warn(f"Skipping {sid}: {p_err}")
+
+    if not results:
+        step_fail("No scenarios were executed.")
+        return 1
 
     generator = ExecutiveReportGenerator(results, args.target)
     overall_score = generator.overall_resilience_score
@@ -177,6 +289,11 @@ def execute_simulate(args: argparse.Namespace) -> int:
         footer="Empirical simulation against landmark real-world enterprise breaches.",
     )
 
+    # Generate METI SCS audit evidence JSON if requested
+    if args.evidence:
+        generate_execution_evidence(runners, results, args.target, output_path=args.evidence)
+        step_ok(f"Saved METI SCS Audit Evidence to {color(args.evidence, BOLD)}")
+
     # Save output if requested
     if args.output or args.format in ("markdown", "json"):
         if args.format == "json":

package/operate/cli/main.py

@@ -7,7 +7,7 @@ import sys
 from typing import List, Optional
 
 from operate.cli import __version__
-from operate.cli.commands import demo, devsecops, fix, init, protect, scan, simulate, test
+from operate.cli.commands import demo, devsecops, evidence, fix, init, inventory, protect, scan, simulate, test
 from operate.cli.ui import print_banner
 
 
@@ -27,6 +27,8 @@ Quick Start Workflow:
   6. Remediate vulnerabilities: komagu fix ./legacy-app --dry-run
   7. Install AI audit agents:   komagu init --ai codex ~/audits/acme
   8. Simulate breach scenarios: komagu simulate --scenario times-car-breach
+  9. Reconcile API inventory:   komagu inventory --routes OUT/routes.json --openapi openapi.json
+  10. Bundle METI SCS evidence: komagu evidence --audit-dir OUT/ --output-dir ./evidence-pack
 
 Documentation & Guides:
   Comprehensive User Guide: docs/cli-guide.md
@@ -62,6 +64,8 @@ Learn more about a specific command:
     fix.register_subcommand(subparsers)
     init.register_subcommand(subparsers)
     simulate.register_subcommand(subparsers)
+    inventory.register_subcommand(subparsers)
+    evidence.register_subcommand(subparsers)
 
     help_parser = subparsers.add_parser(
         "help",
@@ -71,8 +75,8 @@ Learn more about a specific command:
     help_parser.add_argument(
         "topic",
         nargs="?",
-        choices=["scan", "protect", "test", "demo", "devsecops", "fix", "init", "simulate"],
-        help="Subcommand topic to display help for (scan, protect, test, demo, devsecops, fix, init, simulate)",
+        choices=["scan", "protect", "test", "demo", "devsecops", "fix", "init", "simulate", "inventory", "evidence"],
+        help="Subcommand topic to display help for (scan, protect, test, demo, devsecops, fix, init, simulate, inventory, evidence)",
     )
 
     return parser

package/operate/inventory/__init__.py

@@ -0,0 +1,22 @@
+"""API inventory and shadow/zombie API detection package."""
+from operate.inventory.api_inventory import (
+    diff_api_inventory,
+    load_code_routes,
+    load_openapi_spec,
+    load_traffic_log,
+    normalize_path,
+    normalize_route_key,
+    render_inventory_html,
+    render_inventory_md,
+)
+
+__all__ = [
+    "diff_api_inventory",
+    "load_code_routes",
+    "load_openapi_spec",
+    "load_traffic_log",
+    "normalize_path",
+    "normalize_route_key",
+    "render_inventory_html",
+    "render_inventory_md",
+]