@hoanghainh/komagu 2.2.1 → 2.3.0

package/operate/simulate/scenarios/times-car-breach.json

@@ -0,0 +1,74 @@
+{
+  "$schema": "https://raw.githubusercontent.com/hoanghainh1188/komagu/main/contracts/schemas/bas-scenario.schema.json",
+  "id": "times-car-breach",
+  "version": "1.0.0",
+  "title": "Times Car Mass BOLA / IDOR Scraping Simulation",
+  "status": "approved",
+  "approved_by": "sec-team@komagu.io",
+  "approved_at": "2026-03-15T00:00:00Z",
+  "incident": {
+    "name": "Times Car (Times Mobility) 2024 Mass Data Exposure",
+    "date": "2024",
+    "summary": "Simulates automated cross-user account/license harvesting by iterating through consecutive resource IDs with an unauthorized peer credential.",
+    "references": [
+      "https://scan.netsecurity.ne.jp/article/2024/04/17/50873.html",
+      "Times Car 2024 breach: 6.6M customer accounts and ID images exposed."
+    ]
+  },
+  "mappings": {
+    "cwe": ["CWE-639", "CWE-284"],
+    "owasp_api": ["API1:2023"],
+    "mitre_attack": ["T1059", "T1552"]
+  },
+  "parameters": {
+    "endpoint_pattern": {
+      "type": "string",
+      "default": "/api/legacy-orders/{id}",
+      "description": "Target endpoint path containing {id} parameter."
+    },
+    "attacker_token": {
+      "type": "string",
+      "default": "mock-attacker-bob-token",
+      "description": "Bearer token used to test cross-tenant unauthorized access."
+    },
+    "sample_ids": {
+      "type": "list",
+      "default": ["1", "2", "3", "101", "102"],
+      "description": "List of sample resource IDs to probe."
+    }
+  },
+  "steps": [
+    {
+      "name": "Iterate Cross-User Endpoints",
+      "description": "Requests resource IDs using attacker credential, expecting 403 or 404.",
+      "type": "http_request",
+      "destructive": false,
+      "repeat": {
+        "param_name": "sample_ids",
+        "as_var": "id"
+      },
+      "request": {
+        "method": "GET",
+        "path": "{endpoint_pattern}",
+        "headers": {
+          "Authorization": "Bearer {attacker_token}",
+          "User-Agent": "Komagu-BAS-TimesCarSimulator/1.0",
+          "Accept": "application/json"
+        }
+      },
+      "expect_secure": {
+        "status_codes": [403, 404]
+      },
+      "expect_vulnerable": {
+        "status_codes": [200, 201, 204]
+      }
+    }
+  ],
+  "scoring": {
+    "strategy": "ratio",
+    "recommendations": {
+      "remediation": "Enforce Komagu Boilerplate 1 (Object-Level Authorization) with declarative @RequireOwnership decorators or integrate Kong perimeter WAF with strict 404 invariants.",
+      "passed": "Endpoint securely enforces tenant isolation (404 Not Found invariant maintained)."
+    }
+  }
+}

package/operate/simulate/scenarios.py

@@ -107,6 +107,24 @@ class TimesCarBreachScenario(BaseScenario):
         self.sample_ids = sample_ids or ["1", "2", "3", "101", "102"]
 
     def run(self) -> ScenarioResult:
+        from operate.simulate.engine import DeclarativeScenario
+        from operate.simulate.loader import BUILTIN_SCENARIOS_DIR, load_scenario_file
+
+        scenario_path = BUILTIN_SCENARIOS_DIR / "times-car-breach.json"
+        if scenario_path.is_file():
+            data = load_scenario_file(scenario_path)
+            runner = DeclarativeScenario(
+                data,
+                target=self.target,
+                param_overrides={
+                    "endpoint_pattern": self.endpoint_pattern,
+                    "attacker_token": self.attacker_token,
+                    "sample_ids": self.sample_ids,
+                },
+                timeout=self.timeout,
+            )
+            return runner.run()
+
         total = len(self.sample_ids)
         blocked = 0
         leaked = 0
@@ -212,6 +230,24 @@ class MetabaseZeroDayScenario(BaseScenario):
         ]
 
     def run(self) -> ScenarioResult:
+        from operate.simulate.engine import DeclarativeScenario
+        from operate.simulate.loader import BUILTIN_SCENARIOS_DIR, load_scenario_file
+
+        scenario_path = BUILTIN_SCENARIOS_DIR / "metabase-cve-2026-72898.json"
+        if scenario_path.is_file():
+            data = load_scenario_file(scenario_path)
+            runner = DeclarativeScenario(
+                data,
+                target=self.target,
+                param_overrides={
+                    "query_path": self.query_path,
+                    "target_param": self.target_param,
+                    "payloads": self.payloads,
+                },
+                timeout=self.timeout,
+            )
+            return runner.run()
+
         total = len(self.payloads)
         blocked = 0
         leaked = 0
@@ -315,6 +351,22 @@ class IDCFrontierRansomwareScenario(BaseScenario):
         self.target_dir = Path(target_dir).resolve()
 
     def run(self) -> ScenarioResult:
+        from operate.simulate.engine import DeclarativeScenario
+        from operate.simulate.loader import BUILTIN_SCENARIOS_DIR, load_scenario_file
+
+        scenario_path = BUILTIN_SCENARIOS_DIR / "idc-frontier-ransomware.json"
+        if scenario_path.is_file():
+            data = load_scenario_file(scenario_path)
+            runner = DeclarativeScenario(
+                data,
+                target=str(self.target_dir),
+                param_overrides={
+                    "target_dir": str(self.target_dir),
+                },
+                timeout=self.timeout,
+            )
+            return runner.run()
+
         total = 3
         blocked = 0
         leaked = 0

package/operate/tests/test_api_inventory.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+"""Tests for API Inventory Reconciliation & Shadow/Zombie API Detection (Issue #288).
+
+Verifies:
+1. api-inventory-diff.schema.json schema conformity.
+2. Route path parameter normalization (Express, Flask, OpenAPI, integers, UUIDs).
+3. 3-way reconciliation covering all 4 discrepancy cases:
+   - Shadow API: in code but not in spec.
+   - Shadow API: in live traffic but not in spec.
+   - Zombie API: marked deprecated in OpenAPI spec but still receiving traffic.
+   - Documented Missing in Code: in spec but not implemented.
+   - Dormant Code: implemented and documented, but zero traffic.
+4. Kong HTTP-Log (JSON lines) and NGINX Combined access log parsing.
+5. CLI execution and gating flags (--fail-on-shadow, --fail-on-zombie).
+6. Markdown and HTML report generation.
+
+Standard-library-only (Python 3.8+).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from assess.lsa.scripts.merge_findings import load_validator
+from operate.cli.commands.inventory import run_inventory
+from operate.inventory.api_inventory import (
+    diff_api_inventory,
+    load_code_routes,
+    load_openapi_spec,
+    load_traffic_log,
+    normalize_path,
+    normalize_route_key,
+    render_inventory_html,
+    render_inventory_md,
+)
+
+
+class TestApiInventory(unittest.TestCase):
+    def setUp(self) -> None:
+        self.validate_schema = load_validator("api-inventory-diff.schema.json")
+
+    def test_path_normalization(self) -> None:
+        """Verify unified path normalization across diverse frameworks."""
+        self.assertEqual(normalize_path("/api/orders/{id}"), "/api/orders/{}")
+        self.assertEqual(normalize_path("/users/<int:userId>"), "/users/{}")
+        self.assertEqual(normalize_path("/accounts/:accId/profile"), "/accounts/{}/profile")
+        self.assertEqual(normalize_path("/api/orders/42"), "/api/orders/{}")
+        self.assertEqual(
+            normalize_path("/items/12345678-abcd-1234-abcd-123456789abc"),
+            "/items/{}"
+        )
+        self.assertEqual(normalize_path("/trailing/slash/"), "/trailing/slash")
+        self.assertEqual(normalize_route_key("get", "/api/v1/orders/{id}?limit=10"), "GET /api/v1/orders/{}")
+
+    def test_four_way_reconciliation_cases(self) -> None:
+        """Verify detection of Shadow, Zombie, Dead Spec, and Dormant Code."""
+        # 1. Code routes
+        code_routes = [
+            {"method": "GET", "path": "/api/users/{id}", "location": "Controllers/Users.cs:10"},  # Normal
+            {"method": "POST", "path": "/api/internal/debug-dump", "location": "Controllers/Debug.cs:42"},  # Shadow (in code, not spec)
+            {"method": "GET", "path": "/api/legacy-v1/export", "location": "Controllers/Legacy.cs:15"},  # Zombie (deprecated in spec, receiving traffic)
+            {"method": "GET", "path": "/api/dormant-feature", "location": "Controllers/Dormant.cs:88"},  # Dormant (in code & spec, 0 traffic)
+        ]
+
+        # 2. OpenAPI Spec
+        spec_endpoints = [
+            {"method": "GET", "path": "/api/users/{id}", "normalized": "GET /api/users/{}", "deprecated": False},
+            {"method": "GET", "path": "/api/legacy-v1/export", "normalized": "GET /api/legacy-v1/export", "deprecated": True},  # Deprecated!
+            {"method": "GET", "path": "/api/dormant-feature", "normalized": "GET /api/dormant-feature", "deprecated": False},
+            {"method": "DELETE", "path": "/api/planned-ghost", "normalized": "DELETE /api/planned-ghost", "deprecated": False, "summary": "Ghost spec"},  # Missing in code
+        ]
+
+        # 3. Traffic Data
+        traffic_data = {
+            "GET /api/users/{}": {"method": "GET", "path": "/api/users/123", "normalized": "GET /api/users/{}", "traffic_hits": 50},
+            "GET /api/legacy-v1/export": {"method": "GET", "path": "/api/legacy-v1/export", "normalized": "GET /api/legacy-v1/export", "traffic_hits": 12},
+            "POST /api/untracked-admin": {"method": "POST", "path": "/api/untracked-admin", "normalized": "POST /api/untracked-admin", "traffic_hits": 5},  # Shadow in traffic only
+        }
+
+        diff = diff_api_inventory(code_routes, spec_endpoints, traffic_data)
+        errors = self.validate_schema(diff)
+        self.assertEqual(errors, [], f"Schema validation errors: {errors}")
+
+        s = diff["summary"]
+        self.assertEqual(s["shadow_count"], 2)  # debug-dump (code_only) + untracked-admin (traffic_only)
+        self.assertEqual(s["zombie_count"], 1)  # legacy-v1/export
+        self.assertEqual(s["missing_code_count"], 1)  # planned-ghost
+        self.assertEqual(s["dormant_count"], 1)  # dormant-feature
+
+        # Verify Shadow details
+        shadow_keys = {sh["normalized"]: sh["discovered_in"] for sh in diff["shadow_apis"]}
+        self.assertEqual(shadow_keys.get("POST /api/internal/debug-dump"), "code_only")
+        self.assertEqual(shadow_keys.get("POST /api/untracked-admin"), "traffic_only")
+
+        # Verify Zombie details
+        zombie_keys = [z["normalized"] for z in diff["zombie_apis"]]
+        self.assertIn("GET /api/legacy-v1/export", zombie_keys)
+        self.assertEqual(diff["zombie_apis"][0]["traffic_hits"], 12)
+
+        # Verify Missing code details
+        missing_keys = [m["normalized"] for m in diff["documented_missing_code"]]
+        self.assertIn("DELETE /api/planned-ghost", missing_keys)
+
+        # Verify Dormant code details
+        dormant_keys = [d["normalized"] for d in diff["dormant_code"]]
+        self.assertIn("GET /api/dormant-feature", dormant_keys)
+
+    def test_log_parsers(self) -> None:
+        """Verify parsing Kong JSON lines and Combined Nginx log formats."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmp = Path(tmpdir)
+
+            # 1. Kong HTTP-Log NDJSON
+            kong_log = tmp / "kong.log"
+            kong_log.write_text(
+                '{"request": {"method": "GET", "uri": "/api/orders/101"}, "started_at": 1730000000}\n'
+                '{"request": {"method": "GET", "uri": "/api/orders/102"}, "started_at": 1730000001}\n'
+                '{"request": {"method": "POST", "uri": "/api/checkout"}, "started_at": 1730000002}\n'
+            )
+            parsed_kong = load_traffic_log(kong_log)
+            self.assertEqual(parsed_kong["GET /api/orders/{}"]["traffic_hits"], 2)
+            self.assertEqual(parsed_kong["POST /api/checkout"]["traffic_hits"], 1)
+
+            # 2. Combined NGINX log
+            nginx_log = tmp / "nginx.log"
+            nginx_log.write_text(
+                '192.168.1.1 - - [01/Nov/2026:12:00:00 +0000] "GET /api/products/42 HTTP/1.1" 200 450\n'
+                '192.168.1.2 - - [01/Nov/2026:12:00:01 +0000] "DELETE /api/cart HTTP/1.1" 204 0\n'
+            )
+            parsed_nginx = load_traffic_log(nginx_log)
+            self.assertEqual(parsed_nginx["GET /api/products/{}"]["traffic_hits"], 1)
+            self.assertEqual(parsed_nginx["DELETE /api/cart"]["traffic_hits"], 1)
+
+    def test_cli_execution_and_gating(self) -> None:
+        """Verify komagu inventory command execution and gate checks."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmp = Path(tmpdir)
+            routes_file = tmp / "routes.json"
+            routes_file.write_text(json.dumps({
+                "routes": [
+                    {"method": "GET", "path": "/api/users/{id}", "location": "app.py:10"},
+                    {"method": "POST", "path": "/api/shadow-login", "location": "auth.py:50"},
+                ]
+            }))
+
+            spec_file = tmp / "openapi.json"
+            spec_file.write_text(json.dumps({
+                "openapi": "3.0.0",
+                "paths": {
+                    "/api/users/{id}": {"get": {"summary": "Get user"}},
+                    "/api/old-v0/auth": {"post": {"deprecated": True, "summary": "Old auth"}},
+                }
+            }))
+
+            traffic_file = tmp / "traffic.log"
+            traffic_file.write_text('{"request": {"method": "POST", "uri": "/api/old-v0/auth"}}\n')
+
+            out_file = tmp / "diff.json"
+
+            args = argparse.Namespace(
+                routes=routes_file,
+                openapi=spec_file,
+                traffic_log=traffic_file,
+                out=out_file,
+                fail_on_shadow=False,
+                fail_on_zombie=False,
+            )
+
+            # Normal run passes with exit code 0
+            rc = run_inventory(args)
+            self.assertEqual(rc, 0)
+            self.assertTrue(out_file.is_file())
+
+            diff_data = json.loads(out_file.read_text())
+            self.assertEqual(diff_data["summary"]["shadow_count"], 1)
+            self.assertEqual(diff_data["summary"]["zombie_count"], 1)
+
+            # Fail on shadow gate trips exit code 1
+            args_shadow_gate = argparse.Namespace(
+                routes=routes_file,
+                openapi=spec_file,
+                traffic_log=traffic_file,
+                out=out_file,
+                fail_on_shadow=True,
+                fail_on_zombie=False,
+            )
+            self.assertEqual(run_inventory(args_shadow_gate), 1)
+
+            # Fail on zombie gate trips exit code 1
+            args_zombie_gate = argparse.Namespace(
+                routes=routes_file,
+                openapi=spec_file,
+                traffic_log=traffic_file,
+                out=out_file,
+                fail_on_shadow=False,
+                fail_on_zombie=True,
+            )
+            self.assertEqual(run_inventory(args_zombie_gate), 1)
+
+    def test_report_rendering(self) -> None:
+        """Verify Markdown and HTML report generation for API inventory diff."""
+        mock_diff = {
+            "summary": {
+                "total_code_routes": 10,
+                "total_spec_endpoints": 8,
+                "total_traffic_endpoints": 9,
+                "shadow_count": 2,
+                "zombie_count": 1,
+                "missing_code_count": 0,
+                "dormant_count": 1,
+            },
+            "shadow_apis": [
+                {"method": "POST", "path": "/api/secret", "normalized": "POST /api/secret", "discovered_in": "code_only", "traffic_hits": 0, "location": "secret.py:1"}
+            ],
+            "zombie_apis": [
+                {"method": "GET", "path": "/api/v1/old", "normalized": "GET /api/v1/old", "location": "old.py:5", "traffic_hits": 15, "last_seen": "2026-11-01"}
+            ],
+            "documented_missing_code": [],
+            "dormant_code": [],
+        }
+
+        md_en = "\n".join(render_inventory_md(mock_diff, "en"))
+        self.assertIn("API Inventory Reconciliation", md_en)
+        self.assertIn("POST /api/secret", md_en)
+
+        md_ja = "\n".join(render_inventory_md(mock_diff, "ja"))
+        self.assertIn("APIインベントリ照合", md_ja)
+        self.assertIn("Shadow API", md_ja)
+
+        html_card = render_inventory_html(mock_diff)
+        self.assertIn("API Inventory &amp; Shadow / Zombie API Detection", html_card)
+        self.assertIn("POST /api/secret", html_card)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/operate/tests/test_declarative_scenarios.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+"""Comprehensive unit tests for Declarative Incident Scenarios (#317).
+
+Validates:
+1. bas-scenario.schema.json schema validation on built-in and custom scenarios.
+2. Discovery of scenarios from files, directories, and built-ins.
+3. DeclarativeScenario execution of HTTP and storage steps.
+4. Governance safety gates: draft scenarios blocked without --allow-draft.
+5. Destructive step safety gates: destructive steps blocked without --allow-destructive.
+6. Parameter substitutions and CLI --set overrides.
+7. METI SCS audit evidence generation (#271).
+8. CLI commands: --list, --validate, --scenario-file, --evidence.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from operate.cli.main import main as cli_main
+from operate.cross_user_tester.mock_test_server import start_mock_server
+from operate.simulate.engine import (
+    DeclarativeScenario,
+    generate_execution_evidence,
+)
+from operate.simulate.loader import (
+    BUILTIN_SCENARIOS_DIR,
+    discover_scenarios,
+    load_scenario_file,
+    load_validator,
+    validate_scenario,
+)
+
+
+class TestDeclarativeScenarios(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.server, cls.thread, cls.base_url = start_mock_server(port=0)
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cls.server.server_close()
+
+    def test_schema_validates_builtin_scenarios(self) -> None:
+        """All 3 built-in scenario files must strictly conform to bas-scenario.schema.json."""
+        validator = load_validator("bas-scenario.schema.json")
+        for sc_name in ["times-car-breach.json", "metabase-cve-2026-72898.json", "idc-frontier-ransomware.json"]:
+            path = BUILTIN_SCENARIOS_DIR / sc_name
+            self.assertTrue(path.is_file(), f"Missing built-in scenario: {sc_name}")
+            data = load_scenario_file(path)
+            errs = validator(data)
+            self.assertEqual(errs, [], f"Validation errors for {sc_name}: {errs}")
+
+    def test_schema_validation_error_on_invalid_scenario(self) -> None:
+        """Validator must report missing required properties."""
+        invalid_data = {
+            "id": "invalid-scenario",
+            "title": "Invalid",
+            # missing status, version, incident, mappings, etc.
+        }
+        errs = validate_scenario(invalid_data)
+        self.assertTrue(len(errs) > 0)
+        self.assertTrue(any("required property missing" in e or "is a required property" in e for e in errs))
+
+    def test_discovery_scenarios(self) -> None:
+        """discover_scenarios must locate all built-in scenarios and custom ones."""
+        scenarios = discover_scenarios(include_builtins=True)
+        self.assertIn("times-car-breach", scenarios)
+        self.assertIn("metabase-cve-2026-72898", scenarios)
+        self.assertIn("idc-frontier-ransomware", scenarios)
+
+        # Test custom dir discovery
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            custom_path = Path(tmp_dir) / "custom-sc.json"
+            base_sc = dict(scenarios["times-car-breach"])
+            base_sc["id"] = "custom-test-scenario"
+            custom_path.write_text(json.dumps(base_sc), encoding="utf-8")
+
+            discovered = discover_scenarios(scenario_dirs=[tmp_dir], include_builtins=False)
+            self.assertIn("custom-test-scenario", discovered)
+
+    def test_draft_scenario_blocked_without_override(self) -> None:
+        """Scenario with status != 'approved' must fail execution unless allow_draft is set."""
+        data = dict(load_scenario_file(BUILTIN_SCENARIOS_DIR / "times-car-breach.json"))
+        data["status"] = "draft"
+        runner = DeclarativeScenario(data, self.base_url, allow_draft=False)
+        with self.assertRaises(PermissionError):
+            runner.run()
+
+        # Overriding with allow_draft=True must succeed
+        runner_override = DeclarativeScenario(data, self.base_url, allow_draft=True)
+        res = runner_override.run()
+        self.assertIsNotNone(res)
+
+    def test_destructive_step_blocked_without_override(self) -> None:
+        """Destructive steps must be skipped unless allow_destructive is set."""
+        data = dict(load_scenario_file(BUILTIN_SCENARIOS_DIR / "times-car-breach.json"))
+        data["steps"] = [dict(data["steps"][0])]
+        data["steps"][0]["destructive"] = True
+
+        runner_safe = DeclarativeScenario(data, self.base_url, allow_destructive=False)
+        res_safe = runner_safe.run()
+        self.assertTrue(any(d.get("skipped") for d in res_safe.details))
+
+        runner_destr = DeclarativeScenario(data, self.base_url, allow_destructive=True)
+        res_destr = runner_destr.run()
+        self.assertFalse(any(d.get("skipped") for d in res_destr.details))
+
+    def test_parameter_override_and_substitution(self) -> None:
+        """Custom parameter overrides must take precedence over defaults."""
+        data = load_scenario_file(BUILTIN_SCENARIOS_DIR / "times-car-breach.json")
+        runner = DeclarativeScenario(
+            data,
+            self.base_url,
+            param_overrides={
+                "endpoint_pattern": "/api/orders/{id}",
+                "sample_ids": ["999"],
+            },
+        )
+        res = runner.run()
+        self.assertEqual(res.total_attempts, 1)
+        self.assertTrue(res.passed)
+
+    def test_evidence_generation(self) -> None:
+        """generate_execution_evidence produces complete audit record conforming to METI SCS."""
+        data = load_scenario_file(BUILTIN_SCENARIOS_DIR / "times-car-breach.json")
+        runner = DeclarativeScenario(data, self.base_url)
+        res = runner.run()
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            ev_path = Path(tmp_dir) / "evidence.json"
+            ev = generate_execution_evidence([runner], [res], self.base_url, output_path=ev_path)
+            self.assertTrue(ev_path.is_file())
+            self.assertEqual(ev["evidence_type"], "breach_and_attack_simulation_audit")
+            self.assertEqual(len(ev["audit_records"]), 1)
+            self.assertTrue(ev["audit_records"][0]["scenario_hash"].startswith("sha256:"))
+
+    def test_cli_list_and_validate(self) -> None:
+        """Test CLI flags --list and --validate."""
+        # --list
+        exit_code = cli_main(["simulate", "--list"])
+        self.assertEqual(exit_code, 0)
+
+        # --validate valid
+        valid_file = str(BUILTIN_SCENARIOS_DIR / "times-car-breach.json")
+        exit_code = cli_main(["simulate", "--validate", valid_file])
+        self.assertEqual(exit_code, 0)
+
+        # --validate invalid
+        with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
+            f.write("{\"id\": \"incomplete\"}")
+            inv_file = f.name
+        try:
+            exit_code = cli_main(["simulate", "--validate", inv_file])
+            self.assertEqual(exit_code, 2)
+        finally:
+            os.unlink(inv_file)
+
+
+if __name__ == "__main__":
+    unittest.main()