@hoanghainh/komagu 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +1 -1
- package/assess/lsa/examples/eval-run-4/ground-truth.json +823 -0
- package/assess/lsa/examples/eval-run-4/iter0-gptoss-a/ai-run-stats.json +58 -0
- package/assess/lsa/examples/eval-run-4/iter0-gptoss-a/coverage.json +85 -0
- package/assess/lsa/examples/eval-run-4/iter0-gptoss-a/findings/priv-privilege-001.json +822 -0
- package/assess/lsa/examples/eval-run-4/iter0-gptoss-a/findings/priv-privilege-002.json +393 -0
- package/assess/lsa/examples/eval-run-4/iter0-gptoss-a/findings.json +4477 -0
- package/assess/lsa/examples/eval-run-4/iter0-gptoss-a/mass-assignment.candidates.json +232 -0
- package/assess/lsa/examples/eval-run-4/iter0-gptoss-a/routes/routes-001.json +425 -0
- package/assess/lsa/examples/eval-run-4/iter0-gptoss-a/routes/routes-002.json +333 -0
- package/assess/lsa/examples/eval-run-4/iter0-gptoss-a/routes/routes-003.json +400 -0
- package/assess/lsa/examples/eval-run-4/iter0-gptoss-a/routes/routes-004.json +403 -0
- package/assess/lsa/examples/eval-run-4/iter0-gptoss-a/routes/routes-005.json +398 -0
- package/assess/lsa/examples/eval-run-4/iter0-gptoss-a/routes/routes-006.json +63 -0
- package/assess/lsa/examples/eval-run-4/iter0-gptoss-a/score.json +120 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-a/ai-run-stats.json +55 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-a/coverage.json +50 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-a/findings/priv-privilege-001.json +613 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-a/findings/priv-privilege-002.json +287 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-a/findings.json +4870 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-a/mass-assignment.candidates.json +232 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-a/routes/routes-001.json +425 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-a/routes/routes-002.json +333 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-a/routes/routes-003.json +400 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-a/routes/routes-004.json +403 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-a/routes/routes-005.json +398 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-a/routes/routes-006.json +63 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-a/score.json +107 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-b/ai-run-stats.json +58 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-b/coverage.json +92 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-b/findings/priv-privilege-001.json +645 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-b/findings/priv-privilege-002.json +222 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-b/findings.json +4859 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-b/mass-assignment.candidates.json +232 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-b/routes/routes-001.json +425 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-b/routes/routes-002.json +333 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-b/routes/routes-003.json +400 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-b/routes/routes-004.json +403 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-b/routes/routes-005.json +398 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-b/routes/routes-006.json +63 -0
- package/assess/lsa/examples/eval-run-4/iter1-gptoss-b/score.json +121 -0
- package/assess/lsa/examples/eval-run-4/iter1-qwen-a-incomplete/coverage.json +111 -0
- package/assess/lsa/examples/eval-run-4/iter1-qwen-a-incomplete/findings/priv-privilege-001.json +6 -0
- package/assess/lsa/examples/eval-run-4/iter1-qwen-a-incomplete/findings/priv-privilege-002.json +344 -0
- package/assess/lsa/examples/eval-run-4/iter1-qwen-a-incomplete/findings.json +3544 -0
- package/assess/lsa/examples/eval-run-4/iter1-qwen-a-incomplete/mass-assignment.candidates.json +232 -0
- package/assess/lsa/examples/eval-run-4/iter1-qwen-a-incomplete/routes/routes-001.json +425 -0
- package/assess/lsa/examples/eval-run-4/iter1-qwen-a-incomplete/routes/routes-002.json +333 -0
- package/assess/lsa/examples/eval-run-4/iter1-qwen-a-incomplete/routes/routes-003.json +400 -0
- package/assess/lsa/examples/eval-run-4/iter1-qwen-a-incomplete/routes/routes-004.json +403 -0
- package/assess/lsa/examples/eval-run-4/iter1-qwen-a-incomplete/routes/routes-005.json +398 -0
- package/assess/lsa/examples/eval-run-4/iter1-qwen-a-incomplete/routes/routes-006.json +63 -0
- package/assess/lsa/examples/eval-run-4/iter1-qwen-a-incomplete/score.json +123 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/ai-run-stats.json +50 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/coverage.json +42 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/findings/priv-privilege-001.json +540 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/findings/priv-privilege-002.json +238 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/findings.json +4383 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/findings.sarif +8729 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/mass-assignment.candidates.json +232 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/report.ja.md +1911 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/report.md +1911 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/routes/routes-001.json +425 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/routes/routes-002.json +333 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/routes/routes-003.json +400 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/routes/routes-004.json +403 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/routes/routes-005.json +398 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/routes/routes-006.json +63 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/score.json +101 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-a/stack-profile.json +226 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-b/ai-run-stats.json +54 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-b/coverage.json +66 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-b/findings/priv-privilege-001.json +440 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-b/findings/priv-privilege-002.json +229 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-b/findings.json +3424 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-b/mass-assignment.candidates.json +232 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-b/routes/routes-001.json +425 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-b/routes/routes-002.json +333 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-b/routes/routes-003.json +400 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-b/routes/routes-004.json +403 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-b/routes/routes-005.json +398 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-b/routes/routes-006.json +63 -0
- package/assess/lsa/examples/eval-run-4/iter2-gptoss-b/score.json +120 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/ai-run-stats.json +88 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/coverage.json +36 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/findings/priv-privilege-001.json +530 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/findings/priv-privilege-002.json +248 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/findings.json +4766 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/findings.sarif +9326 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/mass-assignment.candidates.json +232 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/report.ja.md +2011 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/report.md +2011 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/routes/routes-001.json +425 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/routes/routes-002.json +333 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/routes/routes-003.json +400 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/routes/routes-004.json +403 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/routes/routes-005.json +398 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/routes/routes-006.json +63 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/score.json +80 -0
- package/assess/lsa/examples/eval-run-5/gptoss-a/stack-profile.json +226 -0
- package/assess/lsa/examples/eval-run-5/gptoss-b/ai-run-stats.json +83 -0
- package/assess/lsa/examples/eval-run-5/gptoss-b/coverage.json +44 -0
- package/assess/lsa/examples/eval-run-5/gptoss-b/findings/priv-privilege-001.json +579 -0
- package/assess/lsa/examples/eval-run-5/gptoss-b/findings/priv-privilege-002.json +268 -0
- package/assess/lsa/examples/eval-run-5/gptoss-b/findings.json +3913 -0
- package/assess/lsa/examples/eval-run-5/gptoss-b/mass-assignment.candidates.json +232 -0
- package/assess/lsa/examples/eval-run-5/gptoss-b/routes/routes-001.json +425 -0
- package/assess/lsa/examples/eval-run-5/gptoss-b/routes/routes-002.json +333 -0
- package/assess/lsa/examples/eval-run-5/gptoss-b/routes/routes-003.json +400 -0
- package/assess/lsa/examples/eval-run-5/gptoss-b/routes/routes-004.json +403 -0
- package/assess/lsa/examples/eval-run-5/gptoss-b/routes/routes-005.json +398 -0
- package/assess/lsa/examples/eval-run-5/gptoss-b/routes/routes-006.json +63 -0
- package/assess/lsa/examples/eval-run-5/gptoss-b/score.json +94 -0
- package/assess/lsa/examples/eval-run-5/ground-truth.json +823 -0
- package/assess/lsa/prompts/35-privilege-auditor.md +3 -1
- package/assess/lsa/tests/EVAL-RESULTS.md +156 -0
- package/assess/lsa/tests/score_findings.py +58 -0
- package/assess/lsa/tests/test_eval_run_4.py +73 -0
- package/assess/lsa/tests/test_eval_run_5.py +73 -0
- package/assess/lsa/tests/test_privilege_tier2.py +30 -0
- package/operate/ai/anchoring.py +130 -0
- package/operate/ai/client.py +13 -0
- package/operate/ai/context.py +105 -0
- package/operate/ai/runner.py +197 -69
- package/operate/cli/__init__.py +1 -1
- package/operate/cli/commands/scan.py +1 -1
- package/operate/tests/test_ai_anchoring.py +184 -0
- package/operate/tests/test_headless_ai.py +148 -1
- package/package.json +1 -1
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Code context for the headless agents: snippets, server-side includes and bound model classes.
|
|
2
|
+
|
|
3
|
+
Standard-library only (Python 3.8+ compatible).
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Dict, List, Optional
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _read_snippet(target_dir: Path, rel_file: str, line: int, radius: int = 15) -> str:
|
|
14
|
+
"""Read a window of code around target line from the audited repository."""
|
|
15
|
+
full_path = target_dir / rel_file
|
|
16
|
+
if not full_path.is_file():
|
|
17
|
+
return f"[File not found: {rel_file}]"
|
|
18
|
+
try:
|
|
19
|
+
lines = full_path.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
20
|
+
start = max(0, line - radius - 1)
|
|
21
|
+
end = min(len(lines), line + radius)
|
|
22
|
+
numbered = [f"{i + 1:4d} | {lines[i]}" for i in range(start, end)]
|
|
23
|
+
return "\n".join(numbered)
|
|
24
|
+
except Exception as e:
|
|
25
|
+
return f"[Error reading {rel_file}: {e}]"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# Server-side includes whose content a verifier must see to judge a control (PHP, Classic ASP, ColdFusion)
|
|
29
|
+
_INCLUDE_RE = re.compile(
|
|
30
|
+
r"(?:require|include)(?:_once)?\s*\(?\s*['\"]([^'\"]+)['\"]"
|
|
31
|
+
r"|<!--\s*#include\s+(?:file|virtual)\s*=\s*\"([^\"]+)\""
|
|
32
|
+
r"|<cfinclude\s+template\s*=\s*\"([^\"]+)\"",
|
|
33
|
+
re.IGNORECASE,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _included_files(target_dir: Path, rel_file: str) -> List[str]:
|
|
38
|
+
"""Repository-relative paths of files that rel_file includes, if they exist."""
|
|
39
|
+
target_dir = target_dir.resolve()
|
|
40
|
+
path = target_dir / rel_file
|
|
41
|
+
if not path.is_file():
|
|
42
|
+
return []
|
|
43
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
44
|
+
found: List[str] = []
|
|
45
|
+
for m in _INCLUDE_RE.finditer(text):
|
|
46
|
+
ref = next(g for g in m.groups() if g)
|
|
47
|
+
candidate = (path.parent / ref).resolve()
|
|
48
|
+
try:
|
|
49
|
+
rel = str(candidate.relative_to(target_dir))
|
|
50
|
+
except ValueError:
|
|
51
|
+
continue
|
|
52
|
+
if candidate.is_file() and rel not in found:
|
|
53
|
+
found.append(rel)
|
|
54
|
+
return found
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
_MODEL_EXTENSIONS = (".java", ".cs", ".vb", ".php", ".py", ".js", ".ts", ".rb", ".cfc")
|
|
58
|
+
_NEW_OBJECT_RE = re.compile(r"\bnew\s+([A-Z][A-Za-z0-9_]*)\s*\(")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _model_source(target_dir: Path, name: Optional[str], near_file: str = "", near_line: int = 1) -> Optional[Dict[str, Any]]:
|
|
62
|
+
"""Definition of the bound model class, so the privilege auditor can see its fields (#319).
|
|
63
|
+
|
|
64
|
+
name is the Tier 1 binding target; when it is unknown, the first `new Foo(` near the binding is used.
|
|
65
|
+
"""
|
|
66
|
+
if not name and near_file:
|
|
67
|
+
m = _NEW_OBJECT_RE.search(_read_snippet(target_dir, near_file, near_line, radius=4))
|
|
68
|
+
name = m.group(1) if m else None
|
|
69
|
+
if not name or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name):
|
|
70
|
+
return None
|
|
71
|
+
class_re = re.compile(rf"\bclass\s+{name}\b")
|
|
72
|
+
# same application and language as the binding: a Node route must not pick up a PHP model of the same name
|
|
73
|
+
suffix = Path(near_file).suffix.lower()
|
|
74
|
+
suffixes = (suffix,) if suffix in _MODEL_EXTENSIONS else _MODEL_EXTENSIONS
|
|
75
|
+
parts = Path(near_file).parts
|
|
76
|
+
root = target_dir / parts[0] if len(parts) > 1 and (target_dir / parts[0]).is_dir() else target_dir
|
|
77
|
+
for path in sorted(root.rglob("*")):
|
|
78
|
+
if path.suffix.lower() not in suffixes or not path.is_file():
|
|
79
|
+
continue
|
|
80
|
+
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
81
|
+
for i, text in enumerate(lines):
|
|
82
|
+
if class_re.search(text):
|
|
83
|
+
rel = str(path.relative_to(target_dir))
|
|
84
|
+
return {"class": name, "file": rel, "code": _read_snippet(target_dir, rel, i + 16, radius=15)}
|
|
85
|
+
return {"class": name, "file": None, "code": "[definition not found in the repository]"}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _verification_context(target_dir: Path, finding: Dict[str, Any]) -> Dict[str, Any]:
|
|
89
|
+
"""Code the verifier needs: the finding's site, its file header, evidence lines and included files."""
|
|
90
|
+
loc = finding.get("location") or {}
|
|
91
|
+
rel = str(loc.get("file") or "")
|
|
92
|
+
line = int(loc.get("start_line") or 1)
|
|
93
|
+
ctx: Dict[str, Any] = {"location": _read_snippet(target_dir, rel, line, radius=20)}
|
|
94
|
+
if line > 40:
|
|
95
|
+
ctx["file_header"] = _read_snippet(target_dir, rel, 13, radius=12)
|
|
96
|
+
others = []
|
|
97
|
+
for ev in finding.get("evidence") or []:
|
|
98
|
+
if isinstance(ev, dict) and ev.get("file") and ev.get("file") != rel:
|
|
99
|
+
others.append({"file": ev["file"], "code": _read_snippet(target_dir, str(ev["file"]), int(ev.get("line") or 1), radius=6)})
|
|
100
|
+
if others:
|
|
101
|
+
ctx["evidence"] = others[:3]
|
|
102
|
+
includes = [{"file": inc, "code": _read_snippet(target_dir, inc, 30, radius=30)} for inc in _included_files(target_dir, rel)[:3]]
|
|
103
|
+
if includes:
|
|
104
|
+
ctx["included_files"] = includes
|
|
105
|
+
return ctx
|
package/operate/ai/runner.py
CHANGED
|
@@ -7,7 +7,7 @@ LLM endpoint (Ollama, vLLM, private OpenAI server):
|
|
|
7
7
|
3. Audits candidate routes for BOLA / IDOR (BOLA Auditor).
|
|
8
8
|
4. Audits candidate queries for SQL/NoSQL Injection (Injection Auditor).
|
|
9
9
|
5. Adversarial Skeptic Verification (Finding Verifier Council).
|
|
10
|
-
6. Merges and schema-validates all findings
|
|
10
|
+
6. Merges and schema-validates all findings; writes OUT/ai-run-stats.json.
|
|
11
11
|
|
|
12
12
|
Standard-library only (Python 3.8+ compatible).
|
|
13
13
|
"""
|
|
@@ -18,30 +18,28 @@ import json
|
|
|
18
18
|
from pathlib import Path
|
|
19
19
|
import subprocess
|
|
20
20
|
import sys
|
|
21
|
+
import time
|
|
21
22
|
from typing import Any, Callable, Dict, List, Optional
|
|
22
23
|
|
|
24
|
+
from operate.ai.anchoring import Anchor, AnchorReport, anchor_findings, load_anchors
|
|
23
25
|
from operate.ai.client import LLMClient
|
|
26
|
+
from operate.ai.context import _included_files, _model_source, _read_snippet, _verification_context # noqa: F401
|
|
24
27
|
|
|
25
28
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
26
29
|
LSA_PROMPTS = REPO_ROOT / "assess" / "lsa" / "prompts"
|
|
27
30
|
DEFAULT_CWE = {"bola": ["CWE-284"], "bfla": ["CWE-285"], "mass_assignment": ["CWE-915"],
|
|
28
31
|
"sql_injection": ["CWE-89"], "nosql_injection": ["CWE-943"]}
|
|
29
32
|
LSA_SCRIPTS = REPO_ROOT / "assess" / "lsa" / "scripts"
|
|
33
|
+
# Valid subtypes per category (finding.schema.json); the first one is used when a model writes an invalid value
|
|
34
|
+
CATEGORY_SUBTYPES = {"bfla": ["missing_role_check", "role_from_request", "client_side_role_check"],
|
|
35
|
+
"mass_assignment": ["whole_object_binding", "privileged_field_writable"]}
|
|
36
|
+
FINDING_STATUSES = {"unverified", "confirmed", "likely", "rejected", "needs_review"}
|
|
30
37
|
|
|
31
38
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
return f"[File not found: {rel_file}]"
|
|
37
|
-
try:
|
|
38
|
-
lines = full_path.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
39
|
-
start = max(0, line - radius - 1)
|
|
40
|
-
end = min(len(lines), line + radius)
|
|
41
|
-
numbered = [f"{i + 1:4d} | {lines[i]}" for i in range(start, end)]
|
|
42
|
-
return "\n".join(numbered)
|
|
43
|
-
except Exception as e:
|
|
44
|
-
return f"[Error reading {rel_file}: {e}]"
|
|
39
|
+
VERIFY_BATCH_SIZE = 5
|
|
40
|
+
AUTH_MODEL_LIMIT = 6000
|
|
41
|
+
RETRY_REMINDER = ("Your previous reply was empty or not valid JSON. Answer again with the JSON object only, "
|
|
42
|
+
"exactly in the format requested above, with no other text.")
|
|
45
43
|
|
|
46
44
|
|
|
47
45
|
class HeadlessAIRunner:
|
|
@@ -58,34 +56,121 @@ class HeadlessAIRunner:
|
|
|
58
56
|
self.out_dir = Path(out_dir).resolve()
|
|
59
57
|
self.client = client
|
|
60
58
|
self.callback = step_callback or (lambda status, msg: None)
|
|
59
|
+
# Per-phase replies that could not be parsed as JSON; written to ai-run-stats.json
|
|
60
|
+
self.parse_failures: Dict[str, List[str]] = {}
|
|
61
|
+
# Per-phase batches whose model call failed (timeout, HTTP error); the run continues without them
|
|
62
|
+
self.call_errors: Dict[str, Dict[str, str]] = {}
|
|
63
|
+
# Per-phase batches whose first reply was empty or unparseable and were asked again
|
|
64
|
+
self.retries: Dict[str, List[str]] = {}
|
|
65
|
+
# Findings whose location was taken from the Tier 1 inventory (#319)
|
|
66
|
+
self.anchor_report = AnchorReport()
|
|
67
|
+
self._anchors: Optional[Dict[str, Anchor]] = None
|
|
68
|
+
self.phase_stats: Dict[str, Dict[str, Any]] = {}
|
|
61
69
|
|
|
62
70
|
def _log(self, status: str, msg: str) -> None:
|
|
63
71
|
self.callback(status, msg)
|
|
64
72
|
|
|
73
|
+
def _anchor(self, findings: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
74
|
+
if self._anchors is None:
|
|
75
|
+
self._anchors = load_anchors(self.out_dir, self.target_dir)
|
|
76
|
+
return anchor_findings(findings, self._anchors, self.target_dir, self.anchor_report)
|
|
77
|
+
|
|
78
|
+
def _chat(self, phase: str, batch_id: str, messages: List[Dict[str, str]]) -> Optional[str]:
|
|
79
|
+
"""One batch call; a failed call is recorded and returns None instead of aborting the run.
|
|
80
|
+
|
|
81
|
+
An empty or unparseable reply is retried once with a reminder to answer with JSON only (#319).
|
|
82
|
+
"""
|
|
83
|
+
try:
|
|
84
|
+
reply = self.client.chat(messages, temperature=0.1)
|
|
85
|
+
except Exception as e:
|
|
86
|
+
self.call_errors.setdefault(phase, {})[batch_id] = str(e)[:300]
|
|
87
|
+
self._log("warn", f"{phase}: model call failed for {batch_id} ({e}); batch left without verdicts")
|
|
88
|
+
return None
|
|
89
|
+
if self._parses(reply):
|
|
90
|
+
return reply
|
|
91
|
+
self.retries.setdefault(phase, []).append(batch_id)
|
|
92
|
+
self._log("warn", f"{phase}: reply for {batch_id} was {'empty' if not (reply or '').strip() else 'not valid JSON'}; retrying once")
|
|
93
|
+
retry_messages = messages + ([{"role": "assistant", "content": reply}] if (reply or "").strip() else []) + [
|
|
94
|
+
{"role": "user", "content": RETRY_REMINDER}]
|
|
95
|
+
try:
|
|
96
|
+
second = self.client.chat(retry_messages, temperature=0.1)
|
|
97
|
+
except Exception as e:
|
|
98
|
+
self._log("warn", f"{phase}: retry for {batch_id} failed ({e})")
|
|
99
|
+
return reply
|
|
100
|
+
return second if self._parses(second) or not (reply or "").strip() else reply
|
|
101
|
+
|
|
102
|
+
def _parses(self, reply: Optional[str]) -> bool:
|
|
103
|
+
if not (reply or "").strip():
|
|
104
|
+
return False
|
|
105
|
+
try:
|
|
106
|
+
self.client.extract_json(reply)
|
|
107
|
+
return True
|
|
108
|
+
except Exception:
|
|
109
|
+
return False
|
|
110
|
+
|
|
111
|
+
def _parse_failed(self, phase: str, batch_id: str, reply: Optional[str] = None) -> None:
|
|
112
|
+
if batch_id in self.call_errors.get(phase, {}):
|
|
113
|
+
return
|
|
114
|
+
self.parse_failures.setdefault(phase, []).append(batch_id)
|
|
115
|
+
if reply is not None:
|
|
116
|
+
raw_dir = self.out_dir / "ai-raw"
|
|
117
|
+
raw_dir.mkdir(parents=True, exist_ok=True)
|
|
118
|
+
(raw_dir / f"{phase}-{batch_id}.txt").write_text(reply, encoding="utf-8")
|
|
119
|
+
self._log("warn", f"{phase}: could not parse the model reply for {batch_id}; batch left without verdicts")
|
|
120
|
+
|
|
121
|
+
def _usage(self) -> Dict[str, Any]:
|
|
122
|
+
return dict(getattr(self.client, "usage", None) or {})
|
|
123
|
+
|
|
124
|
+
def _timed(self, phase: str, fn: Callable[[], Any]) -> None:
|
|
125
|
+
before, started = self._usage(), time.monotonic()
|
|
126
|
+
fn()
|
|
127
|
+
after = self._usage()
|
|
128
|
+
self.phase_stats[phase] = {
|
|
129
|
+
"seconds": round(time.monotonic() - started, 3),
|
|
130
|
+
**{k: after.get(k, 0) - before.get(k, 0) for k in ("calls", "prompt_tokens", "completion_tokens")},
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
def write_run_stats(self) -> Path:
|
|
134
|
+
"""Write model, per-phase durations, token counts and parse failures to OUT/ai-run-stats.json."""
|
|
135
|
+
stats = {
|
|
136
|
+
"model": getattr(self.client, "model", None),
|
|
137
|
+
"endpoint": getattr(self.client, "endpoint", None),
|
|
138
|
+
"phases": self.phase_stats,
|
|
139
|
+
"totals": self._usage(),
|
|
140
|
+
"parse_failures": self.parse_failures,
|
|
141
|
+
"call_errors": self.call_errors,
|
|
142
|
+
"retries": self.retries,
|
|
143
|
+
"anchoring": {"moved": self.anchor_report.moved, "unanchored": self.anchor_report.unanchored},
|
|
144
|
+
}
|
|
145
|
+
path = self.out_dir / "ai-run-stats.json"
|
|
146
|
+
path.write_text(json.dumps(stats, indent=2), encoding="utf-8")
|
|
147
|
+
return path
|
|
148
|
+
|
|
65
149
|
def run(self) -> Dict[str, Any]:
|
|
66
150
|
"""Execute all cognitive AI phases end-to-end."""
|
|
67
151
|
self._log("step", "Starting Tier 2 Cognitive Multi-Agent Audit (Headless BYO-LLM)")
|
|
68
152
|
|
|
69
153
|
# Phase 2: Stack & Auth Model Detector
|
|
70
|
-
self.infer_auth_model
|
|
154
|
+
self._timed("2-stack-detector", self.infer_auth_model)
|
|
71
155
|
|
|
72
156
|
# Phase 3: Route Inventory Mapping
|
|
73
|
-
self.map_routes_inventory
|
|
157
|
+
self._timed("3-route-mapper", self.map_routes_inventory)
|
|
74
158
|
|
|
75
159
|
# Phase 4a: BOLA / IDOR Cognitive Audit
|
|
76
|
-
self.audit_bola_routes
|
|
160
|
+
self._timed("4a-bola-auditor", self.audit_bola_routes)
|
|
77
161
|
|
|
78
162
|
# Phase 4b: SQL / NoSQL Injection Cognitive Audit
|
|
79
|
-
self.audit_injection_candidates
|
|
163
|
+
self._timed("4b-injection-auditor", self.audit_injection_candidates)
|
|
80
164
|
|
|
81
165
|
# Phase 4c: Privilege Audit (BFLA, mass assignment)
|
|
82
|
-
self.audit_privilege
|
|
166
|
+
self._timed("4c-privilege-auditor", self.audit_privilege)
|
|
83
167
|
|
|
84
168
|
# Phase 5: Adversarial Skeptic Verifier
|
|
85
|
-
self.verify_findings_skeptic
|
|
169
|
+
self._timed("5-finding-verifier", self.verify_findings_skeptic)
|
|
86
170
|
|
|
87
171
|
# Phase 6: Merge & Validate Outputs
|
|
88
172
|
self.merge_and_validate()
|
|
173
|
+
self.write_run_stats()
|
|
89
174
|
|
|
90
175
|
self._log("ok", "Tier 2 Cognitive Multi-Agent Audit completed successfully")
|
|
91
176
|
return {"status": "success", "out_dir": str(self.out_dir)}
|
|
@@ -209,6 +294,20 @@ Output MUST be Markdown format adhering to Komagu auth-model.md specification.
|
|
|
209
294
|
"""Normalize finding fields to strictly comply with finding.schema.json."""
|
|
210
295
|
if not isinstance(f, dict):
|
|
211
296
|
return f
|
|
297
|
+
# impact must be a string; small models often return a list of consequences
|
|
298
|
+
if isinstance(f.get("impact"), list):
|
|
299
|
+
f["impact"] = "; ".join(str(i) for i in f["impact"])
|
|
300
|
+
|
|
301
|
+
# a verdict written into subtype belongs in status; fall back to the category's first subtype
|
|
302
|
+
allowed = CATEGORY_SUBTYPES.get(f.get("category"))
|
|
303
|
+
if allowed and f.get("subtype") not in allowed:
|
|
304
|
+
if f.get("subtype") in FINDING_STATUSES:
|
|
305
|
+
f["status"] = f["subtype"]
|
|
306
|
+
f["subtype"] = allowed[0]
|
|
307
|
+
|
|
308
|
+
if f.get("status") not in FINDING_STATUSES:
|
|
309
|
+
f["status"] = "unverified"
|
|
310
|
+
|
|
212
311
|
# recommendation must be a string
|
|
213
312
|
rec = f.get("recommendation")
|
|
214
313
|
if isinstance(rec, dict):
|
|
@@ -227,8 +326,8 @@ Output MUST be Markdown format adhering to Komagu auth-model.md specification.
|
|
|
227
326
|
else:
|
|
228
327
|
for ev in evidence:
|
|
229
328
|
if isinstance(ev, dict):
|
|
230
|
-
if "line"
|
|
231
|
-
ev["line"] = ev.get("start_line", default_line
|
|
329
|
+
if not isinstance(ev.get("line"), int):
|
|
330
|
+
ev["line"] = ev.get("start_line") if isinstance(ev.get("start_line"), int) else default_line
|
|
232
331
|
if "file" not in ev:
|
|
233
332
|
ev["file"] = default_file
|
|
234
333
|
if "why" not in ev:
|
|
@@ -323,13 +422,14 @@ Respond STRICTLY with a valid JSON object matching:
|
|
|
323
422
|
{"role": "user", "content": user_prompt},
|
|
324
423
|
]
|
|
325
424
|
|
|
326
|
-
reply = self.
|
|
425
|
+
reply = self._chat("4a-bola-auditor", r_file.stem, messages)
|
|
327
426
|
try:
|
|
328
427
|
parsed = self.client.extract_json(reply)
|
|
329
428
|
raw_findings = parsed.get("findings", [])
|
|
330
|
-
findings = [self._sanitize_finding(f) for f in raw_findings]
|
|
429
|
+
findings = self._anchor([self._sanitize_finding(f) for f in raw_findings])
|
|
331
430
|
route_verdicts = parsed.get("route_verdicts", [])
|
|
332
431
|
except Exception:
|
|
432
|
+
self._parse_failed("4a-bola-auditor", r_file.stem, reply)
|
|
333
433
|
findings = []
|
|
334
434
|
route_verdicts = []
|
|
335
435
|
|
|
@@ -409,13 +509,14 @@ Respond STRICTLY with a valid JSON object matching:
|
|
|
409
509
|
{"role": "user", "content": user_prompt},
|
|
410
510
|
]
|
|
411
511
|
|
|
412
|
-
reply = self.
|
|
512
|
+
reply = self._chat("4b-injection-auditor", b_file.stem, messages)
|
|
413
513
|
try:
|
|
414
514
|
parsed = self.client.extract_json(reply)
|
|
415
515
|
raw_findings = parsed.get("findings", [])
|
|
416
|
-
findings = [self._sanitize_finding(f) for f in raw_findings]
|
|
516
|
+
findings = self._anchor([self._sanitize_finding(f) for f in raw_findings])
|
|
417
517
|
candidate_verdicts = parsed.get("candidate_verdicts", [])
|
|
418
518
|
except Exception:
|
|
519
|
+
self._parse_failed("4b-injection-auditor", b_file.stem, reply)
|
|
419
520
|
findings = []
|
|
420
521
|
candidate_verdicts = []
|
|
421
522
|
|
|
@@ -467,6 +568,7 @@ Respond STRICTLY with a valid JSON object matching:
|
|
|
467
568
|
"allowlist": c.get("allowlist"),
|
|
468
569
|
"sensitive_fields": c.get("sensitive_fields", []),
|
|
469
570
|
"code_context": _read_snippet(self.target_dir, c.get("file", ""), c.get("line") or 1),
|
|
571
|
+
"model_definition": _model_source(self.target_dir, c.get("target"), c.get("file", ""), c.get("line") or 1),
|
|
470
572
|
} for c in mass]
|
|
471
573
|
|
|
472
574
|
user_prompt = f"""Auth Model:
|
|
@@ -480,20 +582,51 @@ Mass Assignment Candidates:
|
|
|
480
582
|
|
|
481
583
|
Task:
|
|
482
584
|
Judge every route for BFLA and every candidate for mass assignment as described in your role.
|
|
483
|
-
Respond STRICTLY with a valid JSON object
|
|
484
|
-
|
|
585
|
+
Respond STRICTLY with a valid JSON object matching:
|
|
586
|
+
{{
|
|
587
|
+
"batch_id": "{b_file.stem}",
|
|
588
|
+
"privilege_route_verdicts": [
|
|
589
|
+
{{"route_id": "<route_id>", "verdict": "vulnerable|safe|needs_review"}}
|
|
590
|
+
],
|
|
591
|
+
"candidate_verdicts": [
|
|
592
|
+
{{"candidate_id": "<candidate_id>", "verdict": "vulnerable|safe|needs_review"}}
|
|
593
|
+
],
|
|
594
|
+
"findings": [
|
|
595
|
+
{{
|
|
596
|
+
"finding_id": "BFLA-<route_id>-1 or MASS-<candidate_id>-1",
|
|
597
|
+
"category": "bfla|mass_assignment",
|
|
598
|
+
"subtype": "missing_role_check|role_from_request|client_side_role_check (bfla) or whole_object_binding|privileged_field_writable (mass_assignment)",
|
|
599
|
+
"title": "<short title>",
|
|
600
|
+
"route_id": "<route_id, bfla only>",
|
|
601
|
+
"candidate_id": "<candidate_id, mass_assignment only>",
|
|
602
|
+
"location": {{"file": "<file>", "start_line": <line>}},
|
|
603
|
+
"evidence": [{{"file": "<file>", "line": <line>, "why": "<privileged action or binding, and the missing control>"}}],
|
|
604
|
+
"impact": "<one string naming the privileged action or fields>",
|
|
605
|
+
"severity": "critical|high|medium|low",
|
|
606
|
+
"confidence": "high|medium|low",
|
|
607
|
+
"cwe": ["CWE-285"],
|
|
608
|
+
"recommendation": "<one string>",
|
|
609
|
+
"status": "unverified|needs_review"
|
|
610
|
+
}}
|
|
611
|
+
]
|
|
612
|
+
}}
|
|
613
|
+
Write one finding for every "vulnerable" or "needs_review" verdict. A "needs_review" verdict goes in
|
|
614
|
+
"status", never in "subtype".
|
|
615
|
+
For mass assignment, judge the fields from "model_definition". If it says the definition was not found
|
|
616
|
+
(or is null), you cannot know the fields: answer "needs_review", never "safe" for that reason alone.
|
|
485
617
|
"""
|
|
486
618
|
messages = [
|
|
487
619
|
{"role": "system", "content": auditor_prompt},
|
|
488
620
|
{"role": "user", "content": user_prompt},
|
|
489
621
|
]
|
|
490
|
-
reply = self.
|
|
622
|
+
reply = self._chat("4c-privilege-auditor", b_file.stem, messages)
|
|
491
623
|
try:
|
|
492
624
|
parsed = self.client.extract_json(reply)
|
|
493
|
-
findings = [self._sanitize_finding(f) for f in parsed.get("findings", [])]
|
|
625
|
+
findings = self._anchor([self._sanitize_finding(f) for f in parsed.get("findings", [])])
|
|
494
626
|
route_verdicts = parsed.get("privilege_route_verdicts", [])
|
|
495
627
|
candidate_verdicts = parsed.get("candidate_verdicts", [])
|
|
496
628
|
except Exception:
|
|
629
|
+
self._parse_failed("4c-privilege-auditor", b_file.stem, reply)
|
|
497
630
|
findings, route_verdicts, candidate_verdicts = [], [], []
|
|
498
631
|
|
|
499
632
|
out_doc = {
|
|
@@ -528,37 +661,46 @@ Respond STRICTLY with a valid JSON object with keys "batch_id" ("{b_file.stem}")
|
|
|
528
661
|
return
|
|
529
662
|
|
|
530
663
|
verifier_prompt = (LSA_PROMPTS / "50-finding-verifier.md").read_text(encoding="utf-8", errors="replace") if (LSA_PROMPTS / "50-finding-verifier.md").is_file() else ""
|
|
664
|
+
auth_model_path = self.out_dir / "auth-model.md"
|
|
665
|
+
auth_model = auth_model_path.read_text(encoding="utf-8", errors="replace")[:AUTH_MODEL_LIMIT] if auth_model_path.is_file() else ""
|
|
531
666
|
|
|
532
|
-
|
|
533
|
-
batch_size = 10
|
|
667
|
+
batch_size = VERIFY_BATCH_SIZE
|
|
534
668
|
for idx in range(0, len(all_findings), batch_size):
|
|
535
669
|
chunk = all_findings[idx : idx + batch_size]
|
|
536
|
-
|
|
537
|
-
{
|
|
670
|
+
# The headless verifier cannot open files, so it gets the code it must judge (#286)
|
|
671
|
+
to_verify = [{"finding": f, "code_context": _verification_context(self.target_dir, f)} for f in chunk]
|
|
672
|
+
user_prompt = f"""Auth Model:
|
|
673
|
+
{auth_model}
|
|
674
|
+
|
|
675
|
+
Findings to Verify & Challenge (each with the code you may rely on):
|
|
676
|
+
{json.dumps(to_verify, indent=2)}
|
|
538
677
|
|
|
539
678
|
Task:
|
|
540
|
-
You are the skeptical security verifier.
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
679
|
+
You are the skeptical security verifier. You cannot open files: judge only from the code_context given
|
|
680
|
+
with each finding and the Auth Model above.
|
|
681
|
+
For each finding:
|
|
682
|
+
1. Look for a compensating control (role or ownership check, global filter, integer cast, allow-list,
|
|
683
|
+
parameter binding) in the code_context.
|
|
684
|
+
2. Only use reason_code "control_found" when the control is visible in the code_context; quote its
|
|
685
|
+
file and line in "control_found". Never assume a control in a base class, include, config or
|
|
686
|
+
middleware you were not shown.
|
|
687
|
+
3. If the code you would need is not shown, answer verdict "likely" with reason_code
|
|
688
|
+
"code_not_visible" or "config_not_visible".
|
|
689
|
+
Verdicts: "confirmed" | "likely" | "rejected".
|
|
690
|
+
reason_code is one of: "traced", "code_not_visible", "config_not_visible", "data_ownership_unclear",
|
|
691
|
+
"control_found", "evidence_mismatch", "not_executed", "not_reachable", "not_attacker_controlled".
|
|
545
692
|
Respond STRICTLY with a valid JSON object matching verification.schema.json:
|
|
546
693
|
{{
|
|
547
694
|
"batch_id": "verified-{(idx // batch_size) + 1}",
|
|
548
695
|
"verifications": [
|
|
549
696
|
{{
|
|
550
697
|
"finding_id": "<finding_id>",
|
|
551
|
-
"verdict": "confirmed|rejected",
|
|
552
|
-
"reason_code": "
|
|
553
|
-
"confidence": "high",
|
|
554
|
-
"checked": ["
|
|
555
|
-
"
|
|
556
|
-
"
|
|
557
|
-
"score": 0.95,
|
|
558
|
-
"agreement_level": "unanimous",
|
|
559
|
-
"voted_models": ["{self.client.model}"],
|
|
560
|
-
"rationale": "Independently audited and refuted false-alarm candidates."
|
|
561
|
-
}}
|
|
698
|
+
"verdict": "confirmed|likely|rejected",
|
|
699
|
+
"reason_code": "<one of the reason codes above>",
|
|
700
|
+
"confidence": "high|medium|low",
|
|
701
|
+
"checked": ["<what you checked in the code_context>"],
|
|
702
|
+
"control_found": {{"file": "<file>", "line": <line>, "code": "<control>"}} or null,
|
|
703
|
+
"notes": "<rationale for verdict>"
|
|
562
704
|
}}
|
|
563
705
|
]
|
|
564
706
|
}}
|
|
@@ -568,29 +710,15 @@ Respond STRICTLY with a valid JSON object matching verification.schema.json:
|
|
|
568
710
|
{"role": "user", "content": user_prompt},
|
|
569
711
|
]
|
|
570
712
|
|
|
571
|
-
reply = self.
|
|
713
|
+
reply = self._chat("5-finding-verifier", f"verified-{(idx // batch_size) + 1}", messages)
|
|
572
714
|
try:
|
|
573
715
|
parsed = self.client.extract_json(reply)
|
|
574
716
|
verifs = parsed.get("verifications", [])
|
|
575
717
|
except Exception:
|
|
576
|
-
#
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
"verdict": "confirmed",
|
|
581
|
-
"reason_code": "traced",
|
|
582
|
-
"confidence": "high",
|
|
583
|
-
"checked": ["Audited data-flow"],
|
|
584
|
-
"notes": "Traced via headless cognitive consensus.",
|
|
585
|
-
"consensus": {
|
|
586
|
-
"score": 0.90,
|
|
587
|
-
"agreement_level": "majority",
|
|
588
|
-
"voted_models": [self.client.model],
|
|
589
|
-
"rationale": "Verified in headless pipeline.",
|
|
590
|
-
},
|
|
591
|
-
}
|
|
592
|
-
for f in chunk
|
|
593
|
-
]
|
|
718
|
+
# No verdicts: merge_findings.py lists these findings under
|
|
719
|
+
# coverage.findings_without_verification instead of confirming them
|
|
720
|
+
self._parse_failed("5-finding-verifier", f"verified-{(idx // batch_size) + 1}", reply)
|
|
721
|
+
verifs = []
|
|
594
722
|
|
|
595
723
|
verif_doc = {
|
|
596
724
|
"batch_id": f"verified-{(idx // batch_size) + 1}",
|
package/operate/cli/__init__.py
CHANGED
|
@@ -455,7 +455,7 @@ def execute_scan(args: argparse.Namespace) -> int:
|
|
|
455
455
|
f" {color('🚀 Mode: Tier 2 Cognitive Multi-Agent Engine', GREEN, BOLD)} "
|
|
456
456
|
f"(Headless BYO-LLM: {args.headless_ai}, Model: {args.model})"
|
|
457
457
|
)
|
|
458
|
-
print(f" {color('✔ Skeptic Consensus Applied', GREEN)}:
|
|
458
|
+
print(f" {color('✔ Skeptic Consensus Applied', GREEN)}: Findings challenged against compensating controls.\n")
|
|
459
459
|
else:
|
|
460
460
|
print(f" {color('💡 Mode: Tier 1 Deterministic Engine', CYAN, BOLD)} (Regex & AST candidate extraction, no LLM required)")
|
|
461
461
|
print(" To run Tier 2 Cognitive Multi-Agent Audit (BOLA semantic reasoning & Skeptic Verifier):")
|