@hoanghainh/komagu 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/ai-run-stats.json +87 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/auth-model.md +120 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/batches/index.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/batches/injection-001.json +383 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/batches/injection-002.json +205 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/batches/privilege-001.json +838 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/batches/privilege-002.json +124 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/batches/routes-001.json +778 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/batches/routes-002.json +546 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/batches/routes-003.json +661 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/batches/routes-004.json +659 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/batches/routes-005.json +612 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/batches/routes-006.json +106 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/coverage.json +19 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/bola-routes-001.json +516 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/bola-routes-002.json +332 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/bola-routes-003.json +218 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/bola-routes-004.json +561 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/bola-routes-005.json +316 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/bola-routes-006.json +14 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/inj-injection-001.json +782 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/inj-injection-002.json +418 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/priv-privilege-001.json +627 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/priv-privilege-002.json +253 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-1.json +68 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-10.json +68 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-11.json +61 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-12.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-13.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-14.json +66 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-15.json +69 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-16.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-17.json +61 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-18.json +69 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-19.json +64 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-2.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-20.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-21.json +61 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-22.json +50 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-3.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-4.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-5.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-6.json +70 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-7.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-8.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings/verified-9.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/findings.json +4857 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/mass-assignment.candidates.json +232 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/report.ja.md +2112 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/report.md +2112 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/routes/routes-001.json +425 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/routes/routes-002.json +333 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/routes/routes-003.json +400 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/routes/routes-004.json +403 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/routes/routes-005.json +398 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/routes/routes-006.json +63 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/score.json +82 -0
- package/assess/lsa/examples/eval-run-6/gptoss-a/stack-profile.json +225 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/ai-run-stats.json +91 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/auth-model.md +120 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/batches/index.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/batches/injection-001.json +383 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/batches/injection-002.json +205 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/batches/privilege-001.json +838 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/batches/privilege-002.json +124 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/batches/routes-001.json +778 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/batches/routes-002.json +546 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/batches/routes-003.json +661 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/batches/routes-004.json +659 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/batches/routes-005.json +612 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/batches/routes-006.json +106 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/coverage.json +21 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/bola-routes-001.json +593 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/bola-routes-002.json +367 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/bola-routes-003.json +207 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/bola-routes-004.json +477 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/bola-routes-005.json +341 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/bola-routes-006.json +14 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/inj-injection-001.json +821 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/inj-injection-002.json +410 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/priv-privilege-001.json +649 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/priv-privilege-002.json +222 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-1.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-10.json +67 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-11.json +68 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-12.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-13.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-14.json +64 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-15.json +70 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-16.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-17.json +61 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-18.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-19.json +65 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-2.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-20.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-21.json +62 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-22.json +50 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-3.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-4.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-5.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-6.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-7.json +64 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-8.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings/verified-9.json +60 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/findings.json +4966 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/mass-assignment.candidates.json +232 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/routes/routes-001.json +425 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/routes/routes-002.json +333 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/routes/routes-003.json +400 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/routes/routes-004.json +403 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/routes/routes-005.json +398 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/routes/routes-006.json +63 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/score.json +87 -0
- package/assess/lsa/examples/eval-run-6/gptoss-b/stack-profile.json +225 -0
- package/assess/lsa/examples/eval-run-6/ground-truth.json +851 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/ai-run-stats.json +96 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/auth-model.md +191 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/batches/index.json +60 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/batches/injection-001.json +383 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/batches/injection-002.json +205 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/batches/privilege-001.json +838 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/batches/privilege-002.json +124 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/batches/routes-001.json +778 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/batches/routes-002.json +546 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/batches/routes-003.json +661 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/batches/routes-004.json +659 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/batches/routes-005.json +612 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/batches/routes-006.json +106 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/coverage.json +31 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/bola-routes-001.json +527 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/bola-routes-002.json +568 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/bola-routes-003.json +467 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/bola-routes-004.json +342 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/bola-routes-005.json +693 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/bola-routes-006.json +16 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/inj-injection-001.json +1123 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/inj-injection-002.json +391 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/priv-privilege-001.json +726 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/priv-privilege-002.json +306 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-1.json +137 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-10.json +106 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-11.json +81 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-12.json +135 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-13.json +135 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-14.json +135 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-15.json +110 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-16.json +116 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-17.json +141 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-18.json +135 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-19.json +140 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-2.json +135 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-20.json +82 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-21.json +74 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-22.json +99 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-3.json +140 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-4.json +145 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-5.json +135 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-6.json +135 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-7.json +87 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-8.json +125 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings/verified-9.json +135 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/findings.json +6989 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/mass-assignment.candidates.json +232 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/memory.log +36 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/routes/routes-001.json +425 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/routes/routes-002.json +333 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/routes/routes-003.json +400 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/routes/routes-004.json +403 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/routes/routes-005.json +398 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/routes/routes-006.json +63 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/score.json +106 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-nothink/stack-profile.json +225 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-partial/auth-model.md +517 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-partial/findings/bola-routes-001.json +5 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-partial/findings/bola-routes-002.json +428 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-partial/findings/bola-routes-003.json +388 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-partial/findings/bola-routes-004.json +463 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-partial/findings/bola-routes-005.json +5 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-partial/findings/bola-routes-006.json +67 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-partial/findings/inj-injection-001.json +964 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-partial/findings.partial.json +1872 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-partial/memory.log +194 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-partial/score.partial.json +108 -0
- package/assess/lsa/examples/eval-run-6/qwen-16k-partial/stack-profile.json +225 -0
- package/assess/lsa/scripts/laravel_routes.py +102 -0
- package/assess/lsa/scripts/map_routes.py +4 -1
- package/assess/lsa/tests/EVAL-RESULTS.md +126 -0
- package/assess/lsa/tests/test_eval_run_6.py +104 -0
- package/assess/lsa/tests/test_laravel_routes.py +95 -0
- package/operate/ai/client.py +22 -9
- package/operate/ai/runner.py +4 -2
- package/operate/cli/__init__.py +1 -1
- package/operate/cli/commands/scan.py +8 -1
- package/operate/tests/test_ai_reasoning_effort.py +65 -0
- package/operate/tests/test_ai_route_inventory.py +53 -0
- package/package.json +1 -1
|
@@ -24,6 +24,7 @@ from java_routes import (build_class_index, collect_spring_beans, entry_method_l
|
|
|
24
24
|
struts_modules, xml_declared)
|
|
25
25
|
from lsa_common import (JAVA_WEB_ROOT, KIT_VERSION, call_args, compile_id_patterns, is_id_name, # noqa: E402
|
|
26
26
|
iter_source_files, line_of, load_config, read_text, split_top, write_json)
|
|
27
|
+
from laravel_routes import resolve_laravel_handlers, route_action # noqa: E402
|
|
27
28
|
from nextjs_routes import middleware_note, nextjs_declared # noqa: E402
|
|
28
29
|
from privilege import annotate_privilege # noqa: E402
|
|
29
30
|
|
|
@@ -268,7 +269,7 @@ def php_declared(c: Collector, rel: str, text: str, stacks: set) -> bool:
|
|
|
268
269
|
end = text.find(";", m.end())
|
|
269
270
|
stmt = text[m.start(): end if end > 0 else m.end() + 200]
|
|
270
271
|
c.add(stack="php-laravel", kind="declared_route", method=[verb.upper()], path="/" + path.lstrip("/"),
|
|
271
|
-
handler={"file": rel, "line": line, "symbol":
|
|
272
|
+
handler={"file": rel, "line": line, "symbol": route_action(stmt)}, id_params=c.path_params(path),
|
|
272
273
|
guard_hints=c.guards(stmt, line) + c.guards(text[:m.start()][-3000:], 1)[:2],
|
|
273
274
|
data_hints=[], notes="handler is in the controller named in this route")
|
|
274
275
|
found = True
|
|
@@ -744,6 +745,8 @@ def map_routes(target: Path, config: Dict, stack_ids: Optional[set] = None, work
|
|
|
744
745
|
it["id_params"] = c.inputs("java", t)
|
|
745
746
|
it["guard_hints"] = it["guard_hints"] or c.guards(t)
|
|
746
747
|
it["data_hints"] = it["data_hints"] or c.data_ops(t)
|
|
748
|
+
# post-pass 1b: Laravel route declarations -> the controller method they name (#380)
|
|
749
|
+
c.items = resolve_laravel_handlers(c.items, target, enc, c, [rel for _, rel in files])
|
|
747
750
|
# post-pass 2: servlet classes without @WebServlet -> take the path from web.xml
|
|
748
751
|
xml_paths: Dict[str, List[str]] = {}
|
|
749
752
|
for it in c.items:
|
|
@@ -333,3 +333,129 @@ and their tests still score against them.
|
|
|
333
333
|
Run 5 rescored against the new ground truth (for comparison only; findings unchanged): BOLA 28 / 29
|
|
334
334
|
(run A) and 16 / 29 (run B), precision 0.859 and 0.826, still no false positive on SAFE markers. Every
|
|
335
335
|
new marker was already reported by both runs; Tier 1 lists all of them as BOLA route candidates.
|
|
336
|
+
|
|
337
|
+
---
|
|
338
|
+
|
|
339
|
+
## Run 6 — 2026-11-11, platform 2.5.0, kit 0.3.1 (#323 part 4: new harness, 121-item ground truth, redaction)
|
|
340
|
+
|
|
341
|
+
Issue #323. First run on the harness from #334 (no code fields in 4a / 4b templates, quote repair,
|
|
342
|
+
findings from flagged verdicts), the fixtures and ground truth from #336 (121 items: 29 BOLA, 34
|
|
343
|
+
SQL/NoSQL injection, 10 BFLA, 15 mass assignment, 33 SAFE), and secret redaction of code sent to the
|
|
344
|
+
model (v2.4.0). Outputs are archived in [`assess/lsa/examples/eval-run-6/`](../examples/eval-run-6/).
|
|
345
|
+
Runs 4–5 are not directly comparable; the Run 5 rows below are the Run 5 findings rescored against the
|
|
346
|
+
new ground truth.
|
|
347
|
+
|
|
348
|
+
| Item | Value |
|
|
349
|
+
|---|---|
|
|
350
|
+
| Pipeline | `komagu scan <repo> --no-cache --headless-ai http://localhost:11434/v1 --model <model> --ai-timeout 3600` from `main` at v2.5.0 |
|
|
351
|
+
| Hosting | Ollama 0.40.16 on the same Apple M4 Pro, 24 GB RAM, 100% GPU, local only; `gpt-oss:20b` on the llama.cpp runner with 32,768 context (as in Runs 4–5) |
|
|
352
|
+
| Scoring | `tests/score_findings.py --verdicts` (tolerance 3 lines) |
|
|
353
|
+
| Machine | No other heavy work while measuring; up to 5.7 GB swap in use during the `gpt-oss` runs |
|
|
354
|
+
|
|
355
|
+
### Results after verification (`gpt-oss:20b`)
|
|
356
|
+
|
|
357
|
+
| Run | BOLA | Injection | BFLA marker | Mass marker | BFLA verdict | Mass verdict | FP on safe code | Precision | Rejected | Retries · repaired · parse fail | From verdicts | Min · calls · tokens |
|
|
358
|
+
|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
359
|
+
| Run 5 A, rescored | 28/29 | 34/34 | 10/10 | 10/15 | 10/10 | 15/15 | 0 | 0.859 | 2 | 0 · — · 0 | — | 83 · 32 · 328K |
|
|
360
|
+
| Run 5 B, rescored | 16/29 | 32/34 | 9/10 | 14/15 | 10/10 | 15/15 | 0 | 0.826 | 3 | 1 · — · 1 | — | 86 · 31 · 369K |
|
|
361
|
+
| **Run 6 A** | **26/29** | **34/34** | **10/10** | **14/15** | **10/10** | **15/15** | **1** | **0.813** | 2 | 0 · 0 · 0 | 9 (5 BOLA, 4 mass) | 40 · 33 · 315K |
|
|
362
|
+
| **Run 6 B** | **26/29** | **34/34** | **10/10** | **14/15** | **10/10** | **14/15** | **1** | **0.787** | 1 | 1 · 0 · 0 | 14 (5 BOLA, 9 mass) | 56 · 34 · 373K |
|
|
363
|
+
|
|
364
|
+
- **The two runs now agree.** BOLA is 26 / 29 in both (Run 5: 28 and 16). No batch was lost: no parse
|
|
365
|
+
failure, no call error, no batch needed the quote repair, and the one retry (B, `injection-001`)
|
|
366
|
+
succeeded. The `routes-001` batch that failed in Run 4 B and Run 5 B now parses.
|
|
367
|
+
- **Mass assignment:** 14 / 15 as located findings in both runs (Run 5: 10 and 14). Findings written from
|
|
368
|
+
flagged verdicts closed the gap: the four candidates run A of Run 5 judged `vulnerable` without a
|
|
369
|
+
finding (`profile.php` ×2, `CustomerController.cs` ×2) are findings in both runs.
|
|
370
|
+
`php-plain/register.php:4` is still missed. In A the verifier rejected it because the INSERT "only"
|
|
371
|
+
writes `email`, `name` and `role`, missing that `extract($_POST)` can overwrite `$role` (the
|
|
372
|
+
fixture change in #336); in B the auditor did not flag it.
|
|
373
|
+
- **BFLA:** 10 / 10 in both runs, located and by verdict.
|
|
374
|
+
- **Misses:** `perl-cgi/cgi-bin/order.cgi:19` (both runs, missed since Run 4), `node-express/routes/orders.js:29`
|
|
375
|
+
(both), `java-legacy/.../OrderServlet.java:15` (A), `python-flask/app.py:12` (B).
|
|
376
|
+
- **False positives on SAFE markers: 1 per run** (Run 5: 0), both injection:
|
|
377
|
+
- A: `classic-asp/shop/member.asp:6`. The `UPDATE` starts on line 6, whose `name` value is escaped;
|
|
378
|
+
the concatenated `member_id` is on line 7, which is also reported. The finding is anchored one line
|
|
379
|
+
early rather than describing a new flaw, but it is scored as a false positive.
|
|
380
|
+
- B: `python-flask/app.py:12`. `order_id` is an `<int:...>` path parameter, so the `%` formatting
|
|
381
|
+
cannot inject. The model reported the route's missing ownership check as SQL injection, so the
|
|
382
|
+
BOLA marker on the same line was missed.
|
|
383
|
+
- **Precision 0.81 / 0.79.** Five of the unmatched findings per run come from verdicts: the BOLA
|
|
384
|
+
auditor flagged the Laravel routes `/register`, `/posts`, `/teams`, `/account/rename` and
|
|
385
|
+
`/account/profile`, and the generated findings point at their declarations in `routes/web.php`
|
|
386
|
+
(lines 6–10). The triage after Run 5 found these routes act on `$request->user()` or touch no
|
|
387
|
+
record, so they are likely false positives. Without them precision would be 0.85 / 0.83.
|
|
388
|
+
- **Duration:** 40 and 56 minutes (Run 5: 83 and 86). The model wrote fewer tokens (84K and 118K
|
|
389
|
+
completion tokens against about 109K and 111K); the Ollama version also changed (0.40.13 → 0.40.16).
|
|
390
|
+
- **`ai-raw/`** is written only for batches that needed a repair or failed to parse. None did, so neither
|
|
391
|
+
run has an `ai-raw/` folder.
|
|
392
|
+
|
|
393
|
+
### Did redaction reduce recall?
|
|
394
|
+
|
|
395
|
+
No, and on this repo it cannot. Running the redaction used for model context
|
|
396
|
+
(`assess/lsa/scripts/redaction.py`, `redact_lines`) over every file of the eval repo changes **0 lines**:
|
|
397
|
+
the fixtures contain no secrets in the patterns it covers. The prompts sent in Run 6 are therefore the
|
|
398
|
+
same code as without redaction. The BOLA difference from Run 5 A (28 → 26) is run-to-run variance (the
|
|
399
|
+
same model put 16 in Run 5 B). This also means the eval repo does not test whether redaction costs
|
|
400
|
+
recall on code that does contain secrets; that is covered only by the unit tests
|
|
401
|
+
(`operate/tests/test_ai_redaction.py`, `test_no_planted_secret_reaches_the_model`).
|
|
402
|
+
|
|
403
|
+
### `qwen3.5:9b` with a 16K context (not completed)
|
|
404
|
+
|
|
405
|
+
To fit the 24 GB machine, `qwen3.5:9b` was run with `num_ctx 16384` (a local tag built
|
|
406
|
+
`FROM qwen3.5:9b` with `PARAMETER num_ctx 16384`, MLX runner), same command and timeout as above. It did
|
|
407
|
+
not finish: the run was stopped by hand after 3 h 14 min, while the second injection batch was still
|
|
408
|
+
running and before the privilege (4c) and verifier (5) phases. The partial output is archived as
|
|
409
|
+
[`eval-run-6/qwen-16k-partial/`](../examples/eval-run-6/qwen-16k-partial/) (raw batch findings, the
|
|
410
|
+
per-minute memory log, and an indicative score of the unverified findings).
|
|
411
|
+
|
|
412
|
+
| Phase / batch | Time | Result |
|
|
413
|
+
|---|---|---|
|
|
414
|
+
| Stack detection | 2 min | done |
|
|
415
|
+
| BOLA `routes-001` (15 routes) | 60 min | **timed out**, empty |
|
|
416
|
+
| BOLA `routes-002` … `routes-004` | 5, 3 and 2 min | 8, 7 and 9 findings |
|
|
417
|
+
| BOLA `routes-005` | 60 min | **timed out**, empty |
|
|
418
|
+
| BOLA `routes-006` | 1 min | 1 finding |
|
|
419
|
+
| Injection `injection-001` | 11 min | 20 findings |
|
|
420
|
+
| Injection `injection-002` | 49 min+ | still running when stopped |
|
|
421
|
+
|
|
422
|
+
- **Memory:** the runner used 8.1 GB for the first hour, then grew to 13, 18 and finally 20 GB (from about
|
|
423
|
+
10:55, after the first timeout). Swap rose from about 4 GB (other work on the machine) to 12.4 GB of
|
|
424
|
+
13.3 GB; the run was stopped there to keep the machine usable. `gpt-oss:20b` stayed under 5.7 GB of
|
|
425
|
+
swap on the same machine.
|
|
426
|
+
- **Timeouts:** a batch that hits `--ai-timeout` is not retried (only empty or unparseable replies are),
|
|
427
|
+
so two of six BOLA batches were lost outright. The quick batches suggest the model can do the task;
|
|
428
|
+
the timed-out ones look like long reasoning that does not end within the context.
|
|
429
|
+
- **Indicative score** (findings before verification, from the batches that finished): BOLA 12/29,
|
|
430
|
+
injection 22/34, BFLA and mass assignment not reached, precision 0.73, no finding on a SAFE marker.
|
|
431
|
+
These numbers are not comparable with the `gpt-oss` rows: two BOLA batches, half the injection
|
|
432
|
+
candidates and the verifier are missing.
|
|
433
|
+
|
|
434
|
+
Thinking looks like the cause: on this 24 GB machine a 16K context with default reasoning is **not**
|
|
435
|
+
a working setting. The run below with reasoning off is.
|
|
436
|
+
|
|
437
|
+
### `qwen3.5:9b` with a 16K context and reasoning off (#382)
|
|
438
|
+
|
|
439
|
+
Same tag, command and timeout, plus `--ai-reasoning-effort none` (#382), which sends
|
|
440
|
+
`"reasoning_effort": "none"` with every request; Ollama's `/v1` endpoint ignores `think: false` but
|
|
441
|
+
honours this field. One run, archived as [`eval-run-6/qwen-16k-nothink/`](../examples/eval-run-6/qwen-16k-nothink/)
|
|
442
|
+
(with `ai-raw/` for the two repaired verifier replies and the per-minute memory log).
|
|
443
|
+
|
|
444
|
+
| Run | BOLA (29) | Injection (34) | BFLA (10) | Mass (15) | BFLA verdict | Mass verdict | SAFE hit | Precision | Time |
|
|
445
|
+
|---|---|---|---|---|---|---|---|---|---|
|
|
446
|
+
| `gpt-oss:20b` A | 26 | 34 | 10 | 14 | 10 | 15 | 1 | 0.813 | 40 min |
|
|
447
|
+
| `gpt-oss:20b` B | 26 | 34 | 10 | 14 | 10 | 14 | 1 | 0.787 | 56 min |
|
|
448
|
+
| `qwen3.5:9b` 16K, reasoning off | 23 | 29 | 6 | 10 | 8 | 15 | 0 | 0.734 | 35 min |
|
|
449
|
+
|
|
450
|
+
- **Completed:** all phases, 33 calls, no timeout, no parse failure or call error, no retry, all findings
|
|
451
|
+
anchored. Two verifier replies were repaired (`verified-1`, `verified-9`); 11 findings were generated
|
|
452
|
+
from flagged verdicts (2 BOLA, 2 BFLA, 7 mass assignment).
|
|
453
|
+
- **Recall** is lower than `gpt-oss:20b` in every category: BFLA and mass assignment are mostly found as
|
|
454
|
+
verdicts (8/10, 15/15) but placed on the right line less often (6/10, 10/15).
|
|
455
|
+
- **Precision 0.73:** no finding on a SAFE marker, but 25 unmatched findings (23 BOLA, 2 BFLA), not yet triaged;
|
|
456
|
+
none is on `php-laravel/routes/web.php`.
|
|
457
|
+
- **Memory and time:** the runner stayed at 14–17 GB and swap peaked at 6.0 GB (about 3.7 GB before the
|
|
458
|
+
run, from other work), against 20 GB and 12.4 GB with reasoning on. 35 minutes in all, of which the
|
|
459
|
+
verifier took 17.
|
|
460
|
+
- **Minimum to run it:** a 24 GB machine is enough with `num_ctx 16384` and `--ai-reasoning-effort none`.
|
|
461
|
+
With reasoning on it needs more; on this evidence at least 32 GB RAM.
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Keeps the archived Evaluation Run 6 numbers (Issue #323) in step with the scorer and EVAL-RESULTS.md."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import shutil
|
|
8
|
+
import subprocess
|
|
9
|
+
import sys
|
|
10
|
+
import tempfile
|
|
11
|
+
import unittest
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
TESTS_DIR = Path(__file__).resolve().parent
|
|
15
|
+
LSA_ROOT = TESTS_DIR.parent
|
|
16
|
+
EVAL_RUN_6_DIR = LSA_ROOT / "examples" / "eval-run-6"
|
|
17
|
+
GROUND_TRUTH = EVAL_RUN_6_DIR / "ground-truth.json"
|
|
18
|
+
RUNS = ("gptoss-a", "gptoss-b", "qwen-16k-nothink")
|
|
19
|
+
QWEN_PARTIAL = EVAL_RUN_6_DIR / "qwen-16k-partial"
|
|
20
|
+
# (BOLA, injection, BFLA, mass assignment) found by location, then BFLA / mass found by auditor verdict
|
|
21
|
+
EXPECTED = {
|
|
22
|
+
"gptoss-a": {"marker": (26, 34, 10, 14), "verdict": (10, 15)},
|
|
23
|
+
"gptoss-b": {"marker": (26, 34, 10, 14), "verdict": (10, 14)},
|
|
24
|
+
"qwen-16k-nothink": {"marker": (23, 29, 6, 10), "verdict": (8, 15)},
|
|
25
|
+
}
|
|
26
|
+
# Findings that land on a SAFE marker, by category (see EVAL-RESULTS.md, Run 6)
|
|
27
|
+
SAFE_HITS = {
|
|
28
|
+
"gptoss-a": {"injection": ["classic-asp/shop/member.asp:6"]},
|
|
29
|
+
"gptoss-b": {"injection": ["python-flask/app.py:12"]},
|
|
30
|
+
"qwen-16k-nothink": {},
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def score(run: str) -> dict:
|
|
35
|
+
proc = subprocess.run(
|
|
36
|
+
[sys.executable, str(TESTS_DIR / "score_findings.py"), "--verdicts", str(GROUND_TRUTH), str(EVAL_RUN_6_DIR / run)],
|
|
37
|
+
capture_output=True, text=True, check=True,
|
|
38
|
+
)
|
|
39
|
+
return json.loads(proc.stdout)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class TestEvaluationRun6(unittest.TestCase):
|
|
43
|
+
def test_ground_truth_is_the_121_item_set(self) -> None:
|
|
44
|
+
items = json.loads(GROUND_TRUTH.read_text(encoding="utf-8"))["items"]
|
|
45
|
+
self.assertEqual(len(items), 121)
|
|
46
|
+
|
|
47
|
+
def test_artifacts_exist(self) -> None:
|
|
48
|
+
for run in RUNS:
|
|
49
|
+
for name in ("findings.json", "coverage.json", "score.json", "ai-run-stats.json"):
|
|
50
|
+
self.assertTrue((EVAL_RUN_6_DIR / run / name).is_file(), f"{run}/{name}")
|
|
51
|
+
|
|
52
|
+
def test_archived_scores_match_the_scorer(self) -> None:
|
|
53
|
+
for run in RUNS:
|
|
54
|
+
with self.subTest(run=run):
|
|
55
|
+
archived = json.loads((EVAL_RUN_6_DIR / run / "score.json").read_text(encoding="utf-8"))
|
|
56
|
+
self.assertEqual(score(run), archived)
|
|
57
|
+
|
|
58
|
+
def test_numbers_match_eval_results(self) -> None:
|
|
59
|
+
for run, want in EXPECTED.items():
|
|
60
|
+
with self.subTest(run=run):
|
|
61
|
+
report = score(run)
|
|
62
|
+
pc, vl = report["per_category"], report["verdict_level"]
|
|
63
|
+
got = tuple(pc[c]["found"] for c in ("bola", "injection", "bfla", "mass_assignment"))
|
|
64
|
+
self.assertEqual(got, want["marker"])
|
|
65
|
+
self.assertEqual((vl["bfla"]["found"], vl["mass_assignment"]["found"]), want["verdict"])
|
|
66
|
+
|
|
67
|
+
def test_safe_marker_hits_are_the_documented_ones(self) -> None:
|
|
68
|
+
for run in RUNS:
|
|
69
|
+
with self.subTest(run=run):
|
|
70
|
+
report = score(run)
|
|
71
|
+
for cat, row in report["per_category"].items():
|
|
72
|
+
self.assertEqual(row["false_positive_on_safe_code"], SAFE_HITS[run].get(cat, []), cat)
|
|
73
|
+
for cat, row in report["verdict_level"].items():
|
|
74
|
+
self.assertEqual(row["flagged_safe"], [], cat)
|
|
75
|
+
|
|
76
|
+
def test_no_batch_lost_to_parse_or_call_errors(self) -> None:
|
|
77
|
+
for run in RUNS:
|
|
78
|
+
with self.subTest(run=run):
|
|
79
|
+
stats = json.loads((EVAL_RUN_6_DIR / run / "ai-run-stats.json").read_text(encoding="utf-8"))
|
|
80
|
+
self.assertEqual(stats["parse_failures"], {})
|
|
81
|
+
self.assertEqual(stats["call_errors"], {})
|
|
82
|
+
self.assertEqual(stats["anchoring"]["unanchored"], [])
|
|
83
|
+
|
|
84
|
+
def test_qwen_partial_score_matches_eval_results(self) -> None:
|
|
85
|
+
"""The stopped qwen3.5:9b 16K run: indicative score of its unverified findings (EVAL-RESULTS, Run 6)."""
|
|
86
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
87
|
+
shutil.copy(QWEN_PARTIAL / "findings.partial.json", Path(tmp) / "findings.json")
|
|
88
|
+
proc = subprocess.run(
|
|
89
|
+
[sys.executable, str(TESTS_DIR / "score_findings.py"), str(GROUND_TRUTH), tmp],
|
|
90
|
+
capture_output=True, text=True, check=True,
|
|
91
|
+
)
|
|
92
|
+
report = json.loads(proc.stdout)
|
|
93
|
+
archived = json.loads((QWEN_PARTIAL / "score.partial.json").read_text(encoding="utf-8"))
|
|
94
|
+
self.assertEqual(report, archived)
|
|
95
|
+
pc = report["per_category"]
|
|
96
|
+
got = tuple(pc[c]["found"] for c in ("bola", "injection", "bfla", "mass_assignment"))
|
|
97
|
+
self.assertEqual(got, (12, 22, 0, 0))
|
|
98
|
+
self.assertEqual(report["precision"], 0.733)
|
|
99
|
+
for row in pc.values():
|
|
100
|
+
self.assertEqual(row["false_positive_on_safe_code"], [])
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
if __name__ == "__main__":
|
|
104
|
+
unittest.main()
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Laravel routes map to the controller method they name, not to their declaration (#380)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import shutil
|
|
7
|
+
import sys
|
|
8
|
+
import tempfile
|
|
9
|
+
import unittest
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
TESTS_DIR = Path(__file__).resolve().parent
|
|
13
|
+
sys.path.insert(0, str(TESTS_DIR.parent / "scripts"))
|
|
14
|
+
from laravel_routes import route_action # noqa: E402
|
|
15
|
+
from lsa_common import load_config # noqa: E402
|
|
16
|
+
from map_routes import map_routes # noqa: E402
|
|
17
|
+
|
|
18
|
+
ROUTES = """<?php
|
|
19
|
+
use App\\Http\\Controllers\\OrderController;
|
|
20
|
+
use Illuminate\\Support\\Facades\\Route;
|
|
21
|
+
|
|
22
|
+
Route::get('/orders/{id}', [OrderController::class, 'show']);
|
|
23
|
+
Route::post('/orders', 'OrderController@store');
|
|
24
|
+
Route::get('/missing', [GoneController::class, 'index']);
|
|
25
|
+
Route::get('/ping', function () { return 'ok'; });
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
CONTROLLER = """<?php
|
|
29
|
+
namespace App\\Http\\Controllers;
|
|
30
|
+
|
|
31
|
+
use App\\Models\\Order;
|
|
32
|
+
use Illuminate\\Http\\Request;
|
|
33
|
+
|
|
34
|
+
class OrderController extends Controller
|
|
35
|
+
{
|
|
36
|
+
public function show($id)
|
|
37
|
+
{
|
|
38
|
+
return Order::find($id);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
public function store(Request $request)
|
|
42
|
+
{
|
|
43
|
+
return Order::create(['customer_id' => $request->input('customer_id')]);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class TestRouteAction(unittest.TestCase):
|
|
50
|
+
def test_array_and_string_actions(self) -> None:
|
|
51
|
+
self.assertEqual(route_action("Route::get('/a', [OrderController::class, 'show'])"), "OrderController@show")
|
|
52
|
+
self.assertEqual(route_action("Route::get('/a', [\\App\\Http\\Controllers\\OrderController::class, 'show'])"),
|
|
53
|
+
"OrderController@show")
|
|
54
|
+
self.assertEqual(route_action("Route::post('/a', 'App\\Http\\Controllers\\OrderController@store')"),
|
|
55
|
+
"OrderController@store")
|
|
56
|
+
self.assertIsNone(route_action("Route::get('/a', function () { return 1; })"))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class TestLaravelHandlers(unittest.TestCase):
|
|
60
|
+
def setUp(self) -> None:
|
|
61
|
+
self.root = Path(tempfile.mkdtemp(prefix="lsa_laravel_"))
|
|
62
|
+
(self.root / "routes").mkdir()
|
|
63
|
+
(self.root / "app/Http/Controllers").mkdir(parents=True)
|
|
64
|
+
(self.root / "routes/web.php").write_text(ROUTES, encoding="utf-8")
|
|
65
|
+
(self.root / "app/Http/Controllers/OrderController.php").write_text(CONTROLLER, encoding="utf-8")
|
|
66
|
+
|
|
67
|
+
def tearDown(self) -> None:
|
|
68
|
+
shutil.rmtree(self.root, ignore_errors=True)
|
|
69
|
+
|
|
70
|
+
def routes(self) -> dict:
|
|
71
|
+
found = map_routes(self.root, load_config(None), {"php-laravel"}, workers=1)["candidates"]
|
|
72
|
+
return {r["path"]: r for r in found if r["stack"] == "php-laravel"}
|
|
73
|
+
|
|
74
|
+
def test_route_points_at_the_controller_method(self) -> None:
|
|
75
|
+
show = self.routes()["/orders/{id}"]
|
|
76
|
+
self.assertEqual(show["handler"], {"file": "app/Http/Controllers/OrderController.php", "line": 9,
|
|
77
|
+
"symbol": "OrderController@show"})
|
|
78
|
+
self.assertEqual(show["notes"], "declared in routes/web.php:5")
|
|
79
|
+
self.assertIn("id", [p["name"] for p in show["id_params"]])
|
|
80
|
+
|
|
81
|
+
def test_string_action_and_body_inputs(self) -> None:
|
|
82
|
+
store = self.routes()["/orders"]
|
|
83
|
+
self.assertEqual((store["handler"]["file"], store["handler"]["line"]),
|
|
84
|
+
("app/Http/Controllers/OrderController.php", 14))
|
|
85
|
+
self.assertIn("customer_id", [p["name"] for p in store["id_params"]])
|
|
86
|
+
|
|
87
|
+
def test_unresolved_routes_keep_their_declaration(self) -> None:
|
|
88
|
+
routes = self.routes()
|
|
89
|
+
self.assertEqual(routes["/missing"]["handler"], {"file": "routes/web.php", "line": 7,
|
|
90
|
+
"symbol": "GoneController@index"})
|
|
91
|
+
self.assertEqual(routes["/ping"]["handler"]["file"], "routes/web.php")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
if __name__ == "__main__":
|
|
95
|
+
unittest.main()
|
package/operate/ai/client.py
CHANGED
|
@@ -12,6 +12,9 @@ import urllib.error
|
|
|
12
12
|
import urllib.request
|
|
13
13
|
from typing import Any, Dict, List, Optional, Tuple
|
|
14
14
|
|
|
15
|
+
# OpenAI-compatible `reasoning_effort` values; Ollama's /v1 also accepts "none" to turn thinking off (#382)
|
|
16
|
+
REASONING_EFFORTS = ("none", "low", "medium", "high")
|
|
17
|
+
|
|
15
18
|
|
|
16
19
|
class LLMClient:
|
|
17
20
|
"""Standard-library client for OpenAI-compatible /chat/completions endpoints."""
|
|
@@ -22,7 +25,10 @@ class LLMClient:
|
|
|
22
25
|
model: str = "llama3.1",
|
|
23
26
|
api_key: Optional[str] = None,
|
|
24
27
|
timeout: int = 120,
|
|
28
|
+
reasoning_effort: Optional[str] = None,
|
|
25
29
|
) -> None:
|
|
30
|
+
if reasoning_effort is not None and reasoning_effort not in REASONING_EFFORTS:
|
|
31
|
+
raise ValueError(f"reasoning_effort must be one of {', '.join(REASONING_EFFORTS)}")
|
|
26
32
|
raw_endpoint = endpoint.rstrip("/")
|
|
27
33
|
if not raw_endpoint.endswith("/chat/completions"):
|
|
28
34
|
self.endpoint = f"{raw_endpoint}/chat/completions"
|
|
@@ -31,6 +37,7 @@ class LLMClient:
|
|
|
31
37
|
self.model = model
|
|
32
38
|
self.api_key = api_key or "ollama"
|
|
33
39
|
self.timeout = timeout
|
|
40
|
+
self.reasoning_effort = reasoning_effort
|
|
34
41
|
# Running totals across chat() calls, read by the runner for ai-run-stats.json
|
|
35
42
|
self.usage: Dict[str, float] = {"calls": 0, "prompt_tokens": 0, "completion_tokens": 0, "seconds": 0.0}
|
|
36
43
|
|
|
@@ -41,15 +48,7 @@ class LLMClient:
|
|
|
41
48
|
max_tokens: Optional[int] = None,
|
|
42
49
|
) -> str:
|
|
43
50
|
"""Send a chat completion request and return the assistant response content."""
|
|
44
|
-
|
|
45
|
-
"model": self.model,
|
|
46
|
-
"messages": messages,
|
|
47
|
-
"temperature": temperature,
|
|
48
|
-
}
|
|
49
|
-
if max_tokens:
|
|
50
|
-
payload["max_tokens"] = max_tokens
|
|
51
|
-
|
|
52
|
-
data = json.dumps(payload).encode("utf-8")
|
|
51
|
+
data = json.dumps(self.build_payload(messages, temperature, max_tokens)).encode("utf-8")
|
|
53
52
|
headers = {
|
|
54
53
|
"Content-Type": "application/json",
|
|
55
54
|
"Authorization": f"Bearer {self.api_key}",
|
|
@@ -74,6 +73,20 @@ class LLMClient:
|
|
|
74
73
|
except urllib.error.URLError as e:
|
|
75
74
|
raise RuntimeError(f"Failed to connect to LLM endpoint at {self.endpoint}: {e.reason}") from e
|
|
76
75
|
|
|
76
|
+
def build_payload(
|
|
77
|
+
self,
|
|
78
|
+
messages: List[Dict[str, str]],
|
|
79
|
+
temperature: float = 0.1,
|
|
80
|
+
max_tokens: Optional[int] = None,
|
|
81
|
+
) -> Dict[str, Any]:
|
|
82
|
+
"""Request body for /chat/completions; `reasoning_effort` is sent only when it was set."""
|
|
83
|
+
payload: Dict[str, Any] = {"model": self.model, "messages": messages, "temperature": temperature}
|
|
84
|
+
if max_tokens:
|
|
85
|
+
payload["max_tokens"] = max_tokens
|
|
86
|
+
if self.reasoning_effort:
|
|
87
|
+
payload["reasoning_effort"] = self.reasoning_effort
|
|
88
|
+
return payload
|
|
89
|
+
|
|
77
90
|
def _record_usage(self, usage: Dict[str, Any], seconds: float) -> None:
|
|
78
91
|
self.usage = {
|
|
79
92
|
"calls": self.usage["calls"] + 1,
|
package/operate/ai/runner.py
CHANGED
|
@@ -167,6 +167,7 @@ class HeadlessAIRunner:
|
|
|
167
167
|
stats = {
|
|
168
168
|
"model": getattr(self.client, "model", None),
|
|
169
169
|
"endpoint": getattr(self.client, "endpoint", None),
|
|
170
|
+
"reasoning_effort": getattr(self.client, "reasoning_effort", None),
|
|
170
171
|
"phases": self.phase_stats,
|
|
171
172
|
"totals": self._usage(),
|
|
172
173
|
"parse_failures": self.parse_failures,
|
|
@@ -300,8 +301,9 @@ Output MUST be Markdown format adhering to Komagu auth-model.md specification.
|
|
|
300
301
|
{"name": p.get("name", "id"), "source": p.get("source", "path")}
|
|
301
302
|
for p in c.get("id_params", [])
|
|
302
303
|
]
|
|
303
|
-
|
|
304
|
-
|
|
304
|
+
# No detected id input stays empty: inventing a path `id` made the BOLA auditor
|
|
305
|
+
# flag routes that take no object id (#380)
|
|
306
|
+
id_inputs = id_inputs or []
|
|
305
307
|
|
|
306
308
|
mapped_routes.append({
|
|
307
309
|
"route_id": route_id,
|
package/operate/cli/__init__.py
CHANGED
|
@@ -161,6 +161,12 @@ Examples:
|
|
|
161
161
|
default=120,
|
|
162
162
|
help="Timeout in seconds per LLM request (default: 120)",
|
|
163
163
|
)
|
|
164
|
+
parser.add_argument(
|
|
165
|
+
"--ai-reasoning-effort",
|
|
166
|
+
choices=("none", "low", "medium", "high"),
|
|
167
|
+
default=None,
|
|
168
|
+
help="Send reasoning_effort with each LLM request; 'none' turns thinking off on Ollama (default: not sent)",
|
|
169
|
+
)
|
|
164
170
|
parser.set_defaults(func=execute_scan)
|
|
165
171
|
|
|
166
172
|
|
|
@@ -284,7 +290,8 @@ def execute_scan(args: argparse.Namespace) -> int:
|
|
|
284
290
|
step_info(msg)
|
|
285
291
|
|
|
286
292
|
try:
|
|
287
|
-
client = LLMClient(endpoint=endpoint, model=model, api_key=api_key, timeout=timeout
|
|
293
|
+
client = LLMClient(endpoint=endpoint, model=model, api_key=api_key, timeout=timeout,
|
|
294
|
+
reasoning_effort=getattr(args, "ai_reasoning_effort", None))
|
|
288
295
|
runner = HeadlessAIRunner(target_path, out_dir, client, step_callback=_scan_callback)
|
|
289
296
|
runner.run()
|
|
290
297
|
except Exception as e:
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""`--ai-reasoning-effort` reaches each /chat/completions request and ai-run-stats.json (#382)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import json
|
|
8
|
+
import shutil
|
|
9
|
+
import tempfile
|
|
10
|
+
import unittest
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from operate.ai.client import LLMClient
|
|
14
|
+
from operate.ai.runner import HeadlessAIRunner
|
|
15
|
+
from operate.cli.commands.scan import register_subcommand
|
|
16
|
+
|
|
17
|
+
MESSAGES = [{"role": "user", "content": "audit"}]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TestReasoningEffortPayload(unittest.TestCase):
|
|
21
|
+
def test_not_sent_by_default(self) -> None:
|
|
22
|
+
payload = LLMClient(model="m").build_payload(MESSAGES)
|
|
23
|
+
self.assertEqual(payload, {"model": "m", "messages": MESSAGES, "temperature": 0.1})
|
|
24
|
+
|
|
25
|
+
def test_sent_when_set(self) -> None:
|
|
26
|
+
payload = LLMClient(model="m", reasoning_effort="none").build_payload(MESSAGES, max_tokens=50)
|
|
27
|
+
self.assertEqual(payload["reasoning_effort"], "none")
|
|
28
|
+
self.assertEqual(payload["max_tokens"], 50)
|
|
29
|
+
|
|
30
|
+
def test_unknown_value_is_rejected(self) -> None:
|
|
31
|
+
with self.assertRaises(ValueError):
|
|
32
|
+
LLMClient(reasoning_effort="off")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class TestReasoningEffortCli(unittest.TestCase):
|
|
36
|
+
def parse(self, *argv: str) -> argparse.Namespace:
|
|
37
|
+
parser = argparse.ArgumentParser()
|
|
38
|
+
register_subcommand(parser.add_subparsers(dest="command"))
|
|
39
|
+
return parser.parse_args(["scan", ".", *argv])
|
|
40
|
+
|
|
41
|
+
def test_flag_is_optional(self) -> None:
|
|
42
|
+
self.assertIsNone(self.parse().ai_reasoning_effort)
|
|
43
|
+
self.assertEqual(self.parse("--ai-reasoning-effort", "none").ai_reasoning_effort, "none")
|
|
44
|
+
|
|
45
|
+
def test_flag_rejects_unknown_values(self) -> None:
|
|
46
|
+
with self.assertRaises(SystemExit):
|
|
47
|
+
self.parse("--ai-reasoning-effort", "off")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class TestReasoningEffortStats(unittest.TestCase):
|
|
51
|
+
def setUp(self) -> None:
|
|
52
|
+
self.tmp = Path(tempfile.mkdtemp(prefix="komagu_effort_"))
|
|
53
|
+
|
|
54
|
+
def tearDown(self) -> None:
|
|
55
|
+
shutil.rmtree(self.tmp, ignore_errors=True)
|
|
56
|
+
|
|
57
|
+
def test_value_is_recorded(self) -> None:
|
|
58
|
+
runner = HeadlessAIRunner(self.tmp, self.tmp / "out", LLMClient(model="m", reasoning_effort="low"))
|
|
59
|
+
(self.tmp / "out").mkdir(parents=True, exist_ok=True)
|
|
60
|
+
stats = json.loads(runner.write_run_stats().read_text(encoding="utf-8"))
|
|
61
|
+
self.assertEqual(stats["reasoning_effort"], "low")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
if __name__ == "__main__":
|
|
65
|
+
unittest.main()
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Route inventory built by the headless runner from Tier 1 route batches (#380)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import shutil
|
|
8
|
+
import tempfile
|
|
9
|
+
import unittest
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from operate.ai.runner import HeadlessAIRunner
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class _NoCallClient:
|
|
16
|
+
model = "none"
|
|
17
|
+
endpoint = "test://"
|
|
18
|
+
|
|
19
|
+
def chat(self, messages, temperature=0.1): # noqa: ANN001
|
|
20
|
+
raise AssertionError("route inventory must not call the model")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class TestRouteInventory(unittest.TestCase):
|
|
24
|
+
def setUp(self) -> None:
|
|
25
|
+
self.tmp = Path(tempfile.mkdtemp(prefix="komagu_inventory_"))
|
|
26
|
+
self.out = self.tmp / "out"
|
|
27
|
+
(self.out / "batches").mkdir(parents=True)
|
|
28
|
+
|
|
29
|
+
def tearDown(self) -> None:
|
|
30
|
+
shutil.rmtree(self.tmp, ignore_errors=True)
|
|
31
|
+
|
|
32
|
+
def inventory(self, candidates: list) -> list:
|
|
33
|
+
(self.out / "batches" / "routes-001.json").write_text(json.dumps({"candidates": candidates}), encoding="utf-8")
|
|
34
|
+
HeadlessAIRunner(self.tmp, self.out, _NoCallClient()).map_routes_inventory()
|
|
35
|
+
return json.loads((self.out / "routes" / "routes-001.json").read_text(encoding="utf-8"))["routes"]
|
|
36
|
+
|
|
37
|
+
def test_route_without_id_params_gets_no_invented_id(self) -> None:
|
|
38
|
+
routes = self.inventory([{"route_id": "R-0001", "method": ["POST"], "path": "/register",
|
|
39
|
+
"handler": {"file": "app/Http/Controllers/AccountController.php", "line": 11},
|
|
40
|
+
"id_params": []}])
|
|
41
|
+
self.assertEqual(routes[0]["id_inputs"], [])
|
|
42
|
+
|
|
43
|
+
def test_detected_id_params_are_kept(self) -> None:
|
|
44
|
+
routes = self.inventory([{"route_id": "R-0001", "method": ["GET"], "path": "/orders/{id}",
|
|
45
|
+
"handler": {"file": "routes/web.php", "line": 3},
|
|
46
|
+
"id_params": [{"name": "id", "source": "path", "is_id_like": True},
|
|
47
|
+
{"name": "orderId", "source": "query"}]}])
|
|
48
|
+
self.assertEqual(routes[0]["id_inputs"], [{"name": "id", "source": "path"},
|
|
49
|
+
{"name": "orderId", "source": "query"}])
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
if __name__ == "__main__":
|
|
53
|
+
unittest.main()
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hoanghainh/komagu",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.6.0",
|
|
4
4
|
"description": "Security-First Legacy Modernization Platform CLI (LSA Kit, IAM Perimeter, Cross-User Testing)",
|
|
5
5
|
"main": "remediate/authz/nodejs/src/index.js",
|
|
6
6
|
"types": "remediate/authz/nodejs/src/index.d.ts",
|