@hoanghainh/komagu 2.4.0 → 2.6.0

package/assess/lsa/scripts/map_routes.py

@@ -19,8 +19,12 @@ from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
-from lsa_common import (KIT_VERSION, compile_id_patterns, is_id_name, iter_source_files,  # noqa: E402
-                        line_of, load_config, read_text, write_json)
+from java_routes import (build_class_index, collect_spring_beans, entry_method_line, java_declared,  # noqa: E402
+                         struts2_action_extension,
+                         struts_modules, xml_declared)
+from lsa_common import (JAVA_WEB_ROOT, KIT_VERSION, call_args, compile_id_patterns, is_id_name,  # noqa: E402
+                        iter_source_files, line_of, load_config, read_text, split_top, write_json)
+from laravel_routes import resolve_laravel_handlers, route_action  # noqa: E402
 from nextjs_routes import middleware_note, nextjs_declared  # noqa: E402
 from privilege import annotate_privilege  # noqa: E402
 
@@ -250,8 +254,6 @@ def segments(text: str, decl_rx: re.Pattern) -> List[Tuple[re.Match, int, int]]:
 # Declared-route extractors
 # --------------------------------------------------------------------------------------
 PHP_METHOD = re.compile(r"^[ \t]*(?:(public|protected|private)\s+)?(?:static\s+)?function\s+&?(\w+)\s*\(", re.M)
-JAVA_METHOD_ANN = re.compile(r"@(RequestMapping|GetMapping|PostMapping|PutMapping|DeleteMapping|PatchMapping|Path)\b\s*(\(([^)]*)\))?")
-JAVA_HTTP_ANN = re.compile(r"@(GET|POST|PUT|DELETE|PATCH)\b")
 CS_ACTION = re.compile(r"^[ \t]*public\s+(?!record\b|class\b)(?:virtual\s+|async\s+|override\s+|static\s+)*[A-Za-z0-9_<>., \[\]?]+\s+(\w+)\s*\(([^)]*)\)", re.M)
 VB_ACTION = re.compile(r"^[ \t]*Public\s+(?:Overridable\s+|Async\s+)*Function\s+(\w+)\s*\(([^)]*)\)\s+As\s+[\w.(<>) ]+", re.M | re.I)
 WEBMETHOD = re.compile(r"(\[WebMethod[^\]]*\]|<WebMethod[^>]*>|\[OperationContract[^\]]*\]|<OperationContract[^>]*>)[\s\S]{0,300}?"
@@ -267,7 +269,7 @@ def php_declared(c: Collector, rel: str, text: str, stacks: set) -> bool:
         end = text.find(";", m.end())
         stmt = text[m.start(): end if end > 0 else m.end() + 200]
         c.add(stack="php-laravel", kind="declared_route", method=[verb.upper()], path="/" + path.lstrip("/"),
-              handler={"file": rel, "line": line, "symbol": None}, id_params=c.path_params(path),
+              handler={"file": rel, "line": line, "symbol": route_action(stmt)}, id_params=c.path_params(path),
               guard_hints=c.guards(stmt, line) + c.guards(text[:m.start()][-3000:], 1)[:2],
               data_hints=[], notes="handler is in the controller named in this route")
         found = True
@@ -305,107 +307,6 @@ def php_declared(c: Collector, rel: str, text: str, stacks: set) -> bool:
     return found
 
 
-def java_declared(c: Collector, rel: str, text: str) -> bool:
-    found = False
-    cls_prefix = ""
-    cm = re.search(r"@(RequestMapping|Path)\s*\(\s*(?:value\s*=\s*)?\"([^\"]*)\"[^)]*\)\s*(?:@\w+(?:\([^)]*\))?\s*)*(?:public\s+)?(?:abstract\s+)?class", text)
-    if cm:
-        cls_prefix = cm.group(2)
-    is_servlet = re.search(r"extends\s+HttpServlet", text)
-    if is_servlet:
-        ws = re.search(r"@WebServlet\s*\(\s*(?:urlPatterns\s*=\s*|value\s*=\s*)?\{?\s*\"([^\"]+)\"", text)
-        for m in re.finditer(r"(?:public|protected)\s+void\s+(doGet|doPost|doPut|doDelete|service)\s*\(", text):
-            start = m.start()
-            nxt = re.search(r"\n\s*(?:public|protected|private)\s+[\w<>\[\]]+\s+\w+\s*\(", text[m.end():])
-            body = text[start: m.end() + (nxt.start() if nxt else len(text))]
-            line = line_of(text, start)
-            verb = {"doGet": "GET", "doPost": "POST", "doPut": "PUT", "doDelete": "DELETE"}.get(m.group(1), "ANY")
-            c.add(stack="java-servlet", kind="declared_route", method=[verb],
-                  path=ws.group(1) if ws else "<see web.xml servlet-mapping>",
-                  handler={"file": rel, "line": line, "symbol": m.group(1)},
-                  id_params=c.inputs("java", body, line), guard_hints=c.guards(body, line),
-                  data_hints=c.data_ops(body, line), notes="" if ws else "resolve path from web.xml")
-            found = True
-        return found
-    anns = list(JAVA_METHOD_ANN.finditer(text))
-    for m in anns:
-        if cm and m.start() == cm.start():
-            continue
-        after = text[m.end(): m.end() + 600]
-        # skip class-level annotation
-        if re.match(r"\s*(?:@\w+(?:\([^)]*\))?\s*)*(?:public\s+)?(?:abstract\s+)?class\b", after):
-            continue
-        sig = re.search(r"(?:(?:public|protected)\s+[\w<>\[\],.? ]+|fun|(?:override|suspend)\s+fun)\s+(\w+)\s*\(", after)
-        if not sig:
-            continue
-        before = text[:m.start()]
-        cls_m = list(re.finditer(r"@(?:RequestMapping|Path)\s*\(\s*(?:value\s*=\s*|path\s*=\s*)?\"([^\"]*)\"[^)]*\)[\s\S]{0,300}?(?:class\b)", before))
-        method_cls_prefix = cls_m[-1].group(1) if cls_m else cls_prefix
-        args = m.group(3) or ""
-        pv = re.search(r"(?:value\s*=\s*|path\s*=\s*)?\{?\s*\"([^\"]*)\"", args)
-        sub = pv.group(1) if pv else ""
-        verbs = re.findall(r"RequestMethod\.(\w+)", args)
-        ann = m.group(1)
-        if ann.endswith("Mapping") and ann != "RequestMapping":
-            verbs = [ann[:-7].upper()]
-        if ann == "Path":
-            hv = JAVA_HTTP_ANN.search(text[max(0, m.start() - 200): m.end() + 200])
-            verbs = [hv.group(1)] if hv else ["ANY"]
-        path = "/" + "/".join(p.strip("/") for p in (method_cls_prefix, sub) if p.strip("/"))
-        nxt = re.search(r"\n\s*@(RequestMapping|GetMapping|PostMapping|PutMapping|DeleteMapping|PatchMapping|Path)\b(?!Variable|Param)", text[m.end():])
-        body = text[m.start(): m.end() + (nxt.start() if nxt else len(text))]
-        line = line_of(text, m.start())
-        c.add(stack="java-spring-mvc" if ann != "Path" else "java-jaxrs", kind="declared_route", method=verbs or ["ANY"],
-              path=path, handler={"file": rel, "line": line, "symbol": sig.group(1)},
-              id_params=c.path_params(path) + c.inputs("java", body, line),
-              guard_hints=c.guards(text[:m.start()][-1500:], 1)[:2] + c.guards(body, line),
-              data_hints=c.data_ops(body, line), notes="")
-        found = True
-    # Struts 1 / 2 Action classes without annotations are mapped from XML (see xml_declared)
-    return found
-
-
-def xml_declared(c: Collector, rel: str, text: str, class_index: Dict[str, str]) -> bool:
-    found = False
-    if rel.endswith("web.xml"):
-        servlets = dict(re.findall(r"<servlet>\s*<servlet-name>\s*([^<]+?)\s*</servlet-name>[\s\S]*?<servlet-class>\s*([^<]+?)\s*</servlet-class>", text))
-        for m in re.finditer(r"<servlet-mapping>\s*<servlet-name>\s*([^<]+?)\s*</servlet-name>\s*<url-pattern>\s*([^<]+?)\s*</url-pattern>", text):
-            name, pattern = m.group(1), m.group(2)
-            cls = servlets.get(name, "")
-            hfile = class_index.get(cls.split(".")[-1], rel)
-            c.add(stack="java-servlet", kind="declared_route", method=["ANY"], path=pattern,
-                  handler={"file": hfile, "line": 1, "symbol": cls or name}, id_params=[], guard_hints=[],
-                  data_hints=[], notes=f"mapped in {rel}:{line_of(text, m.start())}")
-            found = True
-    if "struts-config" in rel:
-        for m in re.finditer(r"<action\b([^>]*)>", text):
-            attrs = dict(re.findall(r"(\w+)\s*=\s*\"([^\"]*)\"", m.group(1)))
-            if "path" not in attrs:
-                continue
-            cls = attrs.get("type", "")
-            hfile = class_index.get(cls.split(".")[-1], rel)
-            note = f"declared in {rel}:{line_of(text, m.start())}"
-            if attrs.get("parameter"):
-                note += f"; DispatchAction parameter '{attrs['parameter']}' -> one endpoint per method"
-            c.add(stack="java-struts1", kind="declared_route", method=["ANY"], path=attrs["path"] + ".do",
-                  handler={"file": hfile, "line": 1, "symbol": cls}, id_params=[], guard_hints=[], data_hints=[], notes=note)
-            found = True
-    if rel.endswith("struts.xml"):
-        for pm in re.finditer(r"<package\b([^>]*)>([\s\S]*?)</package>", text):
-            pattrs = dict(re.findall(r"(\w+)\s*=\s*\"([^\"]*)\"", pm.group(1)))
-            ns = pattrs.get("namespace", "/").rstrip("/")
-            for am in re.finditer(r"<action\b([^>]*)>", pm.group(2)):
-                attrs = dict(re.findall(r"(\w+)\s*=\s*\"([^\"]*)\"", am.group(1)))
-                cls = attrs.get("class", "")
-                hfile = class_index.get(cls.split(".")[-1], rel)
-                c.add(stack="java-struts2", kind="declared_route", method=["ANY"], path=f"{ns}/{attrs.get('name', '?')}.action",
-                      handler={"file": hfile, "line": 1, "symbol": f"{cls}#{attrs.get('method', 'execute')}"},
-                      id_params=[], guard_hints=[], data_hints=[],
-                      notes=f"declared in {rel}:{line_of(text, pm.start() + am.start())}; action setters are inputs")
-                found = True
-    return found
-
-
 def dotnet_declared(c: Collector, rel: str, text: str) -> bool:
     found = False
     cm = re.search(r"(?:\[ApiController\][\s\S]{0,100}?)?class\s+(\w+?)Controller\s*(?::\s*(\w+))?|Class\s+(\w+?)Controller[\s\S]{0,80}?Inherits\s+(\w+)", text, re.I)
@@ -560,48 +461,6 @@ GO_FRAMEWORKS = [("go-gin", "github.com/gin-gonic/gin"), ("go-echo", "github.com
 GO_PATTERN = re.compile(r"^(?:([A-Z]+)\s+)?(/\S*)$")
 
 
-def call_args(text: str, open_idx: int, limit: int = 4000) -> Tuple[str, int]:
-    """Return (argument text, index after the closing paren) for the call whose '(' is at open_idx."""
-    depth, i, end = 0, open_idx, min(len(text), open_idx + limit)
-    while i < end:
-        ch = text[i]
-        if ch in "\"`":
-            close = text.find(ch, i + 1)
-            while ch == '"' and close != -1 and text[close - 1] == "\\":
-                close = text.find(ch, close + 1)
-            i = close if close != -1 else end
-        elif ch in "([{":
-            depth += 1
-        elif ch in ")]}":
-            depth -= 1
-            if depth == 0:
-                return text[open_idx + 1:i], i + 1
-        i += 1
-    return text[open_idx + 1:end], end
-
-
-def split_top(args: str) -> List[str]:
-    """Split call arguments on top-level commas."""
-    parts, depth, cur, quote = [], 0, [], None
-    for ch in args:
-        if quote:
-            quote = None if ch == quote else quote
-        elif ch in "\"`":
-            quote = ch
-        elif ch in "([{":
-            depth += 1
-        elif ch in ")]}":
-            depth -= 1
-        elif ch == "," and depth == 0:
-            parts.append("".join(cur).strip())
-            cur = []
-            continue
-        cur.append(ch)
-    if "".join(cur).strip():
-        parts.append("".join(cur).strip())
-    return parts
-
-
 def guards_before(c: Collector, text: str, start: int, end: int, span: int = 500) -> List[Dict]:
     ws = max(0, start - span)
     return c.guards(text[ws:end], line_of(text, ws))
@@ -712,6 +571,9 @@ def file_endpoint(c: Collector, rel: str, text: str, lang: str, ext: str, target
         wr = wr.strip("/") + "/"
         if rel.startswith(wr):
             path = "/" + rel[len(wr):]
+    root = JAVA_WEB_ROOT.search(rel) if lang == "java" and path == "/" + rel else None
+    if root:
+        path = "/" + rel[root.end():]
     stack = {"php": "php", "java": "java-jsp", "dotnet": "dotnet-webforms" if ext in (".aspx",) else "dotnet-handlers",
              "asp": "classic-asp", "perl": "perl-cgi", "cfml": "coldfusion"}.get(lang, lang)
     if ext in (".asmx", ".svc"):
@@ -775,7 +637,11 @@ def _scan_route_file_batch(args: Tuple[List[Tuple[str, str]], Dict, set, str]) -
             declared = go_declared(c, rel, t)
         if declared or ext not in FILE_ENDPOINT_EXTS:
             continue
-        if INCLUDE_DIR_RE.search(rel.rsplit("/", 1)[0] + "/" if "/" in rel else ""):
+        if ext == ".jsp":
+            # Tomcat serves any JSP outside WEB-INF, whatever its directory is called; those inside are only forwarded to
+            if "/WEB-INF/" in "/" + rel:
+                continue
+        elif INCLUDE_DIR_RE.search(rel.rsplit("/", 1)[0] + "/" if "/" in rel else ""):
             continue
         if framework_php and lang == "php" and re.search(r"(^|/)(application|system|app|vendor|routes|resources|storage|bootstrap)/", rel):
             continue
@@ -791,15 +657,22 @@ def map_routes(target: Path, config: Dict, stack_ids: Optional[set] = None, work
     files = list(iter_source_files(target, config))
     stack_ids = stack_ids or set()
     framework_php = bool(stack_ids & {"php-codeigniter", "php-cakephp", "php-laravel", "php-zend1"})
-    class_index = {p.stem: rel for p, rel in files if p.suffix == ".java"}
+    class_index = build_class_index(rel for _, rel in files)
     texts: Dict[str, str] = {}
 
-    # 1. XML pre-pass (web.xml / struts.xml declarative mappings)
-    for p, rel in files:
-        if p.suffix.lower() == ".xml":
-            t = read_text(p, enc)
-            if "<servlet" in t or "<action" in t:
-                xml_declared(c, rel, t, class_index)
+    # 1. XML pre-pass (web.xml / Struts / Spring declarative mappings)
+    xml_texts = {rel: read_text(p, enc) for p, rel in files if p.suffix.lower() == ".xml"}
+    modules: Dict[str, str] = {}
+    beans: Dict[str, str] = {}
+    for rel, t in xml_texts.items():
+        if rel.endswith("web.xml"):
+            modules.update(struts_modules(rel, t))
+        if "<bean" in t:
+            beans.update(collect_spring_beans(t))
+    extension = struts2_action_extension(target, config, xml_texts.values(), enc)
+    for rel, t in xml_texts.items():
+        if "<servlet" in t or "<action" in t or "UrlHandlerMapping" in t:
+            xml_declared(c, rel, t, class_index, modules, beans, extension)
 
     xml_items = list(c.items)
 
@@ -865,20 +738,26 @@ def map_routes(target: Path, config: Dict, stack_ids: Optional[set] = None, work
     # post-pass 1: routes declared in XML -> enrich from the handler class file
     for it in c.items:
         hf = it["handler"]["file"]
+        if it["kind"] == "declared_route" and it["handler"]["line"] == 1 and hf in texts and hf.endswith(".java"):
+            it["handler"]["line"] = entry_method_line(texts[hf], it["handler"].get("symbol")) or 1
         if it["kind"] == "declared_route" and not it["id_params"] and hf in texts and hf.endswith(".java"):
             t = texts[hf]
             it["id_params"] = c.inputs("java", t)
             it["guard_hints"] = it["guard_hints"] or c.guards(t)
             it["data_hints"] = it["data_hints"] or c.data_ops(t)
+    # post-pass 1b: Laravel route declarations -> the controller method they name (#380)
+    c.items = resolve_laravel_handlers(c.items, target, enc, c, [rel for _, rel in files])
     # post-pass 2: servlet classes without @WebServlet -> take the path from web.xml
-    xml_paths = {it["handler"]["file"]: it["path"] for it in c.items
-                 if it["stack"] == "java-servlet" and it["notes"].startswith("mapped in ")}
+    xml_paths: Dict[str, List[str]] = {}
+    for it in c.items:
+        if it["stack"] == "java-servlet" and it["notes"].startswith("mapped in "):
+            xml_paths.setdefault(it["handler"]["file"], []).append(it["path"])
     keep = []
     for it in c.items:
         if it["path"] == "<see web.xml servlet-mapping>" and it["handler"]["file"] in xml_paths:
-            it["path"] = xml_paths[it["handler"]["file"]]
-            it["notes"] = "path from web.xml"
-        keep.append(it)
+            keep.extend(dict(it, path=mapped, notes="path from web.xml") for mapped in xml_paths[it["handler"]["file"]])
+        else:
+            keep.append(it)
     mapped_with_methods = {it["handler"]["file"] for it in keep if it["notes"] == "path from web.xml"}
     keep = [it for it in keep if not (it["notes"].startswith("mapped in ") and it["handler"]["file"] in mapped_with_methods)]
     for n, it in enumerate(keep, 1):

package/assess/lsa/scripts/merge_findings.py

@@ -403,6 +403,8 @@ def main() -> int:
                                     "injection_candidates_without_verdict", "bfla_routes_without_verdict",
                                     "mass_candidates_without_verdict", "validation_errors",
                                     "new_issues_from_verifier"))
+    for error in cov.get("validation_errors") or []:
+        print(f"validation error (finding not merged): {error}", file=sys.stderr)
     print(json.dumps({"summary": summary, "coverage_gaps": gaps}, indent=2))
     return 1 if gaps else 0
 

package/assess/lsa/tests/EVAL-RESULTS.md

@@ -333,3 +333,129 @@ and their tests still score against them.
 Run 5 rescored against the new ground truth (for comparison only; findings unchanged): BOLA 28 / 29
 (run A) and 16 / 29 (run B), precision 0.859 and 0.826, still no false positive on SAFE markers. Every
 new marker was already reported by both runs; Tier 1 lists all of them as BOLA route candidates.
+
+---
+
+## Run 6 — 2026-11-11, platform 2.5.0, kit 0.3.1 (#323 part 4: new harness, 121-item ground truth, redaction)
+
+Issue #323. First run on the harness from #334 (no code fields in 4a / 4b templates, quote repair,
+findings from flagged verdicts), the fixtures and ground truth from #336 (121 items: 29 BOLA, 34
+SQL/NoSQL injection, 10 BFLA, 15 mass assignment, 33 SAFE), and secret redaction of code sent to the
+model (v2.4.0). Outputs are archived in [`assess/lsa/examples/eval-run-6/`](../examples/eval-run-6/).
+Runs 4–5 are not directly comparable; the Run 5 rows below are the Run 5 findings rescored against the
+new ground truth.
+
+| Item | Value |
+|---|---|
+| Pipeline | `komagu scan <repo> --no-cache --headless-ai http://localhost:11434/v1 --model <model> --ai-timeout 3600` from `main` at v2.5.0 |
+| Hosting | Ollama 0.40.16 on the same Apple M4 Pro, 24 GB RAM, 100% GPU, local only; `gpt-oss:20b` on the llama.cpp runner with 32,768 context (as in Runs 4–5) |
+| Scoring | `tests/score_findings.py --verdicts` (tolerance 3 lines) |
+| Machine | No other heavy work while measuring; up to 5.7 GB swap in use during the `gpt-oss` runs |
+
+### Results after verification (`gpt-oss:20b`)
+
+| Run | BOLA | Injection | BFLA marker | Mass marker | BFLA verdict | Mass verdict | FP on safe code | Precision | Rejected | Retries · repaired · parse fail | From verdicts | Min · calls · tokens |
+|---|---|---|---|---|---|---|---|---|---|---|---|---|
+| Run 5 A, rescored | 28/29 | 34/34 | 10/10 | 10/15 | 10/10 | 15/15 | 0 | 0.859 | 2 | 0 · — · 0 | — | 83 · 32 · 328K |
+| Run 5 B, rescored | 16/29 | 32/34 | 9/10 | 14/15 | 10/10 | 15/15 | 0 | 0.826 | 3 | 1 · — · 1 | — | 86 · 31 · 369K |
+| **Run 6 A** | **26/29** | **34/34** | **10/10** | **14/15** | **10/10** | **15/15** | **1** | **0.813** | 2 | 0 · 0 · 0 | 9 (5 BOLA, 4 mass) | 40 · 33 · 315K |
+| **Run 6 B** | **26/29** | **34/34** | **10/10** | **14/15** | **10/10** | **14/15** | **1** | **0.787** | 1 | 1 · 0 · 0 | 14 (5 BOLA, 9 mass) | 56 · 34 · 373K |
+
+- **The two runs now agree.** BOLA is 26 / 29 in both (Run 5: 28 and 16). No batch was lost: no parse
+  failure, no call error, no batch needed the quote repair, and the one retry (B, `injection-001`)
+  succeeded. The `routes-001` batch that failed in Run 4 B and Run 5 B now parses.
+- **Mass assignment:** 14 / 15 as located findings in both runs (Run 5: 10 and 14). Findings written from
+  flagged verdicts closed the gap: the four candidates run A of Run 5 judged `vulnerable` without a
+  finding (`profile.php` ×2, `CustomerController.cs` ×2) are findings in both runs.
+  `php-plain/register.php:4` is still missed. In A the verifier rejected it because the INSERT "only"
+  writes `email`, `name` and `role`, missing that `extract($_POST)` can overwrite `$role` (the
+  fixture change in #336); in B the auditor did not flag it.
+- **BFLA:** 10 / 10 in both runs, located and by verdict.
+- **Misses:** `perl-cgi/cgi-bin/order.cgi:19` (both runs, missed since Run 4), `node-express/routes/orders.js:29`
+  (both), `java-legacy/.../OrderServlet.java:15` (A), `python-flask/app.py:12` (B).
+- **False positives on SAFE markers: 1 per run** (Run 5: 0), both injection:
+  - A: `classic-asp/shop/member.asp:6`. The `UPDATE` starts on line 6, whose `name` value is escaped;
+    the concatenated `member_id` is on line 7, which is also reported. The finding is anchored one line
+    early rather than describing a new flaw, but it is scored as a false positive.
+  - B: `python-flask/app.py:12`. `order_id` is an `<int:...>` path parameter, so the `%` formatting
+    cannot inject. The model reported the route's missing ownership check as SQL injection, so the
+    BOLA marker on the same line was missed.
+- **Precision 0.81 / 0.79.** Five of the unmatched findings per run come from verdicts: the BOLA
+  auditor flagged the Laravel routes `/register`, `/posts`, `/teams`, `/account/rename` and
+  `/account/profile`, and the generated findings point at their declarations in `routes/web.php`
+  (lines 6–10). The triage after Run 5 found these routes act on `$request->user()` or touch no
+  record, so they are likely false positives. Without them precision would be 0.85 / 0.83.
+- **Duration:** 40 and 56 minutes (Run 5: 83 and 86). The model wrote fewer tokens (84K and 118K
+  completion tokens against about 109K and 111K); the Ollama version also changed (0.40.13 → 0.40.16).
+- **`ai-raw/`** is written only for batches that needed a repair or failed to parse. None did, so neither
+  run has an `ai-raw/` folder.
+
+### Did redaction reduce recall?
+
+No, and on this repo it cannot. Running the redaction used for model context
+(`assess/lsa/scripts/redaction.py`, `redact_lines`) over every file of the eval repo changes **0 lines**:
+the fixtures contain no secrets in the patterns it covers. The prompts sent in Run 6 are therefore the
+same code as without redaction. The BOLA difference from Run 5 A (28 → 26) is run-to-run variance (the
+same model put 16 in Run 5 B). This also means the eval repo does not test whether redaction costs
+recall on code that does contain secrets; that is covered only by the unit tests
+(`operate/tests/test_ai_redaction.py`, `test_no_planted_secret_reaches_the_model`).
+
+### `qwen3.5:9b` with a 16K context (not completed)
+
+To fit the 24 GB machine, `qwen3.5:9b` was run with `num_ctx 16384` (a local tag built
+`FROM qwen3.5:9b` with `PARAMETER num_ctx 16384`, MLX runner), same command and timeout as above. It did
+not finish: the run was stopped by hand after 3 h 14 min, while the second injection batch was still
+running and before the privilege (4c) and verifier (5) phases. The partial output is archived as
+[`eval-run-6/qwen-16k-partial/`](../examples/eval-run-6/qwen-16k-partial/) (raw batch findings, the
+per-minute memory log, and an indicative score of the unverified findings).
+
+| Phase / batch | Time | Result |
+|---|---|---|
+| Stack detection | 2 min | done |
+| BOLA `routes-001` (15 routes) | 60 min | **timed out**, empty |
+| BOLA `routes-002` … `routes-004` | 5, 3 and 2 min | 8, 7 and 9 findings |
+| BOLA `routes-005` | 60 min | **timed out**, empty |
+| BOLA `routes-006` | 1 min | 1 finding |
+| Injection `injection-001` | 11 min | 20 findings |
+| Injection `injection-002` | 49 min+ | still running when stopped |
+
+- **Memory:** the runner used 8.1 GB for the first hour, then grew to 13, 18 and finally 20 GB (from about
+  10:55, after the first timeout). Swap rose from about 4 GB (other work on the machine) to 12.4 GB of
+  13.3 GB; the run was stopped there to keep the machine usable. `gpt-oss:20b` stayed under 5.7 GB of
+  swap on the same machine.
+- **Timeouts:** a batch that hits `--ai-timeout` is not retried (only empty or unparseable replies are),
+  so two of six BOLA batches were lost outright. The quick batches suggest the model can do the task;
+  the timed-out ones look like long reasoning that does not end within the context.
+- **Indicative score** (findings before verification, from the batches that finished): BOLA 12/29,
+  injection 22/34, BFLA and mass assignment not reached, precision 0.73, no finding on a SAFE marker.
+  These numbers are not comparable with the `gpt-oss` rows: two BOLA batches, half the injection
+  candidates and the verifier are missing.
+
+Thinking looks like the cause: on this 24 GB machine a 16K context with default reasoning is **not**
+a working setting. The run below with reasoning off is.
+
+### `qwen3.5:9b` with a 16K context and reasoning off (#382)
+
+Same tag, command and timeout, plus `--ai-reasoning-effort none` (#382), which sends
+`"reasoning_effort": "none"` with every request; Ollama's `/v1` endpoint ignores `think: false` but
+honours this field. One run, archived as [`eval-run-6/qwen-16k-nothink/`](../examples/eval-run-6/qwen-16k-nothink/)
+(with `ai-raw/` for the two repaired verifier replies and the per-minute memory log).
+
+| Run | BOLA (29) | Injection (34) | BFLA (10) | Mass (15) | BFLA verdict | Mass verdict | SAFE hit | Precision | Time |
+|---|---|---|---|---|---|---|---|---|---|
+| `gpt-oss:20b` A | 26 | 34 | 10 | 14 | 10 | 15 | 1 | 0.813 | 40 min |
+| `gpt-oss:20b` B | 26 | 34 | 10 | 14 | 10 | 14 | 1 | 0.787 | 56 min |
+| `qwen3.5:9b` 16K, reasoning off | 23 | 29 | 6 | 10 | 8 | 15 | 0 | 0.734 | 35 min |
+
+- **Completed:** all phases, 33 calls, no timeout, no parse failure or call error, no retry, all findings
+  anchored. Two verifier replies were repaired (`verified-1`, `verified-9`); 11 findings were generated
+  from flagged verdicts (2 BOLA, 2 BFLA, 7 mass assignment).
+- **Recall** is lower than `gpt-oss:20b` in every category: BFLA and mass assignment are mostly found as
+  verdicts (8/10, 15/15) but placed on the right line less often (6/10, 10/15).
+- **Precision 0.73:** no finding on a SAFE marker, but 25 unmatched findings (23 BOLA, 2 BFLA), not yet triaged;
+  none is on `php-laravel/routes/web.php`.
+- **Memory and time:** the runner stayed at 14–17 GB and swap peaked at 6.0 GB (about 3.7 GB before the
+  run, from other work), against 20 GB and 12.4 GB with reasoning on. 35 minutes in all, of which the
+  verifier took 17.
+- **Minimum to run it:** a 24 GB machine is enough with `num_ctx 16384` and `--ai-reasoning-effort none`.
+  With reasoning on it needs more; on this evidence at least 32 GB RAM.

package/assess/lsa/tests/test_eval_run_6.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Keeps the archived Evaluation Run 6 numbers (Issue #323) in step with the scorer and EVAL-RESULTS.md."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+TESTS_DIR = Path(__file__).resolve().parent
+LSA_ROOT = TESTS_DIR.parent
+EVAL_RUN_6_DIR = LSA_ROOT / "examples" / "eval-run-6"
+GROUND_TRUTH = EVAL_RUN_6_DIR / "ground-truth.json"
+RUNS = ("gptoss-a", "gptoss-b", "qwen-16k-nothink")
+QWEN_PARTIAL = EVAL_RUN_6_DIR / "qwen-16k-partial"
+# (BOLA, injection, BFLA, mass assignment) found by location, then BFLA / mass found by auditor verdict
+EXPECTED = {
+    "gptoss-a": {"marker": (26, 34, 10, 14), "verdict": (10, 15)},
+    "gptoss-b": {"marker": (26, 34, 10, 14), "verdict": (10, 14)},
+    "qwen-16k-nothink": {"marker": (23, 29, 6, 10), "verdict": (8, 15)},
+}
+# Findings that land on a SAFE marker, by category (see EVAL-RESULTS.md, Run 6)
+SAFE_HITS = {
+    "gptoss-a": {"injection": ["classic-asp/shop/member.asp:6"]},
+    "gptoss-b": {"injection": ["python-flask/app.py:12"]},
+    "qwen-16k-nothink": {},
+}
+
+
+def score(run: str) -> dict:
+    proc = subprocess.run(
+        [sys.executable, str(TESTS_DIR / "score_findings.py"), "--verdicts", str(GROUND_TRUTH), str(EVAL_RUN_6_DIR / run)],
+        capture_output=True, text=True, check=True,
+    )
+    return json.loads(proc.stdout)
+
+
+class TestEvaluationRun6(unittest.TestCase):
+    def test_ground_truth_is_the_121_item_set(self) -> None:
+        items = json.loads(GROUND_TRUTH.read_text(encoding="utf-8"))["items"]
+        self.assertEqual(len(items), 121)
+
+    def test_artifacts_exist(self) -> None:
+        for run in RUNS:
+            for name in ("findings.json", "coverage.json", "score.json", "ai-run-stats.json"):
+                self.assertTrue((EVAL_RUN_6_DIR / run / name).is_file(), f"{run}/{name}")
+
+    def test_archived_scores_match_the_scorer(self) -> None:
+        for run in RUNS:
+            with self.subTest(run=run):
+                archived = json.loads((EVAL_RUN_6_DIR / run / "score.json").read_text(encoding="utf-8"))
+                self.assertEqual(score(run), archived)
+
+    def test_numbers_match_eval_results(self) -> None:
+        for run, want in EXPECTED.items():
+            with self.subTest(run=run):
+                report = score(run)
+                pc, vl = report["per_category"], report["verdict_level"]
+                got = tuple(pc[c]["found"] for c in ("bola", "injection", "bfla", "mass_assignment"))
+                self.assertEqual(got, want["marker"])
+                self.assertEqual((vl["bfla"]["found"], vl["mass_assignment"]["found"]), want["verdict"])
+
+    def test_safe_marker_hits_are_the_documented_ones(self) -> None:
+        for run in RUNS:
+            with self.subTest(run=run):
+                report = score(run)
+                for cat, row in report["per_category"].items():
+                    self.assertEqual(row["false_positive_on_safe_code"], SAFE_HITS[run].get(cat, []), cat)
+                for cat, row in report["verdict_level"].items():
+                    self.assertEqual(row["flagged_safe"], [], cat)
+
+    def test_no_batch_lost_to_parse_or_call_errors(self) -> None:
+        for run in RUNS:
+            with self.subTest(run=run):
+                stats = json.loads((EVAL_RUN_6_DIR / run / "ai-run-stats.json").read_text(encoding="utf-8"))
+                self.assertEqual(stats["parse_failures"], {})
+                self.assertEqual(stats["call_errors"], {})
+                self.assertEqual(stats["anchoring"]["unanchored"], [])
+
+    def test_qwen_partial_score_matches_eval_results(self) -> None:
+        """The stopped qwen3.5:9b 16K run: indicative score of its unverified findings (EVAL-RESULTS, Run 6)."""
+        with tempfile.TemporaryDirectory() as tmp:
+            shutil.copy(QWEN_PARTIAL / "findings.partial.json", Path(tmp) / "findings.json")
+            proc = subprocess.run(
+                [sys.executable, str(TESTS_DIR / "score_findings.py"), str(GROUND_TRUTH), tmp],
+                capture_output=True, text=True, check=True,
+            )
+        report = json.loads(proc.stdout)
+        archived = json.loads((QWEN_PARTIAL / "score.partial.json").read_text(encoding="utf-8"))
+        self.assertEqual(report, archived)
+        pc = report["per_category"]
+        got = tuple(pc[c]["found"] for c in ("bola", "injection", "bfla", "mass_assignment"))
+        self.assertEqual(got, (12, 22, 0, 0))
+        self.assertEqual(report["precision"], 0.733)
+        for row in pc.values():
+            self.assertEqual(row["false_positive_on_safe_code"], [])
+
+
+if __name__ == "__main__":
+    unittest.main()