@h2loop/autosar 0.1.2 → 1.1.0

package/python/ecuc_engine/csrc.py

@@ -0,0 +1,189 @@
+"""Local, dependency-free C function index: definitions, bodies, callees, Rte_* calls and the call graph.
+
+Function definitions are found at brace depth 0 in comment/string-stripped text (audit.strip_comments, which
+keeps offsets): a '{' preceded by ')' whose matching '(' follows an identifier is a definition of that
+identifier. Works with AUTOSAR compiler-abstraction macros (FUNC(void, X_CODE) Name(void)) because the name is
+the identifier directly before the parameter list. Preprocessor lines are ignored.
+"""
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass, field
+from typing import Optional
+
+from . import paths
+from .audit import is_generated, iter_calls, strip_comments
+
+KEYWORDS = {"if", "for", "while", "switch", "return", "sizeof", "defined", "else", "do", "case", "goto", "typeof",
+            "_Alignof", "__attribute__", "__asm__", "asm", "offsetof"}
+RTE_CALL = re.compile(r"\b(Rte_\w+)\s*\(")
+ANY_CALL = re.compile(r"\b([A-Za-z_]\w*)\s*\(")
+
+
+@dataclass
+class Func:
+    name: str
+    file: str
+    line: int
+    end_line: int
+    start: int  # offset of the declaration start
+    body_start: int  # offset of '{'
+    end: int  # offset after '}'
+    static: bool
+    signature: str
+    calls: list = field(default_factory=list)  # callee names in order of first use
+    rte: list = field(default_factory=list)  # [(name, line, args)]
+
+    def text(self, src: str) -> str:
+        return src[self.start:self.end]
+
+
+def _blank_preproc(text: str) -> str:
+    out = []
+    cont = False
+    for ln in text.split("\n"):
+        if cont or ln.lstrip().startswith("#"):
+            cont = ln.rstrip().endswith("\\")
+            out.append(" " * len(ln))
+        else:
+            out.append(ln)
+    return "\n".join(out)
+
+
+def _match_open(s: str, close: int) -> int:
+    depth = 0
+    for i in range(close, -1, -1):
+        if s[i] == ")":
+            depth += 1
+        elif s[i] == "(":
+            depth -= 1
+            if depth == 0:
+                return i
+    return -1
+
+
+def functions(path: str, raw: Optional[str] = None) -> tuple:
+    """-> (list of Func, raw source)."""
+    if raw is None:
+        try:
+            raw = open(path, encoding="utf-8", errors="replace").read()
+        except OSError:
+            return [], ""
+    s = _blank_preproc(strip_comments(raw))
+    out = []
+    depth = 0
+    i, n = 0, len(s)
+    last_end = 0  # end of the previous top-level construct (for the declaration start)
+    while i < n:
+        c = s[i]
+        if c == "{":
+            if depth == 0:
+                j = i - 1
+                while j >= 0 and s[j].isspace():
+                    j -= 1
+                fn = None
+                if j >= 0 and s[j] == ")":
+                    o = _match_open(s, j)
+                    k = o - 1
+                    while k >= 0 and s[k].isspace():
+                        k -= 1
+                    m = re.search(r"([A-Za-z_]\w*)$", s[max(0, k - 200):k + 1]) if k >= 0 else None
+                    if m and m.group(1) not in KEYWORDS:
+                        fn = m.group(1)
+                if fn:
+                    # matching close brace
+                    d, e = 0, i
+                    while e < n:
+                        if s[e] == "{":
+                            d += 1
+                        elif s[e] == "}":
+                            d -= 1
+                            if d == 0:
+                                break
+                        e += 1
+                    head_start = max(s.rfind(";", 0, i), s.rfind("}", 0, i), last_end - 1) + 1
+                    while head_start < i and s[head_start].isspace():
+                        head_start += 1
+                    sig = " ".join(s[head_start:i].split())
+                    f = Func(fn, path, s.count("\n", 0, head_start) + 1, s.count("\n", 0, e) + 1, head_start, i,
+                             e + 1, bool(re.match(r"(?:\w+\s+)*?static\b", sig)), sig)
+                    body = s[i:e + 1]
+                    seen = []
+                    for mm in ANY_CALL.finditer(body):
+                        name = mm.group(1)
+                        if name in KEYWORDS or name == fn:
+                            continue
+                        if name not in seen:
+                            seen.append(name)
+                    f.calls = seen
+                    for name, off, _line, args in iter_calls(body, RTE_CALL):
+                        f.rte.append((name, f.line + body.count("\n", 0, off) + (s.count("\n", head_start, i)), args))
+                    out.append(f)
+                    i = e + 1
+                    last_end = i
+                    continue
+            depth += 1
+        elif c == "}":
+            depth = max(0, depth - 1)
+            if depth == 0:
+                last_end = i + 1
+        elif c == ";" and depth == 0:
+            last_end = i + 1
+        i += 1
+    return out, raw
+
+
+class CIndex:
+    """All function definitions under the given C files (in-house first; generated files kept separately)."""
+
+    def __init__(self, root: str, files: Optional[list] = None, include_generated: bool = False):
+        self.root = root
+        all_c = files if files is not None else paths.walk(root, paths.is_c).files
+        self.cfiles = sorted(f for f in all_c if f.lower().endswith(".c"))
+        self.generated = {f for f in self.cfiles if is_generated(f)}
+        self.funcs: dict = {}  # name -> [Func]
+        self.src: dict = {}
+        self.includes: dict = {}
+        for f in self.cfiles:
+            if f in self.generated and not include_generated:
+                continue
+            fs, raw = functions(f)
+            self.src[f] = raw
+            self.includes[f] = re.findall(r'^\s*#\s*include\s*[<"]([^>"]+)[>"]', raw, re.M)
+            for fn in fs:
+                self.funcs.setdefault(fn.name, []).append(fn)
+
+    def get(self, name: str, prefer_file: Optional[str] = None) -> Optional[Func]:
+        lst = self.funcs.get(name) or []
+        if prefer_file:
+            for f in lst:
+                if f.file == prefer_file:
+                    return f
+        nonstatic = [f for f in lst if not f.static]
+        return (nonstatic or lst or [None])[0]
+
+    def callee(self, caller: Func, name: str) -> Optional[Func]:
+        """Resolve a call from ``caller``: a static function in the same file wins, else a global one."""
+        lst = self.funcs.get(name) or []
+        same = [f for f in lst if f.file == caller.file]
+        if same:
+            return same[0]
+        glob = [f for f in lst if not f.static]
+        return glob[0] if glob else None
+
+    def reachable(self, start: Func, limit: int = 2000) -> list:
+        seen, order, stack = {id(start)}, [start], [start]
+        while stack and len(order) < limit:
+            f = stack.pop()
+            for c in f.calls:
+                g = self.callee(f, c)
+                if g is not None and id(g) not in seen:
+                    seen.add(id(g))
+                    order.append(g)
+                    stack.append(g)
+        return order
+
+    def rel(self, f: str) -> str:
+        r = os.path.relpath(f, self.root)
+        return f if r.startswith("..") else r

package/python/ecuc_engine/defs.py

@@ -76,6 +76,11 @@ class DefNode:
     refs: list = field(default_factory=list)  # child ref def paths
     parent: Optional[str] = None
     module: Optional[str] = None  # module def path
+    # Config classes per IMPLEMENTATION-CONFIG-VARIANT ({"VARIANT-PRE-COMPILE": "PRE-COMPILE", ...}).
+    config_classes: Optional[dict] = None  # VALUE-CONFIG-CLASSES (params/refs)
+    mult_classes: Optional[dict] = None  # MULTIPLICITY-CONFIG-CLASSES (containers, params)
+    requires_symbolic: bool = False  # REQUIRES-SYMBOLIC-NAME-VALUE (refs)
+    variants: Optional[list] = None  # SUPPORTED-CONFIG-VARIANTS (module defs)
 
     @property
     def is_container(self) -> bool:
@@ -218,6 +223,13 @@ def _make_def(el, tag: str, sn: str, path: str, path_file: str) -> DefNode:
         d.upper = None
     else:
         d.upper = _int(direct["UPPER-MULTIPLICITY"].text if "UPPER-MULTIPLICITY" in direct else None, 1)
+    for wrap, attr in (("VALUE-CONFIG-CLASSES", "config_classes"), ("MULTIPLICITY-CONFIG-CLASSES", "mult_classes")):
+        w = direct.get(wrap)
+        if w is not None:
+            setattr(d, attr, _config_classes(w))
+    sv = direct.get("SUPPORTED-CONFIG-VARIANTS")
+    if sv is not None:
+        d.variants = [(c.text or "").strip() for c in sv if _local(c.tag) == "SUPPORTED-CONFIG-VARIANT"]
     tr = direct.get("RELATED-TRACE-ITEM-REF")
     if tr is not None:
         d.trace = (tr.text or "").strip()
@@ -256,9 +268,27 @@ def _make_def(el, tag: str, sn: str, path: str, path_file: str) -> DefNode:
             d.with_auto = txt.lower() == "true"
         elif t == "SYMBOLIC-NAME-VALUE":
             d.symbolic_name = txt.lower() == "true"
+        elif t == "REQUIRES-SYMBOLIC-NAME-VALUE":
+            d.requires_symbolic = txt.lower() == "true"
     return d
 
 
+def _config_classes(wrapper) -> dict:
+    """<VALUE-CONFIG-CLASSES>/<MULTIPLICITY-CONFIG-CLASSES> -> {CONFIG-VARIANT: CONFIG-CLASS}."""
+    out = {}
+    for c in wrapper:
+        cls = var = None
+        for f in c:
+            t = _local(f.tag)
+            if t == "CONFIG-CLASS":
+                cls = (f.text or "").strip()
+            elif t == "CONFIG-VARIANT":
+                var = (f.text or "").strip()
+        if cls and var:
+            out[var] = cls
+    return out
+
+
 def load_defs(paths: list[str], use_cache: bool = True) -> DefIndex:
     idx = DefIndex()
     for p in paths:

package/python/ecuc_engine/diff.py

@@ -9,7 +9,13 @@ from __future__ import annotations
 import os
 from typing import Optional
 
-from .model import MODULE_VALUES, CONTAINER_VALUE, _walk_arxml, build_file_model
+from . import paths
+from .model import MODULE_VALUES, CONTAINER_VALUE, build_file_model
+
+
+def _expand(target: str, root: Optional[str] = None) -> list:
+    """A dir (recursive), a glob (** allowed) or a file, relative to root -> ARXML files."""
+    return paths.expand(root or os.getcwd(), [target], paths.is_arxml)
 from .values import parse_int, parse_float
 from .xmltree import Node
 
@@ -42,8 +48,10 @@ def _canon(node: Node, top: bool = True) -> str:
     return f"<{node.tag}[{attrs}]{txt}{''.join(kids)}>"
 
 
-def _side(target: str) -> dict:
-    files = [target] if os.path.isfile(target) else sorted(_walk_arxml(target))
+def _side(target: str, root: Optional[str] = None) -> dict:
+    files = _expand(target, root)
+    if not files:
+        raise FileNotFoundError(f"{target!r} matched no .arxml file (file, directory or glob, relative to {root or os.getcwd()})")
     out: dict = {}
     for f in files:
         with open(f, "rb") as fh:
@@ -82,8 +90,8 @@ def _side(target: str) -> dict:
     return out
 
 
-def diff(a: str, b: str, rel=lambda f: f) -> dict:
-    A, B = _side(a), _side(b)
+def diff(a: str, b: str, rel=lambda f: f, root: Optional[str] = None) -> dict:
+    A, B = _side(a, root), _side(b, root)
     added = [{"path": p, "tag": B[p]["tag"], "at": f"{rel(B[p]['file'])}:{B[p]['line']}"} for p in sorted(set(B) - set(A))]
     removed = [{"path": p, "tag": A[p]["tag"], "at": f"{rel(A[p]['file'])}:{A[p]['line']}"} for p in sorted(set(A) - set(B))]
     changed = []

package/python/ecuc_engine/find.py

@@ -0,0 +1,162 @@
+"""Structured ECUC queries (ecuc_find) and per-module cross-check tables (ecuc_crosscheck)."""
+from __future__ import annotations
+
+import re
+from typing import Optional
+
+from .model import Project
+from .values import parse_int
+
+import operator
+
+CMP = {">": operator.gt, "<": operator.lt, ">=": operator.ge, "<=": operator.le}
+OPS = re.compile(r"^\s*([A-Za-z_]\w*)\s*(!=|>=|<=|=|>|<|~)\s*(.*?)\s*$")
+
+
+def _where(where) -> list:
+    if not where:
+        return []
+    if isinstance(where, dict):
+        return [(k, "=", v) for k, v in where.items()]
+    out = []
+    for part in re.split(r"\s*(?:,|&&|\band\b)\s*", str(where)):
+        m = OPS.match(part)
+        if m:
+            out.append((m.group(1), m.group(2), m.group(3)))
+    return out
+
+
+def _cmp(vals: list, op: str, want) -> bool:
+    w = str(want)
+    w = w[2:] if w.startswith("->") else w
+    for v in vals:
+        s = (v or "").strip()
+        if op == "=" and (s == w or s.lower() == w.lower()):
+            return True
+        if op == "!=" and s != w:
+            return True
+        if op == "~" and re.search(w, s):
+            return True
+        if op in (">", "<", ">=", "<="):
+            a, b = parse_int(s), parse_int(w)
+            if a is not None and b is not None and CMP[op](a, b):
+                return True
+    return op == "!=" and not vals
+
+
+def _own(project: Project, e) -> dict:
+    vals: dict = {}
+    for g in project.find(e.path):
+        for pv in g.params:
+            vals.setdefault(pv.name, []).append(pv.value)
+    return vals
+
+
+def _refs(project: Project, e, depth: int) -> set:
+    """Reference targets of e, following references up to ``depth`` hops."""
+    out, frontier = set(), [e]
+    for _ in range(depth):
+        nxt = []
+        for x in frontier:
+            for g in project.find(x.path):
+                for pv in g.params:
+                    if pv.is_ref and pv.value:
+                        t = pv.value.strip()
+                        if t not in out:
+                            out.add(t)
+                            hit = project.find(t)
+                            if hit:
+                                nxt.append(hit[0])
+        frontier = nxt
+    return out
+
+
+def find(project: Project, module: Optional[str] = None, container_def: Optional[str] = None, where=None,
+         ref_to: Optional[str] = None, referenced_by: Optional[str] = None, limit: int = 50, depth: int = 2) -> dict:
+    from .ecucjson import resolve
+    conds = _where(where)
+    targets = None
+    if ref_to:
+        r = resolve(project, ref_to)
+        if isinstance(r, dict):
+            sys = project.system.get(ref_to)
+            if sys is None:
+                return {"error": r["error"], "didYouMean": r.get("didYouMean", [])}
+            r = ref_to
+        targets = r
+    owner_refs = None
+    if referenced_by:
+        r = resolve(project, referenced_by)
+        if isinstance(r, dict):
+            return {"error": r["error"], "didYouMean": r.get("didYouMean", [])}
+        owner_refs = _refs(project, project.find(r)[0], 1)
+    hits = []
+    total = 0
+    for path, group in project.by_path.items():
+        e = group[0]
+        if e.kind != "container":
+            continue
+        dname = (e.def_ref or "").rsplit("/", 1)[-1]
+        if container_def and not (dname == container_def or (e.def_ref or "").endswith("/" + container_def.strip("/"))):
+            continue
+        if module:
+            m = e
+            while m.parent is not None:
+                m = m.parent
+            if module not in (m.name, (m.def_ref or "").rsplit("/", 1)[-1]):
+                continue
+        if owner_refs is not None and path not in owner_refs:
+            continue
+        own = _own(project, e)
+        if any(not _cmp(own.get(k, []), op, v) for k, op, v in conds):
+            continue
+        if targets is not None:
+            refs = _refs(project, e, depth)
+            if not any(t == targets or t.startswith(targets + "/") for t in refs):
+                continue
+        total += 1
+        if len(hits) < limit:
+            hits.append({"path": path, "def": dname, "at": f"{project.rel(e.file)}:{e.line}",
+                         "values": {k: (v[0] if len(v) == 1 else v) for k, v in own.items()}})
+    return {"count": total, "returned": len(hits), "matches": hits}
+
+
+ELEMENTS = {
+    "Com": ["Com/ComConfig/ComSignal", "Com/ComConfig/ComIPdu"],
+    "PduR": ["PduR/PduRRoutingPaths/PduRRoutingPath"],
+    "EcuC": ["EcuC/EcucConfigSet/EcucPduCollection/Pdu"],
+    "NvM": ["NvM/NvMBlockDescriptor"],
+    "Fee": ["Fee/FeeBlockConfiguration"],
+    "Ea": ["Ea/EaBlockConfiguration"],
+    "CanIf": ["CanIf/CanIfInitCfg/CanIfTxPduCfg", "CanIf/CanIfInitCfg/CanIfRxPduCfg"],
+    "Dem": ["Dem/DemConfigSet/DemEventParameter"],
+}
+
+
+def crosscheck(project: Project, module: str, limit: int = 200) -> dict:
+    from .rules import RuleContext, run_rules
+    mods = [m for m in project.modules if module in (m.name, (m.def_ref or "").rsplit("/", 1)[-1])]
+    if not mods:
+        return {"error": f"module {module!r} is not configured (modules: {', '.join(sorted({m.name for m in project.modules}))})"}
+    roots = [m.path for m in mods]
+    diags = [d for d in run_rules(project) if any(d["path"] == r or d["path"].startswith(r + "/") for r in roots)
+             or d["rule"].endswith("-000")]
+    ctx = RuleContext(project)
+    kinds = ELEMENTS.get((mods[0].def_ref or "").rsplit("/", 1)[-1] or module, [])
+    rows, counts = [], {"checked": 0, "findings": 0}
+    by_path: dict = {}
+    for d in diags:
+        by_path.setdefault(d["path"], []).append(d)
+    for k in kinds:
+        for e in ctx.containers(k):
+            counts["checked"] += 1
+            f = by_path.get(e.path, [])
+            if f:
+                counts["findings"] += 1
+                rows.append({"element": e.path, "def": k.rsplit("/", 1)[-1], "at": f"{project.rel(e.file)}:{e.line}",
+                             "status": "FINDING", "findings": [f"{x['rule']} {x['severity']}: {x['message']}" for x in f]})
+    notes = [d["message"] for d in diags if d["severity"] == "info"]
+    other = [d for d in diags if d["severity"] != "info" and not any(r["element"] == d["path"] for r in rows)]
+    return {"module": module, "elementKinds": [k.rsplit("/", 1)[-1] for k in kinds], "counts": counts,
+            "rows": rows[:limit], "otherFindings": other[:limit], "notes": notes,
+            "summary": f"{counts['checked']} element(s) checked, {counts['findings']} with findings"}

package/python/ecuc_engine/gen.py

@@ -0,0 +1,105 @@
+"""H2L-GEN-001: is the generated code up to date with the configuration?
+
+Runs the project's configured `generate` command in a scratch copy (never in place), then compares the
+copy with the tree. Every file the generator would change is reported with a short diff summary.
+"""
+from __future__ import annotations
+
+import difflib
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+
+from . import paths
+
+DEFAULT_TIMEOUT = 300
+AUTO_LIMIT = 5 * 1024 * 1024  # on by default only when the configuration is smaller than this
+
+
+def generate_cmd(cfg: dict):
+    g = cfg.get("generate")
+    if isinstance(g, str):
+        return g, None, DEFAULT_TIMEOUT
+    if isinstance(g, dict) and g.get("command"):
+        return g["command"], g.get("cwd"), int(g.get("timeoutSec") or DEFAULT_TIMEOUT)
+    return None, None, 0
+
+
+def default_on(project, cfg: dict) -> bool:
+    if "checkGenerated" in cfg:
+        return bool(cfg["checkGenerated"])
+    if not generate_cmd(cfg)[0]:
+        return False
+    return sum(os.path.getsize(f) for f in project.files) < AUTO_LIMIT
+
+
+def _snapshot(root: str) -> dict:
+    out = {}
+    for dp, dns, fns in os.walk(root):
+        dns[:] = [d for d in dns if d not in paths.TOOLING_DIRS]
+        for fn in fns:
+            f = os.path.join(dp, fn)
+            try:
+                st = os.stat(f)
+            except OSError:
+                continue
+            out[os.path.relpath(f, root)] = (st.st_size, st.st_mtime_ns)
+    return out
+
+
+def check(project, cfg: dict, timeout: int = 0) -> list:
+    cmd, cwd, tmo = generate_cmd(cfg)
+    if not cmd:
+        return [_d("info", "", 0, "no `generate` command in .hydron/autosar.json; stale-Output check skipped")]
+    tmo = timeout or tmo
+    root = project.root
+    before = _snapshot(root)
+    t0 = time.time()
+    with tempfile.TemporaryDirectory(prefix="h2l-gen-") as tmp:
+        scratch = os.path.join(tmp, "proj")
+        shutil.copytree(root, scratch, symlinks=True, ignore=shutil.ignore_patterns(*paths.TOOLING_DIRS))
+        try:
+            r = subprocess.run(cmd, shell=True, cwd=os.path.join(scratch, cwd) if cwd else scratch,
+                               capture_output=True, text=True, timeout=tmo)
+        except subprocess.TimeoutExpired:
+            return [_d("info", "", 0, f"NOT CHECKED: generate command timed out after {tmo}s in a scratch copy: {cmd}")]
+        if r.returncode != 0:
+            tail = (r.stderr or r.stdout).strip().splitlines()[-5:]
+            return [_d("warning", "", 0, f"generate command failed in a scratch copy (exit {r.returncode}): {cmd}: "
+                                         + " | ".join(tail) + " — the configuration may not be generatable")]
+        after = _snapshot(scratch)
+        out = []
+        for rel in sorted(set(before) | set(after)):
+            if rel in before and rel not in after:
+                continue  # build artefacts removed by the generator are not staleness
+            a, b = os.path.join(root, rel), os.path.join(scratch, rel)
+            if rel not in before:
+                out.append(_d("error", rel, 0, f"{rel}: the generator creates this file but it is missing from the tree"))
+                continue
+            try:
+                x, y = open(a, "rb").read(), open(b, "rb").read()
+            except OSError:
+                continue
+            if x == y:
+                continue
+            xl = x.decode("utf-8", "replace").splitlines()
+            yl = y.decode("utf-8", "replace").splitlines()
+            diff = [ln for ln in difflib.unified_diff(xl, yl, "tree", "regenerated", n=0, lineterm="")
+                    if ln.startswith(("+", "-")) and not ln.startswith(("+++", "---"))]
+            plus = sum(1 for ln in diff if ln.startswith("+"))
+            minus = len(diff) - plus
+            first = next((i for i, (p, q) in enumerate(zip(xl, yl)) if p != q), min(len(xl), len(yl))) + 1
+            sample = "; ".join(ln[:120] for ln in diff[:4])
+            out.append(_d("error", rel, first, f"{rel} is stale: regenerating changes {minus} line(s) / adds {plus} "
+                                               f"(e.g. {sample})"))
+        if not out:
+            out.append(_d("info", "", 0, f"generated files are up to date (regenerated in {time.time() - t0:.1f}s in a scratch copy)"))
+        return out
+
+
+def _d(sev, file, line, msg):
+    return {"file": file, "line": line, "path": "", "rule": "H2L-GEN-001", "severity": sev,
+            "message": msg + " [TR_Methodology: generated BSW configuration must be regenerated after ECUC changes]",
+            "defRef": None}