@anik-2/autosar 0.0.0-stage → 0.1.0

package/python/ecuc_engine/model.py

@@ -0,0 +1,330 @@
+"""Project discovery + the ECUC value model.
+
+Files are classified by content, not by name or folder: a file holding ECUC-MODULE-DEF is a
+param-def source, one holding ECUC-MODULE-CONFIGURATION-VALUES is configuration, any other .arxml
+is "input" (System extract, SWCs, BSWMD) and only indexed for foreign-reference resolution.
+`.hydron/autosar.json` can pin each set with globs: {"paramDefs": [...], "config": [...], "input": [...]}.
+"""
+from __future__ import annotations
+
+import glob
+import json
+import os
+import re
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
+from typing import Optional
+
+from . import defs as D
+from .xmltree import Node, parse
+
+SKIP_DIRS = {".git", "node_modules", ".hydron", ".venv", "__pycache__", ".cache", ".idea", ".vscode"}
+PROJECT_CONFIG = os.path.join(".hydron", "autosar.json")
+
+MODULE_VALUES = "ECUC-MODULE-CONFIGURATION-VALUES"
+CONTAINER_VALUE = "ECUC-CONTAINER-VALUE"
+PARAM_VALUE_TAGS = {"ECUC-NUMERICAL-PARAM-VALUE", "ECUC-TEXTUAL-PARAM-VALUE", "ECUC-ADD-INFO-PARAM-VALUE"}
+REF_VALUE_TAGS = {"ECUC-REFERENCE-VALUE", "ECUC-INSTANCE-REFERENCE-VALUE"}
+
+
+@dataclass
+class ParamVal:
+    node: Node
+    def_ref: Optional[str]
+    def_dest: Optional[str]
+    tag: str
+    value: Optional[str]  # VALUE text / VALUE-REF text / None
+    value_node: Optional[Node]
+    dest: Optional[str] = None  # VALUE-REF DEST
+    is_auto: bool = False
+
+    @property
+    def name(self) -> str:
+        return (self.def_ref or "?").rsplit("/", 1)[-1]
+
+    @property
+    def is_ref(self) -> bool:
+        return self.tag in REF_VALUE_TAGS
+
+
+@dataclass
+class Elem:
+    """A module configuration (ECUC-MODULE-CONFIGURATION-VALUES) or a container value."""
+    node: Node
+    file: str
+    path: str
+    name: str
+    kind: str  # "module" | "container"
+    def_ref: Optional[str]
+    def_dest: Optional[str]
+    params: list = field(default_factory=list)  # ParamVal (params and refs)
+    children: list = field(default_factory=list)  # Elem
+    parent: Optional["Elem"] = None
+
+    @property
+    def line(self) -> int:
+        return self.node.line
+
+
+@dataclass
+class FileModel:
+    path: str
+    data: bytes
+    root: Node
+    elems: list  # top-level module Elems
+    identifiables: dict  # path -> (tag, line)
+    newline: bytes = b"\n"
+
+
+_MARKERS = {
+    "defs": re.compile(rb"<(?:[\w.-]+:)?ECUC-MODULE-DEF[\s>]"),
+    "config": re.compile(rb"<(?:[\w.-]+:)?ECUC-MODULE-CONFIGURATION-VALUES[\s>]"),
+}
+
+
+def _scan_markers(path: str) -> set:
+    """Which element kinds a file holds (element starts only; DEST="ECUC-MODULE-DEF" text does not count)."""
+    found = set()
+    tail = b""
+    try:
+        with open(path, "rb") as f:
+            while len(found) < len(_MARKERS):
+                chunk = f.read(1 << 20)
+                if not chunk:
+                    break
+                buf = tail + chunk
+                for k, rx in _MARKERS.items():
+                    if k not in found and rx.search(buf):
+                        found.add(k)
+                tail = buf[-96:]
+    except OSError:
+        pass
+    return found
+
+
+def _walk_arxml(root: str):
+    for dp, dns, fns in os.walk(root):
+        dns[:] = [d for d in dns if d not in SKIP_DIRS and not d.startswith(".")]
+        for fn in fns:
+            if fn.lower().endswith(".arxml"):
+                yield os.path.join(dp, fn)
+
+
+def _globs(root: str, pats) -> list:
+    out = []
+    for p in pats or []:
+        p = os.path.expanduser(p)
+        full = p if os.path.isabs(p) else os.path.join(root, p)
+        if os.path.isdir(full):
+            out.extend(_walk_arxml(full))
+        else:
+            out.extend(sorted(glob.glob(full, recursive=True)))
+    seen, res = set(), []
+    for f in out:
+        a = os.path.abspath(f)
+        if a not in seen and os.path.isfile(a):
+            seen.add(a)
+            res.append(a)
+    return res
+
+
+@dataclass
+class Sources:
+    root: str
+    config: list
+    defs: list
+    input: list
+    warnings: list
+    origin: dict  # which mechanism found each set
+
+
+def discover(root: str, config_files: Optional[list] = None, def_files: Optional[list] = None) -> Sources:
+    root = os.path.abspath(root)
+    cfg = {}
+    cfg_path = os.path.join(root, PROJECT_CONFIG)
+    warnings: list = []
+    if os.path.isfile(cfg_path):
+        try:
+            with open(cfg_path) as f:
+                cfg = json.load(f)
+        except Exception as e:
+            warnings.append(f"{PROJECT_CONFIG} is not valid JSON ({e}); ignoring it")
+    origin = {}
+    scanned = None
+
+    def scan():
+        nonlocal scanned
+        if scanned is None:
+            scanned = {f: _scan_markers(f) for f in _walk_arxml(root)}
+        return scanned
+
+    if def_files:
+        dfiles, origin["defs"] = _globs(root, def_files), "argument"
+    elif cfg.get("paramDefs"):
+        dfiles, origin["defs"] = _globs(root, cfg["paramDefs"]), PROJECT_CONFIG
+    else:
+        dfiles, origin["defs"] = [f for f, m in scan().items() if "defs" in m], "auto-discovered"
+        if not dfiles and os.environ.get("AUTOSAR_PARAMDEFS"):
+            dfiles = _globs(root, os.environ["AUTOSAR_PARAMDEFS"].split(os.pathsep))
+            origin["defs"] = "AUTOSAR_PARAMDEFS"
+    if config_files:
+        cfiles, origin["config"] = _globs(root, config_files), "argument"
+    elif cfg.get("config"):
+        cfiles, origin["config"] = _globs(root, cfg["config"]), PROJECT_CONFIG
+    else:
+        cfiles, origin["config"] = [f for f, m in scan().items() if "config" in m], "auto-discovered"
+    if cfg.get("input"):
+        ifiles, origin["input"] = _globs(root, cfg["input"]), PROJECT_CONFIG
+    else:
+        known = set(cfiles) | set(dfiles)
+        ifiles, origin["input"] = [f for f in scan() if f not in known], "auto-discovered"
+    if not dfiles:
+        warnings.append(
+            "No ECUC param definitions found (no ECUC-MODULE-DEF in the project, no paramDefs in "
+            f"{PROJECT_CONFIG}, AUTOSAR_PARAMDEFS unset). Values are readable but NOT validated, and edits "
+            "cannot add new elements. Point paramDefs at the AUTOSAR / vendor BSWMD param-def ARXML.")
+    return Sources(root, cfiles, dfiles, ifiles, warnings, origin)
+
+
+def _dest(n: Optional[Node]) -> Optional[str]:
+    return n.attrs.get("DEST") if n is not None else None
+
+
+def build_file_model(path: str, data: bytes) -> FileModel:
+    root = parse(data)
+    elems: list = []
+    idents: dict = {}
+
+    def visit(node: Node, prefix: str, parent: Optional[Elem]):
+        sn = node.short_name
+        path_here = prefix
+        if sn is not None:
+            path_here = f"{prefix}/{sn}"
+            idents.setdefault(path_here, (node.tag, node.line))
+        if node.tag in (MODULE_VALUES, CONTAINER_VALUE) and sn is not None:
+            dr = node.child("DEFINITION-REF")
+            e = Elem(node=node, file=path, path=path_here, name=sn,
+                     kind="module" if node.tag == MODULE_VALUES else "container",
+                     def_ref=dr.text.strip() if dr is not None else None, def_dest=_dest(dr), parent=parent)
+            for coll in node.children:
+                if coll.tag in ("PARAMETER-VALUES", "REFERENCE-VALUES"):
+                    for pv in coll.children:
+                        e.params.append(_param(pv))
+            if parent is not None:
+                parent.children.append(e)
+            else:
+                elems.append(e)
+            for coll in node.children:
+                if coll.tag in ("CONTAINERS", "SUB-CONTAINERS"):
+                    for c in coll.children:
+                        visit(c, path_here, e)
+            return
+        for c in node.children:
+            if c.tag in ("SHORT-NAME", "DESC", "ADMIN-DATA", "INTRODUCTION", "ANNOTATIONS"):
+                continue
+            visit(c, path_here, parent if sn is None else None)
+
+    visit(root, "", None)
+    nl = b"\r\n" if b"\r\n" in data[:4096] else b"\n"
+    return FileModel(path, data, root, elems, idents, nl)
+
+
+def _param(pv: Node) -> ParamVal:
+    dr = pv.child("DEFINITION-REF")
+    vn = pv.child("VALUE") if pv.tag in PARAM_VALUE_TAGS else pv.child("VALUE-REF") or pv.child("VALUE-IREF")
+    val = None
+    if vn is not None:
+        val = vn.text.strip() if vn.tag != "VALUE-IREF" else "(instance-ref)"
+        if pv.tag == "ECUC-ADD-INFO-PARAM-VALUE":
+            val = " ".join(vn.text.split())
+    auto = (pv.child_text("IS-AUTO-VALUE") or "").lower() == "true"
+    return ParamVal(node=pv, def_ref=dr.text.strip() if dr is not None else None, def_dest=_dest(dr), tag=pv.tag,
+                    value=val, value_node=vn, dest=_dest(vn) if vn is not None and vn.tag == "VALUE-REF" else None,
+                    is_auto=auto)
+
+
+def index_identifiables(path: str) -> dict:
+    """Fast path -> (tag, line=0) index of a non-config ARXML (System, SWC, BSWMD) for foreign refs."""
+    out: dict = {}
+    stack: list = []  # (short_name or None)
+    pending: list = []
+    try:
+        for ev, el in ET.iterparse(path, events=("start", "end")):
+            tag = el.tag.rsplit("}", 1)[-1]
+            if ev == "start":
+                stack.append(None)
+                pending.append(tag)
+            else:
+                if tag == "SHORT-NAME" and len(stack) >= 2:
+                    stack[-2] = (el.text or "").strip()
+                    p = "/" + "/".join(s for s in stack[:-1] if s)
+                    out.setdefault(p, (pending[-2], 0))
+                stack.pop()
+                pending.pop()
+                el.clear()
+    except ET.ParseError:
+        pass
+    return out
+
+
+class Project:
+    def __init__(self, root: str, config_files=None, def_files=None, load_input: bool = True):
+        self.sources = discover(root, config_files, def_files)
+        self.root = self.sources.root
+        self.warnings = list(self.sources.warnings)
+        self.defs = D.load_defs(self.sources.defs) if self.sources.defs else D.DefIndex()
+        if self.sources.defs and not self.defs:
+            self.warnings.append("param-def files were found but contain no ECUC-MODULE-DEF")
+        self.files: dict[str, FileModel] = {}
+        for f in self.sources.config:
+            self._load(f)
+        self.foreign: dict = {}
+        self.input_loaded = False
+        if load_input:
+            for f in self.sources.input:
+                self.foreign.update(index_identifiables(f))
+            self.input_loaded = bool(self.sources.input)
+        self.reindex()
+
+    def _load(self, f: str, data: Optional[bytes] = None):
+        if data is None:
+            with open(f, "rb") as fh:
+                data = fh.read()
+        try:
+            self.files[f] = build_file_model(f, data)
+        except Exception as e:
+            self.warnings.append(f"{self.rel(f)}: XML parse error: {e}")
+
+    def reload(self, f: str, data: bytes):
+        self._load(f, data)
+        self.reindex()
+
+    def reindex(self):
+        self.by_path: dict[str, list] = {}
+        self.modules: list = []
+        self.identifiables: dict = dict(self.foreign)
+        for fm in self.files.values():
+            for p, v in fm.identifiables.items():
+                self.identifiables.setdefault(p, v + (fm.path,))
+            for m in fm.elems:
+                self.modules.append(m)
+                for e in _iter_elems(m):
+                    self.by_path.setdefault(e.path, []).append(e)
+
+    def rel(self, f: str) -> str:
+        try:
+            r = os.path.relpath(f, self.root)
+            return f if r.startswith("..") else r
+        except ValueError:
+            return f
+
+    def find(self, path: str) -> list:
+        return self.by_path.get(path.rstrip("/"), [])
+
+
+def _iter_elems(e: Elem):
+    stack = [e]
+    while stack:
+        x = stack.pop()
+        yield x
+        stack.extend(reversed(x.children))

package/python/ecuc_engine/validate.py

@@ -0,0 +1,222 @@
+"""Mechanical ECUC checks, derived from the param defs only (no hand-written module rules).
+
+Diagnostics are LSP-like: {file, line, path, rule, severity, message, defRef}.
+Rules:
+  ECUC-NO-DEF            module's definition is not among the loaded param defs (module skipped)
+  ECUC-UNKNOWN-DEF       container / parameter / reference whose DEFINITION-REF is not in the module def
+  ECUC-WRONG-PARENT      container's def is not a child of its parent's def
+  ECUC-DEST-MISMATCH     DEFINITION-REF DEST attribute differs from the def element type
+  ECUC-VALUE-TYPE        NUMERICAL vs TEXTUAL value element does not match the def kind
+  ECUC-MULT-CONTAINER    sub-container count outside the def multiplicity (incl. missing mandatory)
+  ECUC-CHOICE            choice container does not hold exactly one choice
+  ECUC-MULT-PARAM        parameter / reference count outside the def multiplicity
+  ECUC-MISSING-PARAM     mandatory parameter / reference absent
+  ECUC-TYPE / ECUC-RANGE / ECUC-ENUM / ECUC-STRING / ECUC-EMPTY-VALUE   value checks
+  ECUC-REF-DANGLING      reference target does not exist
+  ECUC-REF-DEST          reference target is of the wrong definition / metaclass
+  ECUC-REF-UNRESOLVED    foreign reference target not in any loaded file (warning)
+  ECUC-DUP-NAME          two containers with the same AUTOSAR path
+  ECUC-OBSOLETE          value for an obsolete def (warning)
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+from . import defs as D
+from .model import Project
+from .values import check_value, expected_value_tag
+
+GENERIC_CONTAINER_DESTS = {"ECUC-CONTAINER-DEF"}
+SEV_ORDER = {"error": 0, "warning": 1, "info": 2, "hint": 3}
+
+
+class Diags:
+    def __init__(self, project: Project):
+        self.p = project
+        self.items: list = []
+
+    def add(self, file, line, path, rule, severity, message, def_ref=None):
+        self.items.append({"file": self.p.rel(file), "line": line, "path": path, "rule": rule,
+                           "severity": severity, "message": message, "defRef": def_ref})
+
+
+def _same_def(defs: D.DefIndex, a: Optional[str], b: Optional[str]) -> bool:
+    if not a or not b:
+        return False
+    return a == b or defs.rel(a) == defs.rel(b)
+
+
+def _resolve_def(defs: D.DefIndex, ref: Optional[str]) -> Optional[D.DefNode]:
+    if not ref:
+        return None
+    d = defs.get(ref)
+    if d is not None:
+        return d
+    # vendor-relocated path (/Vendor/Com/... vs /AUTOSAR/EcucDefs/Com/...): match by module-relative path
+    rel = defs.rel(ref)
+    for m in defs.modules.values():
+        if rel.split("/", 1)[0] == m.name:
+            return defs.get(m.path + rel[len(m.name):])
+    return None
+
+
+def validate(project: Project, files: Optional[list] = None, module: Optional[str] = None) -> dict:
+    defs = project.defs
+    dg = Diags(project)
+    warned_modules = set()
+    visited = set()
+
+    def fmult(lo, up):
+        return f"{lo}..{'*' if up is None else up}"
+
+    def check_elem_group(path: str):
+        if path in visited:
+            return
+        visited.add(path)
+        group = project.find(path)
+        e0 = group[0]
+        if e0.kind == "container" and len(group) > 1:
+            for e in group[1:]:
+                dg.add(e.file, e.line, path, "ECUC-DUP-NAME", "error",
+                       f"duplicate container path {path} (also at {project.rel(e0.file)}:{e0.line})", e.def_ref)
+        d = _resolve_def(defs, e0.def_ref)
+        if d is None:
+            if e0.kind == "module":
+                key = e0.def_ref or path
+                if key not in warned_modules:
+                    warned_modules.add(key)
+                    dg.add(e0.file, e0.line, path, "ECUC-NO-DEF", "warning",
+                           f"no param def loaded for {e0.def_ref}; module not validated", e0.def_ref)
+            else:
+                dg.add(e0.file, e0.line, path, "ECUC-UNKNOWN-DEF", "error",
+                       f"container definition {e0.def_ref} does not exist in the loaded param defs", e0.def_ref)
+            return
+        if e0.def_dest and e0.def_dest != d.tag and not (e0.def_dest in GENERIC_CONTAINER_DESTS and d.is_container):
+            dg.add(e0.file, e0.line, path, "ECUC-DEST-MISMATCH", "warning",
+                   f"DEFINITION-REF DEST={e0.def_dest} but the def is {d.tag}", d.path)
+        if e0.kind == "container" and e0.parent is not None:
+            pd = _resolve_def(defs, e0.parent.def_ref)
+            if pd is not None and d.parent != pd.path:
+                dg.add(e0.file, e0.line, path, "ECUC-WRONG-PARENT", "error",
+                       f"{d.name} is not a sub-container of {pd.name} (allowed: "
+                       f"{', '.join(defs.by_path[c].name for c in pd.containers) or 'none'})", d.path)
+        if d.obsolete:
+            dg.add(e0.file, e0.line, path, "ECUC-OBSOLETE", "warning", f"{d.name} is obsolete", d.path)
+
+        # children (merged across split fragments)
+        children = [c for e in group for c in e.children]
+        params = [(e, pv) for e in group for pv in e.params]
+        counts: dict = {}
+        seen_child_paths = set()
+        for c in children:
+            if c.path in seen_child_paths:
+                continue
+            seen_child_paths.add(c.path)
+            cd = _resolve_def(defs, c.def_ref)
+            if cd is not None:
+                counts[cd.path] = counts.get(cd.path, 0) + 1
+        if d.tag == D.CHOICE_TAG:
+            n = sum(counts.get(c, 0) for c in d.containers)
+            if n != 1:
+                dg.add(e0.file, e0.line, path, "ECUC-CHOICE", "error",
+                       f"choice container {d.name} holds {n} choices, needs exactly 1 of "
+                       f"{', '.join(defs.by_path[c].name for c in d.containers)}", d.path)
+        else:
+            for cpath in d.containers:
+                cd = defs.by_path[cpath]
+                n = counts.get(cpath, 0)
+                if n < cd.lower or (cd.upper is not None and n > cd.upper):
+                    dg.add(e0.file, e0.line, path, "ECUC-MULT-CONTAINER", "error",
+                           f"{d.name} '{e0.name}': {cd.name} has {n} instance(s), def requires {fmult(cd.lower, cd.upper)}",
+                           cd.path)
+        pcounts: dict = {}
+        for e, pv in params:
+            pd = _resolve_def(defs, pv.def_ref)
+            line = pv.node.line
+            if pd is None or pd.parent != d.path:
+                dg.add(e.file, line, path, "ECUC-UNKNOWN-DEF", "error",
+                       f"parameter/reference {pv.def_ref} is not defined for {d.name}", pv.def_ref)
+                continue
+            pcounts[pd.path] = pcounts.get(pd.path, 0) + 1
+            if pd.obsolete:
+                dg.add(e.file, line, path, "ECUC-OBSOLETE", "warning", f"{pd.name} is obsolete", pd.path)
+            if pv.def_dest and pv.def_dest != pd.tag:
+                dg.add(e.file, line, path, "ECUC-DEST-MISMATCH", "warning",
+                       f"{pd.name}: DEFINITION-REF DEST={pv.def_dest} but the def is {pd.tag}", pd.path)
+            exp = expected_value_tag(pd)
+            if exp and pv.tag != exp:
+                dg.add(e.file, line, path, "ECUC-VALUE-TYPE", "warning",
+                       f"{pd.name} is a {pd.kind} def; value element should be {exp}, found {pv.tag}", pd.path)
+            if D.is_ref_kind(pd.kind or ""):
+                check_ref(e, pv, pd, path, line)
+            elif pd.kind != "ADD-INFO" and not pv.is_auto:
+                for rule, msg in check_value(pd, pv.value):
+                    sev = "warning" if rule == "ECUC-EMPTY-VALUE" else "error"
+                    dg.add(e.file, (pv.value_node or pv.node).line, path, rule, sev, f"{d.name} '{e0.name}': {msg}", pd.path)
+        for ppath in d.params + d.refs:
+            pd = defs.by_path[ppath]
+            n = pcounts.get(ppath, 0)
+            if n == 0 and pd.lower > 0:
+                dg.add(e0.file, e0.line, path, "ECUC-MISSING-PARAM", "error",
+                       f"{d.name} '{e0.name}': mandatory {pd.name} ({pd.kind}, {pd.mult}) is missing", pd.path)
+            elif n < pd.lower or (pd.upper is not None and n > pd.upper):
+                dg.add(e0.file, e0.line, path, "ECUC-MULT-PARAM", "error",
+                       f"{d.name} '{e0.name}': {pd.name} has {n} value(s), def requires {pd.mult}", pd.path)
+        for c in children:
+            check_elem_group(c.path)
+
+    def check_ref(e, pv, pd, path, line):
+        tgt = (pv.value or "").strip()
+        if pd.kind in ("INSTANCE-REFERENCE", "URI-REFERENCE"):
+            return
+        if not tgt:
+            dg.add(e.file, line, path, "ECUC-EMPTY-VALUE", "error", f"{pd.name}: empty reference", pd.path)
+            return
+        if pd.kind == "FOREIGN-REFERENCE":
+            hit = project.identifiables.get(tgt)
+            if hit is None:
+                if project.input_loaded:
+                    dg.add(e.file, line, path, "ECUC-REF-DANGLING", "error",
+                           f"{pd.name} -> {tgt}: target not found in any loaded ARXML", pd.path)
+                else:
+                    dg.add(e.file, line, path, "ECUC-REF-UNRESOLVED", "warning",
+                           f"{pd.name} -> {tgt}: target not in loaded files (add the System/SWC ARXML to "
+                           f"'input' in .hydron/autosar.json)", pd.path)
+                return
+            # DESTINATION-TYPE is often an abstract metaclass (RTE-EVENT for TIMING-EVENT), which needs the
+            # metamodel to check; the VALUE-REF DEST attribute must name the target's concrete type exactly.
+            if pv.dest and pv.dest != hit[0]:
+                dg.add(e.file, line, path, "ECUC-REF-DEST", "error",
+                       f"{pd.name} -> {tgt}: VALUE-REF DEST={pv.dest} but the target is a {hit[0]}", pd.path)
+            return
+        targets = project.find(tgt)
+        if not targets:
+            dg.add(e.file, line, path, "ECUC-REF-DANGLING", "error",
+                   f"{pd.name} -> {tgt}: no ECUC container with that path", pd.path)
+            return
+        t = targets[0]
+        if pd.dest_refs and not any(_same_def(defs, t.def_ref, dr) for dr in pd.dest_refs):
+            dg.add(e.file, line, path, "ECUC-REF-DEST", "error",
+                   f"{pd.name} -> {tgt} is a {t.def_ref.rsplit('/', 1)[-1] if t.def_ref else '?'}, def requires "
+                   f"{' or '.join(r.rsplit('/', 1)[-1] for r in pd.dest_refs)}", pd.path)
+
+    for m in project.modules:
+        if module and m.name != module and (m.def_ref or "").rsplit("/", 1)[-1] != module:
+            continue
+        check_elem_group(m.path)
+
+    items = dg.items
+    if files:
+        want = {project.rel(f) for f in files} | set(files)
+        items = [i for i in items if i["file"] in want]
+    items.sort(key=lambda i: (i["file"], i["line"], SEV_ORDER.get(i["severity"], 9), i["rule"]))
+    summary: dict = {}
+    for i in items:
+        summary[i["severity"]] = summary.get(i["severity"], 0) + 1
+    return {
+        "validated": bool(defs),
+        "defsLoaded": len(defs.modules),
+        "modules": sorted({m.name for m in project.modules}),
+        "summary": summary,
+        "diagnostics": items,
+    }