@h2loop/autosar 0.1.2 → 1.1.0

package/python/ecuc_engine/probe.py

@@ -0,0 +1,229 @@
+"""Compile-only sizeof probes with the project's own compiler and flags.
+
+Nothing is executed: each object's size is recovered from a compiler diagnostic
+(`char (*p)[sizeof(x)] = 1;` makes gcc/clang print "char (*)[N]"). clang and gcc >= 14 report that
+int-to-pointer conversion as an error; gcc <= 13 only as a -Wint-conversion *warning*, so warnings must not be
+silenced (no -w) and -Wint-conversion is forced on. The parser accepts ASCII or UTF-8 quotes, "warning:" or
+"error:", absolute or relative file names. Works for cross compilers.
+
+Compiler and flags: .hydron/autosar.json "probe": {"cc": "...", "flags": [...]} > a Makefile found in the
+project (CC, CFLAGS/CPPFLAGS/INCLUDES/DEFINES) > host `cc` with -I for every directory holding headers.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import shlex
+import shutil
+import subprocess
+import tempfile
+from dataclasses import dataclass, field
+from typing import Optional
+
+from . import paths
+
+KEEP_FLAG = re.compile(r"^(-I|-D|-U|-std=|-m|-f(?!no-?error)|--sysroot|-isystem|-include|-target|--target)")
+
+
+@dataclass
+class Toolchain:
+    cc: list
+    flags: list
+    origin: str
+    notes: list = field(default_factory=list)
+
+
+def _make_vars(path: str) -> dict:
+    """Very small Makefile variable reader (=, :=, ?=, +=, line continuations, $(VAR) expansion)."""
+    raw: dict = {}
+    try:
+        lines = open(path, encoding="utf-8", errors="replace").read().replace("\\\n", " ").split("\n")
+    except OSError:
+        return {}
+    rx = re.compile(r"^\s*(?:override\s+|export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*([:?+]?=)\s*(.*)$")
+    for ln in lines:
+        if ln.startswith("\t"):
+            continue
+        m = rx.match(ln.split("#", 1)[0])
+        if not m:
+            continue
+        name, op, val = m.group(1), m.group(2), m.group(3).strip()
+        if op == "?=" and name in raw:
+            continue
+        raw[name] = (raw.get(name, "") + " " + val).strip() if op == "+=" else val
+
+    def expand(v: str, depth=0) -> str:
+        if depth > 10:
+            return v
+        return re.sub(r"\$[({]([A-Za-z_][A-Za-z0-9_]*)[)}]", lambda m: expand(raw.get(m.group(1), os.environ.get(m.group(1), "")), depth + 1), v)
+
+    return {k: expand(v) for k, v in raw.items()}
+
+
+def _makefiles(root: str) -> list:
+    w = paths.walk(root, lambda n: n in ("Makefile", "makefile", "GNUmakefile") or n.endswith(".mk"))
+    return w.files
+
+
+def toolchain(root: str, cfg: dict) -> Toolchain:
+    pc = cfg.get("probe") if isinstance(cfg.get("probe"), dict) else {}
+    if pc.get("cc"):
+        flags = pc.get("flags") or []
+        flags = shlex.split(flags) if isinstance(flags, str) else list(flags)
+        return Toolchain(shlex.split(pc["cc"]), flags, ".hydron/autosar.json probe")
+    build = cfg.get("build")
+    cmd = build.get("command") if isinstance(build, dict) else build if isinstance(build, str) else ""
+    mk = []
+    m = re.search(r"make\b.*?-C\s+(\S+)", cmd or "")
+    if m and os.path.isfile(os.path.join(root, m.group(1), "Makefile")):
+        mk = [os.path.join(root, m.group(1), "Makefile")]
+    mk = mk or _makefiles(root)
+    for f in mk:
+        v = _make_vars(f)
+        cflags = " ".join(v.get(k, "") for k in ("CPPFLAGS", "CFLAGS"))
+        if not cflags.strip() and not v.get("CC"):
+            continue
+        base = os.path.dirname(f)
+        flags = []
+        toks = shlex.split(cflags) if cflags else []
+        i = 0
+        while i < len(toks):
+            t = toks[i]
+            if t in ("-I", "-isystem", "-include", "-D") and i + 1 < len(toks):
+                t = t + toks[i + 1] if t in ("-I", "-D") else t + " " + toks[i + 1]
+                i += 1
+            if KEEP_FLAG.match(t) and "error" not in t:
+                if t.startswith("-I") and not os.path.isabs(t[2:]):
+                    t = "-I" + os.path.normpath(os.path.join(base, t[2:]))
+                flags.extend(t.split(" ", 1) if t.startswith(("-isystem ", "-include ")) else [t])
+            i += 1
+        cc = shlex.split(v.get("CC") or "cc")
+        if not shutil.which(cc[0]):
+            cc = ["cc"]
+        return Toolchain(cc, flags, os.path.relpath(f, root))
+    inc = sorted({os.path.dirname(f) for f in paths.walk(root, lambda n: n.lower().endswith(".h")).files})
+    return Toolchain(["cc"], [f"-I{d}" for d in inc], "host cc with discovered header dirs")
+
+
+@dataclass
+class Decl:
+    symbol: str
+    type: str
+    text: str  # the declaration, without initializer
+    file: str
+    line: int
+    header: Optional[str]  # a header declaring the symbol (extern) or defining its type
+
+
+def find_decl(symbol: str, cfiles: list, headers: list) -> Optional[Decl]:
+    rx = re.compile(rf"^[ \t]*((?:(?:static|extern|volatile|const)\s+)*)([A-Za-z_]\w*(?:\s*\*)?)\s+{re.escape(symbol)}\s*(\[[^\]]*\])?\s*(=|;)", re.M)
+    best = None
+    for f in headers + cfiles:
+        try:
+            text = open(f, encoding="utf-8", errors="replace").read()
+        except OSError:
+            continue
+        m = rx.search(text)
+        if not m or m.group(2) in ("return", "VAR", "CONST", "P2VAR"):
+            continue
+        line = text.count("\n", 0, m.start()) + 1
+        decl = f"extern {m.group(2)} {symbol}{m.group(3) or ''};"
+        d = Decl(symbol, m.group(2).strip(), decl, f, line, f if f.endswith(".h") else None)
+        if d.header:
+            return d
+        best = best or d
+    return best
+
+
+def type_header(type_name: str, headers: list) -> Optional[str]:
+    rx = re.compile(rf"(\}}\s*{re.escape(type_name)}\s*;|typedef\s+[^;]*\b{re.escape(type_name)}\s*;)")
+    for f in headers:
+        try:
+            if rx.search(open(f, encoding="utf-8", errors="replace").read()):
+                return f
+        except OSError:
+            continue
+    return None
+
+
+def sizes(root: str, cfg: dict, decls: dict, timeout: int = 60) -> tuple:
+    """decls: key -> Decl. Returns ({key: size}, notes, toolchain). Missing keys = not determined."""
+    tc = toolchain(root, cfg)
+    if not decls:
+        return {}, [], tc
+    cc = tc.cc
+    if not shutil.which(cc[0]):
+        return {}, [f"no C compiler '{cc[0]}' on PATH; sizeof probe skipped (set .hydron/autosar.json probe.cc)"], tc
+    headers = sorted({d.header for d in decls.values() if d.header})
+    head = [f'#include "{h}"' for h in headers]
+    keys = list(decls)
+    body = []
+    line_of: dict = {}
+    for i, k in enumerate(keys):
+        d = decls[k]
+        if not d.header:
+            body.append(d.text)  # declare it ourselves (type comes from the included headers)
+        body.append(f"char (*h2l_probe_{i})[sizeof({d.symbol})] = 1;")
+        line_of[i] = len(head) + len(body)
+    src = "\n".join(head + body) + "\n"
+    with tempfile.TemporaryDirectory(prefix="h2l-probe-") as tmp:
+        f = os.path.join(tmp, "h2l_probe.c")
+        open(f, "w").write(src)
+        kind = _kind(cc)
+        syntax = ["-fsyntax-only"] if kind else ["-c", "-o", os.path.join(tmp, "p.o")]
+        # never -w: on gcc <= 13 the probe diagnostic is a warning and -w would swallow it
+        syntax += {"clang": ["-ferror-limit=0", "-Wint-conversion"], "gcc": ["-fmax-errors=0", "-Wint-conversion"]}.get(kind, [])
+        env = dict(os.environ, LC_ALL="C", LANG="C")  # ASCII quotes, untranslated messages
+        try:
+            r = subprocess.run(cc + tc.flags + syntax + [f], capture_output=True, text=True, timeout=timeout, cwd=root, env=env)
+        except (OSError, subprocess.TimeoutExpired) as e:
+            return {}, [f"sizeof probe could not run {cc[0]}: {e}"], tc
+        out = r.stdout + r.stderr
+        found = parse_probe_output(out, {line_of[i]: k for i, k in enumerate(keys)})
+        notes = []
+        fatal = [ln for ln in out.splitlines() if re.search(r"fatal error|error:.*(not found|No such file|unknown type|undeclared)", ln)]
+        if fatal and not found:
+            notes.append("sizeof probe did not compile with " + " ".join(cc) + f" ({tc.origin}): " + "; ".join(fatal[:3]))
+        missing = [k for k in keys if k not in found]
+        if missing and not (fatal and not found):
+            notes.append(f"compiler output did not reveal sizes for {len(missing)} object(s)"
+                         + (f" ({'; '.join(fatal[:2])})" if fatal else "")
+                         + "; use .hydron/autosar.json probe.cc with gcc/clang-compatible diagnostics")
+        return found, notes, tc
+
+
+PROBE_DIAG = re.compile(r"h2l_probe\.c:(\d+):(?:\d+:)?[^\n]*?char \(\*\)\s*\[(\d+)[uUlL]*\]")
+
+
+def parse_probe_output(out: str, key_of_line: dict) -> dict:
+    """{key: size} from compiler output; key_of_line maps the probe source line to its key.
+
+    gcc:   h2l_probe.c:2:40: warning: initialization of ‘char (*)[8]’ from ‘int’ makes pointer ... [-Wint-conversion]
+    gcc14: h2l_probe.c:2:40: error: initialization of 'char (*)[8]' from 'int' makes pointer ... [-Wint-conversion]
+    clang: h2l_probe.c:2:8: error: incompatible integer to pointer conversion initializing 'char (*)[8]' with ...
+    """
+    found: dict = {}
+    for m in PROBE_DIAG.finditer(out):
+        k = key_of_line.get(int(m.group(1)))
+        if k is not None and k not in found:
+            found[k] = int(m.group(2))
+    return found
+
+
+def _kind(cc: list) -> str:
+    """'clang' | 'gcc' | '' (other compiler) from `cc --version`."""
+    try:
+        v = subprocess.run(cc + ["--version"], capture_output=True, text=True, timeout=10).stdout.lower()
+    except (OSError, subprocess.TimeoutExpired):
+        return ""
+    return "clang" if "clang" in v else "gcc" if ("gcc" in v or "free software foundation" in v) else ""
+
+
+def load_cfg(root: str) -> dict:
+    try:
+        with open(os.path.join(root, ".hydron", "autosar.json")) as fh:
+            d = json.load(fh)
+            return d if isinstance(d, dict) else {}
+    except (OSError, ValueError):
+        return {}