@h2loop/autosar 1.0.1 → 1.2.0

package/python/ecuc_engine/cli.py

@@ -1,4 +1,5 @@
-"""CLI: python -m ecuc_engine [--root DIR] <info|read|edit|validate|diff> ... --json
+"""CLI: python -m ecuc_engine [--root DIR] <info|read|edit|validate|diff|audit|codegen|codegen-diff|rte-api|verify-rte|
+                                          build-context> ... --json
 
 Always prints one JSON object on stdout. {"ok": false, "error": ...} on failure (exit code 0 for
 handled errors, so callers parse stdout rather than the status).
@@ -49,16 +50,73 @@ def main(argv=None) -> int:
     v = sub.add_parser("validate")
     v.add_argument("--file", action="append")
     v.add_argument("--module")
+    v.add_argument("--check-generated", dest="check_generated", action="store_true", default=None,
+                   help="regenerate in a scratch copy and report stale generated files (H2L-GEN-001)")
+    v.add_argument("--no-check-generated", dest="check_generated", action="store_false")
+    v.add_argument("--no-code", dest="code", action="store_false", help="skip the C-usage completeness check")
     v.add_argument("--max", type=int, default=0, help="cap the number of diagnostics returned (0 = all)")
     au = sub.add_parser("audit")
     au.add_argument("paths", nargs="*", help="C files/dirs to scan (default: audit.paths in .hydron/autosar.json, else Source/)")
     au.add_argument("--headers", action="append", help="generated header dirs for symbols and RTE macros (default Output/)")
+    asg = sub.add_parser("add-signal")
+    asg.add_argument("--name", required=True)
+    asg.add_argument("--ipdu", required=True)
+    asg.add_argument("--bit-size", dest="bit_size", type=int)
+    asg.add_argument("--type", dest="signal_type", default="UINT8")
+    asg.add_argument("--bit-position", dest="bit_position", type=int)
+    asg.add_argument("--endianness", default="LITTLE_ENDIAN")
+    asg.add_argument("--init", default="0")
+    asg.add_argument("--transfer-property", dest="transfer_property")
+    asg.add_argument("--dry-run", action="store_true")
+    ae = sub.add_parser("arxml-edit")
+    ae.add_argument("--ops", required=True)
+    ae.add_argument("--dry-run", action="store_true")
+    fd = sub.add_parser("find")
+    fd.add_argument("--module")
+    fd.add_argument("--container-def", dest="container_def")
+    fd.add_argument("--where", help='"Param=Value, Other>3" or a JSON object')
+    fd.add_argument("--ref-to", dest="ref_to")
+    fd.add_argument("--referenced-by", dest="referenced_by")
+    fd.add_argument("--limit", type=int, default=50)
+    cx = sub.add_parser("crosscheck")
+    cx.add_argument("module")
     so = sub.add_parser("signal-owners")
     so.add_argument("paths", nargs="+")
+    vs = sub.add_parser("value-sites", help="per AUTOSAR path: each value with its VALUE element file:line, dec/hex")
+    vs.add_argument("paths", nargs="+")
+    si = sub.add_parser("spec-ids", help="check requirement IDs against the configured spec text docs")
+    si.add_argument("ids", nargs="+")
+    si.add_argument("--docs", action="append", help="spec text file/dir/glob (default: specDocs / $AUTOSAR_SPEC_DOCS)")
     d = sub.add_parser("diff")
     d.add_argument("a")
     d.add_argument("b")
-    for p in (r, e, v, d, au, so):  # allow --json after the subcommand too
+    cg = sub.add_parser("codegen", help="deterministic config code generation")
+    cg.add_argument("--module", action="append", help="module to generate (repeatable; default: all registered ones configured)")
+    cg.add_argument("--out", help="output dir (default: .hydron/codegen/out scratch; 'project' = codegen.outDir, else Output)")
+    cg.add_argument("--no-write-back", action="store_true", help="do not write assigned handle IDs into the ECUC ARXML")
+    cg.add_argument("--binding", help="static-code binding name ('none' = generic tables only)")
+    cd = sub.add_parser("codegen-diff", help="semantic diff of generated C against a reference dir")
+    cd.add_argument("reference")
+    cd.add_argument("generated", nargs="?", help="default: the codegen scratch dir")
+    cd.add_argument("--module", action="append")
+    cd.add_argument("--max", type=int, default=50, help="max element mismatches listed per object")
+    ra = sub.add_parser("rte-api", help="expected RTE API per SW-C / runnable (SWS_RTE naming and signatures)")
+    ra.add_argument("--swc")
+    ra.add_argument("--runnable")
+    ra.add_argument("--no-header", action="store_true", help="skip matching against Rte_<Swc>.h")
+    vr = sub.add_parser("verify-rte", help="classify Rte_* calls in C per runnable: OK / NOT-ALLOWED / INVENTED")
+    vr.add_argument("paths", nargs="*", help="C files/dirs/globs (default: all in-house C)")
+    vr.add_argument("--diff-against", dest="diff_against", help="only calls on lines changed since this git ref")
+    vr.add_argument("--swc")
+    vr.add_argument("--runnable")
+    vr.add_argument("--no-check-ops", dest="check_ops", action="store_false")
+    bc = sub.add_parser("build-context", help="deterministic context for implementing/changing a runnable")
+    bc.add_argument("--swc", required=True)
+    bc.add_argument("--runnable")
+    bc.add_argument("--task", required=True, help="task text, @file, or - for stdin")
+    bc.add_argument("--budget-tokens", dest="budget_tokens", type=int, default=24000)
+    bc.add_argument("--no-history", dest="history", action="store_false")
+    for p in (r, e, v, d, au, so, vs, si, fd, cx, ae, asg, cg, cd, ra, vr, bc):  # allow --json after the subcommand too
         p.add_argument("--json", action="store_true", help=argparse.SUPPRESS)
     a = ap.parse_args(argv)
     t0 = time.time()
@@ -74,7 +132,7 @@ def main(argv=None) -> int:
 
 
 def run(a) -> dict:
-    if a.cmd == "edit":
+    if a.cmd in ("edit", "arxml-edit"):
         try:
             a.ops = _ops(a.ops)
         except (ValueError, OSError) as ex:
@@ -87,8 +145,70 @@ def run(a) -> dict:
             return diff(a.a, a.b, rel, root)
         except FileNotFoundError as ex:
             return {"error": str(ex)}
+    if a.cmd == "codegen-diff":
+        from .codegen import SCRATCH
+        from .codegen.cdiff import diff_dirs
+        root = os.path.abspath(a.root)
+        ab = lambda p: p if os.path.isabs(p) else os.path.join(root, p)  # noqa: E731
+        ref, gen = ab(a.reference), ab(a.generated or SCRATCH)
+        for p in (ref, gen):
+            if not os.path.exists(p):
+                return {"error": f"{p} does not exist" + (" (run codegen first)" if p == gen else "")}
+        res = diff_dirs(ref, gen, modules=a.module, max_items=a.max)
+        res.update({"reference": ref, "generated": gen})
+        return res
+    if a.cmd == "rte-api":
+        from .rte_api import rte_api
+        return rte_api(os.path.abspath(a.root), swc=a.swc, runnable=a.runnable, header=not a.no_header)
+    if a.cmd == "verify-rte":
+        from .verify_rte import verify
+        return verify(os.path.abspath(a.root), files=a.paths or None, diff_against=a.diff_against, swc=a.swc,
+                      runnable=a.runnable, check_ops=a.check_ops)
+    if a.cmd == "build-context":
+        from .context import build_context
+        task = a.task
+        if task == "-":
+            task = sys.stdin.read()
+        elif task.startswith("@"):
+            with open(task[1:]) as fh:
+                task = fh.read()
+        return build_context(os.path.abspath(a.root), a.swc, task, runnable=a.runnable,
+                             budget_tokens=a.budget_tokens, history=a.history)
+    if a.cmd == "spec-ids":
+        from .specids import check_ids
+        return check_ids(os.path.abspath(a.root), a.ids, docs=a.docs)
     from .model import Project
-    project = Project(a.root, config_files=a.config, def_files=a.defs, load_input=a.cmd in ("validate", "edit"))
+    project = Project(a.root, config_files=a.config, def_files=a.defs, load_input=a.cmd in ("validate", "edit", "find", "crosscheck", "signal-owners", "arxml-edit", "add-signal"))
+    if a.cmd == "add-signal":
+        from .add_signal import AddSignalError, add_signal
+        try:
+            return add_signal(project, a.name, a.ipdu, bit_size=a.bit_size, signal_type=a.signal_type,
+                              bit_position=a.bit_position, endianness=a.endianness, init=a.init,
+                              transfer_property=a.transfer_property, dry_run=a.dry_run)
+        except AddSignalError as ex:
+            return {"error": str(ex), "written": False}
+    if a.cmd == "arxml-edit":
+        from .arxml_edit import arxml_edit
+        return arxml_edit(project, a.ops, dry_run=a.dry_run)
+    if a.cmd == "find":
+        from .find import find
+        where = a.where
+        if where and where.strip().startswith("{"):
+            where = json.loads(where)
+        return find(project, module=a.module, container_def=a.container_def, where=where, ref_to=a.ref_to,
+                    referenced_by=a.referenced_by, limit=a.limit)
+    if a.cmd == "crosscheck":
+        from .find import crosscheck
+        return crosscheck(project, a.module)
+    if a.cmd == "codegen":
+        from .codegen import generate
+        res = generate(project, modules=a.module, out_dir=a.out, write_back=not a.no_write_back, binding=a.binding)
+        if project.warnings:
+            res["warnings"] = project.warnings
+        return res
+    if a.cmd == "value-sites":
+        from .ecucjson import value_sites
+        return {"sites": value_sites(project, a.paths)}
     if a.cmd == "signal-owners":
         from .audit import signal_owners
         return {"owners": signal_owners(project, a.paths)}
@@ -115,7 +235,7 @@ def run(a) -> dict:
         return res
     if a.cmd == "validate":
         from .validate import validate
-        res = validate(project, files=a.file, module=a.module)
+        res = validate(project, files=a.file, module=a.module, check_generated=a.check_generated, code=a.code)
         if a.max and len(res["diagnostics"]) > a.max:
             res["truncated"] = len(res["diagnostics"]) - a.max
             res["diagnostics"] = res["diagnostics"][: a.max]

package/python/ecuc_engine/codegen/__init__.py

@@ -0,0 +1,233 @@
+"""Deterministic ECUC -> C configuration code generation.
+
+Same ECUC ARXML + same param defs -> byte-identical C. No LLM is involved: every line comes from
+the param defs (generic emitter), a module generator (codegen/modules/, only for what the defs
+cannot express), or a static-code binding (codegen/bindings/*.json, the vendor struct layout).
+
+Entry point: generate(project, modules=None, out_dir=None, write_back=True).
+Project options (.hydron/autosar.json, key "codegen"):
+  {"outDir": "Output",             # the project's generated-code folder (used when out_dir="project")
+   "binding": "sip-standin",       # force a static-code binding ("none" = generic tables only)
+   "mip": {"Eep": "Eep_21_LDExt"}, # module implementation prefixes (vendorId/vendorApiInfix)
+   "memmap": {"pragmas": {"CODE": {"start": "#pragma ...", "stop": "#pragma ..."}}},
+   "mcal": {"silicon": "tc4d", "pinMap": "mcal/pins.json"}}
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import time
+from typing import Optional
+
+from .. import paths as P
+from ..model import PROJECT_CONFIG, Project
+from . import binding as B
+from . import handles as H
+from . import schm_memmap as S
+from .emit import Ctx
+from .model import CgModel
+from .registry import generator_for, registered
+
+SCRATCH = os.path.join(".hydron", "codegen", "out")
+
+
+def project_options(root: str) -> dict:
+    f = os.path.join(root, PROJECT_CONFIG)
+    if not os.path.isfile(f):
+        return {}
+    try:
+        with open(f) as fh:
+            return json.load(fh).get("codegen", {}) or {}
+    except (OSError, ValueError):
+        return {}
+
+
+def _scan_c(root: str, exclude: list) -> tuple:
+    """(headers basename->path, source files) of the project's static/in-house C, excluding generator output."""
+    from ..audit import is_generated
+    ex = [os.path.realpath(e) + os.sep for e in exclude if e]
+    w = P.walk(root, lambda n: n.endswith((".h", ".c")))
+    headers: dict = {}
+    sources: list = []
+    for f in w.files:
+        rf = os.path.realpath(f)
+        if any(rf.startswith(e) for e in ex):
+            continue
+        if f.endswith(".h"):
+            headers.setdefault(os.path.basename(f), f)
+        if not is_generated(f):
+            sources.append(f)
+    return headers, sources
+
+
+def resolve_out(root: str, out_dir: Optional[str], opts: dict) -> str:
+    if not out_dir:
+        return os.path.join(root, SCRATCH)
+    if out_dir == "project":
+        out_dir = opts.get("outDir", "Output")
+    return out_dir if os.path.isabs(out_dir) else os.path.join(root, out_dir)
+
+
+def generate(project: Project, modules: Optional[list] = None, out_dir: Optional[str] = None,
+             write_back: bool = True, binding: Optional[str] = None) -> dict:
+    t0 = time.time()
+    root = project.root
+    opts = project_options(root)
+    out = resolve_out(root, out_dir, opts)
+    binding = binding or opts.get("binding")
+    timings: dict = {}
+
+    cg = CgModel(project, opts.get("mip"))
+    configured = sorted(cg.modules)
+    if modules:
+        want = []
+        for m in modules:
+            hit = [x for x in configured if x.lower() == m.lower()]
+            if not hit:
+                return {"ok": False, "error": f"module {m!r} is not configured (configured: {', '.join(configured)})"}
+            want.append(hit[0])
+    else:
+        want = [m for m in registered() if m in cg.modules]
+    if not project.defs:
+        return {"ok": False, "error": "code generation needs the ECUC param defs (paramDefs in .hydron/autosar.json "
+                                      "or $AUTOSAR_PARAMDEFS): config classes and types come from them"}
+    timings["load_ms"] = int((time.time() - t0) * 1000)
+
+    # 1. handle IDs (TPS_ECUC_02107): assign, write back (or patch in memory only)
+    t1 = time.time()
+    pl = H.plan(cg, want)
+    if pl["errors"]:
+        return {"ok": False, "error": "handle-ID conflicts; fix the configuration first", "handleErrors": pl["errors"]}
+    wb = H.apply(project, pl["assign"], write=write_back)
+    if pl["assign"]:
+        cg = CgModel(project, opts.get("mip"))
+    # handle layout the static code indexes by (binding "handleLayout"), only for withAuto handle params
+    proj_out = resolve_out(root, "project", opts)
+    headers, sources = _scan_c(root, [out, proj_out, os.path.join(root, ".hydron")])
+    bindings = B.load_all(root)
+    lay = H.layout(cg, want, {m: B.select(bindings, m, headers, binding) for m in want})
+    if lay["errors"]:
+        return {"ok": False, "error": "handle IDs do not fit the static code's table layout", "handleErrors": lay["errors"]}
+    if lay["assign"]:
+        wb2 = H.apply(project, lay["assign"], write=write_back)
+        wb = {"written": wb["written"] or wb2["written"], "changes": wb["changes"] + wb2["changes"]}
+        pl["assign"] += lay["assign"]
+        cg = CgModel(project, opts.get("mip"))
+    pl["notes"] += lay["notes"]
+    timings["handles_ms"] = int((time.time() - t1) * 1000)
+
+    # 2. emit
+    t2 = time.time()
+    ctx = Ctx(cg=cg, headers=headers, bindings=bindings, binding_name=binding, release=cg.release(),
+              sources=sources)
+    ctx.options = opts
+    ctx.generating = set(want)
+    ctx.available = set(headers) | (set(os.listdir(proj_out)) if os.path.isdir(proj_out) else set())
+    ctx.out_headers = {f: os.path.join(proj_out, f) for f in sorted(os.listdir(proj_out))
+                       if f.endswith(".h")} if os.path.isdir(proj_out) else {}
+    files: dict = {}
+    report: dict = {}
+    memmaps = []
+    notes = list(pl["notes"])
+    errors: dict = {}
+    outputs = []
+    for name in want:
+        mc = cg.module(name)
+        gen = generator_for(name)(ctx, mc)
+        mo = gen.run()
+        if mo.errors:
+            errors[name] = mo.errors
+            continue
+        outputs.append(mo)
+        rendered = mo.render()
+        n, text = S.schm_header(ctx, mo)
+        rendered[n] = text
+        mm = f"{mo.mip}_MemMap.h"
+        if mm in headers:
+            notes.append(f"{name}: project provides {P_rel(root, headers[mm])}; {mm} not generated")
+        else:
+            n, text, sections = S.module_memmap(ctx, mo, (opts.get("memmap") or {}).get("pragmas", {}))
+            rendered[n] = text
+            memmaps.append((mo.mip, sections))
+        files.update(rendered)
+        report[name] = {
+            "variant": mc.variant,
+            "binding": (mo.binding or {}).get("_binding"),
+            "files": sorted(rendered),
+            "symbols": len(mo.symbols),
+            "notes": mo.notes,
+        }
+    if memmaps and "MemMap.h" not in headers:
+        files["MemMap.h"] = S.aggregate_memmap(ctx, memmaps)
+    elif memmaps:
+        notes.append(f"project MemMap.h ({P_rel(root, headers['MemMap.h'])}) kept; generated code includes "
+                     "<Mip>_MemMap.h (SWS_MemMap_00032)")
+    timings["emit_ms"] = int((time.time() - t2) * 1000)
+    if errors:
+        return {"ok": False, "error": "the configuration cannot be represented by the static code's binding; "
+                                      "nothing was written", "moduleErrors": errors, "notes": notes,
+                "handles": {"assigned": pl["assign"], "writtenBack": wb["written"], "changes": wb["changes"]}}
+    notes += dangling_symbols(ctx, outputs)
+
+    # 3. write
+    os.makedirs(out, exist_ok=True)
+    manifest = []
+    for name in sorted(files):
+        data = files[name].encode("utf-8")
+        path = os.path.join(out, name)
+        with open(path, "wb") as fh:
+            fh.write(data)
+        manifest.append({"file": name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()[:16]})
+    std = _std_types_release(headers)
+    if std and std[:2] != ctx.release[:2]:
+        notes.append(f"Std_Types.h in the project is AUTOSAR {std[0]}.{std[1]}.x but the configuration is "
+                     f"{ctx.release[0]}.{ctx.release[1]}.x (schema); the release check against <Mip>.h may fail")
+    timings["total_ms"] = int((time.time() - t0) * 1000)
+    return {"ok": True, "outDir": out, "modules": report, "files": manifest,
+            "handles": {"assigned": pl["assign"], "writtenBack": wb["written"], "changes": wb["changes"]},
+            "notes": notes, "timings": timings, "release": ".".join(map(str, ctx.release))}
+
+
+def dangling_symbols(ctx, outputs: list) -> list:
+    """Symbolic names (<Mip>Conf_<Def>_<Name>) that the project's C uses but the configuration no longer has.
+    The old generated header may still define them, so the stale use only shows once this output replaces it."""
+    defined = {s for mo in outputs for s, _, _ in mo.symbols}
+    rx = re.compile(r"\b(" + "|".join(sorted(re.escape(mo.mip) for mo in outputs)) + r")Conf_\w+") if outputs else None
+    out = []
+    if rx is None:
+        return out
+    seen: dict = {}
+    for f in ctx.sources:
+        try:
+            with open(f, errors="replace") as fh:
+                text = fh.read()
+        except OSError:
+            continue
+        for m in rx.finditer(text):
+            if m.group(0) not in defined:
+                seen.setdefault(m.group(0), ctx.cg.p.rel(f))
+    for sym in sorted(seen):
+        out.append(f"{seen[sym]} uses {sym}, which the configuration does not define (container deleted or "
+                   "renamed?); the build will fail on it")
+    return out
+
+
+def P_rel(root: str, f: str) -> str:
+    r = os.path.relpath(f, root)
+    return f if r.startswith("..") else r
+
+
+def _std_types_release(headers: dict) -> Optional[tuple]:
+    f = headers.get("Std_Types.h")
+    if not f:
+        return None
+    try:
+        with open(f, errors="replace") as fh:
+            t = fh.read()
+    except OSError:
+        return None
+    m1 = re.search(r"STD_TYPES_AR_RELEASE_MAJOR_VERSION\s+\(?(\d+)", t)
+    m2 = re.search(r"STD_TYPES_AR_RELEASE_MINOR_VERSION\s+\(?(\d+)", t)
+    return (int(m1.group(1)), int(m2.group(1))) if m1 and m2 else None