@h2loop/autosar 0.1.2 → 1.1.0

package/python/ecuc_engine/history.py

@@ -0,0 +1,146 @@
+"""Git-history-lite: "similar past changes" for a SW-C, without embeddings.
+
+Takes ``git log`` (a bounded number of commits) touching the SW-C's ARXML and C files and those of similar
+SW-Cs, summarises each commit as an ARXML delta (the semantic ``arxml_diff`` of the before/after blobs) plus the
+C diff hunks, and ranks by overlap with the task's terms (ports, data elements, words of the task) and recency.
+Everything is bounded: commits scanned, hunks per commit, characters per summary, results returned.
+"""
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import tempfile
+from typing import Optional
+
+MAX_COMMITS = 60
+MAX_HUNK_CHARS = 1500
+MAX_DELTA_ITEMS = 12
+MAX_COMMIT_CHARS = 4000  # all C hunks of one commit together
+MAX_FILES_PER_COMMIT = 6
+
+
+def _git(root: str, *args, timeout: int = 60) -> Optional[str]:
+    try:
+        r = subprocess.run(["git", "-C", root, *args], capture_output=True, timeout=timeout)
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    return r.stdout.decode("utf-8", "replace") if r.returncode == 0 else None
+
+
+def is_repo(root: str) -> bool:
+    return (_git(root, "rev-parse", "--is-inside-work-tree") or "").strip() == "true"
+
+
+def terms_of(text: str) -> set:
+    """Identifier-ish terms, with CamelCase / snake_case parts, lower-cased; short noise words dropped."""
+    out = set()
+    for tok in re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}", text or ""):
+        out.add(tok.lower())
+        for part in re.findall(r"[A-Z]+(?![a-z])|[A-Z]?[a-z]+|\d+", tok):
+            if len(part) >= 3:
+                out.add(part.lower())
+    return out - STOP
+
+
+STOP = {"the", "and", "for", "with", "from", "this", "that", "into", "when", "add", "use", "new", "void", "rte",
+        "std", "returntype", "uint8", "uint16", "uint32", "boolean", "return", "data", "static", "const", "then",
+        "should", "must", "set", "get", "true", "false", "null", "ptr", "null_ptr", "not", "are", "has", "its"}
+
+
+def _blob(root: str, rev: str, rel: str) -> Optional[bytes]:
+    try:
+        r = subprocess.run(["git", "-C", root, "show", f"{rev}:./{rel}"], capture_output=True, timeout=60)
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    return r.stdout if r.returncode == 0 else None
+
+
+def arxml_delta(root: str, commit: str, rel: str) -> dict:
+    """Semantic delta of one ARXML file across ``commit`` (parent -> commit), via diff.diff on temp copies."""
+    from .diff import diff
+    before, after = _blob(root, f"{commit}^", rel), _blob(root, commit, rel)
+    if before is None and after is None:
+        return {"file": rel, "error": "blob not readable"}
+    if before is None or after is None:
+        return {"file": rel, "status": "added" if before is None else "deleted"}
+    with tempfile.TemporaryDirectory(prefix="h2l-hist-") as tmp:
+        a, b = os.path.join(tmp, "a.arxml"), os.path.join(tmp, "b.arxml")
+        open(a, "wb").write(before)
+        open(b, "wb").write(after)
+        try:
+            d = diff(a, b, root=tmp)
+        except Exception as ex:  # noqa: BLE001
+            return {"file": rel, "error": f"arxml_diff failed: {ex}"}
+    items = [f"+ {x['path']} ({x['tag']})" for x in d["added"]] + [f"- {x['path']} ({x['tag']})" for x in d["removed"]]
+    for x in d["changed"]:
+        vals = x.get("values")
+        items.append(f"~ {x['path']}" + (": " + ", ".join(f"{k} {v[0]}->{v[1]}" for k, v in sorted(vals.items()))
+                                          if vals else " (content)"))
+    return {"file": rel, "summary": d["summary"], "items": items[:MAX_DELTA_ITEMS],
+            "more": max(0, len(items) - MAX_DELTA_ITEMS)}
+
+
+def c_hunks(root: str, commit: str, rel: str) -> str:
+    out = _git(root, "show", "--format=", "--no-color", "--no-ext-diff", "-U2", commit, "--", rel) or ""
+    body = "\n".join(ln for ln in out.splitlines() if not ln.startswith(("diff --git", "index ", "--- ", "+++ ")))
+    return body if len(body) <= MAX_HUNK_CHARS else body[:MAX_HUNK_CHARS] + "\n… (hunk truncated)"
+
+
+def similar_changes(root: str, files: list, related: list, task_terms: set, limit: int = 5,
+                    max_commits: int = MAX_COMMITS) -> dict:
+    """files: the SW-C's own ARXML/C files (abs); related: those of similar SW-Cs. -> {changes, scanned, notes}."""
+    if not is_repo(root):
+        return {"changes": [], "scanned": 0, "notes": ["not a git repository; no history"]}
+    rels = sorted({os.path.relpath(f, root) for f in files + related if f and os.path.exists(f)})
+    own = {os.path.relpath(f, root) for f in files if f}
+    if not rels:
+        return {"changes": [], "scanned": 0, "notes": ["no files to look up in history"]}
+    log = _git(root, "log", f"-n{max_commits}", "--format=%x1e%H%x1f%ct%x1f%s", "--name-only", "--", *rels) or ""
+    commits = []
+    for chunk in log.split("\x1e"):
+        if not chunk.strip():
+            continue
+        head, *names = chunk.strip("\n").split("\n")
+        h, ct, subj = (head.split("\x1f") + ["", ""])[:3]
+        commits.append((h, int(ct or 0), subj, [n for n in names if n.strip() in rels]))
+    scored = []
+    for i, (h, ct, subj, touched) in enumerate(commits):
+        text = subj + "\n" + "\n".join(touched)
+        overlap = terms_of(text) & task_terms
+        scored.append((i, h, ct, subj, touched, overlap))
+    # cheap first pass by subject/file overlap + recency; expensive deltas only for the best candidates
+    pre = sorted(scored, key=lambda x: (-(len(x[5]) * 2 + (1 if set(x[4]) & own else 0)), x[0]))[:limit * 3]
+    out = []
+    for i, h, ct, subj, touched, _ov in pre:
+        deltas, hunks = [], []
+        budget = MAX_COMMIT_CHARS
+        # the SW-C's own files first, then the rest, at most MAX_FILES_PER_COMMIT
+        touched = sorted(touched, key=lambda r: (r not in own, r))[:MAX_FILES_PER_COMMIT]
+        for rel in touched:
+            if rel.lower().endswith(".arxml"):
+                deltas.append(arxml_delta(root, h, rel))
+            elif rel.lower().endswith((".c", ".h")) and budget > 0:
+                d = c_hunks(root, h, rel)
+                if len(d) > budget:
+                    d = d[:budget] + "\n… (commit diff truncated)"
+                budget -= len(d)
+                hunks.append({"file": rel, "diff": d})
+        full = subj + "\n" + "\n".join(json_lines(deltas)) + "\n" + "\n".join(x["diff"] for x in hunks)
+        overlap = sorted(terms_of(full) & task_terms)
+        recency = 1.0 / (1 + i)
+        score = round(len(overlap) + recency + (0.5 if set(touched) & own else 0.0), 4)
+        out.append({"commit": h[:12], "subject": subj, "score": score, "overlap": overlap[:20],
+                    "ownFiles": sorted(set(touched) & own), "arxml": deltas, "c": hunks, "_order": i})
+    out.sort(key=lambda x: (-x["score"], x["_order"]))
+    for x in out:
+        x.pop("_order")
+    return {"changes": out[:limit], "scanned": len(commits), "notes": []}
+
+
+def json_lines(deltas: list) -> list:
+    lines = []
+    for d in deltas:
+        lines.append(d.get("file", ""))
+        lines.extend(d.get("items", []))
+    return lines

package/python/ecuc_engine/model.py

@@ -16,9 +16,10 @@ from dataclasses import dataclass, field
 from typing import Optional
 
 from . import defs as D
+from . import paths
 from .xmltree import Node, parse
 
-SKIP_DIRS = {".git", "node_modules", ".hydron", ".venv", "__pycache__", ".cache", ".idea", ".vscode"}
+SKIP_DIRS = paths.TOOLING_DIRS  # kept for importers; discovery lives in paths.py
 PROJECT_CONFIG = os.path.join(".hydron", "autosar.json")
 
 MODULE_VALUES = "ECUC-MODULE-CONFIGURATION-VALUES"
@@ -102,30 +103,28 @@ def _scan_markers(path: str) -> set:
     return found
 
 
-def _walk_arxml(root: str):
-    # Sorted walk: os.walk order is filesystem-dependent (macOS and Linux differ),
-    # and load order decides which definition of a path wins.
-    for dp, dns, fns in os.walk(root):
-        dns[:] = sorted(d for d in dns if d not in SKIP_DIRS and not d.startswith("."))
-        for fn in sorted(fns):
-            if fn.lower().endswith(".arxml"):
-                yield os.path.join(dp, fn)
+def _walk_arxml(root: str, notes: Optional[list] = None):
+    """Every ARXML file under root (any depth/folder name/extension case); sorted, deterministic."""
+    w = paths.walk(root, paths.is_arxml)
+    if notes is not None:
+        notes.extend(w.notes(root))
+    return w.files
 
 
-def _globs(root: str, pats) -> list:
+def _globs(root: str, pats, warnings: Optional[list] = None, what: str = "") -> list:
+    """Files/dirs/globs (relative or absolute, trailing slash optional) -> ARXML files on disk."""
+    pats = [pats] if isinstance(pats, str) else list(pats or [])
     out = []
-    for p in pats or []:
-        p = os.path.expanduser(p)
-        full = p if os.path.isabs(p) else os.path.join(root, p)
-        if os.path.isdir(full):
-            out.extend(_walk_arxml(full))
-        else:
-            out.extend(sorted(glob.glob(full, recursive=True)))
+    for p in paths.patterns(root, pats):
+        got = [f for f in p.expand(paths.is_arxml) if paths.is_arxml(f)]
+        if not got and warnings is not None:
+            warnings.append(f"{what or 'path'} {p.raw!r} matched no .arxml file under {root}")
+        out.extend(got)
     seen, res = set(), []
     for f in out:
-        a = os.path.abspath(f)
-        if a not in seen and os.path.isfile(a):
-            seen.add(a)
+        a, real = os.path.abspath(f), os.path.realpath(f)
+        if real not in seen and os.path.isfile(a):
+            seen.add(real)
             res.append(a)
     return res
 
@@ -138,6 +137,7 @@ class Sources:
     input: list
     warnings: list
     origin: dict  # which mechanism found each set
+    notes: list = field(default_factory=list)  # info: hidden dirs included, cycles, caps
 
 
 def discover(root: str, config_files: Optional[list] = None, def_files: Optional[list] = None) -> Sources:
@@ -153,30 +153,31 @@ def discover(root: str, config_files: Optional[list] = None, def_files: Optional
             warnings.append(f"{PROJECT_CONFIG} is not valid JSON ({e}); ignoring it")
     origin = {}
     scanned = None
+    notes: list = []
 
     def scan():
         nonlocal scanned
         if scanned is None:
-            scanned = {f: _scan_markers(f) for f in _walk_arxml(root)}
+            scanned = {f: _scan_markers(f) for f in _walk_arxml(root, notes)}
         return scanned
 
     if def_files:
-        dfiles, origin["defs"] = _globs(root, def_files), "argument"
+        dfiles, origin["defs"] = _globs(root, def_files, warnings, "--defs"), "argument"
     elif cfg.get("paramDefs"):
-        dfiles, origin["defs"] = _globs(root, cfg["paramDefs"]), PROJECT_CONFIG
+        dfiles, origin["defs"] = _globs(root, cfg["paramDefs"], warnings, "paramDefs"), PROJECT_CONFIG
     else:
         dfiles, origin["defs"] = [f for f, m in scan().items() if "defs" in m], "auto-discovered"
         if not dfiles and os.environ.get("AUTOSAR_PARAMDEFS"):
-            dfiles = _globs(root, os.environ["AUTOSAR_PARAMDEFS"].split(os.pathsep))
+            dfiles = _globs(root, os.environ["AUTOSAR_PARAMDEFS"].split(os.pathsep), warnings, "AUTOSAR_PARAMDEFS")
             origin["defs"] = "AUTOSAR_PARAMDEFS"
     if config_files:
-        cfiles, origin["config"] = _globs(root, config_files), "argument"
+        cfiles, origin["config"] = _globs(root, config_files, warnings, "--config"), "argument"
     elif cfg.get("config"):
-        cfiles, origin["config"] = _globs(root, cfg["config"]), PROJECT_CONFIG
+        cfiles, origin["config"] = _globs(root, cfg["config"], warnings, "config"), PROJECT_CONFIG
     else:
         cfiles, origin["config"] = [f for f, m in scan().items() if "config" in m], "auto-discovered"
     if cfg.get("input"):
-        ifiles, origin["input"] = _globs(root, cfg["input"]), PROJECT_CONFIG
+        ifiles, origin["input"] = _globs(root, cfg["input"], warnings, "input"), PROJECT_CONFIG
     else:
         known = set(cfiles) | set(dfiles)
         ifiles, origin["input"] = [f for f in scan() if f not in known], "auto-discovered"
@@ -185,7 +186,7 @@ def discover(root: str, config_files: Optional[list] = None, def_files: Optional
             "No ECUC param definitions found (no ECUC-MODULE-DEF in the project, no paramDefs in "
             f"{PROJECT_CONFIG}, AUTOSAR_PARAMDEFS unset). Values are readable but NOT validated, and edits "
             "cannot add new elements. Point paramDefs at the AUTOSAR / vendor BSWMD param-def ARXML.")
-    return Sources(root, cfiles, dfiles, ifiles, warnings, origin)
+    return Sources(root, cfiles, dfiles, ifiles, warnings, origin, notes)
 
 
 def _dest(n: Optional[Node]) -> Optional[str]:
@@ -282,9 +283,12 @@ class Project:
             self._load(f)
         self.foreign: dict = {}
         self.input_loaded = False
+        from .sysmodel import SystemIndex
+        self.system = SystemIndex()
         if load_input:
             for f in self.sources.input:
-                self.foreign.update(index_identifiables(f))
+                self.system.add_file(f)
+            self.foreign = self.system.foreign()
             self.input_loaded = bool(self.sources.input)
         self.reindex()
 

package/python/ecuc_engine/paths.py

@@ -0,0 +1,221 @@
+"""Layout-agnostic file discovery and path filters.
+
+Nothing here assumes Config/, Input/, Source/ or Output/:
+- walk() finds files at any depth, skips only tooling dirs, includes other hidden dirs (and says so),
+  follows symlinked dirs once with cycle protection, and stops at a cap with a clear message.
+- Pattern() matches a user path argument against files: an exact file, a directory (= path prefix,
+  with or without trailing slash), a glob with ** (relative to the root, or anywhere when it has no
+  slash), relative or absolute.
+"""
+from __future__ import annotations
+
+import fnmatch
+import os
+import re
+from dataclasses import dataclass, field
+from typing import Callable, Iterable, Optional
+
+TOOLING_DIRS = {".git", ".hg", ".svn", ".hydron", ".venv", "venv", "node_modules", "__pycache__", ".cache",
+                ".idea", ".vscode", ".pytest_cache", ".mypy_cache", ".tox"}
+MAX_FILES = int(os.environ.get("AUTOSAR_MAX_FILES", "200000"))
+GLOB_CHARS = set("*?[")
+
+
+def is_arxml(name: str) -> bool:
+    return name.lower().endswith(".arxml")  # .arxml, .ARXML, .ecuc.arxml, ...
+
+
+def is_c(name: str) -> bool:
+    return name.lower().endswith((".c", ".h"))
+
+
+@dataclass
+class Walk:
+    files: list = field(default_factory=list)
+    hidden: list = field(default_factory=list)  # hidden dirs that contributed files
+    cycles: list = field(default_factory=list)  # symlinked dirs skipped as already visited
+    truncated: bool = False
+    visited_dirs: int = 0
+
+    def notes(self, root: str) -> list:
+        out = []
+        for d in self.hidden:
+            out.append(f"included hidden directory {os.path.relpath(d, root)}/ (it contains matching files)")
+        for d in self.cycles[:5]:
+            out.append(f"symlinked directory {os.path.relpath(d, root)} skipped: already visited (cycle)")
+        if self.truncated:
+            out.append(f"file walk stopped at {MAX_FILES} files; narrow the scan with .hydron/autosar.json globs "
+                       "or raise AUTOSAR_MAX_FILES")
+        return out
+
+
+def walk(root: str, want: Callable[[str], bool], cap: int = 0) -> Walk:
+    """All files under ``root`` (any depth) for which ``want(name)`` is true, sorted per directory."""
+    cap = cap or MAX_FILES
+    res = Walk()
+    seen: set = set()
+    hidden_hits: dict = {}
+
+    def visit(d: str, hidden_anc: Optional[str]):
+        real = os.path.realpath(d)
+        if real in seen:
+            res.cycles.append(d)
+            return
+        seen.add(real)
+        res.visited_dirs += 1
+        try:
+            entries = sorted(os.scandir(d), key=lambda e: e.name)
+        except OSError:
+            return
+        subdirs = []
+        for e in entries:
+            if res.truncated:
+                return
+            try:
+                if e.is_dir(follow_symlinks=True):
+                    if e.name not in TOOLING_DIRS:
+                        subdirs.append(e)
+                elif e.is_file(follow_symlinks=True) and want(e.name):
+                    res.files.append(e.path)
+                    if hidden_anc:
+                        hidden_hits.setdefault(hidden_anc, True)
+                    if len(res.files) >= cap:
+                        res.truncated = True
+                        return
+            except OSError:
+                continue
+        for e in subdirs:
+            anc = hidden_anc or (e.path if e.name.startswith(".") else None)
+            visit(e.path, anc)
+
+    if os.path.isdir(root):
+        visit(root, None)
+    res.hidden = sorted(hidden_hits)
+    return res
+
+
+def _glob_regex(pat: str) -> re.Pattern:
+    """Glob with ** (any number of dirs), * and ? (within a segment), [..] classes."""
+    out, i = [], 0
+    while i < len(pat):
+        if pat.startswith("**/", i):
+            out.append("(?:.*/)?")
+            i += 3
+        elif pat.startswith("**", i):
+            out.append(".*")
+            i += 2
+        elif pat[i] == "*":
+            out.append("[^/]*")
+            i += 1
+        elif pat[i] == "?":
+            out.append("[^/]")
+            i += 1
+        elif pat[i] == "[":
+            j = pat.find("]", i)
+            if j < 0:
+                out.append(re.escape(pat[i]))
+                i += 1
+            else:
+                out.append(fnmatch.translate(pat[i:j + 1])[4:-3])
+                i = j + 1
+        else:
+            out.append(re.escape(pat[i]))
+            i += 1
+    return re.compile("".join(out) + r"\Z", re.IGNORECASE)  # *.ARXML == *.arxml
+
+
+def _norm(p: str) -> str:
+    return p.replace("\\", "/")
+
+
+class Pattern:
+    """One user-supplied path argument, resolved against ``root``."""
+
+    def __init__(self, root: str, raw: str):
+        self.raw = raw
+        p = os.path.expanduser(str(raw).strip())
+        stripped = p.rstrip("/\\") or p
+        self.is_glob = any(ch in GLOB_CHARS for ch in stripped)
+        full = stripped if os.path.isabs(stripped) else os.path.join(root, stripped)
+        self.abs = os.path.normpath(full)
+        self.real = os.path.realpath(self.abs)
+        self.root = os.path.realpath(root)
+        if self.is_glob:
+            self.kind = "glob"
+            self.rx = _glob_regex(_norm(stripped))
+            self.anywhere = "/" not in _norm(stripped).lstrip("./")  # "*.arxml" or "Com*.arxml": match basenames
+        elif os.path.isdir(self.abs):
+            self.kind = "dir"
+        elif os.path.isfile(self.abs):
+            self.kind = "file"
+        else:
+            self.kind = "missing"
+
+    def matches(self, path: str) -> bool:
+        a = os.path.normpath(os.path.abspath(path))
+        r = os.path.realpath(a)
+        if self.kind == "file":
+            return a == self.abs or r == self.real
+        if self.kind == "dir":
+            return any(x.startswith(base.rstrip(os.sep) + os.sep) for x in (a, r) for base in (self.abs, self.real))
+        if self.kind == "glob":
+            rel = _norm(os.path.relpath(r, self.root))
+            cands = [rel, _norm(a), _norm(r), os.path.basename(a)] if self.anywhere else [rel, _norm(a), _norm(r)]
+            return any(self.rx.match(c) for c in cands) or (self.anywhere and any(self.rx.match(c.split("/")[-1]) for c in cands))
+        # missing on disk: maybe a path relative to root with odd separators, or a basename
+        name = _norm(str(self.raw)).strip("/")
+        rel = _norm(os.path.relpath(r, self.root))
+        return rel == name or rel.startswith(name + "/") or os.path.basename(a) == name
+
+    def expand(self, want: Callable[[str], bool]) -> list:
+        """Files on disk this pattern names (dirs walked recursively)."""
+        if self.kind == "file":
+            return [self.abs]
+        if self.kind == "dir":
+            return walk(self.abs, want).files
+        if self.kind == "glob":
+            # Walk (cycle-safe, tooling dirs skipped) from the glob's fixed prefix instead of glob.glob,
+            # which follows symlink loops and is case-sensitive.
+            stripped = _norm(os.path.expanduser(str(self.raw).strip())).rstrip("/")
+            fixed = []
+            for seg in stripped.split("/"):
+                if any(ch in GLOB_CHARS for ch in seg):
+                    break
+                fixed.append(seg)
+            prefix = "/".join(fixed)
+            base = prefix if os.path.isabs(stripped) else os.path.join(self.root, prefix) if prefix else self.root
+            if self.anywhere:
+                base = self.root
+            if not os.path.isdir(base):
+                return []
+            return [f for f in walk(base, want).files if self.matches(f)]
+        return []
+
+
+def patterns(root: str, raws: Optional[Iterable]) -> list:
+    return [Pattern(root, r) for r in (raws or []) if str(r).strip()]
+
+
+def expand(root: str, raws: Optional[Iterable], want: Callable[[str], bool]) -> list:
+    """Files named by the patterns, each real file once (symlinked routes collapse)."""
+    seen, out = set(), []
+    for p in patterns(root, raws):
+        for f in p.expand(want):
+            real = os.path.realpath(f)
+            if real not in seen:
+                seen.add(real)
+                out.append(os.path.abspath(f))
+    return out
+
+
+def select(root: str, raws: Optional[Iterable], files: Iterable[str]) -> tuple:
+    """(matched files, patterns that matched nothing) for a filter over already-loaded files."""
+    pats = patterns(root, raws)
+    files = list(files)
+    hit, empty = [], []
+    for p in pats:
+        m = [f for f in files if p.matches(f)]
+        if not m:
+            empty.append(p.raw)
+        hit.extend(m)
+    return sorted(set(hit)), empty