@hypersoniclabs/helix-cli 0.1.7 → 0.1.24-helix3.438

package/plugins/beamng-vehicle/bin/beamng_materials.py

@@ -0,0 +1,261 @@
+"""Pure BeamNG material declaration and texture-resolution helpers.
+
+This module deliberately has no Blender dependency so the importer can test the
+source decision model without starting an editor. The Blender converter consumes
+the same structures rather than maintaining a second first-stage-only parser.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+from pathlib import Path
+
+
+MAP_ROLES = {
+    "basecolormap": "baseColor",
+    "colormap": "baseColor",
+    "diffusemap": "baseColor",
+    "normalmap": "normal",
+    "roughnessmap": "roughness",
+    "metallicmap": "metallic",
+    "ambientocclusionmap": "occlusion",
+    "opacitymap": "opacity",
+    "emissivemap": "emissive",
+    "detailmap": "detail",
+    "detailnormalmap": "detailNormal",
+    "specularmap": "specular",
+    "colorpalettemap": "colorPalette",
+    "clearcoatmap": "clearCoat",
+}
+FACTOR_KEYS = {
+    "baseColorFactor", "diffuseColor", "metallicFactor", "roughnessFactor",
+    "opacityFactor", "emissiveFactor", "clearCoatFactor", "clearCoatRoughnessFactor",
+}
+TEXTURE_EXTENSIONS = (".dds", ".png", ".jpg", ".jpeg", ".tga", ".bmp")
+
+
+def _strip_json_comments(text: str) -> str:
+    output, index, quoted, escaped = [], 0, False, False
+    while index < len(text):
+        char = text[index]
+        if quoted:
+            output.append(char)
+            if escaped:
+                escaped = False
+            elif char == "\\":
+                escaped = True
+            elif char == '"':
+                quoted = False
+            index += 1
+            continue
+        if char == '"':
+            quoted = True
+            output.append(char)
+            index += 1
+        elif text.startswith("//", index):
+            end = text.find("\n", index)
+            index = len(text) if end < 0 else end
+        elif text.startswith("/*", index):
+            end = text.find("*/", index + 2)
+            index = len(text) if end < 0 else end + 2
+        else:
+            output.append(char)
+            index += 1
+    return "".join(output)
+
+
+def _load_json(path: Path) -> dict:
+    text = _strip_json_comments(path.read_text(errors="ignore"))
+    text = re.sub(r",\s*([}\]])", r"\1", text)
+    value = json.loads(text)
+    return value if isinstance(value, dict) else {}
+
+
+def _portable(path: Path, root: Path) -> str:
+    return str(path.relative_to(root)).replace(os.sep, "/")
+
+
+def _normalize_map(name: str) -> str:
+    return name.strip().strip('"').replace("\\", "/").lstrip("/")
+
+
+def is_dynamic_texture(name: object) -> bool:
+    return isinstance(name, str) and name.strip().startswith("@")
+
+
+def is_sentinel_map(name: object) -> bool:
+    if not isinstance(name, str):
+        return False
+    base = os.path.basename(_normalize_map(name)).lower()
+    stem = base.split(".")[0]
+    return stem == "null" or stem.startswith("null_")
+
+
+def _alpha(raw: dict) -> dict:
+    translucent = bool(raw.get("translucent"))
+    alpha_test = bool(raw.get("alphaTest"))
+    if translucent and alpha_test:
+        ref = raw.get("alphaRef")
+        cutoff = float(ref) / 255.0 if isinstance(ref, (int, float)) and ref else 0.5
+        return {"mode": "MASK", "cutoff": cutoff, "source": "materials.json",
+                "declaredBy": {key: raw[key] for key in
+                               ("translucent", "alphaTest", "alphaRef", "translucentBlendOp") if key in raw}}
+    return {"mode": "BLEND" if translucent else "OPAQUE", "source": "materials.json",
+            "declaredBy": {key: raw[key] for key in
+                           ("translucent", "alphaTest", "alphaRef", "translucentBlendOp") if key in raw}}
+
+
+def _material(key: str, raw: dict, path: Path, root: Path, tier: str) -> tuple[dict, dict[str, list]]:
+    stages = raw.get("Stages") if isinstance(raw.get("Stages"), list) else []
+    maps, ordered_maps, factors = {}, [], {}
+    diagnostics = {"stageConflicts": [], "dynamicTextures": [], "sentinels": [], "unsupportedMaps": []}
+    identity = str(raw.get("mapTo") or raw.get("name") or key).strip()
+    for stage_index, stage in enumerate(stages):
+        if not isinstance(stage, dict):
+            continue
+        for source_key, value in stage.items():
+            lower = source_key.lower()
+            role = MAP_ROLES.get(lower)
+            if role is None and lower.endswith("map"):
+                role = source_key[:-3]
+            if role and isinstance(value, str) and value.strip():
+                name = value.strip()
+                event = {
+                    "material": identity, "role": role, "name": name, "stage": stage_index,
+                    "declarationTier": tier, "declarationFile": _portable(path, root),
+                    "declarationKey": key, "sourceKey": source_key,
+                }
+                if is_sentinel_map(name):
+                    diagnostics["sentinels"].append(event)
+                    continue
+                if is_dynamic_texture(name):
+                    diagnostics["dynamicTextures"].append(event)
+                    continue
+                clean = _normalize_map(name)
+                previous = maps.get(role)
+                if previous is not None and previous != clean:
+                    diagnostics["stageConflicts"].append({
+                        "material": identity, "role": role, "kept": clean, "dropped": previous,
+                        "stage": stage_index, "declarationTier": tier,
+                        "declarationFile": _portable(path, root), "declarationKey": key,
+                        "why": "later BeamNG material stage wins; both source declarations remain in provenance",
+                    })
+                maps[role] = clean
+                ordered_maps.append({"role": role, "name": clean, "stage": stage_index, "sourceKey": source_key})
+                if role not in {"baseColor", "normal", "roughness", "metallic", "occlusion", "opacity", "emissive", "colorPalette"}:
+                    diagnostics["unsupportedMaps"].append(event)
+            if source_key in FACTOR_KEYS:
+                factors[source_key] = {"value": value, "stage": stage_index, "source": "materials.json"}
+    row = {
+        "key": key,
+        "mapTo": identity,
+        "stageCount": len(stages),
+        "stages": stages,
+        "maps": maps,
+        "orderedMaps": ordered_maps,
+        "factors": factors,
+        "alpha": _alpha(raw),
+        "doubleSided": bool(raw.get("doubleSided")) if "doubleSided" in raw else None,
+        "provenance": {
+            "declarationTier": tier,
+            "declarationFile": _portable(path, root),
+            "declarationKey": key,
+            "mapTo": identity,
+        },
+    }
+    for field, role in (("base", "baseColor"), ("paletteMask", "colorPalette"), ("normal", "normal"),
+                        ("roughnessMap", "roughness"), ("metallicMap", "metallic")):
+        row[field] = maps.get(role)
+    for field in ("metallic", "roughness"):
+        source = factors.get(f"{field}Factor")
+        row[field] = source.get("value") if source else None
+    color = factors.get("baseColorFactor") or factors.get("diffuseColor")
+    row["color"] = color.get("value") if color else None
+    row["alphaDeclared"] = row["alpha"]["mode"] != "OPAQUE"
+    return row, diagnostics
+
+
+def parse_material_tree(root: str | Path, default_tier: str = "mod", vehicle_root: str | Path | None = None) -> dict:
+    root_path = Path(root).resolve()
+    vehicle_path = Path(vehicle_root).resolve() if vehicle_root else None
+    materials, aliases = {}, {}
+    diagnostics = {
+        "parseFailures": [], "shadowedDeclarations": [], "stageConflicts": [],
+        "dynamicTextures": [], "sentinels": [], "unsupportedMaps": [],
+    }
+    paths = sorted(root_path.rglob("*.materials.json")) if root_path.is_dir() else []
+    for path in paths:
+        try:
+            document = _load_json(path)
+        except Exception as error:
+            diagnostics["parseFailures"].append({"file": _portable(path, root_path), "error": str(error)})
+            continue
+        tier = "vehicle" if vehicle_path and (path == vehicle_path or vehicle_path in path.parents) else default_tier
+        for key, raw in document.items():
+            if not isinstance(raw, dict):
+                continue
+            row, row_diagnostics = _material(str(key), raw, path, root_path, tier)
+            identity = row["mapTo"].lower()
+            existing = materials.get(identity)
+            if existing:
+                ranks = {"basegame": 0, "mod": 1, "vehicle": 2}
+                existing_rank = ranks.get(existing["provenance"]["declarationTier"], 0)
+                current_rank = ranks.get(tier, 0)
+                if existing_rank > current_rank:
+                    diagnostics["shadowedDeclarations"].append({
+                        "material": identity, "kept": existing["provenance"], "ignored": row["provenance"]})
+                    continue
+                diagnostics["shadowedDeclarations"].append({
+                    "material": identity, "kept": row["provenance"], "ignored": existing["provenance"]})
+            materials[identity] = row
+            aliases[str(key).lower()] = identity
+            aliases[row["mapTo"].lower()] = identity
+            for name, values in row_diagnostics.items():
+                diagnostics[name].extend(values)
+    return {"materials": materials, "aliases": aliases, "diagnostics": diagnostics}
+
+
+class TextureChain:
+    """Most-specific-first texture lookup with explicit ambiguity and provenance."""
+
+    def __init__(self, roots: list[tuple[str, str | Path]]):
+        self.roots = [(tier, Path(root).resolve()) for tier, root in roots if Path(root).is_dir()]
+        self.index: dict[str, dict[str, list[Path]]] = {}
+        for tier, root in self.roots:
+            tier_index: dict[str, list[Path]] = {}
+            for path in sorted(root.rglob("*")):
+                if not path.is_file() or path.suffix.lower() not in TEXTURE_EXTENSIONS:
+                    continue
+                rel = _portable(path, root).lower()
+                tier_index.setdefault(rel, []).append(path)
+                tier_index.setdefault(path.name.lower(), []).append(path)
+            self.index[tier] = tier_index
+
+    def resolve(self, raw: object) -> dict:
+        if not isinstance(raw, str) or not raw.strip():
+            return {"resolved": False, "reason": "empty"}
+        if is_dynamic_texture(raw):
+            return {"resolved": False, "dynamic": True, "name": raw}
+        if is_sentinel_map(raw):
+            return {"resolved": False, "sentinel": True, "name": raw}
+        requested = _normalize_map(raw).lower()
+        stem, extension = os.path.splitext(requested)
+        full_candidates = [requested]
+        if extension not in TEXTURE_EXTENSIONS:
+            full_candidates.extend(requested + suffix for suffix in TEXTURE_EXTENSIONS)
+            full_candidates.extend(stem + suffix for suffix in TEXTURE_EXTENSIONS)
+        basename_candidates = [os.path.basename(value) for value in full_candidates]
+        for tier, _root in self.roots:
+            tier_index = self.index[tier]
+            for candidate_group in (full_candidates, basename_candidates):
+                hits = sorted({path for candidate in candidate_group for path in tier_index.get(candidate, [])})
+                if len(hits) == 1:
+                    hit = hits[0]
+                    return {"resolved": True, "name": raw, "path": hit, "tier": tier,
+                            "matched": candidate_group[0], "corrected": hit.name.lower() != os.path.basename(requested)}
+                if len(hits) > 1:
+                    return {"resolved": False, "name": raw, "tier": tier, "reason": "ambiguous",
+                            "candidates": [str(path) for path in hits]}
+        return {"resolved": False, "name": raw, "reason": "missing", "searched": [tier for tier, _ in self.roots],
+                "remedy": "supply the exact texture in the source or digest-bound BeamNG common dependency"}

package/plugins/beamng-vehicle/bin/beamng_stock_events.py

@@ -0,0 +1,171 @@
+"""Resolve a BeamNG stock FMOD event (``event:>Engine>Starter>v8flat_1986_eng``) to the recording the game plays.
+
+BeamNG.drive ships its FMOD Studio 2 banks in ``content/art_sound.zip`` under ``art/sound/fmod/desktop/``:
+``main_preload.strings.bank`` (every event path -> GUID), metadata banks (``vehicle.bank``, ``vehicle_preload.bank``,
+``main_preload.bank``: event -> instruments -> waveform GUIDs) and sample banks (``*.assets.bank`` /
+``*.streams.bank``: waveform GUID -> FSB5 subsound). This module extracts those banks once into a cache directory,
+indexes them, and decodes the one subsound a slot needs.
+
+An event with several instruments (a horn's start/loop/end, four ratchet variations) yields one recording: the loop
+for sustained sounds, else the first in natural name order; the others are listed as alternatives.
+"""
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import zipfile
+from pathlib import Path
+from typing import Any
+
+import fmod_studio
+import fsb5
+from sim_audio import normalize_wav, sha256_file, staged_name
+
+BANK_DIR = "art/sound/fmod/desktop/"
+LOOP_SLOTS = {"horn", "transmissionWhine", "turboSpool", "tyreScrub", "tyreSpin", "tyreRollSlow", "tyreRollFast",
+              "brakeSqueal", "reverseBeep", "intake", "tyreSqueal"}
+
+
+def _natural(name: str) -> list[Any]:
+    return [int(t) if t.isdigit() else t.lower() for t in re.split(r"(\d+)", name)]
+
+
+def default_cache_root() -> Path:
+    base = os.environ.get("HELIX_CACHE_DIR") or os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
+    return Path(base) / "helix-cli" / "beamng-fmod"
+
+
+class StockEvents:
+    def __init__(self, beamng_root: Path, staging: Path, vgmstream: str | None = None, cache: Path | None = None):
+        archive = beamng_root / "content" / "art_sound.zip"
+        if not archive.is_file():
+            raise ValueError(f"{beamng_root} has no content/art_sound.zip (is it the BeamNG.drive install folder?)")
+        info = archive.stat()
+        self.archive = archive
+        self.cache = (cache or default_cache_root()) / f"{info.st_size}-{int(info.st_mtime)}"
+        self.staging = staging
+        self.staging.mkdir(parents=True, exist_ok=True)
+        self.vgmstream = vgmstream
+        self._extract_banks()
+        desktop = self.cache / BANK_DIR
+        self.paths: dict[str, bytes] = {}
+        for strings in sorted(desktop.glob("*.strings.bank")):
+            self.paths.update(fmod_studio.read_strings_bank(strings))
+        by_guid = {guid: path for path, guid in self.paths.items()}
+        self.lower = {path.lower(): guid for path, guid in self.paths.items()}
+        self.events: dict[bytes, fmod_studio.EventInfo] = {}
+        self.metadata_banks: list[str] = []
+        for bank in sorted(desktop.glob("*.bank")):
+            name = bank.name
+            if name.endswith((".strings.bank", ".assets.bank", ".streams.bank")):
+                continue
+            try:
+                meta = fmod_studio.read_metadata(bank, by_guid)
+            except fmod_studio.FmodMetadataError:
+                continue
+            self.metadata_banks.append(name)
+            for event in meta.events:
+                self.events.setdefault(event.guid, event)
+        self.waveforms: dict[bytes, tuple[Path, int, int]] = {}
+        for bank in sorted(desktop.glob("*.bank")):
+            if bank.name.endswith((".assets.bank", ".streams.bank")):
+                for guid, (block, index) in fmod_studio.read_waveform_index(bank).items():
+                    self.waveforms.setdefault(guid, (bank, block, index))
+        self._banks: dict[Path, fsb5.FsbBank] = {}
+        self.extracted: list[dict[str, Any]] = []
+
+    def _extract_banks(self) -> None:
+        marker = self.cache / ".complete"
+        if marker.is_file():
+            return
+        target = self.cache
+        target.mkdir(parents=True, exist_ok=True)
+        root = target.resolve()
+        with zipfile.ZipFile(self.archive) as handle:
+            for member in handle.infolist():
+                name = member.filename.replace("\\", "/")
+                if not name.lower().startswith(BANK_DIR) or not name.lower().endswith(".bank") or "/" in name[len(BANK_DIR):]:
+                    continue
+                destination = (target / name).resolve()
+                if root not in destination.parents:
+                    continue
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                partial = destination.with_suffix(".part")
+                with handle.open(member) as source, partial.open("wb") as out:
+                    shutil.copyfileobj(source, out, 4 * 1024 * 1024)
+                partial.replace(destination)
+        marker.write_text("ok\n")
+
+    def bank(self, path: Path) -> fsb5.FsbBank:
+        if path not in self._banks:
+            self._banks[path] = fsb5.read_bank(path)
+        return self._banks[path]
+
+    def samples(self, event_path: str) -> list[tuple[fsb5.FsbBank, fsb5.FsbSample]]:
+        guid = self.lower.get(event_path.lower())
+        if guid is None or guid not in self.events:
+            return []
+        return self._samples(guid, 0)
+
+    def _samples(self, guid: bytes, depth: int) -> list[tuple[fsb5.FsbBank, fsb5.FsbSample]]:
+        event = self.events[guid]
+        found: list[tuple[fsb5.FsbBank, fsb5.FsbSample]] = []
+        for instrument in event.instruments:
+            located = self.waveforms.get(instrument.waveform or b"")
+            if not located:
+                continue
+            bank_path, block, index = located
+            bank = self.bank(bank_path)
+            sample = next((s for s in bank.samples if s.block == block and s.local_index == index), None)
+            if sample and all(s.index != sample.index or b.path != bank.path for b, s in found):
+                found.append((bank, sample))
+        if depth < 4:
+            # Event instruments play other (often unnamed, embedded) events: follow them by GUID.
+            for nested in event.nested:
+                if nested in self.events and nested != guid:
+                    for bank, sample in self._samples(nested, depth + 1):
+                        if all(s.index != sample.index or b.path != bank.path for b, s in found):
+                            found.append((bank, sample))
+        return found
+
+    def extract(self, event_path: str, slot: str | None = None, prefer: str | None = None,
+                avoid: str | None = None, exclude: set[str] | None = None) -> dict[str, Any] | None:
+        candidates = self.samples(event_path)
+        if not candidates:
+            if self.lower.get(event_path.lower()) is None:
+                raise ValueError("no such event in the game's strings bank")
+            return None
+        if avoid:
+            kept = [c for c in candidates if not re.search(avoid, c[1].name, re.I)]
+            candidates = kept or candidates
+        if exclude:
+            kept = [c for c in candidates if c[1].name not in exclude]
+            candidates = kept or candidates
+        loopish = slot in LOOP_SLOTS
+        candidates.sort(key=lambda bs: (0 if prefer and re.search(prefer, bs[1].name, re.I) else 1,
+                                        (0 if "loop" in bs[1].name.lower() else 1) if loopish else 0,
+                                        -bs[1].duration if loopish else 0, _natural(bs[1].name)))
+        bank, sample = candidates[0]
+        raw = self.staging / f"stock-{sample.index}-{bank.path.stem}.wav"
+        decoder = fsb5.decode_sample(bank, sample, raw, self.vgmstream)
+        normalized = self.staging / f"stock-norm-{sample.index}-{bank.path.stem}.wav"
+        transforms = normalize_wav(raw, normalized)
+        if sample.codec != "pcm16":
+            transforms = [{"kind": "decode", "detail": f"FMOD {sample.codec} subsound -> 16-bit PCM WAV"}, *transforms]
+        digest = sha256_file(normalized)
+        final = self.staging / staged_name(sample.name, digest)
+        if not final.exists():
+            shutil.copyfile(normalized, final)
+        rel = f"content/art_sound.zip:{BANK_DIR}{bank.path.name}"
+        self.extracted.append({"event": event_path, "sample": sample.name, "bank": bank.path.name})
+        return {"wav": str(final),
+                "alternatives": [s.name for _, s in candidates[1:]][:16],
+                "provenance": {"kind": "sim-official", "sim": "beamng", "mod": "BeamNG.drive", "sourceFile": rel,
+                               "sourceSha256": bank.sha256, "event": event_path, "sample": sample.name,
+                               "subsong": sample.index, "extractor": f"helix beamng_stock_events + {decoder}",
+                               **({"transforms": transforms} if transforms else {})}}
+
+    def report(self) -> dict[str, Any]:
+        return {"strings": len(self.paths), "events": len(self.events), "waveforms": len(self.waveforms),
+                "metadataBanks": self.metadata_banks, "cache": str(self.cache), "extracted": self.extracted}