gelang 0.1.0

package/python/pyeffic/modules.py

@@ -0,0 +1,220 @@
+"""Module/import system for GE.
+
+Resolves `from module import function` statements by finding the
+corresponding `.ge.py` file and merging its functions into the
+current compilation unit.
+
+Supported import syntax:
+    from mymodule import my_function
+    from mymodule import func1, func2
+    from .mymodule import func  (relative import)
+
+The module file is searched in:
+1. The same directory as the source file
+2. The GE standard library directory (future)
+"""
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+from .analyzer import FuncUnit, parse_source_full, ClassUnit, collect_constants, collect_preamble
+from .frontends import frontend_for, python_flavour_of
+
+
+#: Modules provided by the compiler itself. Importing them is how a source
+#: file selects a backend decorator; they must never be resolved as user code.
+_COMPILER_MODULES = ("pyeffic", "__future__")
+
+
+def _is_compiler_module(module_name: str) -> bool:
+    """True if the import names a compiler-provided module."""
+    head = module_name.lstrip(".").split(".")[0]
+    return head in _COMPILER_MODULES
+
+
+def _lower_source(source: str, path: Path | None = None) -> str:
+    flavour = frontend_for(path) if path is not None else "python"
+    if flavour == "typescript":
+        from .frontends.typescript import ts_to_python
+        return ts_to_python(source)
+    if flavour == "hybrid":
+        from .frontends.hybrid import hybrid_to_python
+        return hybrid_to_python(source)
+    return source
+
+
+def _parse_source(source: str, path: Path | None = None) -> tuple[list[FuncUnit], list[ClassUnit]]:
+    """Parse source using the frontend that matches the file flavour."""
+    if path is not None and frontend_for(path) != "python":
+        from .analyzer import parse_source_full as _parse_py
+        return _parse_py(_lower_source(source, path))
+    return parse_source_full(source)
+
+
+def _collect_constants(source: str, path: Path | None = None) -> dict[str, object]:
+    """Collect module-level constants using the matching frontend."""
+    if path is not None and frontend_for(path) != "python":
+        return collect_constants(_lower_source(source, path))
+    return collect_constants(source)
+
+
+def _collect_preamble(source: str, path: Path | None = None) -> dict[str, str]:
+    """Collect ge_preamble/gePreamble blocks using the matching frontend."""
+    if path is not None and frontend_for(path) != "python":
+        return collect_preamble(_lower_source(source, path))
+    return collect_preamble(source)
+
+
+def resolve_imports(source: str, source_path: Path) -> tuple[list[FuncUnit], list[ClassUnit], list[str]]:
+    """Parse source, resolve all imports (transitively), and return merged units.
+
+    Imports are followed recursively so a module that only re-exports from
+    other modules (an aggregator) still contributes the full function set.
+    Each module file is visited once; cycles are broken by the visited set.
+
+    Returns:
+        (units, classes, warnings)
+    """
+    warnings: list[str] = []
+    source_dir = source_path.parent if source_path else Path(".")
+
+    units: list[FuncUnit] = []
+    classes: list[ClassUnit] = []
+    constants: dict[str, object] = {}
+    preamble: dict[str, str] = {}
+    seen_modules: set[str] = set()
+    seen_units: set[str] = set()
+    seen_classes: set[str] = set()
+
+    def _merge_preamble(src_preamble: dict[str, str]) -> None:
+        for bk, code in src_preamble.items():
+            if bk in preamble:
+                preamble[bk] += "\n" + code
+            else:
+                preamble[bk] = code
+
+    def _visit(module_source: str, base_dir: Path, module_key: str, depth: int,
+               module_path: Path | None = None) -> None:
+        """Parse one module and recurse into its imports (depth-first)."""
+        if module_key in seen_modules or depth > 64:
+            return
+        seen_modules.add(module_key)
+
+        # Non-Python flavours are lowered before the import scan; the same
+        # frontend also builds the IR below.
+        scan_source = _lower_source(module_source, module_path)
+        tree = ast.parse(scan_source)
+
+        # 1) recurse into this module's imports first (dependencies before dependents)
+        for node in tree.body:
+            if not isinstance(node, ast.ImportFrom):
+                continue
+            module_name = node.module or ""
+            if not module_name:
+                continue
+            if _is_compiler_module(module_name):
+                # GE runtime modules (`from pyeffic.backends import rust`) are
+                # compiler intrinsics, not user code — resolving them would
+                # pull the compiler's own sources into the program.
+                continue
+            dep_path = _find_module(module_name, base_dir)
+            if dep_path is None or not dep_path.exists():
+                if depth == 0:
+                    warnings.append(f"could not find module '{module_name}' at line {node.lineno}")
+                continue
+            dep_source = dep_path.read_text(encoding="utf-8")
+            dep_key = str(dep_path.resolve())
+            _visit(dep_source, dep_path.parent, dep_key, depth + 1, dep_path)
+
+        # 2) then add this module's own definitions
+        mod_units, mod_classes = _parse_source(module_source, module_path)
+        for u in mod_units:
+            if u.name not in seen_units:
+                seen_units.add(u.name)
+                units.append(u)
+        for c in mod_classes:
+            if c.name not in seen_classes:
+                seen_classes.add(c.name)
+                classes.append(c)
+        constants.update(_collect_constants(module_source, module_path))
+        _merge_preamble(_collect_preamble(module_source, module_path))
+
+    _visit(source, source_dir, str(source_path.resolve()) if source_path else "<source>", 0,
+           source_path)
+
+    globals()["_LAST_CONSTANTS"] = constants
+    globals()["_LAST_PREAMBLE"] = preamble
+
+    return units, classes, warnings
+
+
+def get_last_constants() -> dict[str, object]:
+    """Return the constants collected during the last resolve_imports call."""
+    return globals().get("_LAST_CONSTANTS", {})
+
+
+def get_last_preamble() -> dict[str, str]:
+    """Return the preamble collected during the last resolve_imports call."""
+    return globals().get("_LAST_PREAMBLE", {})
+
+
+def _find_module(module_name: str, source_dir: Path) -> Path | None:
+    """Find a GE source file for the given module name.
+
+    Both flavours are searched, Python-like first:
+        app.main -> app/main.ge.py, app/main.ts.ge.py, app/main.ge.ts
+    """
+    # handle relative imports (from .module import ...)
+    if module_name.startswith("."):
+        module_name = module_name.lstrip(".")
+
+    # handle dotted module names: app.main -> app/main.ge.py
+    parts = module_name.split(".")
+    base = parts[-1]
+    # candidate file names for the module leaf, in preference order.
+    # `.ge` (hybrid) is canonical; the single-flavour variants are honoured
+    # next, and a plain .py is the last resort.
+    leaf_names = (f"{base}.ge", f"{base}.ge.py", f"{base}.ge.ts",
+                  f"{base}.ts.ge.py", f"{base}.py")
+
+    # search in source_dir and parent directories (up to 3 levels)
+    search_dirs = [source_dir]
+    parent = source_dir.parent
+    for _ in range(3):
+        search_dirs.append(parent)
+        parent = parent.parent
+
+    for search_dir in search_dirs:
+        if len(parts) > 1:
+            # dotted name: only resolve relative to the package path.
+            # app.main -> search_dir/app/main.ge.py
+            # Never fall back to matching just the leaf, or `app.main` would
+            # wrongly resolve to a sibling `main.ge.py`.
+            pkg_dir = search_dir.joinpath(*parts[:-1])
+            for leaf in leaf_names:
+                candidate = pkg_dir / leaf
+                if candidate.exists():
+                    return candidate
+            # package directory: app/main/__init__.ge
+            for leaf in ("__init__.ge", "__init__.ge.py", "__init__.ge.ts",
+                         "__init__.py"):
+                candidate = pkg_dir / leaf
+                if candidate.exists():
+                    return candidate
+            continue
+
+        # simple name: search_dir/<name>.<flavour>
+        for leaf in leaf_names:
+            candidate = search_dir / leaf
+            if candidate.exists():
+                return candidate
+
+        # search_dir / name / __init__.ge
+        pkg_dir = search_dir / module_name
+        for leaf in ("__init__.ge", "__init__.ge.py", "__init__.ge.ts",
+                     "__init__.py"):
+            candidate = pkg_dir / leaf
+            if candidate.exists():
+                return candidate
+
+    return None

package/python/pyeffic/packer.py

@@ -0,0 +1,222 @@
+"""GE packer: bundle artifacts into a single .ge distribution package.
+
+Like npm: `ge pack` builds everything and bundles it into one `.ge` file.
+`ge install package.ge` unpacks and compiles for any target (Windows, Android).
+
+The .ge file is a zip archive containing:
+  package.json    — metadata (name, version, ffi_exports, targets)
+  source.ge.py    — the original GE source (for recompilation on any platform)
+  native/
+    ge_logic.rs   — generated Rust source (for cross-compilation)
+    ge_logic.dll  — pre-compiled Windows native lib (if available)
+  dart/
+    bindings.dart — generated Dart FFI bindings
+    main.dart     — generated Flutter UI
+    pubspec.yaml  — Flutter project config
+
+Compression shrinks DISTRIBUTION SIZE, not runtime speed.
+"""
+from __future__ import annotations
+
+import json
+import re
+import zlib
+import zipfile
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class FileSize:
+    path: Path
+    raw: int
+    packed: int
+
+    @property
+    def ratio(self) -> float:
+        return self.packed / self.raw if self.raw else 0.0
+
+
+@dataclass
+class PackReport:
+    package: Path | None = None
+    files: list[FileSize] = field(default_factory=list)
+    total_raw: int = 0
+    total_packed: int = 0
+
+    @property
+    def ratio(self) -> float:
+        return self.total_packed / self.total_raw if self.total_raw else 0.0
+
+
+@dataclass
+class PackageMeta:
+    """Metadata stored inside the .ge package (package.json)."""
+    name: str = ""
+    version: str = "0.1.0"
+    ge_version: str = "0.1.0"
+    app_name: str = ""
+    lib_name: str = "ge_logic"
+    backend: str = "rust"
+    ffi_exports: list[str] = field(default_factory=list)
+    targets: list[str] = field(default_factory=lambda: ["windows", "android"])
+    source_file: str = "source.ge.py"
+
+    def to_dict(self) -> dict:
+        return {
+            "name": self.name, "version": self.version,
+            "ge_version": self.ge_version, "app_name": self.app_name,
+            "lib_name": self.lib_name, "backend": self.backend,
+            "ffi_exports": self.ffi_exports, "targets": self.targets,
+            "source_file": self.source_file,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "PackageMeta":
+        return cls(
+            name=d.get("name", ""), version=d.get("version", "0.1.0"),
+            ge_version=d.get("ge_version", "0.1.0"),
+            app_name=d.get("app_name", ""), lib_name=d.get("lib_name", "ge_logic"),
+            backend=d.get("backend", "rust"),
+            ffi_exports=d.get("ffi_exports", []),
+            targets=d.get("targets", ["windows", "android"]),
+            source_file=d.get("source_file", "source.ge.py"),
+        )
+
+
+def minify_source(text: str) -> str:
+    """Strip comments and collapse runs of whitespace. Preserves strings."""
+    out_lines = []
+    for line in text.splitlines():
+        stripped = re.sub(r'(?:"[^"]*")|(\s*//[^\n]*)', lambda m: m.group(1) or "", line)
+        stripped = re.sub(r"\s+", " ", stripped).strip()
+        if stripped:
+            out_lines.append(stripped)
+    return "\n".join(out_lines)
+
+
+def strip_binary_size(path: Path) -> int:
+    return path.stat().st_size if path.exists() else 0
+
+
+# ---- .ge package format (zip-based, npm-like) ----
+
+def pack_ge(source: Path, meta: PackageMeta, artifacts: dict[str, Path],
+            out_path: Path, minify: bool = True) -> PackReport:
+    """Bundle source + generated artifacts + metadata into a single .ge zip.
+
+    artifacts: dict mapping archive path -> local file path, e.g.:
+        {"native/ge_logic.rs": Path("ge_build/myrent/lib/ge_logic.rs"),
+         "dart/bindings.dart": Path("ge_build/myrent/lib/bindings.dart"), ...}
+    """
+    report = PackReport(package=out_path)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    with zipfile.ZipFile(out_path, "w", zipfile.ZIP_DEFLATED, compresslevel=9) as zf:
+        # 1) metadata
+        meta_json = json.dumps(meta.to_dict(), indent=2)
+        zf.writestr("package.json", meta_json)
+        report.files.append(FileSize(Path("package.json"), len(meta_json), len(meta_json.encode())))
+        report.total_raw += len(meta_json)
+        report.total_packed += len(meta_json.encode())
+
+        # 2) GE source
+        raw = source.read_bytes()
+        if minify:
+            text = raw.decode("utf-8", errors="ignore")
+            mini = minify_source(text)
+            data = mini.encode("utf-8")
+        else:
+            data = raw
+        zf.writestr(meta.source_file, data)
+        report.files.append(FileSize(source, len(raw), len(data)))
+        report.total_raw += len(raw)
+        report.total_packed += len(data)
+
+        # 3) artifacts (native lib, Rust source, Dart files, pubspec)
+        for arc_name, local_path in artifacts.items():
+            if not local_path or not local_path.exists():
+                continue
+            raw = local_path.read_bytes()
+            # only minify the GE source (.py); native source (.rs/.cpp) and Dart
+            # must be preserved exactly for cross-compilation and Flutter
+            if minify and local_path.suffix == ".py":
+                text = raw.decode("utf-8", errors="ignore")
+                mini = minify_source(text)
+                data = mini.encode("utf-8")
+            else:
+                data = raw
+            zf.writestr(arc_name, data)
+            report.files.append(FileSize(local_path, len(raw), len(data)))
+            report.total_raw += len(raw)
+            report.total_packed += len(data)
+
+    report.total_packed = out_path.stat().st_size
+    return report
+
+
+def pack_simple(source: Path, out_path: Path, app_name: str = "",
+                backend: str = "rust", minify: bool = True) -> PackReport:
+    """Convenience: pack a single source file into a .ge package.
+
+    This is the simplest API for the common case of bundling one .ge.py file.
+    """
+    meta = PackageMeta(
+        name=app_name or source.stem.replace(".ge", ""),
+        app_name=app_name or source.stem.replace(".ge", ""),
+        backend=backend,
+    )
+    return pack_ge(source, meta, {}, out_path, minify=minify)
+
+
+def unpack_ge(pkg_path: Path, dest_dir: Path) -> PackageMeta:
+    """Extract a .ge package to dest_dir. Returns the parsed metadata."""
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    with zipfile.ZipFile(pkg_path, "r") as zf:
+        zf.extractall(dest_dir)
+    meta_path = dest_dir / "package.json"
+    if meta_path.exists():
+        return PackageMeta.from_dict(json.loads(meta_path.read_text(encoding="utf-8")))
+    return PackageMeta()
+
+
+# ---- legacy .gepkg format (backward compat) ----
+
+def pack(files: list[Path] | Path, out_path: Path, minify: bool = True) -> PackReport:
+    """Bundle files into a zlib-compressed .gepkg. Returns a size report.
+
+    Accepts either a single Path or a list of Paths for convenience.
+    """
+    if isinstance(files, Path):
+        files = [files]
+    report = PackReport(package=out_path)
+    blob = bytearray()
+    for f in files:
+        if not f.exists():
+            continue
+        raw = f.read_bytes()
+        if minify and f.suffix in (".rs", ".cpp", ".dart", ".py"):
+            text = raw.decode("utf-8", errors="ignore")
+            mini = minify_source(text)
+            data = mini.encode("utf-8")
+        else:
+            data = raw
+        name = f.name.encode("utf-8")
+        blob += len(name).to_bytes(4, "little") + name
+        blob += len(data).to_bytes(4, "little") + data
+        report.files.append(FileSize(f, len(raw), len(data)))
+        report.total_raw += len(raw)
+        report.total_packed += len(data)
+
+    compressed = zlib.compress(bytes(blob), level=9)
+    out_path.write_bytes(compressed)
+    report.total_packed = len(compressed)
+    return report
+
+
+def fmt_size(n: int) -> str:
+    if n < 1024:
+        return f"{n} B"
+    if n < 1024 * 1024:
+        return f"{n/1024:.1f} KB"
+    return f"{n/(1024*1024):.2f} MB"