frida 16.2.1 → 16.2.3

package/releng/meson_configure.py

@@ -0,0 +1,506 @@
+import argparse
+import os
+from pathlib import Path
+import pickle
+import platform
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+from typing import Any, Callable, Optional
+
+RELENG_DIR = Path(__file__).resolve().parent
+SCRIPTS_DIR = RELENG_DIR / "meson-scripts"
+
+sys.path.insert(0, str(RELENG_DIR / "meson"))
+import mesonbuild.interpreter
+from mesonbuild.coredata import UserArrayOption, UserBooleanOption, \
+        UserComboOption, UserFeatureOption, UserOption, UserStringOption
+
+from . import deps, env
+from .machine_spec import MachineSpec
+from .progress import ProgressCallback, print_progress
+
+
+def main():
+    default_sourcedir = Path(sys.argv.pop(1))
+    sourcedir = Path(os.environ.get("MESON_SOURCE_ROOT", default_sourcedir)).resolve()
+
+    workdir = Path(os.getcwd())
+    if workdir == sourcedir:
+        default_builddir = sourcedir / "build"
+    else:
+        default_builddir = workdir
+    builddir = Path(os.environ.get("MESON_BUILD_ROOT", default_builddir)).resolve()
+
+    parser = argparse.ArgumentParser(prog="configure",
+                                     add_help=False)
+    opts = parser.add_argument_group(title="generic options")
+    opts.add_argument("-h", "--help",
+                      help="show this help message and exit",
+                      action="help")
+    opts.add_argument("--prefix",
+                      help="install files in PREFIX",
+                      metavar="PREFIX",
+                      type=parse_prefix)
+    opts.add_argument("--build",
+                      help="configure for building on BUILD",
+                      metavar="BUILD",
+                      type=MachineSpec.parse)
+    opts.add_argument("--host",
+                      help="cross-compile to build binaries to run on HOST",
+                      metavar="HOST",
+                      type=MachineSpec.parse)
+    opts.add_argument("--enable-symbols",
+                      help="build binaries with debug symbols included (default: disabled)",
+                      action="store_true")
+    opts.add_argument("--enable-shared",
+                      help="enable building shared libraries (default: disabled)",
+                      action="store_true")
+    opts.add_argument("--with-meson",
+                      help="which Meson implementation to use (default: internal)",
+                      choices=["internal", "system"],
+                      dest="meson",
+                      default="internal")
+    opts.add_argument(f"--without-prebuilds",
+                      help="do not make use of prebuilt bundles",
+                      metavar="{" + ",".join(query_supported_bundle_types(include_wildcards=True)) + "}",
+                      type=parse_bundle_type_set,
+                      default=set())
+    opts.add_argument("extra_meson_options",
+                      nargs="*",
+                      help=argparse.SUPPRESS)
+
+    meson_options_file = sourcedir / "meson.options"
+    if not meson_options_file.exists():
+        meson_options_file = sourcedir / "meson_options.txt"
+    if meson_options_file.exists():
+        meson_group = parser.add_argument_group(title="project-specific options")
+        meson_opts = register_meson_options(meson_options_file, meson_group)
+
+    options = parser.parse_args()
+
+    if builddir.exists():
+        if (builddir / "build.ninja").exists():
+            print(f"Already configured. Wipe .{os.sep}{builddir.relative_to(workdir)} to reconfigure.",
+                  file=sys.stderr)
+            sys.exit(1)
+
+    default_library = "shared" if options.enable_shared else "static"
+
+    allowed_prebuilds = set(query_supported_bundle_types(include_wildcards=False)) - options.without_prebuilds
+
+    try:
+        configure(sourcedir,
+                  builddir,
+                  options.prefix,
+                  options.build,
+                  options.host,
+                  os.environ,
+                  "included" if options.enable_symbols else "stripped",
+                  default_library,
+                  allowed_prebuilds,
+                  options.meson,
+                  collect_meson_options(options))
+    except Exception as e:
+        print(e, file=sys.stderr)
+        if isinstance(e, subprocess.CalledProcessError):
+            for label, data in [("Output", e.output),
+                                ("Stderr", e.stderr)]:
+                if data:
+                    print(f"{label}:\n\t| " + "\n\t| ".join(data.strip().split("\n")), file=sys.stderr)
+        sys.exit(1)
+
+
+def configure(sourcedir: Path,
+              builddir: Path,
+              prefix: Optional[str] = None,
+              build_machine: Optional[MachineSpec] = None,
+              host_machine: Optional[MachineSpec] = None,
+              environ: dict[str, str] = os.environ,
+              debug_symbols: str = "stripped",
+              default_library: str = "static",
+              allowed_prebuilds: set[str] = None,
+              meson: str = "internal",
+              extra_meson_options: list[str] = [],
+              call_meson: Callable = env.call_meson,
+              on_progress: ProgressCallback = print_progress):
+    if prefix is None:
+        prefix = env.detect_default_prefix()
+
+    project_vscrt = detect_project_vscrt(sourcedir)
+
+    if build_machine is None:
+        build_machine = MachineSpec.make_from_local_system()
+    build_machine = build_machine.default_missing(recommended_vscrt=project_vscrt)
+
+    if host_machine is None:
+        host_machine = build_machine
+    else:
+        host_machine = host_machine.default_missing(recommended_vscrt=project_vscrt)
+
+    if host_machine.os == "windows":
+        vs_arch = environ.get("VSCMD_ARG_TGT_ARCH")
+        if vs_arch == "x86":
+            host_machine = host_machine.evolve(arch=vs_arch)
+
+    build_machine = build_machine.maybe_adapt_to_host(host_machine)
+
+    if allowed_prebuilds is None:
+        allowed_prebuilds = set(query_supported_bundle_types(include_wildcards=False))
+
+    call_selected_meson = lambda argv, *args, **kwargs: call_meson(argv,
+                                                                   use_submodule=meson == "internal",
+                                                                   *args,
+                                                                   **kwargs)
+
+    meson_options = [
+        f"-Dprefix={prefix}",
+        f"-Ddefault_library={default_library}",
+        *host_machine.meson_optimization_options,
+    ]
+    if debug_symbols == "stripped" and host_machine.toolchain_can_strip:
+        meson_options += ["-Dstrip=true"]
+
+    deps_dir = deps.detect_cache_dir(sourcedir)
+
+    allow_prebuilt_toolchain = "toolchain" in allowed_prebuilds
+    if allow_prebuilt_toolchain:
+        try:
+            toolchain_prefix, _ = deps.ensure_toolchain(build_machine, deps_dir, on_progress=on_progress)
+        except deps.BundleNotFoundError as e:
+            raise_toolchain_not_found(e)
+    else:
+        if project_depends_on_vala_compiler(sourcedir):
+            toolchain_prefix = deps.query_toolchain_prefix(build_machine, deps_dir)
+            vala_compiler = env.detect_toolchain_vala_compiler(toolchain_prefix, build_machine)
+            if vala_compiler is None:
+                build_vala_compiler(toolchain_prefix, deps_dir, call_selected_meson)
+        else:
+            toolchain_prefix = None
+
+    is_cross_build = host_machine != build_machine
+
+    build_sdk_prefix = None
+    required = {"sdk:build"}
+    if not is_cross_build:
+        required.add("sdk:host")
+    if allowed_prebuilds.issuperset(required):
+        try:
+            build_sdk_prefix, _ = deps.ensure_sdk(build_machine, deps_dir, on_progress=on_progress)
+        except deps.BundleNotFoundError as e:
+            raise_sdk_not_found(e, "build", build_machine)
+
+    host_sdk_prefix = None
+    if is_cross_build and "sdk:host" in allowed_prebuilds:
+        try:
+            host_sdk_prefix, _ = deps.ensure_sdk(host_machine, deps_dir, on_progress=on_progress)
+        except deps.BundleNotFoundError as e:
+            raise_sdk_not_found(e, "host", host_machine)
+
+    build_config, host_config = \
+            env.generate_machine_configs(build_machine,
+                                         host_machine,
+                                         environ,
+                                         toolchain_prefix,
+                                         build_sdk_prefix,
+                                         host_sdk_prefix,
+                                         call_selected_meson,
+                                         default_library,
+                                         builddir)
+
+    meson_options += [f"--native-file={build_config.machine_file}"]
+    if host_config is not build_config:
+        meson_options += [f"--cross-file={host_config.machine_file}"]
+
+    setup_env = host_config.make_merged_environment(environ)
+    setup_env["FRIDA_ALLOWED_PREBUILDS"] = ",".join(allowed_prebuilds)
+
+    call_selected_meson(["setup"] + meson_options + extra_meson_options + [builddir],
+                        cwd=sourcedir,
+                        env=setup_env,
+                        check=True)
+
+    shutil.copy(SCRIPTS_DIR / "BSDmakefile", builddir)
+    (builddir / "Makefile").write_text(generate_out_of_tree_makefile(sourcedir), encoding="utf-8")
+    if platform.system() == "Windows":
+        (builddir / "make.bat").write_text(generate_out_of_tree_make_bat(sourcedir), encoding="utf-8")
+
+    (builddir / "frida-env.dat").write_bytes(pickle.dumps({
+        "meson": meson,
+        "build": build_config,
+        "host": host_config if host_config is not build_config else None,
+        "allowed_prebuilds": allowed_prebuilds,
+        "deps": deps_dir,
+    }))
+
+
+def parse_prefix(raw_prefix: str) -> Path:
+    prefix = Path(raw_prefix)
+    if not prefix.is_absolute():
+        prefix = Path(os.getcwd()) / prefix
+    return prefix
+
+
+def query_supported_bundle_types(include_wildcards: bool) -> list[str]:
+    for e in deps.Bundle:
+        identifier = e.name.lower()
+        if e == deps.Bundle.SDK:
+            if include_wildcards:
+                yield identifier
+            yield identifier + ":build"
+            yield identifier + ":host"
+        else:
+            yield identifier
+
+
+def query_supported_bundle_type_values() -> list[deps.Bundle]:
+    return [e for e in deps.Bundle]
+
+
+def parse_bundle_type_set(raw_array: str) -> list[str]:
+    supported_types = list(query_supported_bundle_types(include_wildcards=True))
+    result = set()
+    for element in raw_array.split(","):
+        bundle_type = element.strip()
+        if bundle_type not in supported_types:
+            pretty_choices = "', '".join(supported_types)
+            raise argparse.ArgumentTypeError(f"invalid bundle type: '{bundle_type}' (choose from '{pretty_choices}')")
+        if bundle_type == "sdk":
+            result.add("sdk:build")
+            result.add("sdk:host")
+        else:
+            result.add(bundle_type)
+    return result
+
+
+def raise_toolchain_not_found(e: Exception):
+    raise ToolchainNotFoundError("\n".join([
+        f"Unable to download toolchain: {e}",
+        "",
+        "Specify --without-prebuilds=toolchain to only use tools on your PATH.",
+        "",
+        "Another option is to do what Frida's CI does:",
+        "",
+        "    ./releng/deps.py build --bundle=toolchain",
+        "",
+        "This produces a tarball in ./deps which gets picked up if you retry `./configure`.",
+        "You may also want to make a backup of it for future reuse.",
+    ]))
+
+
+def raise_sdk_not_found(e: Exception, kind: str, machine: MachineSpec):
+    raise SDKNotFoundError("\n".join([
+        f"Unable to download SDK: {e}",
+        "",
+        f"Specify --without-prebuilds=sdk:{kind} to build dependencies from source code.",
+        "",
+        "Another option is to do what Frida's CI does:",
+        "",
+        f"    ./releng/deps.py build --bundle=sdk --host={machine.identifier}",
+        "",
+        "This produces a tarball in ./deps which gets picked up if you retry `./configure`.",
+        "You may also want to make a backup of it for future reuse.",
+    ]))
+
+
+def generate_out_of_tree_makefile(sourcedir: Path) -> str:
+    m = ((SCRIPTS_DIR / "Makefile").read_text(encoding="utf-8")
+            .replace("sys.argv[1]", "r'" + str(RELENG_DIR.parent) + "'")
+            .replace('"$(shell pwd)"', shlex.quote(str(sourcedir)))
+            .replace("./build", "."))
+    return re.sub(r"git-submodules:.+?(?=\.PHONY:)", "", m, flags=re.MULTILINE | re.DOTALL)
+
+
+def generate_out_of_tree_make_bat(sourcedir: Path) -> str:
+    m = ((SCRIPTS_DIR / "make.bat").read_text(encoding="utf-8")
+            .replace("sys.argv[1]", "r'" + str(RELENG_DIR.parent) + "'")
+            .replace('"%dp0%"', '"' + str(sourcedir) + '"')
+            .replace('.\\build', "\"%dp0%\""))
+    return re.sub(r"if not exist .+?(?=endlocal)", "", m, flags=re.MULTILINE | re.DOTALL)
+
+
+def register_meson_options(meson_option_file: Path, group: argparse._ArgumentGroup):
+    interpreter = mesonbuild.optinterpreter.OptionInterpreter(subproject="")
+    interpreter.process(meson_option_file)
+
+    for key, opt in interpreter.options.items():
+        name = key.name
+        pretty_name = name.replace("_", "-")
+
+        if isinstance(opt, UserFeatureOption):
+            if opt.value != "enabled":
+                action = "enable"
+                value_to_set = "enabled"
+            else:
+                action = "disable"
+                value_to_set = "disabled"
+            group.add_argument(f"--{action}-{pretty_name}",
+                               action="append_const",
+                               const=f"-D{name}={value_to_set}",
+                               dest="main_meson_options",
+                               **parse_option_meta(name, action, opt))
+            if opt.value == "auto":
+                group.add_argument(f"--disable-{pretty_name}",
+                                   action="append_const",
+                                   const=f"-D{name}=disabled",
+                                   dest="main_meson_options",
+                                   **parse_option_meta(name, "disable", opt))
+        elif isinstance(opt, UserBooleanOption):
+            if not opt.value:
+                action = "enable"
+                value_to_set = "true"
+            else:
+                action = "disable"
+                value_to_set = "false"
+            group.add_argument(f"--{action}-{pretty_name}",
+                               action="append_const",
+                               const=f"-D{name}={value_to_set}",
+                               dest="main_meson_options",
+                               **parse_option_meta(name, action, opt))
+        elif isinstance(opt, UserComboOption):
+            group.add_argument(f"--with-{pretty_name}",
+                               choices=opt.choices,
+                               dest="meson_option:" + name,
+                               **parse_option_meta(name, "with", opt))
+        elif isinstance(opt, UserArrayOption):
+            group.add_argument(f"--with-{pretty_name}",
+                               dest="meson_option:" + name,
+                               type=make_array_option_value_parser(opt),
+                               **parse_option_meta(name, "with", opt))
+        else:
+            group.add_argument(f"--with-{pretty_name}",
+                               dest="meson_option:" + name,
+                               **parse_option_meta(name, "with", opt))
+
+
+def parse_option_meta(name: str,
+                      action: str,
+                      opt: UserOption[Any]):
+    params = {}
+
+    if isinstance(opt, UserStringOption):
+        default_value = repr(opt.value)
+        metavar = name.upper()
+    elif isinstance(opt, UserArrayOption):
+        default_value = ",".join(opt.value)
+        metavar = "{" + ",".join(opt.choices) + "}"
+    elif isinstance(opt, UserComboOption):
+        default_value = opt.value
+        metavar = "{" + "|".join(opt.choices) + "}"
+    else:
+        default_value = str(opt.value).lower()
+        metavar = name.upper()
+
+    if not (isinstance(opt, UserFeatureOption) \
+            and opt.value == "auto" \
+            and action == "disable"):
+        text = f"{help_text_from_meson(opt.description)} (default: {default_value})"
+        if action == "disable":
+            text = "do not " + text
+        params["help"] = text
+    params["metavar"] = metavar
+
+    return params
+
+
+def help_text_from_meson(description: str) -> str:
+    if description:
+        return description[0].lower() + description[1:]
+    return description
+
+
+def collect_meson_options(options: argparse.Namespace) -> list[str]:
+    result = []
+
+    for raw_name, raw_val in vars(options).items():
+        if raw_val is None:
+            continue
+        if raw_name == "main_meson_options":
+            result += raw_val
+        if raw_name.startswith("meson_option:"):
+            name = raw_name[13:]
+            val = raw_val if isinstance(raw_val, str) else ",".join(raw_val)
+            result += [f"-D{name}={val}"]
+
+    result += options.extra_meson_options
+
+    return result
+
+
+def make_array_option_value_parser(opt: UserOption[Any]) -> Callable[[str], list[str]]:
+    return lambda v: parse_array_option_value(v, opt)
+
+
+def parse_array_option_value(v: str, opt: UserArrayOption) -> list[str]:
+    vals = [v.strip() for v in v.split(",")]
+
+    choices = opt.choices
+    for v in vals:
+        if v not in choices:
+            pretty_choices = "', '".join(choices)
+            raise argparse.ArgumentTypeError(f"invalid array value: '{v}' (choose from '{pretty_choices}')")
+
+    return vals
+
+
+def detect_project_vscrt(sourcedir: Path) -> Optional[str]:
+    m = next(re.finditer(r"project\(([^)]+\))", read_meson_build(sourcedir)), None)
+    if m is not None:
+        project_args = m.group(1)
+        m = next(re.finditer("'b_vscrt=([^']+)'", project_args), None)
+        if m is not None:
+            return m.group(1)
+    return None
+
+
+def project_depends_on_vala_compiler(sourcedir: Path) -> bool:
+    return "'vala'" in read_meson_build(sourcedir)
+
+
+def read_meson_build(sourcedir: Path) -> str:
+    return (sourcedir / "meson.build").read_text(encoding="utf-8")
+
+
+def build_vala_compiler(toolchain_prefix: Path, deps_dir: Path, call_selected_meson: Callable):
+    print("Building Vala compiler...", flush=True)
+
+    workdir = deps_dir / "src"
+    workdir.mkdir(parents=True, exist_ok=True)
+
+    git = lambda *args, **kwargs: subprocess.run(["git", *args],
+                                                 **kwargs,
+                                                 capture_output=True,
+                                                 encoding="utf-8")
+    vala_checkout = workdir / "vala"
+    if vala_checkout.exists():
+        shutil.rmtree(vala_checkout)
+    vala_pkg = deps.load_dependency_parameters().packages["vala"]
+    deps.clone_shallow(vala_pkg, vala_checkout, git)
+
+    run_kwargs = {
+        "stdout": subprocess.PIPE,
+        "stderr": subprocess.STDOUT,
+        "encoding": "utf-8",
+        "check": True,
+    }
+    call_selected_meson([
+                            "setup",
+                            f"--prefix={toolchain_prefix}",
+                            "-Doptimization=2",
+                            "build",
+                        ],
+                        cwd=vala_checkout,
+                        **run_kwargs)
+    call_selected_meson(["install"],
+                        cwd=vala_checkout / "build",
+                        **run_kwargs)
+
+
+class ToolchainNotFoundError(Exception):
+    pass
+
+
+class SDKNotFoundError(Exception):
+    pass

package/releng/meson_make.py

@@ -0,0 +1,131 @@
+import argparse
+import os
+from pathlib import Path
+import pickle
+import shlex
+import shutil
+import sys
+from typing import Callable
+
+from . import env
+from .meson_configure import configure
+
+
+STANDARD_TARGET_NAMES = ["all", "clean", "distclean", "install", "test"]
+
+
+def main():
+    default_sourcedir = Path(sys.argv.pop(1)).resolve()
+    sourcedir = Path(os.environ.get("MESON_SOURCE_ROOT", default_sourcedir)).resolve()
+
+    default_builddir = Path(sys.argv.pop(1)).resolve()
+    builddir = Path(os.environ.get("MESON_BUILD_ROOT", default_builddir)).resolve()
+
+    parser = argparse.ArgumentParser(prog="make")
+    parser.add_argument("targets",
+                        help="Targets to build, e.g.: " + ", ".join(STANDARD_TARGET_NAMES),
+                        nargs="*",
+                        default="all")
+    options = parser.parse_args()
+
+    targets = options.targets
+    if isinstance(targets, str):
+        targets = [targets]
+
+    try:
+        make(sourcedir, builddir, targets)
+    except Exception as e:
+        print(e, file=sys.stderr)
+        sys.exit(1)
+
+
+def make(sourcedir: Path,
+         builddir: Path,
+         targets: list[str],
+         environ: dict[str, str] = os.environ,
+         call_meson: Callable = env.call_meson):
+    if not (builddir / "build.ninja").exists():
+        configure(sourcedir, builddir, environ=environ)
+
+    compile_options = []
+    if environ.get("V") == "1":
+        compile_options += ["-v"]
+
+    test_options = shlex.split(environ.get("FRIDA_TEST_OPTIONS", "-v"))
+
+    standard_targets = {
+        "all": ["compile"] + compile_options,
+        "clean": ["compile", "--clean"] + compile_options,
+        "distclean": lambda: distclean(sourcedir, builddir),
+        "install": ["install"],
+        "test": ["test"] + test_options,
+    }
+
+    env_state = pickle.loads((builddir / "frida-env.dat").read_bytes())
+
+    machine_config = env_state["host"]
+    if machine_config is None:
+        machine_config = env_state["build"]
+    meson_env = machine_config.make_merged_environment(environ)
+    meson_env["FRIDA_ALLOWED_PREBUILDS"] = ",".join(env_state["allowed_prebuilds"])
+    meson_env["FRIDA_DEPS"] = str(env_state["deps"])
+
+    def do_meson_command(args):
+        call_meson(args,
+                   use_submodule=env_state["meson"] == "internal",
+                   cwd=builddir,
+                   env=meson_env,
+                   check=True)
+
+    pending_targets = targets.copy()
+    pending_compile = None
+
+    while pending_targets:
+        target = pending_targets.pop(0)
+
+        action = standard_targets.get(target, None)
+        if action is None:
+            meson_command = "compile"
+        elif not callable(action):
+            meson_command = action[0]
+        else:
+            meson_command = None
+
+        if meson_command == "compile":
+            if pending_compile is None:
+                pending_compile = ["compile"]
+            if action is not None:
+                pending_compile += action[1:]
+            else:
+                pending_compile += [target]
+            continue
+
+        if pending_compile is not None:
+            do_meson_command(pending_compile)
+            pending_compile = None
+
+        if meson_command is not None:
+            do_meson_command(action)
+        else:
+            action()
+
+    if pending_compile is not None:
+        do_meson_command(pending_compile)
+
+
+def distclean(sourcedir: Path, builddir: Path):
+    items_to_delete = []
+
+    if not builddir.is_relative_to(sourcedir):
+        items_to_delete += list(builddir.iterdir())
+
+    items_to_delete += [
+        sourcedir / "build",
+        sourcedir / "deps",
+    ]
+
+    for item in items_to_delete:
+        try:
+            shutil.rmtree(item)
+        except:
+            pass