frida 16.2.1 → 16.2.2

package/releng/mkdevkit.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+import argparse
+import hashlib
+from pathlib import Path
+import subprocess
+import sys
+from typing import Optional
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT))
+from releng import devkit, env, machine_spec
+
+
+def main():
+    raw_args: list[str] = []
+    ool_optvals: dict[str, list[str]] = {}
+    pending_raw_args = sys.argv[1:]
+    while len(pending_raw_args) > 0:
+        cur = pending_raw_args.pop(0)
+        if cur == ">>>":
+            ool_hash = hashlib.sha256()
+            ool_strv = []
+            while True:
+                cur = pending_raw_args.pop(0)
+                if cur == "<<<":
+                    break
+                ool_hash.update(cur.encode("utf-8"))
+                ool_strv.append(cur)
+            val_id = "ool:" + ool_hash.hexdigest()
+            ool_optvals[val_id] = ool_strv
+            raw_args.append(val_id)
+        else:
+            raw_args.append(cur)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("kit")
+    parser.add_argument("machine",
+                        type=machine_spec.MachineSpec.parse)
+    parser.add_argument("outdir",
+                        type=Path)
+    parser.add_argument("-t", "--thin",
+                        help="build without cross-arch support",
+                        action="store_const",
+                        dest="flavor",
+                        const="_thin",
+                        default="")
+    parser.add_argument("--cc",
+                        help="C compiler to use",
+                        type=lambda v: parse_array_option_value(v, ool_optvals))
+    machine_options = dict.fromkeys(["c_args", "lib", "libtool", "ar", "nm", "objcopy", "pkg_config", "pkg_config_path"])
+    for name in machine_options.keys():
+        pretty_name = name.replace("_", "-")
+        parser.add_argument("--" + pretty_name,
+                            help=f"The {pretty_name} to use",
+                            type=lambda v: parse_array_option_value(v, ool_optvals))
+
+    options = parser.parse_args(raw_args)
+
+    kit = options.kit
+    machine = options.machine
+    outdir = options.outdir.resolve()
+    flavor = options.flavor
+
+    cc = options.cc
+    if cc is not None:
+        meson_config = {"c": cc}
+        for k, v in vars(options).items():
+            if k in machine_options and v is not None:
+                name = "pkg-config" if k == "pkg_config" else k
+                meson_config[name] = v
+    else:
+        build_dir = REPO_ROOT / "build"
+
+        if flavor == "":
+            fat_machine_file = env.query_machine_file_path(machine, flavor, build_dir)
+            if not fat_machine_file.exists() \
+                    and env.query_machine_file_path(machine, "_thin", build_dir).exists():
+                flavor = "_thin"
+
+        meson_config = env.load_meson_config(machine, flavor, build_dir)
+        assert meson_config is not None
+
+    try:
+        app = devkit.CompilerApplication(kit, machine, meson_config, outdir)
+        app.run()
+    except subprocess.CalledProcessError as e:
+        print(e, file=sys.stderr)
+        if e.output:
+            print("Stdout:", e.output, file=sys.stderr)
+        if e.stderr:
+            print("Stderr:", e.stderr, file=sys.stderr)
+        sys.exit(1)
+
+
+def parse_array_option_value(val: str, ool_optvals: dict[str, list[str]]) -> Optional[list[str]]:
+    if val == "":
+        return None
+    if val.startswith("ool:"):
+        ool_val = ool_optvals.get(val)
+        if ool_val is not None:
+            return ool_val
+    return [val]
+
+
+if __name__ == "__main__":
+    main()

package/releng/mkfatmacho.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+import os
+import shutil
+import struct
+import sys
+
+
+def make_fat_macho(output_path, input_paths):
+    """
+    Used to create a FAT Mach-O when Apple's lipo tool refuses to do so, such as
+    when needing two arm64e slices to support both the new and the old arm64e ABI.
+    """
+    input_slices = []
+    offset = 0x8000
+    slice_alignment = 0x4000
+    for input_path in input_paths:
+        delta = offset % slice_alignment
+        if delta != 0:
+            offset += slice_alignment - delta
+
+        offset_bits = bin(offset)
+        alignment = len(offset_bits[offset_bits.rfind("1") + 1:])
+
+        f = open(input_path, "rb+")
+
+        f.seek(4)
+        cpu_type, cpu_subtype = struct.unpack("<II", f.read(8))
+
+        f.seek(0, os.SEEK_END)
+        size = f.tell()
+
+        input_slices.append((f, cpu_type, cpu_subtype, offset, size, alignment))
+
+        offset += size
+
+    with open(output_path, "wb") as output_file:
+        header = struct.pack(">II", 0xcafebabe, len(input_slices))
+        output_file.write(header)
+
+        for (_, cpu_type, cpu_subtype, offset, size, alignment) in input_slices:
+            slice_spec = struct.pack(">IIIII", cpu_type, cpu_subtype, offset, size, alignment)
+            output_file.write(slice_spec)
+
+        for (input_file, _, _, offset, _, _) in input_slices:
+            input_file.seek(0)
+            output_file.seek(offset)
+            shutil.copyfileobj(input_file, output_file)
+
+
+if __name__ == '__main__':
+    output_path = sys.argv[1]
+    input_paths = sys.argv[2:]
+    make_fat_macho(output_path, input_paths)

package/releng/post-process-oabi.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+import argparse
+from pathlib import Path
+import shutil
+import struct
+import subprocess
+import tarfile
+import tempfile
+import urllib.request
+
+
+ARM64E_URL = "https://build.frida.re/deps/{version}/sdk-ios-arm64e.tar.xz"
+
+
+class CommandError(Exception):
+    pass
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--bundle", required=True)
+    parser.add_argument("--host", required=True)
+    parser.add_argument("--artifact", required=True)
+    parser.add_argument("--version", required=True)
+    args = parser.parse_args()
+
+    if args.bundle != "sdk":
+        raise CommandError("wrong bundle")
+    if args.host != "ios-arm64eoabi":
+        raise CommandError("wrong host")
+
+    arm64e_sdk_url = ARM64E_URL.format(version=args.version)
+
+    print(f"Downloading {arm64e_sdk_url}")
+    with urllib.request.urlopen(arm64e_sdk_url) as response, \
+            tempfile.NamedTemporaryFile(suffix=".tar.xz") as archive:
+        shutil.copyfileobj(response, archive)
+        archive.flush()
+        arm64e_artifact_path = Path(archive.name)
+
+        with tempfile.TemporaryDirectory() as patched_artifact_dir:
+            patched_artifact_file = Path(patched_artifact_dir) / "patched.tar.xz"
+
+            with tempfile.TemporaryDirectory() as artifact_extracted_dir, \
+                    tempfile.TemporaryDirectory() as arm64e_extracted_dir:
+                artifact_extracted_path = Path(artifact_extracted_dir)
+                arm64e_extracted_path = Path(arm64e_extracted_dir)
+
+                with tarfile.open(arm64e_artifact_path, "r:xz") as arm64e_tar:
+                    arm64e_tar.extractall(arm64e_extracted_path)
+
+                    artifact_path = Path(args.artifact)
+                    with tarfile.open(artifact_path, "r:xz") as tar:
+                        tar.extractall(artifact_extracted_path)
+
+                        print("Patching libffi.a...")
+                        steal_object(artifact_extracted_path / "lib" / "libffi.a",
+                                     arm64e_extracted_path / "lib" / "libffi.a")
+                        with tarfile.open(patched_artifact_file, "w:xz") as patched_tar:
+                            patched_tar.add(artifact_extracted_path, arcname="./")
+
+            print(f"Overwriting {artifact_path}")
+            shutil.copy(patched_artifact_file, artifact_path)
+
+
+def steal_object(arm64eoabi_libffi_a_path: Path, arm64e_libffi_a_path: Path):
+    """
+    Steal just the aarch64_sysv.S.o object file from the arm64e libffi.a in
+    order to get the CIE info from the future compiler. Then patch the Mach-O
+    header of the stolen object to match the old arm64e ABI. It works because
+    the __text section is exactly the same.
+    """
+    if not arm64eoabi_libffi_a_path.exists():
+        raise RuntimeError("input arm64eoabi libffi.a not found")
+    if not arm64e_libffi_a_path.exists():
+        raise RuntimeError("input arm64e libffi.a not found")
+
+    with tempfile.TemporaryDirectory() as oabi_dir, tempfile.TemporaryDirectory() as nabi_dir:
+        perform("ar", "-x", arm64eoabi_libffi_a_path.absolute(), cwd=oabi_dir)
+        perform("ar", "-x", arm64e_libffi_a_path.absolute(), cwd=nabi_dir)
+        dst = Path(oabi_dir) / "aarch64_sysv.S.o"
+        dst.unlink()
+        shutil.copy(Path(nabi_dir) / "aarch64_sysv.S.o", dst)
+        with dst.open("rb+") as f:
+            f.seek(0xb)
+            f.write(struct.pack("B", 0))
+
+        perform("ar", "-r", arm64eoabi_libffi_a_path.absolute(), dst.name, cwd=oabi_dir)
+
+
+def perform(*args, **kwargs):
+    print(">", " ".join([str(arg) for arg in args]), flush=True)
+    return subprocess.run(args, check=True, **kwargs)
+
+
+if __name__ == "__main__":
+    main()

package/releng/progress.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+from typing import Callable
+
+
+@dataclass
+class Progress:
+    message: str
+
+
+ProgressCallback = Callable[[Progress], None]
+
+
+def print_progress(progress: Progress):
+    print(f"{progress.message}...", flush=True)

package/releng/sync-from-upstream.py

@@ -0,0 +1,185 @@
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+
+
+def make_gnome_url(repo_name):
+    return "https://gitlab.gnome.org/GNOME/{}.git".format(repo_name)
+
+
+upstreams = {
+    "meson": ("https://github.com/mesonbuild/meson.git", "master"),
+    "termux-elf-cleaner": "https://github.com/termux/termux-elf-cleaner.git",
+    "libiconv": "https://git.savannah.gnu.org/git/libiconv.git",
+    "zlib": "https://github.com/madler/zlib.git",
+    "brotli": "https://github.com/google/brotli.git",
+    "minizip": "https://github.com/zlib-ng/minizip-ng.git",
+    "libffi": "https://github.com/libffi/libffi.git",
+    "libunwind": "https://github.com/libunwind/libunwind.git",
+    "glib": make_gnome_url("glib"),
+    "glib-networking": make_gnome_url("glib-networking"),
+    "libnice": "https://gitlab.freedesktop.org/libnice/libnice.git",
+    "usrsctp": "https://github.com/sctplab/usrsctp.git",
+    "libgee": make_gnome_url("libgee"),
+    "json-glib": make_gnome_url("json-glib"),
+    "libpsl": "https://github.com/rockdaboot/libpsl.git",
+    "libxml2": make_gnome_url("libxml2"),
+    "libsoup": make_gnome_url("libsoup"),
+    "vala": make_gnome_url("vala"),
+    "xz": "https://git.tukaani.org/xz.git",
+    "pkg-config": "https://gitlab.freedesktop.org/pkg-config/pkg-config.git",
+    "quickjs": ("https://github.com/bellard/quickjs.git", "master"),
+    "gn": "https://gn.googlesource.com/gn",
+    "v8": "https://chromium.googlesource.com/v8/v8",
+    "capstone": ("https://github.com/capstone-engine/capstone.git", "v5"),
+    "tinycc": "https://repo.or.cz/tinycc.git",
+}
+
+
+def sync(repo_path):
+    repo_name = os.path.basename(repo_path)
+
+    patches_path = os.path.join(str(Path.home()), ".frida-sync-" + re.sub(r"[^\w\d]", "-", repo_path.lower()).lstrip("-"))
+    if os.path.exists(patches_path):
+        patches = PendingPatches.load(patches_path)
+
+        print("Applying {} pending patches".format(patches.count))
+    else:
+        entry = upstreams.get(repo_name, None)
+        if entry is None:
+            raise UnknownUpstreamError("Unknown upstream: {}".format(repo_name))
+        if isinstance(entry, tuple):
+            upstream_url, upstream_branch = entry
+        else:
+            upstream_url = entry
+            upstream_branch = "main"
+        upstream_target = "upstream/" + upstream_branch
+
+        print("Synchronizing with {}".format(upstream_url))
+
+        subprocess.run(["git", "checkout", "main"], cwd=repo_path, capture_output=True, check=True)
+        subprocess.run(["git", "pull"], cwd=repo_path, capture_output=True, check=True)
+        result = subprocess.run(["git", "status"], cwd=repo_path, capture_output=True, check=True, encoding='utf-8')
+        if not "working tree clean" in result.stdout:
+            raise WorkingTreeDirtyError("Working tree is dirty")
+
+        subprocess.run(["git", "remote", "add", "upstream", upstream_url], cwd=repo_path, capture_output=True)
+        subprocess.run(["git", "fetch", "upstream"], cwd=repo_path, check=True)
+
+        patches, base = list_our_patches(repo_path)
+        print("We have {} patches on top of upstream".format(patches.count))
+
+        new_entries = list_upstream_changes(repo_path, upstream_target, base)
+        if len(new_entries) == 0:
+            print("Already up-to-date")
+            return
+
+        print("Upstream has {} new commits".format(len(new_entries)))
+
+        print("Merging...")
+        subprocess.run(["git", "merge", "-s", "ours", upstream_target], cwd=repo_path, capture_output=True, check=True)
+        subprocess.run(["git", "checkout", "--detach", upstream_target], cwd=repo_path, capture_output=True, check=True)
+        subprocess.run(["git", "reset", "--soft", "main"], cwd=repo_path, capture_output=True, check=True)
+        subprocess.run(["git", "checkout", "main"], cwd=repo_path, capture_output=True, check=True)
+        subprocess.run(["git", "commit", "--amend", "-C", "HEAD"], cwd=repo_path, capture_output=True, check=True)
+
+        patches.save(patches_path)
+
+    while True:
+        index, cid, message = patches.try_pop()
+        if index is None:
+            break
+
+        print("Cherry-picking {}/{}: {}".format(index + 1, patches.count, message))
+        try:
+            subprocess.run(["git", "cherry-pick", cid], cwd=repo_path, capture_output=True, encoding='utf-8', check=True)
+        except subprocess.CalledProcessError as e:
+            patches.save(patches_path)
+
+            print("\n*** Unable to apply this patch:")
+            print(e.stderr)
+            print("Run `git cherry-pick --abort` and re-run script to skip it.")
+
+            return
+
+    os.remove(patches_path)
+    print("Done!")
+
+def list_our_patches(repo_path):
+    items = []
+    base = None
+    entries = list_recent_commits(repo_path, "--max-count=1000")
+    for index, entry in enumerate(entries):
+        cid, message = entry
+        if message.startswith("Merge"):
+            base = entries[index + 1][0]
+            break
+        items.append(("pending", cid, message))
+    items.reverse()
+    return (PendingPatches(items), base)
+
+def list_upstream_changes(repo_path, upstream_target, since):
+    return list(reversed(list_recent_commits(repo_path, since + ".." + upstream_target)))
+
+def list_recent_commits(repo_path, *args):
+    result = subprocess.run(["git", "log", "--pretty=oneline", "--abbrev-commit", "--topo-order"] + list(args),
+        cwd=repo_path, capture_output=True, check=True, encoding='utf-8', errors='surrogateescape')
+    return [line.split(" ", 1) for line in result.stdout.rstrip().split("\n")]
+
+
+class PendingPatches(object):
+    def __init__(self, items):
+        self._items = items
+
+        offset = 0
+        for status, cid, message in items:
+            if status == "applied":
+                offset += 1
+            else:
+                break
+        self._offset = offset
+
+    @property
+    def count(self):
+        return len(self._items)
+
+    def try_pop(self):
+        index = self._offset
+        if index == len(self._items):
+            return (None, None, None)
+
+        _, cid, message = self._items[index]
+        self._items[index] = ("applied", cid, message)
+        self._offset += 1
+
+        return (index, cid, message)
+
+    @classmethod
+    def load(cls, path):
+        with open(path, "r", encoding='utf-8') as f:
+            data = f.read()
+
+        items = []
+        for line in data.strip().split("\n"):
+            status, cid, message = line.split(" ", maxsplit=2)
+            items.append((status, cid, message))
+        return PendingPatches(items)
+
+    def save(self, path):
+        data = "\n".join([" ".join(item) for item in self._items]) + "\n"
+        with open(path, "w", encoding='utf-8') as f:
+            f.write(data)
+
+
+class WorkingTreeDirtyError(Exception):
+    pass
+
+
+class UnknownUpstreamError(Exception):
+    pass
+
+
+if __name__ == '__main__':
+    sync(os.path.abspath(sys.argv[1]))

package/releng/tomlkit/tomlkit/__init__.py

@@ -0,0 +1,59 @@
+from tomlkit.api import TOMLDocument
+from tomlkit.api import aot
+from tomlkit.api import array
+from tomlkit.api import boolean
+from tomlkit.api import comment
+from tomlkit.api import date
+from tomlkit.api import datetime
+from tomlkit.api import document
+from tomlkit.api import dump
+from tomlkit.api import dumps
+from tomlkit.api import float_
+from tomlkit.api import inline_table
+from tomlkit.api import integer
+from tomlkit.api import item
+from tomlkit.api import key
+from tomlkit.api import key_value
+from tomlkit.api import load
+from tomlkit.api import loads
+from tomlkit.api import nl
+from tomlkit.api import parse
+from tomlkit.api import register_encoder
+from tomlkit.api import string
+from tomlkit.api import table
+from tomlkit.api import time
+from tomlkit.api import unregister_encoder
+from tomlkit.api import value
+from tomlkit.api import ws
+
+
+__version__ = "0.12.4"
+__all__ = [
+    "aot",
+    "array",
+    "boolean",
+    "comment",
+    "date",
+    "datetime",
+    "document",
+    "dump",
+    "dumps",
+    "float_",
+    "inline_table",
+    "integer",
+    "item",
+    "key",
+    "key_value",
+    "load",
+    "loads",
+    "nl",
+    "parse",
+    "string",
+    "table",
+    "time",
+    "TOMLDocument",
+    "value",
+    "ws",
+    "register_encoder",
+    "unregister_encoder",
+]

package/releng/tomlkit/tomlkit/_compat.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+import contextlib
+import sys
+
+from typing import Any
+
+
+PY38 = sys.version_info >= (3, 8)
+
+
+def decode(string: Any, encodings: list[str] | None = None):
+    if not isinstance(string, bytes):
+        return string
+
+    encodings = encodings or ["utf-8", "latin1", "ascii"]
+
+    for encoding in encodings:
+        with contextlib.suppress(UnicodeEncodeError, UnicodeDecodeError):
+            return string.decode(encoding)
+
+    return string.decode(encodings[0], errors="ignore")

package/releng/tomlkit/tomlkit/_types.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from typing import Any
+from typing import TypeVar
+
+
+WT = TypeVar("WT", bound="WrapperType")
+
+if TYPE_CHECKING:  # pragma: no cover
+    # Define _CustomList and _CustomDict as a workaround for:
+    # https://github.com/python/mypy/issues/11427
+    #
+    # According to this issue, the typeshed contains a "lie"
+    # (it adds MutableSequence to the ancestry of list and MutableMapping to
+    # the ancestry of dict) which completely messes with the type inference for
+    # Table, InlineTable, Array and Container.
+    #
+    # Importing from builtins is preferred over simple assignment, see issues:
+    # https://github.com/python/mypy/issues/8715
+    # https://github.com/python/mypy/issues/10068
+    from builtins import dict as _CustomDict  # noqa: N812
+    from builtins import float as _CustomFloat  # noqa: N812
+    from builtins import int as _CustomInt  # noqa: N812
+    from builtins import list as _CustomList  # noqa: N812
+    from typing import Callable
+    from typing import Concatenate
+    from typing import ParamSpec
+    from typing import Protocol
+
+    P = ParamSpec("P")
+
+    class WrapperType(Protocol):
+        def _new(self: WT, value: Any) -> WT:
+            ...
+
+else:
+    from collections.abc import MutableMapping
+    from collections.abc import MutableSequence
+    from numbers import Integral
+    from numbers import Real
+
+    class _CustomList(MutableSequence, list):
+        """Adds MutableSequence mixin while pretending to be a builtin list"""
+
+        def __add__(self, other):
+            new_list = self.copy()
+            new_list.extend(other)
+            return new_list
+
+        def __iadd__(self, other):
+            self.extend(other)
+            return self
+
+    class _CustomDict(MutableMapping, dict):
+        """Adds MutableMapping mixin while pretending to be a builtin dict"""
+
+        def __or__(self, other):
+            new_dict = self.copy()
+            new_dict.update(other)
+            return new_dict
+
+        def __ior__(self, other):
+            self.update(other)
+            return self
+
+    class _CustomInt(Integral, int):
+        """Adds Integral mixin while pretending to be a builtin int"""
+
+    class _CustomFloat(Real, float):
+        """Adds Real mixin while pretending to be a builtin float"""
+
+
+def wrap_method(
+    original_method: Callable[Concatenate[WT, P], Any],
+) -> Callable[Concatenate[WT, P], Any]:
+    def wrapper(self: WT, *args: P.args, **kwargs: P.kwargs) -> Any:
+        result = original_method(self, *args, **kwargs)
+        if result is NotImplemented:
+            return result
+        return self._new(result)
+
+    return wrapper