firedrill-run 2.2.4 → 3.0.0

package/dist/cli/runtime/source_snapshot.py

@@ -0,0 +1,391 @@
+"""Original local intake. Never imports/builds the selected project or contacts a service.
+
+Only the explicit inspect/snapshot command invokes this bundled stdlib helper.
+The manifest is customer-runner evidence, not independent source attestation.
+"""
+import argparse
+import gzip
+import hashlib
+import io
+import json
+import os
+from pathlib import Path, PurePosixPath
+import re
+import select
+import shutil
+import signal
+import stat
+import subprocess
+import sys
+import tarfile
+import tempfile
+import time
+
+EXCLUDED = {".git", ".hg", ".svn", ".aws", ".ssh", ".gcloud", "node_modules", ".venv", "venv",
+    "__pycache__", ".next", ".turbo", "coverage", ".firedrill-simulation"}
+PRIVATE_SUFFIXES = {".pem", ".key", ".p12", ".pfx", ".sqlite", ".sqlite3", ".db", ".kdbx", ".log",
+    ".sqlite-wal", ".sqlite-shm", ".sqlite-journal", ".sqlite3-wal", ".sqlite3-shm", ".sqlite3-journal",
+    ".db-wal", ".db-shm", ".db-journal"}
+PRIVATE_NAMES = {".npmrc", ".pypirc", ".netrc", "credentials.json", "id_rsa", "id_ed25519"}
+SECRET_PATTERNS = [re.compile(value) for value in (
+    r"\bfd_tool_[a-f0-9-]{36}\.[A-Za-z0-9_-]{64}",
+    r"-----BEGIN (?:RSA |EC |DSA |ENCRYPTED |OPENSSH )?PRIVATE KEY-----",
+    r"\b(?:sk-ant-api\d+-|sk-proj-|sk-svcacct-|gh[pousr]_|github_pat_|xox[baprs]-)[A-Za-z0-9_-]{16,}",
+    r"\bAKIA[A-Z0-9]{16}\b", r'"type"\s*:\s*"service_account"[\s\S]{0,4096}"private_key"',
+)]
+VARIABLE_PATTERNS = [re.compile(value) for value in (
+    r"(?:process\.env\.|import\.meta\.env\.)([A-Za-z_][A-Za-z0-9_]*)",
+    r"(?:process\.env|os\.environ)\s*\[\s*['\"]([A-Za-z_][A-Za-z0-9_]*)['\"]\s*\]",
+    r"(?:os\.(?:getenv|environ\.get)|env)\s*\(\s*['\"]([A-Za-z_][A-Za-z0-9_]*)['\"]",
+)]
+MAX_FILES, MAX_FILE_BYTES, MAX_BYTES = 100000, 50 * 1024**2, 2 * 1024**3
+
+
+class IntakeError(Exception):
+    def __init__(self, code, message):
+        self.code, self.message = code, message
+
+
+def canonical(value):
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
+
+
+def safe_path(raw):
+    try:
+        raw.encode("utf-8")
+    except UnicodeError:
+        raise IntakeError("source_path_invalid", "A source path is not valid Unicode.") from None
+    path = PurePosixPath(raw)
+    if (not raw or len(raw) > 1024 or "\\" in raw or any(ord(c) < 32 or ord(c) == 127 for c in raw)
+            or path.is_absolute() or ".." in path.parts or not path.parts or ":" in path.parts[0]
+            or path.as_posix() != raw):
+        raise IntakeError("source_path_invalid", "A source path is not supported.")
+    return path
+
+
+def excluded(path):
+    return (bool(EXCLUDED.intersection(path.parts)) or path.suffix.lower() in PRIVATE_SUFFIXES
+        or path.name.startswith(".env") or path.name in PRIVATE_NAMES)
+
+
+def git(root, *args, optional=False):
+    executable = shutil.which("git")
+    if not executable:
+        if optional:
+            return None
+        raise IntakeError("git_required", "Install Git to retain repository identity for this source.")
+    executable = str(Path(executable).resolve(strict=True))
+    if Path(executable) == root or root in Path(executable).parents:
+        raise IntakeError("git_runtime_refused", "Use Git installed outside the project. A project-supplied executable was not run.")
+    environment = {key: os.environ[key] for key in ("PATH", "SYSTEMROOT") if key in os.environ}
+    environment.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0", GIT_OPTIONAL_LOCKS="0")
+    try:
+        # Never run a repository hook/fsmonitor, diff driver, filter, checkout,
+        # submodule command or network action. stderr may contain local values.
+        with subprocess.Popen([executable, "--no-optional-locks", "-c", "core.fsmonitor=false", "-c",
+                "core.hooksPath=" + os.devnull, "-c", "core.untrackedCache=false", "-C", str(root), *args],
+                stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=environment) as process:
+            output, deadline = bytearray(), time.monotonic() + 30
+            try:
+                while True:
+                    remaining = deadline - time.monotonic()
+                    if remaining <= 0 or not select.select([process.stdout], [], [], remaining)[0]:
+                        raise IntakeError("git_metadata_unavailable", "Repository metadata did not finish within the read deadline.")
+                    chunk = os.read(process.stdout.fileno(), 65536)
+                    if not chunk:
+                        break
+                    output.extend(chunk)
+                    if len(output) > 32 * 1024**2:
+                        raise IntakeError("source_capacity_required", "Repository metadata exceeds local intake capacity. Select a smaller project root.")
+                returncode = process.wait(timeout=max(0.01, deadline - time.monotonic()))
+            finally:
+                if process.poll() is None:
+                    process.kill()
+                    process.wait()
+        if returncode:
+            if optional:
+                return None
+            raise IntakeError("git_metadata_unavailable", "Repository metadata could not be read safely.")
+        return bytes(output)
+    except (OSError, subprocess.TimeoutExpired):
+        raise IntakeError("git_metadata_unavailable", "Repository metadata could not be read safely.") from None
+
+
+def repository(root):
+    top = git(root, "rev-parse", "--show-toplevel", optional=True)
+    if top is None:
+        # A .git marker with a failed Git read must not silently become a plain
+        # folder. Check parents for actual repository markers without reading keys.
+        if any((parent / ".git").exists() for parent in (root, *root.parents)):
+            raise IntakeError("git_metadata_unavailable", "Repository identity could not be confirmed. No folder fallback was used.")
+        return None, None, {}
+    try:
+        git_root = Path(os.fsdecode(top.rstrip(b"\r\n"))).resolve(strict=True)
+        relative = root.relative_to(git_root).as_posix()
+        head = git(root, "rev-parse", "--verify", "HEAD", optional=True)
+        commit = head.strip().decode("ascii") if head else None
+        if commit is not None and not re.fullmatch(r"[a-f0-9]{40}|[a-f0-9]{64}", commit):
+            raise ValueError()
+        listed = git(root, "ls-files", "--cached", "--others", "--exclude-standard", "-z")
+        paths = sorted({safe_path(os.fsdecode(raw)).as_posix() for raw in listed.split(b"\0") if raw})
+        if len(paths) > MAX_FILES:
+            raise IntakeError("source_capacity_required", "Select a project root with fewer than 100,000 source entries.")
+        baseline = {}
+        if commit:
+            tree = git(git_root, "ls-tree", "-r", "-z", commit, "--", relative)
+            prefix = "" if relative == "." else relative + "/"
+            for record in tree.split(b"\0"):
+                if not record:
+                    continue
+                header, raw = record.split(b"\t", 1)
+                mode, kind, oid = header.decode("ascii").split(" ")
+                name = os.fsdecode(raw)
+                if not name.startswith(prefix):
+                    raise ValueError()
+                path = safe_path(name[len(prefix):])
+                if excluded(path):
+                    continue
+                if not re.fullmatch(r"[a-f0-9]{40}|[a-f0-9]{64}", oid) or len(oid) != len(commit):
+                    raise ValueError()
+                if kind == "blob" and mode in {"100644", "100755"}:
+                    baseline[path.as_posix()] = {"oid": oid, "mode": 0o755 if mode == "100755" else 0o644}
+                elif kind not in {"blob", "commit"}:
+                    raise ValueError()
+                if len(baseline) > MAX_FILES:
+                    raise IntakeError("source_capacity_required", "The base revision exceeds local inspection capacity. Select a smaller root.")
+        return {"base_commit": commit, "project_root": "" if relative == "." else relative,
+            "selection": "git_index_and_unignored_files", "unborn": commit is None}, paths, baseline
+    except (ValueError, UnicodeError, OSError):
+        raise IntakeError("git_metadata_unavailable", "Repository identity or paths could not be read safely.") from None
+
+
+def directory_paths(root_fd):
+    paths, omitted = [], 0
+
+    def visit(fd, prefix=""):
+        nonlocal omitted
+        with os.scandir(fd) as entries:
+            names = []
+            for entry in entries:
+                names.append(entry.name)
+                if len(names) > MAX_FILES:
+                    raise IntakeError("source_capacity_required", "Select a smaller source folder.")
+            names.sort()
+        for name in names:
+            relative = prefix + name
+            path = safe_path(relative)
+            if excluded(path):
+                omitted += 1
+                continue
+            metadata = os.stat(name, dir_fd=fd, follow_symlinks=False)
+            if stat.S_ISDIR(metadata.st_mode):
+                child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
+                try:
+                    visit(child, relative + "/")
+                finally:
+                    os.close(child)
+            else:
+                paths.append(relative)
+                if len(paths) > MAX_FILES:
+                    raise IntakeError("source_capacity_required", "Select a smaller source folder.")
+    visit(root_fd)
+    return paths, omitted
+
+
+def opened(root_fd, path):
+    parent = os.dup(root_fd)
+    try:
+        for name in path.parts[:-1]:
+            child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent)
+            os.close(parent)
+            parent = child
+        metadata = os.stat(path.name, dir_fd=parent, follow_symlinks=False)
+        if not stat.S_ISREG(metadata.st_mode):
+            return None, metadata
+        fd = os.open(path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent)
+        actual = os.fstat(fd)
+        if not stat.S_ISREG(actual.st_mode) or (actual.st_dev, actual.st_ino) != (metadata.st_dev, metadata.st_ino):
+            os.close(fd)
+            raise IntakeError("source_changed", "A source file changed while it was being opened. Create a new snapshot.")
+        return fd, actual
+    finally:
+        os.close(parent)
+
+
+def inspect(root, copy_to=None):
+    if os.name != "posix" or not hasattr(os, "O_NOFOLLOW"):
+        raise IntakeError("local_source_platform_unsupported", "Safe local intake currently requires macOS or Linux. Use source upload in the web app on this platform.")
+    root = root.resolve(strict=True)
+    if root == Path(root.anchor) or root == Path.home().resolve() or not root.is_dir():
+        raise IntakeError("source_root_invalid", "Choose a project folder, not the filesystem root or home folder.")
+    root_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+    try:
+        root_identity = os.fstat(root_fd)
+        identity, paths, baseline = repository(root)
+        current_root = root.lstat()
+        if (current_root.st_dev, current_root.st_ino) != (root_identity.st_dev, root_identity.st_ino):
+            raise IntakeError("source_changed", "The project directory changed during capture.")
+        omitted = 0
+        if paths is None:
+            paths, omitted = directory_paths(root_fd)
+        files, variables, changes, skipped, total = [], {}, [], {"private_or_generated": omitted, "non_regular": 0, "missing": 0}, 0
+        observed = set()
+        for raw in paths:
+            path = safe_path(raw)
+            if excluded(path):
+                skipped["private_or_generated"] += 1
+                continue
+            try:
+                fd, before = opened(root_fd, path)
+            except FileNotFoundError:
+                skipped["missing"] += 1
+                continue
+            if fd is None:
+                skipped["non_regular"] += 1
+                continue
+            try:
+                if before.st_size > MAX_FILE_BYTES or total + before.st_size > MAX_BYTES:
+                    raise IntakeError("source_capacity_required", "Source exceeds local inspection capacity. Select a smaller root or request support.")
+                with os.fdopen(fd, "rb", closefd=False) as incoming:
+                    body = incoming.read(MAX_FILE_BYTES + 1)
+                after = os.fstat(fd)
+                if (before.st_size, before.st_mtime_ns, before.st_ctime_ns) != (after.st_size, after.st_mtime_ns, after.st_ctime_ns) or len(body) != before.st_size:
+                    raise IntakeError("source_changed", "Source changed during capture. Create a new snapshot rather than reusing this one.")
+            finally:
+                os.close(fd)
+            text = body.decode("utf-8", errors="replace")
+            if any(pattern.search(text) for pattern in SECRET_PATTERNS):
+                raise IntakeError("source_credentials_detected", "A possible credential or private key was detected. Move it to local configuration or Vault before capturing source.")
+            for pattern in VARIABLE_PATTERNS:
+                for match in pattern.finditer(text):
+                    name = match.group(1)
+                    if len(name) > 128 or name not in variables and len(variables) >= 10000:
+                        raise IntakeError("source_capacity_required", "Detected configuration names exceed local inspection capacity. Select a smaller root or request support.")
+                    if name not in variables:
+                        variables[name] = {"name": name, "source_path": raw, "line": text.count("\n", 0, match.start()) + 1,
+                            "phase": "unresolved", "required": None, "has_value": False}
+            mode = 0o755 if before.st_mode & 0o111 else 0o644
+            observed.add(raw)
+            if identity:
+                base = baseline.get(raw)
+                algorithm = "sha256" if len(identity["base_commit"] or "") == 64 else "sha1"
+                oid = hashlib.new(algorithm, b"blob " + str(len(body)).encode("ascii") + b"\0" + body).hexdigest()
+                change = "added" if base is None else "modified" if base["oid"] != oid else "mode_changed" if base["mode"] != mode else None
+                if change:
+                    changes.append({"path": raw, "kind": change})
+            files.append({"path": raw, "size": len(body), "sha256": hashlib.sha256(body).hexdigest(), "mode": mode})
+            total += len(body)
+            if copy_to is not None:
+                destination = copy_to.joinpath(*path.parts)
+                destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+                with destination.open("xb") as target:
+                    target.write(body)
+                destination.chmod(mode & 0o700)
+        if not files:
+            raise IntakeError("source_root_empty", "No eligible source files were found.")
+        if identity:
+            # Check the same base identity again; retained file hashes, not a
+            # clean/dirty label, define captured working-copy content.
+            last = git(root, "rev-parse", "--verify", "HEAD", optional=True)
+            if (last.strip().decode("ascii") if last else None) != identity["base_commit"]:
+                raise IntakeError("source_changed", "The repository revision changed during capture. Capture it again.")
+            for raw in sorted(set(baseline) - observed):
+                try:
+                    fd, _ = opened(root_fd, safe_path(raw))
+                except FileNotFoundError:
+                    changes.append({"path": raw, "kind": "deleted"})
+                else:
+                    if fd is not None:
+                        os.close(fd)
+                    # Present but unselected/unsupported is not a deletion.
+                    skipped["non_regular"] += 1
+        ordered = sorted(files, key=lambda item: item["path"])
+        current_root = root.lstat()
+        if (current_root.st_dev, current_root.st_ino) != (root_identity.st_dev, root_identity.st_ino):
+            raise IntakeError("source_changed", "The project directory changed during capture.")
+        result = {"schema": "firedrill.local-source.v1", "files": ordered, "expanded_bytes": total,
+            "content_digest": hashlib.sha256(canonical({"schema": "firedrill.local-source-content.v1", "files": ordered})).hexdigest(),
+            "repository": identity, "variables": sorted(variables.values(), key=lambda item: item["name"]), "excluded": skipped,
+            "excluded_unit": "observed_entries_not_recursive_file_totals", "git_ignored_entries_enumerated": False,
+            "changes_from_base": sorted(changes, key=lambda item: item["path"]),
+            "state": "captured_working_copy", "capture_boundary": "per_file_consistency_not_filesystem_atomic",
+            "provenance": "customer_device", "uploaded": False,
+            "disclosure": "Source pattern screening is not complete secret or personal-data detection. Review selected files before any explicit transfer. No environment values were read."}
+        if len(canonical(result)) > 24 * 1024**2:
+            raise IntakeError("source_capacity_required", "The local source manifest is too large. Select a smaller root or request support.")
+        return result
+    finally:
+        os.close(root_fd)
+
+
+def snapshot(root, storage):
+    metadata = storage.lstat()
+    if (not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_uid != os.getuid()
+            or metadata.st_mode & 0o077):
+        raise IntakeError("source_storage_unsafe", "Local source storage must be private and owned by this OS user.")
+    temporary = Path(tempfile.mkdtemp(prefix="source-", dir=storage))
+    try:
+        files = temporary / "files"
+        files.mkdir(mode=0o700)
+        manifest = inspect(root, files)
+        if manifest["excluded"]["non_regular"]:
+            raise IntakeError("source_entries_unsupported", "This project contains selected links, submodules or unsupported files. Inspect the source and choose a self-contained project root before creating a snapshot.")
+        if shutil.disk_usage(storage).free < manifest["expanded_bytes"] + 32 * 1024**2:
+            raise IntakeError("source_storage_full", "Free disk space before creating the source archive.")
+        archive_path = temporary / "source.tar.gz"
+        with archive_path.open("xb") as outgoing, gzip.GzipFile(filename="", mode="wb", fileobj=outgoing, mtime=0) as compressed:
+            with tarfile.open(mode="w|", fileobj=compressed, format=tarfile.PAX_FORMAT) as archive:
+                for record in manifest["files"]:
+                    source = files.joinpath(*PurePosixPath(record["path"]).parts)
+                    with source.open("rb") as incoming:
+                        body = incoming.read(MAX_FILE_BYTES + 1)
+                        if len(body) != record["size"] or hashlib.sha256(body).hexdigest() != record["sha256"]:
+                            raise IntakeError("source_changed", "The private source copy changed. Capture source again.")
+                        info = tarfile.TarInfo("project/" + record["path"])
+                        info.size, info.mode, info.mtime = record["size"], record["mode"], 0
+                        archive.addfile(info, io.BytesIO(body))
+        archive_path.chmod(0o600)
+        with archive_path.open("rb") as incoming:
+            manifest["archive"] = {"filename": "source.tar.gz", "size": archive_path.stat().st_size,
+                "sha256": hashlib.file_digest(incoming, "sha256").hexdigest()}
+        with (temporary / "manifest.json").open("xb") as target:
+            target.write(canonical(manifest))
+            target.flush()
+            os.fsync(target.fileno())
+        (temporary / "manifest.json").chmod(0o600)
+        return {**manifest, "local_snapshot_path": str(temporary)}
+    except BaseException:
+        # Only the exact mkdtemp directory created here is removed on failure.
+        shutil.rmtree(temporary)
+        raise
+
+
+def main():
+    try:
+        sys.stdout.reconfigure(encoding="utf-8", errors="strict")
+        if not (3, 12) <= sys.version_info[:2] < (3, 14):
+            raise IntakeError("python_runtime_required", "Local source intake requires Python 3.12 or 3.13. No runtime was installed.")
+        if os.name != "posix" or not hasattr(os, "O_NOFOLLOW"):
+            raise IntakeError("local_source_platform_unsupported", "Safe local intake currently requires macOS or Linux. Use source upload in the web app on this platform.")
+        parser = argparse.ArgumentParser()
+        parser.add_argument("action", choices=("inspect", "snapshot"))
+        parser.add_argument("--root", required=True)
+        parser.add_argument("--storage")
+        args = parser.parse_args()
+        def interrupted(*_):
+            raise IntakeError("source_canceled", "Source capture stopped. No upload or target execution occurred.")
+        signal.signal(signal.SIGTERM, interrupted)
+        signal.signal(signal.SIGINT, interrupted)
+        result = inspect(Path(args.root)) if args.action == "inspect" else snapshot(Path(args.root), Path(args.storage))
+        print(json.dumps({"ok": True, "data": result}, ensure_ascii=False, separators=(",", ":")))
+    except IntakeError as error:
+        print(json.dumps({"ok": False, "error": {"code": error.code, "message": error.message}}))
+        return 2
+    except Exception:
+        print(json.dumps({"ok": False, "error": {"code": "source_capture_failed", "message": "Source capture could not finish safely. No upload or execution occurred."}}))
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())