@kinosec/cli 0.10.3 → 0.11.2

package/dist/vendor/ctxpack.py

@@ -0,0 +1,701 @@
+#!/usr/bin/env python3
+"""ctxpack -- source-context ingestion for graybox CTF/pentest agents.
+
+Turns a black-box agent into a graybox one by packing a challenge's SOURCE CODE
+into a single inject-ready bundle. The only real problem it solves is that source
+archives are padded with non-logic files (CSS frameworks, images, fonts, vendored
+deps, lockfiles, editor junk); a cheap extension/path FILTER -- not a code graph,
+embeddings, or a whitebox agent -- is what makes "just paste the source" work.
+
+Pipeline: FILTER (KEEP vs DROP) -> GATE (sum kept-source est-tokens vs budget)
+-> ROUTE (paste bundle if it fits, else a bodiless retrieve outline) -> MANIFEST
+(every included + every excluded file, with reasons; no silent truncation).
+
+Metric: tokens of logic-bearing source, estimated as bytes / 3.7 (no tokenizer
+dependency). Threshold: --budget (default 120000). Both are tunable.
+
+Usage:
+  ctxpack <SOURCE> [-o OUT] [--budget N] [--bytes-per-token F] [--stdout]
+          [--emit-context-file PATH] [--stage-retrieve] [--stage-dir DIR]
+          [--ctags] [--fork-diff [UPSTREAM_REF]]
+
+Examples:
+  ctxpack ./challenge-src
+  ctxpack handout.zip -o /tmp/ctx --budget 120000
+  ctxpack ./big-repo --stage-retrieve --stage-dir /tmp/ctxpack_src
+  ctxpack . --fork-diff origin/main        # emit planted delta vs upstream
+
+Zero third-party dependencies -- Python standard library only.
+"""
+
+import argparse
+import fnmatch
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import zipfile
+
+# ============================ CONFIG (edit freely) ============================
+# Token estimate = bytes / BYTES_PER_TOKEN. The repo habit elsewhere is 4.0; we
+# default to 3.7 (slightly more conservative) and expose --bytes-per-token.
+BYTES_PER_TOKEN = 3.7
+# Route paste vs retrieve on the sum of KEEP-file est-tokens (not raw bytes).
+DEFAULT_BUDGET_TOKENS = 120_000
+
+# --- DROP by extension -> reason group (checked case-insensitively). ---
+DROP_EXT = {
+    "image":  {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"},
+    "font":   {".woff", ".woff2", ".ttf", ".eot", ".otf"},
+    "style":  {".css", ".scss", ".sass", ".less"},
+    "media":  {".mp4", ".mp3", ".pdf"},
+    "binary": {".zip", ".tar", ".gz", ".so", ".a", ".o", ".pyc",
+               ".class", ".wasm", ".bin"},
+}
+# Compound suffixes checked BEFORE single-extension rules: minified + sourcemaps.
+DROP_SUFFIX_MINIFIED = (".min.js", ".min.css", ".map")
+
+# --- DROP by exact basename -> reason. ---
+DROP_FILENAMES = {
+    "package-lock.json": "lockfile", "yarn.lock": "lockfile",
+    "poetry.lock": "lockfile", "Pipfile.lock": "lockfile",
+    "Cargo.lock": "lockfile", "composer.lock": "lockfile", "go.sum": "lockfile",
+    ".DS_Store": "editor-junk", "Thumbs.db": "editor-junk",
+}
+
+# --- DROP by path segment (any path component match) -> reason path:<seg>. ---
+DROP_PATH_SEGMENTS = {
+    "node_modules", "vendor", "dist", "build", "__pycache__", "third_party", ".git",
+}
+
+# --- KEEP hints. Anything not DROPped is kept regardless; these drive the
+#     logic-first ORDERING (and document intent). ---
+KEEP_SOURCE_EXT = {
+    ".py", ".js", ".ts", ".go", ".rb", ".php", ".java", ".c", ".cpp", ".h",
+    ".rs", ".sh", ".jsx", ".tsx", ".cc", ".hpp", ".pyi", ".mjs", ".cjs",
+}
+KEEP_TEMPLATE_EXT = {
+    ".html", ".htm", ".jinja", ".j2", ".ejs", ".erb", ".twig", ".mako",
+    ".hbs", ".vue", ".svelte",
+}
+# Behavior-carrying config: exact names + glob patterns (matched on basename).
+KEEP_CONFIG_NAMES = {"docker-compose.yml", "docker-compose.yaml", "package.json"}
+KEEP_CONFIG_GLOBS = ("Dockerfile*", "docker-compose.y*ml", "*.conf",
+                     "requirements*.txt", "nginx*.conf", "supervisor*.conf",
+                     "*.env.example")
+
+# Entrypoint/route/service hints -> logic-first tier 1 (matched as path tokens).
+ENTRYPOINT_HINTS = {
+    "app", "main", "index", "server", "wsgi", "asgi", "manage", "run",
+    "route", "routes", "view", "views", "controller", "controllers",
+    "handler", "handlers", "api", "service", "services", "url", "urls",
+    "endpoint", "endpoints", "middleware",
+}
+
+# Known-CTF marker filename tokens + content regex (SIGNAL only; never acted on).
+CTF_MARKER_TOKENS = {"flag", "flags", "challenge", "benchmark", "solve",
+                     "solver", "exploit", "writeup"}
+CTF_CONTENT_RE = re.compile(rb"(FLAG|CTF|HTB|flag)\{")
+# ============================================================================
+
+TIER_RANK = {"entrypoint": 0, "source": 1, "template": 2, "config": 3, "other": 4}
+_TOKEN_RE = re.compile(r"[^a-z0-9]+")
+_CHUNK = 8192
+# Printable text bytes for the binary-content heuristic.
+_TEXT_BYTES = bytes(range(0x20, 0x7F)) + b"\n\r\t\f\b\x1b"
+
+
+def _dbg(on, msg):
+    """Per-decision trace to stderr (opt-in via --debug) so the filter is
+    auditable mid-event: exactly which files were kept/dropped and why."""
+    if on:
+        sys.stderr.write("[ctxpack] " + msg + "\n")
+
+
+# ------------------------------- classification -----------------------------
+def _path_tokens(rel_path):
+    return {t for t in _TOKEN_RE.split(rel_path.lower()) if t}
+
+
+def _is_config(name):
+    if name in KEEP_CONFIG_NAMES:
+        return True
+    return any(fnmatch.fnmatchcase(name, pat) or fnmatch.fnmatchcase(name.lower(), pat.lower())
+               for pat in KEEP_CONFIG_GLOBS)
+
+
+def classify(rel_path):
+    """Partition a file into KEEP or DROP.
+
+    Returns (keep: bool, tag: str). For DROP, tag is the reason code. For KEEP,
+    tag is the ordering tier (entrypoint/source/template/config/other).
+    Check order per spec: exact filename -> minified suffix -> extension ->
+    path segment -> else KEEP.
+    """
+    parts = rel_path.replace("\\", "/").split("/")
+    name = parts[-1]
+    lname = name.lower()
+
+    if name in DROP_FILENAMES:
+        return False, DROP_FILENAMES[name]
+
+    for suf in DROP_SUFFIX_MINIFIED:
+        if lname.endswith(suf):
+            return False, "ext:minified"
+
+    ext = os.path.splitext(lname)[1]
+    for group, exts in DROP_EXT.items():
+        if ext in exts:
+            return False, "ext:" + group
+
+    for seg in parts:
+        if seg in DROP_PATH_SEGMENTS:
+            return False, "path:" + seg
+
+    # KEEP -- assign a logic-first tier.
+    tokens = _path_tokens(rel_path)
+    if ext in KEEP_SOURCE_EXT:
+        return True, "entrypoint" if tokens & ENTRYPOINT_HINTS else "source"
+    if ext in KEEP_TEMPLATE_EXT:
+        return True, "template"
+    if _is_config(name):
+        return True, "config"
+    return True, "other"
+
+
+def looks_binary(chunk):
+    """Classic file(1)/git heuristic: NUL byte, or >30% non-text bytes."""
+    if not chunk:
+        return False
+    if b"\x00" in chunk:
+        return True
+    nontext = chunk.translate(None, _TEXT_BYTES)
+    return len(nontext) / len(chunk) > 0.30
+
+
+# --------------------------------- scanning ---------------------------------
+def _iter_files(root):
+    """Yield relative POSIX paths under root, deterministically, not following
+    symlinked directories."""
+    for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
+        dirnames.sort()
+        for fn in sorted(filenames):
+            ap = os.path.join(dirpath, fn)
+            if os.path.islink(ap):
+                continue  # skip symlinked files (avoid escaping the tree)
+            rel = os.path.relpath(ap, root).replace(os.sep, "/")
+            yield rel, ap
+
+
+def detect_stack(root, included):
+    """Best-effort web-framework fingerprint of the KEEP set, so a graybox launcher can
+    sanity-check the bundle against the live target before spending a run. Returns a short
+    lowercase name ("nextjs", "react", "express", "flask", "django", "php", ...) or "" when
+    nothing is recognized. Heuristic and intentionally conservative (only positive signals)."""
+    lowers = [r["path"].lower() for r in included]
+    bases = {os.path.basename(p) for p in lowers}
+
+    def _head(rel):
+        try:
+            with open(os.path.join(root, rel), "rb") as fh:
+                return fh.read(_CHUNK).decode("utf-8", "replace").lower()
+        except OSError:
+            return ""
+
+    # --- Node/JS family: trust package.json dependencies first.
+    pkg_rel = next((r["path"] for r in included
+                    if os.path.basename(r["path"]).lower() == "package.json"), None)
+    has_next_cfg = any(b.startswith("next.config.") for b in bases)
+    if pkg_rel:
+        txt = _head(pkg_rel)
+        if '"next"' in txt or has_next_cfg:
+            return "nextjs"
+        if '"express"' in txt:
+            return "express"
+        if '"@angular/core"' in txt:
+            return "angular"
+        if '"vue"' in txt:
+            return "vue"
+        if '"svelte"' in txt:
+            return "svelte"
+        if ('"react"' in txt or '"react-dom"' in txt
+                or any(p.endswith((".tsx", ".jsx")) for p in lowers)):
+            return "react"
+        return "node"
+    if has_next_cfg:
+        return "nextjs"
+
+    # --- Python family: requirements first, then a bounded source sniff.
+    req_rels = [r["path"] for r in included
+                if os.path.basename(r["path"]).lower().startswith("requirements")]
+    reqtext = " ".join(_head(p) for p in req_rels)
+    if "manage.py" in bases or "django" in reqtext:
+        return "django"
+    if "fastapi" in reqtext:
+        return "fastapi"
+    if "flask" in reqtext:
+        return "flask"
+    sniffed = 0
+    for r in included:
+        if sniffed >= 20:
+            break
+        if r["path"].lower().endswith(".py"):
+            sniffed += 1
+            t = _head(r["path"])
+            if "from flask" in t or "import flask" in t:
+                return "flask"
+            if "from fastapi" in t or "import fastapi" in t:
+                return "fastapi"
+            if "from django" in t or "import django" in t:
+                return "django"
+
+    # --- Other ecosystems (coarse).
+    if any(p.endswith(".php") for p in lowers):
+        return "php"
+    if "gemfile" in bases or any(p.endswith(".rb") for p in lowers):
+        return "rails"
+    return ""
+
+
+def scan(root, bytes_per_token, debug=False):
+    """Walk root, classify every file, and collect included/excluded records
+    plus signal evidence. Returns a dict."""
+    included, excluded = [], []
+    git_present = os.path.isdir(os.path.join(root, ".git"))
+    ctf_markers = set()
+    binary_artifact_seen = False
+
+    for rel, ap in _iter_files(root):
+        try:
+            size = os.path.getsize(ap)
+        except OSError:
+            excluded.append({"path": rel, "bytes": 0, "reason": "unreadable"})
+            _dbg(debug, "DROP unreadable        {}".format(rel))
+            continue
+
+        base_tokens = _path_tokens(os.path.basename(rel))
+        if base_tokens & CTF_MARKER_TOKENS:
+            ctf_markers.add("name:" + rel)
+
+        keep, tag = classify(rel)
+        if not keep:
+            if tag == "ext:binary":
+                binary_artifact_seen = True
+            if ".git" in rel.split("/"):
+                git_present = True
+            excluded.append({"path": rel, "bytes": size, "reason": tag})
+            _dbg(debug, "DROP {:<17} {}".format(tag, rel))
+            continue
+
+        # KEEP candidate: read head once for binary + CTF-content detection.
+        try:
+            with open(ap, "rb") as fh:
+                head = fh.read(_CHUNK)
+        except OSError:
+            excluded.append({"path": rel, "bytes": size, "reason": "unreadable"})
+            _dbg(debug, "DROP unreadable        {}".format(rel))
+            continue
+
+        if looks_binary(head):
+            binary_artifact_seen = True
+            excluded.append({"path": rel, "bytes": size, "reason": "binary-content"})
+            _dbg(debug, "DROP binary-content    {}".format(rel))
+            continue
+
+        if CTF_CONTENT_RE.search(head):
+            ctf_markers.add("content:" + rel)
+
+        rec = {
+            "path": rel,
+            "bytes": size,
+            "est_tokens": round(size / bytes_per_token),
+            "tier": tag,
+        }
+        if tag == "other":
+            rec["note"] = "kept:default-unknown-ext"
+        included.append(rec)
+        _dbg(debug, "KEEP {:<17} {}".format(tag, rel))
+
+    binaries_only = (not included) and binary_artifact_seen
+    _dbg(debug, "scan done: {} kept, {} dropped, {} scanned".format(
+        len(included), len(excluded), len(included) + len(excluded)))
+    return {
+        "included": included,
+        "excluded": excluded,
+        "signals": {
+            "git_present": bool(git_present),
+            "binaries_only": bool(binaries_only),
+            "known_ctf_markers": sorted(ctf_markers),
+            "stack": detect_stack(root, included),
+        },
+    }
+
+
+# --------------------------------- rendering --------------------------------
+def _order_logic_first(included):
+    return sorted(included, key=lambda r: (TIER_RANK.get(r["tier"], 9), r["path"]))
+
+
+def _read_text(ap):
+    with open(ap, "rb") as fh:
+        return fh.read().decode("utf-8", errors="replace")
+
+
+def render_paste(root, included, kept_tokens, budget, source_label):
+    ordered = _order_logic_first(included)
+    pct = (100 * kept_tokens // budget) if budget else 0
+    out = [
+        "# ctxpack bundle -- mode=paste -- {n} files, ~{t} tokens "
+        "(budget {b}, ~{p}%) -- {s}".format(
+            n=len(ordered), t=kept_tokens, b=budget, p=pct, s=source_label),
+        "# Each file is prefixed with '===== <relative/path> ====='. "
+        "Ordering: entrypoints/routes/services, then source, templates, config.",
+        "",
+    ]
+    for rec in ordered:
+        out.append("===== {} =====".format(rec["path"]))
+        out.append(_read_text(os.path.join(root, rec["path"].replace("/", os.sep))))
+        out.append("")
+    return "\n".join(out)
+
+
+def _build_tree(paths):
+    tree = {}
+    for p in paths:
+        node = tree
+        for part in p.split("/"):
+            node = node.setdefault(part, {})
+    lines = []
+
+    def walk(node, prefix=""):
+        entries = sorted(node.items())
+        for i, (name, child) in enumerate(entries):
+            last = i == len(entries) - 1
+            lines.append(prefix + ("└── " if last else "├── ") + name)
+            if child:
+                walk(child, prefix + ("    " if last else "│   "))
+
+    walk(tree)
+    return "\n".join(lines)
+
+
+def render_retrieve(included, kept_tokens, budget, source_label, staged_dir=None):
+    ordered = _order_logic_first(included)
+    where = (" Files staged under {} -- grep/read there.".format(staged_dir)
+             if staged_dir else "")
+    out = [
+        "# ctxpack outline -- mode=retrieve -- kept ~{t} tokens > budget {b} -- {s}".format(
+            t=kept_tokens, b=budget, s=source_label),
+        "# NO file bodies below.{} Grep/read these {} paths on demand.".format(
+            where, len(ordered)),
+        "",
+        "## FILE TREE (KEEP set)",
+        _build_tree([r["path"] for r in ordered]),
+        "",
+        "## KEEP PATHS (logic-first; path<TAB>bytes<TAB>est_tokens)",
+    ]
+    for rec in ordered:
+        out.append("{}\t{}\t{}".format(rec["path"], rec["bytes"], rec["est_tokens"]))
+    return "\n".join(out)
+
+
+def render_manifest_md(manifest):
+    m = manifest
+    t = m["totals"]
+    sig = m["signals"]
+    lines = [
+        "# ctxpack manifest",
+        "",
+        "- **source:** `{}`  ({})".format(m["source"], m["source_type"]),
+        "- **mode:** `{}`  (fits budget: {})".format(m["mode"], m["fits"]),
+        "- **budget:** {} tokens   **bytes/token:** {}".format(
+            m["budget_tokens"], m["bytes_per_token"]),
+        "- **kept source:** {} files, {} bytes, ~{} tokens".format(
+            t["included_files"], t["included_bytes"], t["included_est_tokens"]),
+        "- **excluded:** {} files, {} bytes".format(
+            t["excluded_files"], t["excluded_bytes"]),
+        "- **scanned total:** {} files".format(t["scanned_files"]),
+        "",
+        "## Signals",
+        "- git_present: {}".format(sig["git_present"]),
+        "- binaries_only: {}".format(sig["binaries_only"]),
+        "- stack: {}".format(sig.get("stack") or "(unknown)"),
+        "- known_ctf_markers: {}".format(", ".join(sig["known_ctf_markers"]) or "(none)"),
+        "",
+        "## Included ({}) -- path | bytes | est_tokens | tier".format(t["included_files"]),
+    ]
+    for r in _order_logic_first(m["included"]):
+        note = "  _({})_".format(r["note"]) if r.get("note") else ""
+        lines.append("- `{}` | {} | {} | {}{}".format(
+            r["path"], r["bytes"], r["est_tokens"], r["tier"], note))
+    lines += ["", "## Excluded ({}) -- path | bytes | reason".format(t["excluded_files"])]
+    for r in sorted(m["excluded"], key=lambda x: (x["reason"], x["path"])):
+        lines.append("- `{}` | {} | {}".format(r["path"], r["bytes"], r["reason"]))
+    lines.append("")
+    return "\n".join(lines)
+
+
+# ------------------------------ archive input -------------------------------
+def _validate_member(dest, member_name):
+    dest_abs = os.path.abspath(dest)
+    target = os.path.abspath(os.path.join(dest, member_name))
+    if target != dest_abs and not target.startswith(dest_abs + os.sep):
+        raise ValueError("unsafe path in archive: {!r}".format(member_name))
+
+
+def extract_archive(path, dest, zip_password=None):
+    """Safely extract a .zip/.tar[.gz/.bz2/.xz] into dest. Returns 'zip'/'tar'.
+    Password-encrypted zips need zip_password (stdlib supports classic ZipCrypto
+    only, not WinZip-AES)."""
+    if zipfile.is_zipfile(path):
+        with zipfile.ZipFile(path) as zf:
+            for n in zf.namelist():
+                _validate_member(dest, n)
+            encrypted = any(zi.flag_bits & 0x1 for zi in zf.infolist())
+            if encrypted and not zip_password:
+                raise ValueError("zip is password-encrypted; pass --zip-password PW "
+                                 "(stdlib supports classic ZipCrypto, not AES)")
+            pwd = zip_password.encode() if zip_password else None
+            try:
+                zf.extractall(dest, pwd=pwd)
+            except RuntimeError as e:
+                raise ValueError("zip extraction failed (wrong password?): {}".format(e))
+        return "zip"
+    if tarfile.is_tarfile(path):
+        with tarfile.open(path) as tf:
+            try:
+                tf.extractall(dest, filter="data")  # py>=3.12 safe filter
+            except TypeError:
+                for m in tf.getmembers():
+                    _validate_member(dest, m.name)
+                tf.extractall(dest)
+        return "tar"
+    raise ValueError("not a directory or supported archive: {}".format(path))
+
+
+# --------------------------------- stretch ----------------------------------
+def _stage_files(root, included, stage_dir, clean):
+    """Copy KEEP files into stage_dir (preserving relative layout). Only wipe the
+    dir first when it is the tool-owned default (clean=True); NEVER delete a
+    user-supplied --stage-dir wholesale (it is overlaid, not cleared)."""
+    if clean and os.path.isdir(stage_dir):
+        shutil.rmtree(stage_dir)
+    os.makedirs(stage_dir, exist_ok=True)
+    for rec in included:
+        src = os.path.join(root, rec["path"].replace("/", os.sep))
+        dst = os.path.join(stage_dir, rec["path"].replace("/", os.sep))
+        os.makedirs(os.path.dirname(dst) or stage_dir, exist_ok=True)
+        shutil.copy2(src, dst)
+    return stage_dir
+
+
+def _run_ctags(stage_dir, out_dir):
+    if not shutil.which("ctags"):
+        return {"ctags": "skipped: ctags not found on PATH"}
+    tags_path = os.path.join(out_dir, "tags")
+    try:
+        subprocess.run(["ctags", "-R", "-f", tags_path, stage_dir],
+                       check=True, capture_output=True)
+        return {"ctags": os.path.relpath(tags_path, out_dir)}
+    except subprocess.CalledProcessError as e:
+        return {"ctags": "failed: {}".format(e.stderr.decode("utf-8", "replace")[:200])}
+
+
+def _fork_diff_bundle(source_dir, upstream_ref, source_label):
+    """Emit git log + diff vs upstream instead of the full bundle. Degrades to
+    None (caller falls back to a normal bundle) if not a usable git repo."""
+    if not shutil.which("git") or not os.path.isdir(os.path.join(source_dir, ".git")):
+        return None
+
+    def git(*args):
+        return subprocess.run(["git", "-C", source_dir, *args],
+                              capture_output=True, text=True).stdout.strip()
+
+    ref = upstream_ref if isinstance(upstream_ref, str) else ""
+    if not ref:
+        for cand in ("@{upstream}", "origin/HEAD", "origin/main", "origin/master", "main"):
+            if git("rev-parse", "--verify", "--quiet", cand):
+                ref = cand
+                break
+    if not ref:
+        return None
+    log = git("log", "--oneline", "{}..HEAD".format(ref))
+    diff = git("diff", "{}...HEAD".format(ref))
+    return "\n".join([
+        "# ctxpack fork-diff -- planted delta vs {} -- {}".format(ref, source_label),
+        "", "## git log ({}..HEAD)".format(ref), log or "(no new commits)",
+        "", "## git diff ({}...HEAD)".format(ref), diff or "(no diff)", "",
+    ])
+
+
+# ----------------------------------- core -----------------------------------
+def pack(source, out_dir, budget_tokens=DEFAULT_BUDGET_TOKENS,
+         bytes_per_token=BYTES_PER_TOKEN, stage_retrieve=False, stage_dir=None,
+         do_ctags=False, fork_diff=None, emit_context_file=None, to_stdout=False,
+         debug=False, zip_password=None):
+    """Run the full FILTER->GATE->ROUTE->MANIFEST pipeline. Writes bundle.txt,
+    manifest.json and manifest.md into out_dir. Returns the manifest dict."""
+    source = os.path.abspath(source)
+    tmp_root = None
+    try:
+        if os.path.isdir(source):
+            root, source_type = source, "dir"
+        elif os.path.isfile(source):
+            tmp_root = tempfile.mkdtemp(prefix="ctxpack_x_")
+            source_type = extract_archive(source, tmp_root, zip_password=zip_password)
+            root = tmp_root
+            _dbg(debug, "extracted {} archive -> {}".format(source_type, root))
+        else:
+            raise ValueError("source not found: {}".format(source))
+
+        source_label = os.path.basename(source.rstrip("/\\")) or source
+        scanned = scan(root, bytes_per_token, debug=debug)
+        included, excluded = scanned["included"], scanned["excluded"]
+
+        inc_bytes = sum(r["bytes"] for r in included)
+        kept_tokens = sum(r["est_tokens"] for r in included)
+        exc_bytes = sum(r["bytes"] for r in excluded)
+        fits = kept_tokens <= budget_tokens
+        mode = "paste" if fits else "retrieve"
+
+        os.makedirs(out_dir, exist_ok=True)
+        extras = {}
+        staged_at = None
+
+        # --fork-diff overrides the bundle body (both modes) when usable.
+        fork_body = _fork_diff_bundle(root, fork_diff, source_label) if fork_diff else None
+
+        if fork_body is not None:
+            bundle = fork_body
+            extras["fork_diff"] = "emitted"
+        elif mode == "paste":
+            bundle = render_paste(root, included, kept_tokens, budget_tokens, source_label)
+        else:
+            if stage_retrieve or stage_dir or do_ctags:
+                target_stage = stage_dir or os.path.join(out_dir, "keep_files")
+                staged_at = _stage_files(root, included, target_stage,
+                                         clean=(stage_dir is None))
+            bundle = render_retrieve(included, kept_tokens, budget_tokens,
+                                     source_label, staged_dir=staged_at)
+            if do_ctags:
+                extras.update(_run_ctags(staged_at, out_dir))
+
+        manifest = {
+            "tool": "ctxpack",
+            "version": "1.0",
+            "source": source,
+            "source_type": source_type,
+            "budget_tokens": budget_tokens,
+            "bytes_per_token": bytes_per_token,
+            "mode": mode,
+            "fits": fits,
+            "totals": {
+                "included_files": len(included),
+                "included_bytes": inc_bytes,
+                "included_est_tokens": kept_tokens,
+                "excluded_files": len(excluded),
+                "excluded_bytes": exc_bytes,
+                "scanned_files": len(included) + len(excluded),
+            },
+            "included": included,
+            "excluded": excluded,
+            "signals": scanned["signals"],
+            "outputs": dict({"bundle": "bundle.txt",
+                             "manifest_json": "manifest.json",
+                             "manifest_md": "manifest.md"}, **extras),
+        }
+        if staged_at:
+            manifest["outputs"]["staged_dir"] = staged_at
+
+        bundle_path = os.path.join(out_dir, "bundle.txt")
+        _write(bundle_path, bundle)
+        _write(os.path.join(out_dir, "manifest.json"),
+               json.dumps(manifest, indent=2, sort_keys=False) + "\n")
+        _write(os.path.join(out_dir, "manifest.md"), render_manifest_md(manifest))
+        if emit_context_file:
+            os.makedirs(os.path.dirname(os.path.abspath(emit_context_file)) or ".",
+                        exist_ok=True)
+            _write(emit_context_file, bundle)
+        if to_stdout:
+            sys.stdout.write(bundle)
+            if not bundle.endswith("\n"):
+                sys.stdout.write("\n")
+        return manifest
+    finally:
+        if tmp_root:
+            shutil.rmtree(tmp_root, ignore_errors=True)
+
+
+def _write(path, text):
+    with open(path, "w", encoding="utf-8", newline="\n") as fh:
+        fh.write(text)
+
+
+# ----------------------------------- CLI ------------------------------------
+def build_parser():
+    p = argparse.ArgumentParser(
+        prog="ctxpack",
+        description="Pack a challenge's source into an inject-ready graybox bundle.")
+    p.add_argument("source", help="path to a source directory or archive (.zip/.tar[.gz])")
+    p.add_argument("-o", "--out", default="./ctxpack-out",
+                   help="output directory (default: ./ctxpack-out)")
+    p.add_argument("--budget", type=int, default=DEFAULT_BUDGET_TOKENS,
+                   help="token budget for paste vs retrieve (default: %(default)s)")
+    p.add_argument("--bytes-per-token", type=float, default=BYTES_PER_TOKEN,
+                   help="token estimate divisor (default: %(default)s)")
+    p.add_argument("--stdout", action="store_true", help="also print the bundle to stdout")
+    p.add_argument("--emit-context-file", metavar="PATH",
+                   help="also write the bundle to this exact path (for direct wiring)")
+    p.add_argument("--stage-retrieve", action="store_true",
+                   help="in retrieve mode, also copy KEEP files to a staging dir")
+    p.add_argument("--stage-dir", metavar="DIR",
+                   help="override staging location (default: OUT/keep_files)")
+    p.add_argument("--ctags", action="store_true",
+                   help="[stretch] run ctags -R over KEEP files in retrieve mode")
+    p.add_argument("--fork-diff", nargs="?", const=True, metavar="UPSTREAM_REF",
+                   help="[stretch] emit git log/diff vs upstream instead of the bundle")
+    p.add_argument("--zip-password", metavar="PW",
+                   help="password for an encrypted zip handout (classic ZipCrypto)")
+    p.add_argument("--debug", action="store_true",
+                   help="trace every KEEP/DROP decision to stderr")
+    return p
+
+
+def main(argv=None):
+    args = build_parser().parse_args(argv)
+    try:
+        m = pack(
+            args.source, args.out, budget_tokens=args.budget,
+            bytes_per_token=args.bytes_per_token, stage_retrieve=args.stage_retrieve,
+            stage_dir=args.stage_dir, do_ctags=args.ctags, fork_diff=args.fork_diff,
+            emit_context_file=args.emit_context_file, to_stdout=args.stdout,
+            debug=args.debug, zip_password=args.zip_password,
+        )
+    except ValueError as e:
+        sys.stderr.write("ctxpack: error: {}\n".format(e))
+        return 2
+
+    t = m["totals"]
+    sig = m["signals"]
+    flags = [k for k in ("git_present", "binaries_only") if sig[k]]
+    if sig.get("stack"):
+        flags.append("stack={}".format(sig["stack"]))
+    if sig["known_ctf_markers"]:
+        flags.append("ctf_markers={}".format(len(sig["known_ctf_markers"])))
+    sys.stderr.write(
+        "ctxpack: mode={mode} kept={files} files ~{tok} tokens "
+        "(budget {bud}) excluded={exc}{sig}\n".format(
+            mode=m["mode"], files=t["included_files"], tok=t["included_est_tokens"],
+            bud=m["budget_tokens"], exc=t["excluded_files"],
+            sig=("  signals[{}]".format(",".join(flags)) if flags else "")))
+    sys.stderr.write("ctxpack: wrote {0}/bundle.txt , {0}/manifest.json , "
+                     "{0}/manifest.md\n".format(os.path.abspath(args.out)))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())