@getmagical/cli 0.4.0 → 0.7.0

package/dist/network-recording-archive.py

@@ -0,0 +1,519 @@
+#!/usr/bin/env python3
+"""Bounded offline reader for Magical mitmproxy recording archives."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import sys
+import urllib.parse
+import zlib
+from datetime import datetime, timezone
+from typing import Any, BinaryIO
+
+EXPECTED_MITMPROXY_VERSION = "12.2.3"
+MAX_REQUEST_BYTES = 64 * 1024
+READ_CHUNK_BYTES = 64 * 1024
+EXCHANGE_REFERENCE_PATTERN = re.compile(
+    r"^mitm-v1:([0-9a-f]{64}):([0-9]+)$"
+)
+
+
+class ArchiveError(Exception):
+    def __init__(self, code: str, message: str):
+        super().__init__(message)
+        self.code = code
+
+
+class BoundedFlowFile:
+    """Limit individual parser reads before tnetstring allocates a flow record."""
+
+    def __init__(self, archive: BinaryIO, max_read_bytes: int):
+        self.archive = archive
+        self.max_read_bytes = max_read_bytes
+
+    def read(self, size: int = -1) -> bytes:
+        if size < 0:
+            raise ArchiveError(
+                "UNSUPPORTED_ARCHIVE_FORMAT",
+                "Expected a mitmproxy flow archive, not a JSON archive.",
+            )
+        if size > self.max_read_bytes:
+            raise ArchiveError(
+                "FLOW_TOO_LARGE", "A serialized archive flow exceeds the size limit."
+            )
+        return self.archive.read(size)
+
+    def peek(self, size: int = 0) -> bytes:
+        if size > self.max_read_bytes:
+            raise ArchiveError(
+                "FLOW_TOO_LARGE", "A serialized archive flow exceeds the size limit."
+            )
+        peek = getattr(self.archive, "peek", None)
+        return peek(size) if peek is not None else b""
+
+    def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
+        return self.archive.seek(offset, whence)
+
+    def tell(self) -> int:
+        return self.archive.tell()
+
+
+def emit(value: dict[str, Any]) -> None:
+    json.dump(value, sys.stdout, separators=(",", ":"))
+    sys.stdout.write("\n")
+
+
+def fail(code: str, message: str) -> None:
+    emit({"ok": False, "error": {"code": code, "message": message}})
+
+
+def read_request() -> dict[str, Any]:
+    request_bytes = sys.stdin.buffer.read(MAX_REQUEST_BYTES + 1)
+    if len(request_bytes) > MAX_REQUEST_BYTES:
+        raise ArchiveError("INVALID_REQUEST", "Parser request is too large.")
+    try:
+        request = json.loads(request_bytes)
+    except (UnicodeDecodeError, json.JSONDecodeError) as error:
+        raise ArchiveError("INVALID_REQUEST", "Parser request is not valid JSON.") from error
+    if not isinstance(request, dict):
+        raise ArchiveError("INVALID_REQUEST", "Parser request must be an object.")
+    return request
+
+
+def require_string(request: dict[str, Any], key: str) -> str:
+    value = request.get(key)
+    if not isinstance(value, str) or not value:
+        raise ArchiveError("INVALID_REQUEST", f"{key} must be a non-empty string.")
+    return value
+
+
+def require_positive_integer(request: dict[str, Any], key: str) -> int:
+    value = request.get(key)
+    if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+        raise ArchiveError("INVALID_REQUEST", f"{key} must be a positive integer.")
+    return value
+
+
+def require_positive_number(request: dict[str, Any], key: str) -> float:
+    value = request.get(key)
+    if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
+        raise ArchiveError("INVALID_REQUEST", f"{key} must be a positive number.")
+    return float(value)
+
+
+def archive_identity(archive_path: str, max_archive_bytes: int) -> tuple[int, str]:
+    try:
+        size_bytes = os.path.getsize(archive_path)
+    except OSError as error:
+        raise ArchiveError("ARCHIVE_UNAVAILABLE", "Recording archive is unavailable.") from error
+    if size_bytes > max_archive_bytes:
+        raise ArchiveError("ARCHIVE_TOO_LARGE", "Recording archive exceeds the size limit.")
+
+    digest = hashlib.sha256()
+    try:
+        with open(archive_path, "rb") as archive:
+            while chunk := archive.read(1024 * 1024):
+                digest.update(chunk)
+    except OSError as error:
+        raise ArchiveError("ARCHIVE_UNAVAILABLE", "Recording archive could not be read.") from error
+    return size_bytes, digest.hexdigest()
+
+
+def projected_origin(flow: Any) -> str:
+    request = flow.request
+    try:
+        split = urllib.parse.urlsplit(request.url)
+        scheme = split.scheme.lower()
+        hostname = split.hostname or request.host
+        port = split.port
+    except (TypeError, ValueError):
+        scheme = str(request.scheme).lower()
+        hostname = str(request.host)
+        port = request.port
+    default_port = (scheme == "http" and port == 80) or (
+        scheme == "https" and port == 443
+    )
+    authority_host = f"[{hostname}]" if ":" in hostname else hostname
+    host = (
+        authority_host
+        if port is None or default_port
+        else f"{authority_host}:{port}"
+    )
+    origin = f"{scheme}://{host}"
+    if len(origin) > 512 or any(character < " " for character in origin):
+        raise ArchiveError("UNSUPPORTED_EXTRACTION", "Request origin is invalid.")
+    return origin
+
+
+def normalized_media_type(value: str | None) -> str | None:
+    if value is None:
+        return None
+    if any(character < " " for character in value):
+        return None
+    media_type = value.partition(";")[0].strip().lower()
+    token = r"[!#$%&'*+.^_`|~0-9a-z-]+"
+    return media_type if re.fullmatch(rf"{token}/{token}", media_type) else None
+
+
+def request_timestamp(flow: Any) -> str | None:
+    try:
+        return datetime.fromtimestamp(
+            flow.request.timestamp_start, timezone.utc
+        ).isoformat().replace("+00:00", "Z")
+    except (OverflowError, OSError, TypeError, ValueError):
+        return None
+
+
+def body_metadata(response: Any) -> dict[str, Any]:
+    raw_content = response.raw_content
+    if raw_content is None:
+        return {"state": "MISSING", "encodedSizeBytes": None}
+    if not raw_content:
+        return {"state": "EMPTY", "encodedSizeBytes": 0}
+    return {"state": "PRESENT", "encodedSizeBytes": len(raw_content)}
+
+
+def response_metadata(response: Any) -> dict[str, Any] | None:
+    if response is None:
+        return None
+    return {
+        "status": response.status_code,
+        "contentType": normalized_media_type(response.headers.get("content-type")),
+        "body": body_metadata(response),
+    }
+
+
+def read_next_flow(iterator: Any) -> Any | None:
+    try:
+        return next(iterator)
+    except StopIteration:
+        return None
+    except Exception as error:
+        from mitmproxy import exceptions
+
+        if isinstance(error, exceptions.FlowReadException):
+            raise ArchiveError("MALFORMED_ARCHIVE", "Recording archive is malformed.") from error
+        raise
+
+
+def inspect_archive(
+    request: dict[str, Any], archive_path: str, size_bytes: int, archive_sha256: str
+) -> dict[str, Any]:
+    from mitmproxy import http, io
+
+    max_flows = require_positive_integer(request, "maxFlows")
+    max_exchanges = require_positive_integer(request, "maxExchanges")
+    max_serialized_flow_bytes = require_positive_integer(
+        request, "maxSerializedFlowBytes"
+    )
+    exchanges: list[dict[str, Any]] = []
+    scanned_flow_count = 0
+    complete = False
+    truncation_reason: str | None = None
+
+    try:
+        with open(archive_path, "rb") as archive:
+            bounded_archive = BoundedFlowFile(archive, max_serialized_flow_bytes)
+            iterator = iter(io.FlowReader(bounded_archive).stream())
+            while scanned_flow_count < max_flows:
+                flow = read_next_flow(iterator)
+                if flow is None:
+                    complete = True
+                    break
+                flow_index = scanned_flow_count
+                scanned_flow_count += 1
+                if not isinstance(flow, http.HTTPFlow) or flow.request is None:
+                    continue
+                if len(exchanges) >= max_exchanges:
+                    truncation_reason = "RESULT_LIMIT"
+                    break
+                method = flow.request.method
+                if len(method) > 32 or not re.fullmatch(
+                    r"[!#$%&'*+.^_`|~0-9A-Za-z-]+", method
+                ):
+                    method = "UNKNOWN"
+                exchanges.append(
+                    {
+                        "exchange": f"mitm-v1:{archive_sha256}:{flow_index}",
+                        "index": flow_index,
+                        "method": method,
+                        "origin": projected_origin(flow),
+                        "timestamp": request_timestamp(flow),
+                        "response": response_metadata(flow.response),
+                    }
+                )
+            else:
+                if archive.peek(1):
+                    truncation_reason = "FLOW_LIMIT"
+                else:
+                    complete = True
+    except ArchiveError:
+        raise
+    except OSError as error:
+        raise ArchiveError("ARCHIVE_UNAVAILABLE", "Recording archive could not be read.") from error
+
+    return {
+        "archive": {"sha256": archive_sha256, "sizeBytes": size_bytes},
+        "exchanges": exchanges,
+        "scan": {
+            "scannedFlowCount": scanned_flow_count,
+            "returnedExchangeCount": len(exchanges),
+            "complete": complete,
+            "resultLimit": max_exchanges,
+            "flowLimit": max_flows,
+            "truncationReason": truncation_reason,
+        },
+    }
+
+
+def decode_zlib(content: bytes, max_output_bytes: int, window_bits: int) -> bytes:
+    decoder = zlib.decompressobj(window_bits)
+    output = bytearray()
+    for offset in range(0, len(content), READ_CHUNK_BYTES):
+        pending = content[offset : offset + READ_CHUNK_BYTES]
+        while pending:
+            remaining = max_output_bytes - len(output)
+            decoded = decoder.decompress(pending, remaining + 1)
+            output.extend(decoded)
+            if len(output) > max_output_bytes:
+                raise ArchiveError(
+                    "DECODED_BODY_TOO_LARGE",
+                    "Decoded response body exceeds the size limit.",
+                )
+            pending = decoder.unconsumed_tail
+    output.extend(decoder.flush(max_output_bytes - len(output) + 1))
+    if len(output) > max_output_bytes:
+        raise ArchiveError(
+            "DECODED_BODY_TOO_LARGE", "Decoded response body exceeds the size limit."
+        )
+    if not decoder.eof or decoder.unused_data:
+        raise zlib.error("incomplete stream or trailing data")
+    return bytes(output)
+
+
+def decode_one(content: bytes, encoding: str, max_output_bytes: int) -> bytes:
+    if encoding in {"", "identity"}:
+        if len(content) > max_output_bytes:
+            raise ArchiveError(
+                "DECODED_BODY_TOO_LARGE", "Decoded response body exceeds the size limit."
+            )
+        return content
+    try:
+        if encoding == "gzip":
+            return decode_zlib(content, max_output_bytes, 31)
+        if encoding == "deflate":
+            try:
+                return decode_zlib(content, max_output_bytes, zlib.MAX_WBITS)
+            except zlib.error:
+                return decode_zlib(content, max_output_bytes, -zlib.MAX_WBITS)
+    except ArchiveError:
+        raise
+    except Exception as error:
+        raise ArchiveError(
+            "INVALID_CONTENT_ENCODING", "Response content encoding is invalid."
+        ) from error
+    raise ArchiveError(
+        "UNSUPPORTED_ENCODING",
+        f"Response content encoding {encoding!r} is unsupported.",
+    )
+
+
+def decode_content(
+    content: bytes,
+    content_encoding: str | None,
+    max_decoded_body_bytes: int,
+    max_expansion_ratio: float,
+) -> bytes:
+    expansion_limit = int(max(len(content), 1) * max_expansion_ratio)
+    output_limit = min(max_decoded_body_bytes, expansion_limit)
+    encodings = [
+        encoding.strip().lower()
+        for encoding in (content_encoding or "identity").split(",")
+    ]
+    if len(encodings) != 1:
+        raise ArchiveError(
+            "UNSUPPORTED_ENCODING",
+            "Stacked response content encodings are unsupported.",
+        )
+    decoded = content
+    try:
+        for encoding in reversed(encodings):
+            decoded = decode_one(decoded, encoding, output_limit)
+    except ArchiveError as error:
+        if (
+            error.code == "DECODED_BODY_TOO_LARGE"
+            and expansion_limit < max_decoded_body_bytes
+        ):
+            raise ArchiveError(
+                "DECOMPRESSION_RATIO_EXCEEDED",
+                "Decoded response body exceeds the decompression ratio limit.",
+            ) from error
+        raise
+    if len(decoded) > max_decoded_body_bytes:
+        raise ArchiveError(
+            "DECODED_BODY_TOO_LARGE", "Decoded response body exceeds the size limit."
+        )
+    if len(decoded) > max(len(content), 1) * max_expansion_ratio:
+        raise ArchiveError(
+            "DECOMPRESSION_RATIO_EXCEEDED",
+            "Decoded response body exceeds the decompression ratio limit.",
+        )
+    return decoded
+
+
+def write_private_file(output_path: str, content: bytes) -> None:
+    descriptor: int | None = None
+    try:
+        descriptor = os.open(output_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with os.fdopen(descriptor, "wb") as output:
+            descriptor = None
+            output.write(content)
+            output.flush()
+            os.fsync(output.fileno())
+    except FileExistsError as error:
+        raise ArchiveError("OUTPUT_EXISTS", "Response output already exists.") from error
+    except OSError as error:
+        try:
+            os.unlink(output_path)
+        except OSError:
+            pass
+        raise ArchiveError("OUTPUT_WRITE_FAILED", "Response output could not be written.") from error
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+
+
+def extract_response(
+    request: dict[str, Any], archive_path: str, size_bytes: int, archive_sha256: str
+) -> dict[str, Any]:
+    from mitmproxy import http, io
+
+    reference = require_string(request, "exchange")
+    match = EXCHANGE_REFERENCE_PATTERN.fullmatch(reference)
+    if match is None:
+        raise ArchiveError("INVALID_EXCHANGE_REFERENCE", "Exchange reference is invalid.")
+    expected_sha256, index_text = match.groups()
+    if expected_sha256 != archive_sha256:
+        raise ArchiveError(
+            "ARCHIVE_MISMATCH", "Exchange reference belongs to a different archive."
+        )
+    target_index = int(index_text)
+    max_flows = require_positive_integer(request, "maxFlows")
+    max_serialized_flow_bytes = require_positive_integer(
+        request, "maxSerializedFlowBytes"
+    )
+
+    try:
+        with open(archive_path, "rb") as archive:
+            bounded_archive = BoundedFlowFile(archive, max_serialized_flow_bytes)
+            iterator = iter(io.FlowReader(bounded_archive).stream())
+            flow = None
+            current_index = 0
+            while current_index < max_flows:
+                current_flow = read_next_flow(iterator)
+                if current_flow is None:
+                    break
+                if current_index == target_index:
+                    flow = current_flow
+                current_index += 1
+            if current_index == max_flows and archive.peek(1):
+                raise ArchiveError(
+                    "FLOW_LIMIT_EXCEEDED", "Recording archive exceeds the flow limit."
+                )
+    except ArchiveError:
+        raise
+    except OSError as error:
+        raise ArchiveError("ARCHIVE_UNAVAILABLE", "Recording archive could not be read.") from error
+
+    if flow is None:
+        raise ArchiveError("EXCHANGE_NOT_FOUND", "Exchange was not found.")
+    if not isinstance(flow, http.HTTPFlow) or flow.request is None:
+        raise ArchiveError("UNSUPPORTED_FLOW", "Exchange is not an HTTP request.")
+    if flow.response is None:
+        raise ArchiveError("MISSING_RESPONSE", "Exchange has no recorded response.")
+    if flow.response.raw_content is None:
+        raise ArchiveError("MISSING_BODY", "Exchange response body was not recorded.")
+
+    encoded_content = flow.response.raw_content
+    max_encoded_body_bytes = require_positive_integer(request, "maxEncodedBodyBytes")
+    if len(encoded_content) > max_encoded_body_bytes:
+        raise ArchiveError(
+            "ENCODED_BODY_TOO_LARGE", "Encoded response body exceeds the size limit."
+        )
+    decoded_content = decode_content(
+        encoded_content,
+        flow.response.headers.get("content-encoding"),
+        require_positive_integer(request, "maxDecodedBodyBytes"),
+        require_positive_number(request, "maxExpansionRatio"),
+    )
+    output_path = require_string(request, "outputPath")
+    write_private_file(output_path, decoded_content)
+    content_encoding = flow.response.headers.get("content-encoding")
+    normalized_encodings = [
+        encoding.strip().lower()
+        for encoding in (content_encoding or "").split(",")
+        if encoding.strip()
+    ]
+    return {
+        "archive": {"sha256": archive_sha256, "sizeBytes": size_bytes},
+        "exchange": reference,
+        "index": target_index,
+        "response": {
+            "contentType": normalized_media_type(
+                flow.response.headers.get("content-type")
+            ),
+            "sourceContentEncoding": ", ".join(normalized_encodings) or None,
+            "representation": "decoded-entity-body",
+            "encodedSizeBytes": len(encoded_content),
+            "sizeBytes": len(decoded_content),
+            "sha256": hashlib.sha256(decoded_content).hexdigest(),
+        },
+        "outputPath": output_path,
+    }
+
+
+def main() -> int:
+    try:
+        if sys.version_info < (3, 12):
+            raise ArchiveError(
+                "RUNTIME_VERSION_MISMATCH", "Python 3.12 or newer is required."
+            )
+        request = read_request()
+        try:
+            from mitmproxy import version
+        except ImportError as error:
+            raise ArchiveError(
+                "RUNTIME_UNAVAILABLE",
+                "Python environment does not contain mitmproxy 12.2.3.",
+            ) from error
+        if version.VERSION != EXPECTED_MITMPROXY_VERSION:
+            raise ArchiveError(
+                "RUNTIME_VERSION_MISMATCH",
+                f"Expected mitmproxy {EXPECTED_MITMPROXY_VERSION}.",
+            )
+        archive_path = require_string(request, "archivePath")
+        size_bytes, archive_sha256 = archive_identity(
+            archive_path, require_positive_integer(request, "maxArchiveBytes")
+        )
+        operation = require_string(request, "operation")
+        if operation == "inspect":
+            result = inspect_archive(request, archive_path, size_bytes, archive_sha256)
+        elif operation == "extract":
+            result = extract_response(request, archive_path, size_bytes, archive_sha256)
+        else:
+            raise ArchiveError("INVALID_REQUEST", "Parser operation is invalid.")
+        emit({"ok": True, "result": result})
+        return 0
+    except ArchiveError as error:
+        fail(error.code, str(error))
+        return 1
+    except Exception:
+        fail("PARSER_FAILED", "Recording archive parser failed.")
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@getmagical/cli",
-  "version": "0.4.0",
+  "version": "0.7.0",
   "private": false,
   "type": "module",
   "description": "Magical CLI for shared MCP operations",
@@ -23,13 +23,16 @@
     "magical": "./bin/mgcl.mjs"
   },
   "dependencies": {
+    "@inquirer/prompts": "^7.10.1",
     "commander": "^14.0.3",
     "keytar": "^7.9.0",
+    "mime-types": "^2.1.35",
     "zod": "^4.1.12"
   },
   "devDependencies": {
     "@magical/common": "0.0.0",
     "@magical/mcp-registry": "0.0.0",
+    "@types/mime-types": "^2.1.4",
     "esbuild": "^0.28.0",
     "vitest": "^4.1.7"
   },