@rulemetric/proxy 0.6.9 → 0.7.1

package/addon/allow_hosts.json

@@ -0,0 +1,49 @@
+{
+  "_doc": [
+    "Canonical interception scope for the capture proxy (mitmdump --allow-hosts).",
+    "Each entry is a Python regex matched by mitmproxy against 'host:port' of a",
+    "CONNECT. Hosts that match NOTHING here are tunneled untouched — no TLS",
+    "interception, no CA involvement — so bundled-CA tools (pip/certifi, gcloud,",
+    "node without NODE_EXTRA_CA_CERTS) keep working while the machine-wide",
+    "HTTPS_PROXY is pinned. The list must cover every host detect_provider()",
+    "(providers.py) and the usage-research adapters can capture; the pytest",
+    "sync-guard test_allow_hosts.py enforces that. Escape hatches (read by",
+    "`rulemetric proxy start`): RULEMETRIC_PROXY_ALL_HOSTS=1 restores",
+    "intercept-everything (needed for generic OpenAI-compatible endpoints on",
+    "arbitrary self-hosted domains — the detect_provider catch-all);",
+    "RULEMETRIC_PROXY_EXTRA_HOSTS=<comma-separated regexes> extends this list."
+  ],
+  "allow": [
+    "(^|\\.)api\\.anthropic\\.com:",
+    "(^|\\.)generativelanguage\\.googleapis\\.com:",
+    "bedrock-runtime[^:]*:",
+    "\\.snowflakecomputing\\.com:",
+    "(^|\\.)copilot-proxy\\.githubusercontent\\.com:",
+    "\\.openai\\.azure\\.com:",
+    "\\.models\\.ai\\.azure\\.com:",
+    "(^|\\.)api\\.cloudflare\\.com:",
+    "\\.cloud\\.databricks\\.com:",
+    "(^|\\.)api\\.openai\\.com:",
+    "(^|\\.)api\\.groq\\.com:",
+    "(^|\\.)api\\.together\\.xyz:",
+    "(^|\\.)api\\.fireworks\\.ai:",
+    "(^|\\.)api\\.deepseek\\.com:",
+    "(^|\\.)api\\.perplexity\\.ai:",
+    "(^|\\.)api\\.cerebras\\.ai:",
+    "(^|\\.)api\\.sambanova\\.ai:",
+    "(^|\\.)api\\.x\\.ai:",
+    "(^|\\.)api\\.mistral\\.ai:",
+    "(^|\\.)integrate\\.api\\.nvidia\\.com:",
+    "(^|\\.)api\\.ai21\\.com:",
+    "(^|\\.)api\\.cohere\\.com:",
+    "(^|\\.)openrouter\\.ai:",
+    "(^|\\.)openai-proxy\\.replicate\\.com:",
+    "(^|\\.)api\\.githubcopilot\\.com:",
+    "(^|\\.)api\\.individual\\.githubcopilot\\.com:",
+    "(^|\\.)api\\.business\\.githubcopilot\\.com:",
+    "(^|\\.)api\\.enterprise\\.githubcopilot\\.com:",
+    "(^|\\.)chatgpt\\.com:",
+    "^localhost:",
+    "^127\\.0\\.0\\.1:"
+  ]
+}

package/addon/reporter.py

@@ -40,15 +40,60 @@ import threading
 import time
 from typing import Any
 from urllib.error import HTTPError, URLError
-from urllib.request import Request, urlopen
+import ssl
+from urllib.request import (
+    HTTPSHandler,
+    ProxyHandler,
+    Request,
+    build_opener,
+    install_opener,
+    urlopen,
+)
 
 from secret_redactor import redact_snapshot
 
+
+def _make_ssl_context() -> ssl.SSLContext:
+    """Default-verify TLS context with a certifi fallback.
+
+    The prod-default upload target makes TLS verification load-bearing, and
+    some Python builds (pyenv/homebrew) ship OpenSSL with NO default CA
+    paths — there, every https upload would fail CERTIFICATE_VERIFY_FAILED
+    and spool forever. mitmproxy depends on certifi, so load its bundle
+    whenever the interpreter's default store comes up empty. Verification is
+    never weakened — this only ADDs roots when there are none.
+    """
+    ctx = ssl.create_default_context()
+    try:
+        if ctx.cert_store_stats().get("x509_ca", 0) == 0:
+            import certifi
+
+            ctx.load_verify_locations(certifi.where())
+    except Exception:  # pragma: no cover — verification stays at default
+        pass
+    return ctx
+
+
+# The reporter's OWN uploads to the RuleMetric API must go DIRECT, never through
+# the capture proxy. The mitmproxy process inherits a machine-wide HTTPS_PROXY
+# (installMacOSProxyEnv runs `launchctl setenv HTTPS_PROXY http://localhost:8787`),
+# and Python's default urlopen honors it — so without this, every upload to
+# https://rulemetric.com would loop back through the gateway → mitmproxy, get
+# re-signed with mitmproxy's CA (which urllib does not trust), fail TLS
+# verification, and be dropped. Installing a no-proxy opener PROCESS-WIDE (the
+# urllib equivalent of the bash hooks' `curl --noproxy '*'`) fixes every
+# urlopen() upload in both reporter.py and rulemetric_addon.py while keeping the
+# call sites (and their test mocks of `reporter.urlopen`) unchanged.
+install_opener(
+    build_opener(ProxyHandler({}), HTTPSHandler(context=_make_ssl_context()))
+)
+
 logger = logging.getLogger("rulemetric.reporter")
 
 DEFAULT_TIMEOUT = 5  # seconds (was 2 — too aggressive for cold starts)
 MAX_RETRIES = 2  # 3 total attempts
 RETRY_BACKOFF = 0.5  # seconds — exponential backoff base
+RETRY_AFTER_CAP = 10  # seconds — cap server-provided Retry-After (mirrors _post.sh)
 
 SPOOL_MAX_BYTES = 100 * 1024 * 1024  # 100 MB cap before FIFO eviction
 SPOOL_EVICT_FRACTION = 0.10  # evict the oldest 10% when over cap
@@ -94,13 +139,70 @@ def get_stats() -> dict[str, dict[str, int]]:
         return {kind: dict(values) for kind, values in _stats.items()}
 
 
+def _env_file_values() -> dict[str, str]:
+    """Config fallback from ~/.config/rulemetric/env, mirroring the CLI
+    (loadEnvFileAuth, api-client.ts) and the bash hooks (_config.sh). The
+    mitmproxy process is frequently spawned WITHOUT RULEMETRIC_API_URL / _API_KEY
+    in its own environment, so without this fallback the reporter would default
+    to a dead localhost and silently drop every capture. Cheap (a <10-line file),
+    read per-call so a mid-session `auth login` is picked up. Never raises."""
+    values: dict[str, str] = {}
+    try:
+        with open(os.path.join(_config_dir(), "env"), "r", encoding="utf-8") as fh:
+            for raw in fh:
+                line = raw.strip()
+                if not line or line.startswith("#"):
+                    continue
+                if line.startswith("export "):
+                    line = line[len("export "):]
+                key, sep, val = line.partition("=")
+                if sep and key.strip() in (
+                    "RULEMETRIC_API_URL",
+                    "RULEMETRIC_API_KEY",
+                    "RULEMETRIC_ACCESS_TOKEN",
+                ):
+                    values[key.strip()] = val.strip().strip('"').strip("'")
+    except OSError:
+        pass
+    return values
+
+
 def _get_api_url() -> str:
-    return os.environ.get("RULEMETRIC_API_URL", "http://localhost:3000")
+    # env > env-file > prod (NOT a dead localhost). Mirrors api-client.ts:70 and
+    # _config.sh:78 so the proxy agrees with the CLI + hooks. Dev still opts into
+    # localhost by exporting RULEMETRIC_API_URL.
+    return (
+        os.environ.get("RULEMETRIC_API_URL")
+        or _env_file_values().get("RULEMETRIC_API_URL")
+        or "https://rulemetric.com"
+    )
 
 
-def _get_auth_headers() -> dict[str, str]:
-    api_key = os.environ.get("RULEMETRIC_API_KEY", "")
-    token = os.environ.get("RULEMETRIC_ACCESS_TOKEN", "")
+def _get_auth_headers(prefer_file: bool = False) -> dict[str, str]:
+    env_api_key = os.environ.get("RULEMETRIC_API_KEY", "")
+    env_token = os.environ.get("RULEMETRIC_ACCESS_TOKEN", "")
+
+    # Env-file fallback (same file the CLI + bash hooks read). Two cases:
+    # the mitmproxy process may carry NO credential in its own env, or — the
+    # nastier one — only an EXPIRING access token frozen at spawn time. This
+    # is a long-running daemon: an inherited JWT dies within ~an hour and
+    # every upload then 401s forever. The env file is re-read per call, so a
+    # later `rulemetric auth login` (which writes a non-expiring rm_ API key)
+    # heals a running proxy. Precedence: an API key always beats a token
+    # (keys don't expire); `prefer_file=True` (the 401-refresh path in
+    # _post_with_retry) additionally lets FILE values beat the frozen spawn
+    # env within each credential type.
+    env_file: dict[str, str] = {}
+    if prefer_file or not env_api_key or not env_token:
+        env_file = _env_file_values()
+    file_api_key = env_file.get("RULEMETRIC_API_KEY", "")
+    file_token = env_file.get("RULEMETRIC_ACCESS_TOKEN", "")
+    if prefer_file:
+        api_key = file_api_key or env_api_key
+        token = file_token or env_token
+    else:
+        api_key = env_api_key or file_api_key
+        token = env_token or file_token
 
     headers: dict[str, str] = {"Content-Type": "application/json"}
     if api_key:
@@ -181,8 +283,10 @@ def _post_with_retry(
     """
     last_error: str | None = None
     is_permanent = False
+    auth_refreshed = False
 
     for attempt in range(1 + MAX_RETRIES):
+        sleep_override: float | None = None
         try:
             req = Request(url, data=data, headers=headers, method="POST")
             with urlopen(req, timeout=DEFAULT_TIMEOUT) as resp:
@@ -196,7 +300,25 @@ def _post_with_retry(
                 )
         except HTTPError as exc:
             last_error = f"HTTP {exc.code}: {exc.reason}"
-            if 400 <= exc.code < 500:
+            # 401/403 with credentials frozen at spawn time: the env file may
+            # hold FRESHER credentials (a later `auth login` rewrites it while
+            # this daemon keeps running). Rebuild headers file-first and retry
+            # once; if the file offers nothing different, the rejection is
+            # real and falls through to the permanent branch below.
+            if exc.code in (401, 403) and not auth_refreshed:
+                auth_refreshed = True
+                fresh = _get_auth_headers(prefer_file=True)
+                if (
+                    fresh.get("Authorization")
+                    and fresh.get("Authorization") != headers.get("Authorization")
+                ):
+                    logger.warning(
+                        "%s: HTTP %d — retrying with refreshed env-file credentials",
+                        log_label, exc.code,
+                    )
+                    headers = {**headers, "Authorization": fresh["Authorization"]}
+                    continue
+            if 400 <= exc.code < 500 and exc.code not in (408, 429):
                 # 409 from a duplicate replay is "success" — receiver already
                 # has this row. Treat as posted, not failed.
                 if exc.code == 409:
@@ -207,6 +329,19 @@ def _post_with_retry(
                 )
                 is_permanent = True
                 return False, last_error, True
+            # 429 is the API's DESIGNED backpressure: the ingest admission
+            # gate sheds bursts on exactly the two endpoints we post to with
+            # 429 + Retry-After (ingest-load-shed.ts), and its safety
+            # argument assumes clients retry — the bash hooks do; so must
+            # we. 408 is a timeout, transient by definition. Honor
+            # Retry-After (capped) when the server supplies it.
+            if exc.code == 429:
+                hdr = exc.headers.get("Retry-After") if exc.headers else None
+                try:
+                    if hdr is not None:
+                        sleep_override = min(max(int(hdr), 0), RETRY_AFTER_CAP)
+                except (TypeError, ValueError):
+                    sleep_override = None
             logger.warning(
                 "%s: HTTP %d on attempt %d/%d: %s",
                 log_label, exc.code, attempt + 1, 1 + MAX_RETRIES, exc.reason,
@@ -219,7 +354,11 @@ def _post_with_retry(
             )
 
         if attempt < MAX_RETRIES:
-            time.sleep(RETRY_BACKOFF * (2 ** attempt))
+            time.sleep(
+                sleep_override
+                if sleep_override is not None
+                else RETRY_BACKOFF * (2 ** attempt)
+            )
 
     return False, last_error, is_permanent
 
@@ -559,6 +698,7 @@ def _post_snapshot(session_id: str, snapshot: dict) -> None:
                 _linked_sessions.add(session_id)
                 threading.Thread(
                     target=_trigger_link_instructions,
+                    args=(session_id,),
                     daemon=True,
                     name=f"rulemetric-link-{session_id[:8]}",
                 ).start()
@@ -569,14 +709,19 @@ def _post_snapshot(session_id: str, snapshot: dict) -> None:
     _spool_append("snapshot", session_id, snapshot)
 
 
-def _trigger_link_instructions() -> None:
-    """Call POST /api/sessions/link-instructions to auto-link captured context."""
+def _trigger_link_instructions(session_id: str) -> None:
+    """POST /api/sessions/link-instructions scoped to one session.
+
+    The session id is required: the API answers an unscoped body as a no-op
+    (whole-account scans are gated behind an explicit backfill flag).
+    """
     api_url = _get_api_url()
     url = f"{api_url}/api/sessions/link-instructions"
     headers = _get_auth_headers()
+    data = json.dumps({"sessionId": session_id}).encode("utf-8")
 
     try:
-        req = Request(url, data=b"{}", headers=headers, method="POST")
+        req = Request(url, data=data, headers=headers, method="POST")
         with urlopen(req, timeout=10) as resp:
             if resp.status < 300:
                 body = json.loads(resp.read())

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulemetric/proxy",
-  "version": "0.6.9",
+  "version": "0.7.1",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",