hyperbrane 0.1.1 → 0.2.0

package/dist/portal-gateway/imageFiles.generated.js

@@ -1,8 +1,8 @@
 // Generated by scripts/embed-image-files.ts from portal-gateway/image/ — do not edit; run `bun run embed:images`.
 export const IMAGE_FILES = {
     "Dockerfile": "FROM debian:13-slim\nRUN apt-get update \\\n && apt-get install -y --no-install-recommends bash sudo git jq ca-certificates iproute2 nftables python3 python3-venv \\\n && python3 -m venv /opt/mitmproxy \\\n && /opt/mitmproxy/bin/pip install --no-cache-dir mitmproxy \\\n && ln -s /opt/mitmproxy/bin/mitmdump /usr/local/bin/mitmdump \\\n && mitmdump --version \\\n && rm -rf /var/lib/apt/lists/*\nCOPY entry.sh /usr/local/bin/hyb-gateway\nCOPY wire-portal.sh /usr/local/bin/hyb-wire-portal\nCOPY traffic_rules.py /opt/hyperbrane/traffic_rules.py\nCOPY upstreams.py /opt/hyperbrane/upstreams.py\nRUN chmod 0755 /usr/local/bin/hyb-gateway /usr/local/bin/hyb-wire-portal\nCMD [\"hyb-gateway\"]\n",
-    "entry.sh": "#!/bin/sh\n# Gateway container entry: redirects everything arriving over a portal into mitmproxy, then becomes\n# mitmproxy. If the redirect can't be installed, or mitmproxy exits, the container exits: a gateway\n# that isn't intercepting must not look healthy.\nset -eu\n\nSTATE=/state\nPROXY_PORT=8080\nDNS_PORT=5353\n\n# Portals end in \"hvg*\" interfaces. All TCP goes to the transparent proxy and DNS to its resolver;\n# nothing is forwarded, so there is no path out that bypasses mitmproxy. mitmproxy's own upstream\n# connections are locally generated and never pass PREROUTING, so it doesn't intercept itself.\nnft -f - <<EOF\ntable ip hyperbrane\ndelete table ip hyperbrane\ntable ip hyperbrane {\n  chain prerouting {\n    type nat hook prerouting priority dstnat; policy accept;\n    iifname \"hvg*\" meta l4proto tcp redirect to :$PROXY_PORT\n    iifname \"hvg*\" udp dport 53 redirect to :$DNS_PORT\n  }\n  chain input {\n    type filter hook input priority filter; policy accept;\n    iifname \"hvg*\" tcp dport $PROXY_PORT accept\n    iifname \"hvg*\" udp dport $DNS_PORT accept\n    iifname \"hvg*\" drop\n  }\n  chain forward {\n    type filter hook forward priority filter; policy accept;\n    iifname \"hvg*\" drop\n  }\n}\nEOF\n\n# An exit code of 0 is not proof: read the rules back.\nrules=\"$(nft list table ip hyperbrane)\"\ncase \"$rules\" in\n  *\"redirect to :$PROXY_PORT\"*\"redirect to :$DNS_PORT\"*) ;;\n  *) echo \"gateway: redirect rules are not installed\" >&2; exit 1 ;;\nesac\n\n# mitmproxy runs as the owner of the state directory, so everything it writes there (the CA) stays\n# readable and removable by the host service.\nowner_uid=\"$(stat -c %u \"$STATE\")\"\nowner_gid=\"$(stat -c %g \"$STATE\")\"\nexport HOME=/tmp\n# Eager: connect upstream as soon as the client connects, or protocols where the server speaks first\n# (SSH, SMTP) wait forever for a client that is waiting for them.\nexec setpriv --reuid=\"$owner_uid\" --regid=\"$owner_gid\" --clear-groups -- \\\n  mitmdump \\\n  --mode \"transparent@$PROXY_PORT\" \\\n  --mode \"dns@$DNS_PORT\" \\\n  --set \"confdir=$STATE/mitmproxy\" \\\n  --set connection_strategy=eager \\\n  -s /opt/hyperbrane/traffic_rules.py \\\n  -s /opt/hyperbrane/upstreams.py\n",
-    "wire-portal.sh": "#!/bin/sh\n# Wires a portal: a veth pair with one end in the dimension's network namespace and the other in the\n# gateway's, and a default route down it. A dimension with several portals has one default route per\n# portal; the lowest ROUTE_METRIC wins, and when its veth goes away (its gateway stopped) so does the\n# route, so traffic falls over to the next portal.\n# Runs in a privileged helper container sharing the host's network and pid namespaces, with the\n# host's /run at /hostrun.\n#\n# Inputs: PORTAL, DIMENSION_PID, GATEWAY_PID, DIMENSION_IF, GATEWAY_IF, DIMENSION_IP, GATEWAY_IP,\n# ROUTE_METRIC.\n# Safe to re-run: an already correct wiring is left alone, anything else is torn down and rebuilt.\nset -eu\n\n# Concurrent wirings of one portal would delete each other's veths; the lock lives on the host so it\n# is shared by every helper container.\nexec 9>\"/hostrun/hyperbrane-portal-$PORTAL.lock\"\nflock -w 30 9 || { echo \"timed out waiting for the wiring lock of $PORTAL\" >&2; exit 1; }\n\nmkdir -p /var/run/netns\nln -sf \"/proc/$DIMENSION_PID/ns/net\" /var/run/netns/dimension\nln -sf \"/proc/$GATEWAY_PID/ns/net\" /var/run/netns/gateway\ntrap 'rm -f /var/run/netns/dimension /var/run/netns/gateway' EXIT\n\nwired() {\n  ip -n dimension addr show dev \"$DIMENSION_IF\" 2>/dev/null | grep -q \"inet $DIMENSION_IP/30\" \\\n    && ip -n dimension route show default 2>/dev/null | grep -q \"via $GATEWAY_IP dev $DIMENSION_IF metric $ROUTE_METRIC\" \\\n    && ip -n gateway addr show dev \"$GATEWAY_IF\" 2>/dev/null | grep -q \"inet $GATEWAY_IP/30\"\n}\nwired && exit 0\n\n# A stale end can be in either namespace, or left on the host when a container died mid-wiring.\nip -n dimension link del \"$DIMENSION_IF\" 2>/dev/null || true\nip -n gateway link del \"$GATEWAY_IF\" 2>/dev/null || true\nip link del \"$DIMENSION_IF\" 2>/dev/null || true\nip link del \"$GATEWAY_IF\" 2>/dev/null || true\n\nip link add \"$DIMENSION_IF\" type veth peer name \"$GATEWAY_IF\"\nip link set \"$DIMENSION_IF\" netns dimension\nip link set \"$GATEWAY_IF\" netns gateway\nip -n dimension addr replace \"$DIMENSION_IP/30\" dev \"$DIMENSION_IF\"\nip -n dimension link set \"$DIMENSION_IF\" up\nip -n gateway addr replace \"$GATEWAY_IP/30\" dev \"$GATEWAY_IF\"\nip -n gateway link set \"$GATEWAY_IF\" up\nip -n dimension route replace default via \"$GATEWAY_IP\" dev \"$DIMENSION_IF\" metric \"$ROUTE_METRIC\"\n",
+    "entry.sh": "#!/bin/sh\nset -eu\n\nSTATE=/state\nPROXY_PORT=8080\nDNS_PORT=5353\n\nnft -f - <<EOF\ntable ip hyperbrane\ndelete table ip hyperbrane\ntable ip hyperbrane {\n  chain prerouting {\n    type nat hook prerouting priority dstnat; policy accept;\n    iifname \"hvg*\" meta l4proto tcp redirect to :$PROXY_PORT\n    iifname \"hvg*\" udp dport 53 redirect to :$DNS_PORT\n  }\n  chain input {\n    type filter hook input priority filter; policy accept;\n    iifname \"hvg*\" tcp dport $PROXY_PORT accept\n    iifname \"hvg*\" udp dport $DNS_PORT accept\n    iifname \"hvg*\" drop\n  }\n  chain forward {\n    type filter hook forward priority filter; policy accept;\n    iifname \"hvg*\" drop\n  }\n}\nEOF\n\nrules=\"$(nft list table ip hyperbrane)\"\ncase \"$rules\" in\n  *\"redirect to :$PROXY_PORT\"*\"redirect to :$DNS_PORT\"*) ;;\n  *) echo \"gateway: redirect rules are not installed\" >&2; exit 1 ;;\nesac\n\nowner_uid=\"$(stat -c %u \"$STATE\")\"\nowner_gid=\"$(stat -c %g \"$STATE\")\"\nexport HOME=/tmp\nexec setpriv --reuid=\"$owner_uid\" --regid=\"$owner_gid\" --clear-groups -- \\\n  mitmdump \\\n  --mode \"transparent@$PROXY_PORT\" \\\n  --mode \"dns@$DNS_PORT\" \\\n  --set \"confdir=$STATE/mitmproxy\" \\\n  --set connection_strategy=eager \\\n  -s /opt/hyperbrane/traffic_rules.py \\\n  -s /opt/hyperbrane/upstreams.py\n",
+    "wire-portal.sh": "#!/bin/sh\nset -eu\n\nexec 9>\"/hostrun/hyperbrane-portal-$PORTAL.lock\"\nflock -w 30 9 || { echo \"timed out waiting for the wiring lock of $PORTAL\" >&2; exit 1; }\n\nmkdir -p /var/run/netns\nln -sf \"/proc/$DIMENSION_PID/ns/net\" /var/run/netns/dimension\nln -sf \"/proc/$GATEWAY_PID/ns/net\" /var/run/netns/gateway\ntrap 'rm -f /var/run/netns/dimension /var/run/netns/gateway' EXIT\n\nwired() {\n  ip -n dimension addr show dev \"$DIMENSION_IF\" 2>/dev/null | grep -q \"inet $DIMENSION_IP/30\" \\\n    && ip -n dimension route show default 2>/dev/null | grep -q \"via $GATEWAY_IP dev $DIMENSION_IF metric $ROUTE_METRIC\" \\\n    && ip -n gateway addr show dev \"$GATEWAY_IF\" 2>/dev/null | grep -q \"inet $GATEWAY_IP/30\"\n}\nwired && exit 0\n\nip -n dimension link del \"$DIMENSION_IF\" 2>/dev/null || true\nip -n gateway link del \"$GATEWAY_IF\" 2>/dev/null || true\nip link del \"$DIMENSION_IF\" 2>/dev/null || true\nip link del \"$GATEWAY_IF\" 2>/dev/null || true\n\nip link add \"$DIMENSION_IF\" type veth peer name \"$GATEWAY_IF\"\nip link set \"$DIMENSION_IF\" netns dimension\nip link set \"$GATEWAY_IF\" netns gateway\nip -n dimension addr replace \"$DIMENSION_IP/30\" dev \"$DIMENSION_IF\"\nip -n dimension link set \"$DIMENSION_IF\" up\nip -n gateway addr replace \"$GATEWAY_IP/30\" dev \"$GATEWAY_IF\"\nip -n gateway link set \"$GATEWAY_IF\" up\nip -n dimension route replace default via \"$GATEWAY_IP\" dev \"$DIMENSION_IF\" metric \"$ROUTE_METRIC\"\n",
     "traffic_rules.py": "# mitmproxy addon applying the gateway's traffic rules to everything arriving through a portal.\n#\n# Rules are the JSON files the gateway service writes to /state/traffic-rules, re-read every time, so\n# a change takes effect without a reload. A rule's urlMask is a glob, and its scheme says what it\n# applies to:\n#   tcp://<address>:<port>  every TCP connection, by its destination, before it is made upstream.\n#                           address is \"*\", an IPv4 address or a CIDR; port is \"*\" or a number.\n#                           Hostnames aren't known at that point, so rules don't use them.\n#   anything else           every HTTP(S) request, matched as a glob against host:port/path?query.\n#                           The scheme doesn't matter (http, https or none match the same), and a\n#                           mask without a port matches any port, one without a path any path:\n#                           \"github.com\" is the whole host, \"github.com:80/*\" only its plain HTTP.\n# Rules are checked in name order and the first match decides:\n#   BLOCK  refuse: a 403 for a request, a closed connection for TCP\n#   DELAY  hold for delayMs, then carry on\n#   ASK    hold until the service records an answer for the rule in /state/traffic-responses (only\n#          answers given after the hold began count); DENY refuses, ALLOW carries on\n\nimport asyncio\nimport fnmatch\nimport ipaddress\nimport json\nimport os\nimport time\n\nfrom mitmproxy import http\nfrom mitmproxy.proxy import server_hooks\n\nSTATE = \"/state\"\nRULES = os.path.join(STATE, \"traffic-rules\")\nRESPONSES = os.path.join(STATE, \"traffic-responses\")\nASK_POLL_SECONDS = 0.2\nTCP_SCHEME = \"tcp://\"\n\n\ndef _load(path):\n    try:\n        with open(path) as f:\n            return json.load(f)\n    except (OSError, ValueError):\n        return None\n\n\ndef _rules(tcp):\n    try:\n        files = sorted(f for f in os.listdir(RULES) if f.endswith(\".json\"))\n    except OSError:\n        return []\n    rules = (_load(os.path.join(RULES, f)) for f in files)\n    return [r for r in rules if r and r.get(\"urlMask\", \"\").startswith(TCP_SCHEME) == tcp]\n\n\ndef _request_target(request):\n    \"\"\"host:port/path?query: the port always written out, the scheme left out.\"\"\"\n    return f\"{request.pretty_host.lower()}:{request.port}{request.path}\"\n\n\ndef _url_mask(mask):\n    \"\"\"The mask in _request_target's form: no scheme, and \"any\" for a port or path it leaves out.\"\"\"\n    if \"://\" in mask:\n        mask = mask.split(\"://\", 1)[1]\n    host, slash, path = mask.partition(\"/\")\n    if \":\" not in host:\n        host += \":*\"\n    return f\"{host.lower()}/{path if slash else '*'}\"\n\n\ndef _matching_url_rule(request):\n    target = _request_target(request)\n    for rule in _rules(tcp=False):\n        if fnmatch.fnmatchcase(target, _url_mask(rule[\"urlMask\"])):\n            return rule\n    return None\n\n\ndef _tcp_mask_matches(mask, ip, port):\n    address, _, mask_port = mask[len(TCP_SCHEME):].rpartition(\":\")\n    if mask_port != \"*\" and mask_port != str(port):\n        return False\n    if address == \"*\":\n        return True\n    try:\n        return ipaddress.ip_address(ip) in ipaddress.ip_network(address, strict=False)\n    except ValueError:\n        return False\n\n\ndef _matching_tcp_rule(ip, port):\n    for rule in _rules(tcp=True):\n        if _tcp_mask_matches(rule[\"urlMask\"], ip, port):\n            return rule\n    return None\n\n\nasync def _await_answer(name):\n    \"\"\"The ALLOW/DENY given for the rule after now, or None once the rule is no longer an ASK.\"\"\"\n    asked_at = time.time() * 1000\n    while True:\n        answer = _load(os.path.join(RESPONSES, f\"{name}.json\"))\n        if answer and answer.get(\"at\", 0) >= asked_at:\n            return answer.get(\"action\")\n        rule = _load(os.path.join(RULES, f\"{name}.json\"))\n        if not rule or rule.get(\"action\") != \"ASK\":\n            return None\n        # Never time.sleep: this runs on mitmproxy's event loop, shared by every connection.\n        await asyncio.sleep(ASK_POLL_SECONDS)\n\n\nasync def _verdict(rule):\n    \"\"\"Applies the rule's hold, if any; returns the refusal message, or None to carry on.\"\"\"\n    action = rule.get(\"action\")\n    if action == \"BLOCK\":\n        return f\"blocked by gateway rule {rule['name']}\"\n    if action == \"DELAY\":\n        await asyncio.sleep(max(0, rule.get(\"delayMs\", 0)) / 1000)\n    elif action == \"ASK\" and await _await_answer(rule[\"name\"]) == \"DENY\":\n        return f\"denied by gateway rule {rule['name']}\"\n    return None\n\n\nasync def request(flow: http.HTTPFlow):\n    rule = _matching_url_rule(flow.request)\n    refusal = rule and await _verdict(rule)\n    if refusal:\n        flow.response = http.Response.make(403, f\"{refusal}\\n\", {\"content-type\": \"text/plain\"})\n\n\nasync def server_connect(data: server_hooks.ServerConnectionHookData):\n    if not data.server.address:\n        return\n    rule = _matching_tcp_rule(data.server.address[0], data.server.address[1])\n    refusal = rule and await _verdict(rule)\n    if refusal:\n        data.server.error = refusal\n\n",
-    "upstreams.py": "# mitmproxy addon applying the gateway's upstreams (see portal-gateway/upstream.ts) to traffic from\n# portals. Definitions are the JSON files the gateway service writes to /state/upstreams; this addon\n# keeps each upstream's OAuth credential in /state/credentials/<upstream>.oauth.json and reads its\n# API keys from /state/credentials/<upstream>.api-keys.json. Everything is re-read per request, so a\n# change takes effect without a reload.\n#\n# Clients sign in from dimensions, through the proxy:\n#   token endpoint   without a usable credential, requests go through and the response is kept as\n#                    the credential, reaching the dimension with fakes in place of the tokens. With\n#                    one, the gateway answers from it: a sign-in (whatever code it carries) and a\n#                    refresh made with a fake refresh token. A client's refresh goes upstream: the\n#                    gateway replays it (encoding, fields, User-Agent) with the real refresh token,\n#                    one at a time, and refreshes arriving together share its result. Every client\n#                    holds the same fakes, so they all get the new tokens. A sign-in is answered from\n#                    the credential, refreshed first if expired by replaying the last client refresh.\n#                    An expired credential it can't refresh (no client refresh to replay yet, or\n#                    upstream refused) counts as none: a sign-in goes to the upstream again.\n#   answers          other sign-in steps (a device flow's poll) answered as the definition says, once\n#                    there is a credential\n#   other requests   a fake in the mode's `present` location is replaced by the real secret\n\nimport asyncio\nimport http.client as http_client\nimport json\nimport os\nimport secrets\nimport string\nimport time\nimport urllib.parse\n\nfrom mitmproxy import http\n\nSTATE = \"/state\"\nUPSTREAMS = os.path.join(STATE, \"upstreams\")\nCREDENTIALS = os.path.join(STATE, \"credentials\")\n# A token this close to expiry counts as expired.\nREFRESH_MARGIN_MS = 60_000\n# Refreshes arriving within this long of the last one get its tokens instead of going upstream again.\nREFRESH_SHARED_MS = 5_000\nFAKE_ALPHABET = string.ascii_letters + string.digits + \"-_\"\nTOKEN_FIELDS = ((\"access\", \"access_token\"), (\"refresh\", \"refresh_token\"), (\"id\", \"id_token\"))\n\n# One refresh per upstream at a time: concurrent ones would each spend the rotating refresh token.\n_refresh_locks: dict[str, asyncio.Lock] = {}\n\n\ndef _now_ms():\n    return int(time.time() * 1000)\n\n\ndef _load(path):\n    try:\n        with open(path) as f:\n            return json.load(f)\n    except (OSError, ValueError):\n        return None\n\n\ndef _save(path, data):\n    tmp = f\"{path}.{secrets.token_hex(4)}.tmp\"\n    with open(tmp, \"w\") as f:\n        json.dump(data, f, indent=2)\n    os.replace(tmp, path)\n\n\ndef _definitions():\n    try:\n        files = sorted(f for f in os.listdir(UPSTREAMS) if f.endswith(\".json\"))\n    except OSError:\n        return []\n    definitions = (_load(os.path.join(UPSTREAMS, f)) for f in files)\n    return [d for d in definitions if d and d.get(\"name\")]\n\n\ndef _oauth_path(name):\n    return os.path.join(CREDENTIALS, f\"{name}.oauth.json\")\n\n\ndef _keys_path(name):\n    return os.path.join(CREDENTIALS, f\"{name}.api-keys.json\")\n\n\ndef _base(url):\n    return url.split(\"?\", 1)[0].split(\"#\", 1)[0]\n\n\ndef _hosts(mode):\n    return {h.lower() for h in mode.get(\"hosts\", [])}\n\n\n# --- fakes -----------------------------------------------------------------------------------------\n\n\ndef _random(length):\n    return \"\".join(secrets.choice(FAKE_ALPHABET) for _ in range(length))\n\n\ndef _issue_fake(shape, real):\n    if shape.get(\"jwt\"):\n        signature = real.rsplit(\".\", 1)[-1] if real.count(\".\") == 2 else \"\"\n        return {\"jwtSignature\": _random(max(len(signature), 43))}\n    prefix = shape.get(\"prefix\", \"\")\n    return {\"value\": prefix + _random(max(len(real) - len(prefix), 16))}\n\n\ndef _fake_value(stored, real):\n    if \"jwtSignature\" in stored:\n        header_and_claims = real.rsplit(\".\", 1)[0] if real.count(\".\") == 2 else real\n        return f\"{header_and_claims}.{stored['jwtSignature']}\"\n    return stored[\"value\"]\n\n\ndef _is_fake(stored, candidate):\n    if not stored or not candidate:\n        return False\n    if \"jwtSignature\" in stored:\n        return candidate.count(\".\") == 2 and candidate.rsplit(\".\", 1)[1] == stored[\"jwtSignature\"]\n    return candidate == stored[\"value\"]\n\n\n# --- presenting secrets ----------------------------------------------------------------------------\n\n\ndef _presented(request, present):\n    if \"query\" in present:\n        return request.query.get(present[\"query\"])\n    value = request.headers.get(present[\"header\"])\n    scheme = present.get(\"scheme\")\n    if value and scheme:\n        prefix = f\"{scheme} \"\n        return value[len(prefix):] if value.lower().startswith(prefix.lower()) else None\n    return value\n\n\ndef _present(request, present, secret):\n    if \"query\" in present:\n        request.query[present[\"query\"]] = secret\n    elif present.get(\"scheme\"):\n        request.headers[present[\"header\"]] = f\"{present['scheme']} {secret}\"\n    else:\n        request.headers[present[\"header\"]] = secret\n\n\ndef _force(request, force):\n    if not force:\n        return\n    if force.get(\"target\"):\n        target = urllib.parse.urlsplit(force[\"target\"])\n        request.scheme = target.scheme or request.scheme\n        request.host = target.hostname or request.host\n        request.port = target.port or (443 if request.scheme == \"https\" else 80)\n    if force.get(\"path\"):\n        request.path = force[\"path\"].rstrip(\"/\") + request.path\n    for key, value in force.get(\"query\", {}).items():\n        request.query[key] = value\n    for key, value in force.get(\"headers\", {}).items():\n        request.headers[key] = value\n\n\ndef _swap_oauth(request, name, mode):\n    credential = _load(_oauth_path(name))\n    presented = _presented(request, mode[\"present\"])\n    if credential and _is_fake(credential[\"fakes\"].get(\"access\"), presented):\n        _present(request, mode[\"present\"], credential[\"access\"])\n        return True\n    return False\n\n\ndef _swap_api_key(request, name, mode):\n    stored = _load(_keys_path(name)) or {}\n    presented = _presented(request, mode[\"present\"])\n    for key in stored.get(\"keys\", {}).values():\n        if presented and presented == key[\"fake\"]:\n            _present(request, mode[\"present\"], key[\"key\"])\n            return True\n    return False\n\n\n# --- the token endpoint ----------------------------------------------------------------------------\n\n\ndef _parse_form_or_json(text, content_type):\n    if not text:\n        return {}\n    if \"json\" in (content_type or \"\"):\n        try:\n            value = json.loads(text)\n            return value if isinstance(value, dict) else {}\n        except ValueError:\n            return {}\n    return dict(urllib.parse.parse_qsl(text))\n\n\ndef _refresh_request(flow, fields):\n    \"\"\"How a client refreshes, to replay with the real refresh token: everything but that token.\"\"\"\n    return {\n        \"contentType\": flow.request.headers.get(\"content-type\", \"\"),\n        \"userAgent\": flow.request.headers.get(\"user-agent\"),\n        \"fields\": {k: v for k, v in fields.items() if k != \"refresh_token\"},\n    }\n\n\ndef _store_tokens(name, mode, body, refresh_request):\n    \"\"\"Keeps a token response as the upstream's credential and returns it.\"\"\"\n    path = _oauth_path(name)\n    credential = dict(_load(path) or {})\n    if refresh_request:\n        credential[\"refreshRequest\"] = refresh_request\n    now = _now_ms()\n    for key, field in TOKEN_FIELDS:\n        if body.get(field):\n            credential[key] = body[field]\n    if \"expires_in\" in body:\n        credential[\"expiresAt\"] = now + int(body[\"expires_in\"]) * 1000\n    if \"refresh_token_expires_in\" in body:\n        credential[\"refreshExpiresAt\"] = now + int(body[\"refresh_token_expires_in\"]) * 1000\n    if body.get(\"scope\"):\n        credential[\"scopes\"] = body[\"scope\"].split()\n    credential[\"lastRefresh\"] = now\n    fakes = dict(credential.get(\"fakes\") or {})\n    for key, _ in TOKEN_FIELDS:\n        shape = mode[\"fakes\"].get(key)\n        if shape and credential.get(key) and key not in fakes:\n            fakes[key] = _issue_fake(shape, credential[key])\n    credential[\"fakes\"] = fakes\n    stripped = set((mode.get(\"strip\") or {}).get(\"fields\", []))\n    credential[\"response\"] = {k: v for k, v in body.items() if k not in stripped}\n    _save(path, credential)\n    return credential\n\n\ndef _client_body(mode, credential):\n    \"\"\"The token response the dimension gets: the last real one, with fakes for every token.\"\"\"\n    body = dict(credential.get(\"response\") or {})\n    fakes = credential[\"fakes\"]\n    for key, field in TOKEN_FIELDS:\n        if field not in body:\n            continue\n        if key in fakes and credential.get(key):\n            body[field] = _fake_value(fakes[key], credential[key])\n        else:\n            del body[field]  # a token with no fake shape never reaches a dimension\n    now = _now_ms()\n    if credential.get(\"expiresAt\") and \"expires_in\" in body:\n        body[\"expires_in\"] = max(0, (credential[\"expiresAt\"] - now) // 1000)\n    if credential.get(\"refreshExpiresAt\") and \"refresh_token_expires_in\" in body:\n        body[\"refresh_token_expires_in\"] = max(0, (credential[\"refreshExpiresAt\"] - now) // 1000)\n    return body\n\n\ndef _refresh_upstream(mode, credential):\n    \"\"\"Replays the client's refresh request with the real refresh token; returns (status, parsed body\n    or raw bytes). http.client, unlike urllib, adds no User-Agent of its own: a client that sent none\n    stays that way.\"\"\"\n    template = credential[\"refreshRequest\"]\n    fields = dict(template[\"fields\"], refresh_token=credential[\"refresh\"])\n    content_type = template[\"contentType\"] or \"application/x-www-form-urlencoded\"\n    data = json.dumps(fields) if \"json\" in content_type else urllib.parse.urlencode(fields)\n    headers = {\"content-type\": content_type, \"accept\": \"application/json\"}\n    if template.get(\"userAgent\"):\n        headers[\"user-agent\"] = template[\"userAgent\"]\n    url = urllib.parse.urlsplit(mode[\"tokenUrl\"])\n    connection_type = http_client.HTTPSConnection if url.scheme == \"https\" else http_client.HTTPConnection\n    connection = connection_type(url.hostname, url.port, timeout=30)\n    try:\n        connection.request(\"POST\", url.path + (f\"?{url.query}\" if url.query else \"\"), body=data.encode(), headers=headers)\n        response = connection.getresponse()\n        raw = response.read()\n        try:\n            return response.status, json.loads(raw)\n        except ValueError:\n            return response.status, raw\n    finally:\n        connection.close()\n\n\nasync def _answer_from_credential(flow, name, mode, refresh_request):\n    \"\"\"Answers a sign-in, or a refresh made with a fake (`refresh_request`), with the credential's\n    fakes. A refresh is replayed upstream first unless another one just was; a sign-in only if the\n    credential has expired. Returns False, answering nothing, when a sign-in can't be given valid\n    tokens: expired, and no client refresh to replay yet or upstream refused it. A client's refresh\n    gets upstream's refusal instead.\"\"\"\n    lock = _refresh_locks.setdefault(name, asyncio.Lock())\n    async with lock:\n        credential = _load(_oauth_path(name))\n        if refresh_request and credential.get(\"refreshRequest\") != refresh_request:\n            credential[\"refreshRequest\"] = refresh_request\n            _save(_oauth_path(name), credential)\n        now = _now_ms()\n        expired = not credential.get(\"expiresAt\") or credential[\"expiresAt\"] - now < REFRESH_MARGIN_MS\n        just_refreshed = now - credential.get(\"lastRefresh\", 0) < REFRESH_SHARED_MS\n        if expired or (refresh_request and not just_refreshed):\n            if not (credential.get(\"refresh\") and credential.get(\"refreshRequest\")):\n                if not refresh_request:\n                    return False\n                # A client's refresh with nothing to refresh with: all there is to give it.\n                flow.response = http.Response.make(\n                    200, json.dumps(_client_body(mode, credential)), {\"content-type\": \"application/json\"}\n                )\n                return True\n            status, body = await asyncio.to_thread(_refresh_upstream, mode, credential)\n            if status != 200 or not isinstance(body, dict) or \"access_token\" not in body:\n                if not refresh_request:\n                    return False\n                content = body if isinstance(body, bytes) else json.dumps(body).encode()\n                flow.response = http.Response.make(status, content, {\"content-type\": \"application/json\"})\n                return True\n            credential = _store_tokens(name, mode, body, None)\n    flow.response = http.Response.make(\n        200, json.dumps(_client_body(mode, credential)), {\"content-type\": \"application/json\"}\n    )\n    return True\n\n\nasync def request(flow: http.HTTPFlow):\n    if flow.response:  # already answered, e.g. by a traffic rule\n        return\n    host = flow.request.pretty_host.lower()\n    url = _base(flow.request.pretty_url)\n    for definition in _definitions():\n        name = definition[\"name\"]\n        oauth = definition.get(\"modes\", {}).get(\"oauth\")\n        api_key = definition.get(\"modes\", {}).get(\"apiKey\")\n        credential = _load(_oauth_path(name)) if oauth else None\n        if oauth and url == _base(oauth[\"tokenUrl\"]):\n            fields = _parse_form_or_json(flow.request.get_text(strict=False), flow.request.headers.get(\"content-type\"))\n            refreshing = fields.get(\"grant_type\") == \"refresh_token\"\n            refresh_request = _refresh_request(flow, fields) if refreshing else None\n            # A refresh token the gateway didn't issue is a client that signed in before the upstream\n            # was registered (or elsewhere): let it through, and keep what it gets.\n            foreign_refresh = refreshing and not _is_fake(\n                (credential or {}).get(\"fakes\", {}).get(\"refresh\"), fields.get(\"refresh_token\")\n            )\n            if credential and not foreign_refresh and await _answer_from_credential(flow, name, oauth, refresh_request):\n                return\n            flow.metadata[\"hyperbrane_token\"] = (name, refresh_request)\n            return\n        if credential:\n            for answer in oauth.get(\"answers\", []):\n                if url == _base(answer[\"url\"]):\n                    flow.response = http.Response.make(\n                        answer.get(\"status\", 200), json.dumps(answer[\"json\"]), {\"content-type\": \"application/json\"}\n                    )\n                    return\n        for mode, swap in ((oauth, _swap_oauth), (api_key, _swap_api_key)):\n            if mode and host in _hosts(mode):\n                swap(flow.request, name, mode)\n                _force(flow.request, mode.get(\"force\"))\n\n\ndef responseheaders(flow: http.HTTPFlow):\n    # Only token responses are rewritten; everything else streams through as it arrives. mitmproxy\n    # otherwise holds each response until its body is complete, so a model's streamed reply (SSE)\n    # reached the dimension in one piece at the end of the turn.\n    if not flow.metadata.get(\"hyperbrane_token\"):\n        flow.response.stream = True\n\n\ndef response(flow: http.HTTPFlow):\n    token = flow.metadata.get(\"hyperbrane_token\")\n    if not token:\n        return\n    name, refresh_request = token\n    definition = next((d for d in _definitions() if d[\"name\"] == name), None)\n    oauth = (definition or {}).get(\"modes\", {}).get(\"oauth\")\n    if not oauth:\n        return\n    body = _parse_form_or_json(flow.response.get_text(strict=False), \"json\")\n    if flow.response.status_code == 200 and body.get(\"access_token\"):\n        credential = _store_tokens(name, oauth, body, refresh_request)\n        flow.response.text = json.dumps(_client_body(oauth, credential))\n    for header in (oauth.get(\"strip\") or {}).get(\"headers\", []):\n        flow.response.headers.pop(header, None)\n",
+    "upstreams.py": "# mitmproxy addon applying the gateway's upstreams (see portal-gateway/upstream.ts) to traffic from\n# portals. Definitions are the JSON files the gateway service writes to /state/upstreams; this addon\n# keeps each upstream's OAuth credential in /state/credentials/<upstream>.oauth.json and reads its\n# API keys from /state/credentials/<upstream>.api-keys.json. Everything is re-read per request, so a\n# change takes effect without a reload.\n#\n# Clients sign in from dimensions, through the proxy:\n#   token endpoint   without a usable credential, requests go through and the response is kept as\n#                    the credential, reaching the dimension with fakes in place of the tokens. With\n#                    one, the gateway answers from it: a sign-in (whatever code it carries) and a\n#                    refresh made with a fake refresh token. A client's refresh goes upstream: the\n#                    gateway replays it (encoding, fields, User-Agent) with the real refresh token,\n#                    one at a time, and refreshes arriving together share its result. Every client\n#                    holds the same fakes, so they all get the new tokens. A sign-in is answered from\n#                    the credential, refreshed first if expired by replaying the last client refresh.\n#                    An expired credential it can't refresh (no client refresh to replay yet, or\n#                    upstream refused) counts as none: a sign-in goes to the upstream again.\n#   answers          other sign-in steps (a device flow's poll) answered as the definition says, once\n#                    there is a credential\n#   other requests   a fake in the mode's `present` location is replaced by the real secret\n\nimport asyncio\nimport http.client as http_client\nimport json\nimport os\nimport secrets\nimport string\nimport time\nimport urllib.parse\n\nfrom mitmproxy import http\n\nSTATE = \"/state\"\nUPSTREAMS = os.path.join(STATE, \"upstreams\")\nCREDENTIALS = os.path.join(STATE, \"credentials\")\n# A token this close to expiry counts as expired.\nREFRESH_MARGIN_MS = 60_000\n# Refreshes arriving within this long of the last one get its tokens instead of going upstream again.\nREFRESH_SHARED_MS = 5_000\nFAKE_ALPHABET = string.ascii_letters + string.digits + \"-_\"\nTOKEN_FIELDS = ((\"access\", \"access_token\"), (\"refresh\", \"refresh_token\"), (\"id\", \"id_token\"))\n\n# One refresh per upstream at a time: concurrent ones would each spend the rotating refresh token.\n_refresh_locks: dict[str, asyncio.Lock] = {}\n\n\ndef _now_ms():\n    return int(time.time() * 1000)\n\n\ndef _load(path):\n    try:\n        with open(path) as f:\n            return json.load(f)\n    except (OSError, ValueError):\n        return None\n\n\ndef _save(path, data):\n    tmp = f\"{path}.{secrets.token_hex(4)}.tmp\"\n    with open(tmp, \"w\") as f:\n        json.dump(data, f, indent=2)\n    os.replace(tmp, path)\n\n\ndef _definitions():\n    try:\n        files = sorted(f for f in os.listdir(UPSTREAMS) if f.endswith(\".json\"))\n    except OSError:\n        return []\n    definitions = (_load(os.path.join(UPSTREAMS, f)) for f in files)\n    return [d for d in definitions if d and d.get(\"name\")]\n\n\ndef _oauth_path(name):\n    return os.path.join(CREDENTIALS, f\"{name}.oauth.json\")\n\n\ndef _keys_path(name):\n    return os.path.join(CREDENTIALS, f\"{name}.api-keys.json\")\n\n\ndef _base(url):\n    return url.split(\"?\", 1)[0].split(\"#\", 1)[0]\n\n\ndef _hosts(mode):\n    return {h.lower() for h in mode.get(\"hosts\", [])}\n\n\n# --- fakes -----------------------------------------------------------------------------------------\n\n\ndef _random(length):\n    return \"\".join(secrets.choice(FAKE_ALPHABET) for _ in range(length))\n\n\ndef _issue_fake(shape, real):\n    if shape.get(\"jwt\"):\n        signature = real.rsplit(\".\", 1)[-1] if real.count(\".\") == 2 else \"\"\n        return {\"jwtSignature\": _random(max(len(signature), 43))}\n    prefix = shape.get(\"prefix\", \"\")\n    return {\"value\": prefix + _random(max(len(real) - len(prefix), 16))}\n\n\ndef _fake_value(stored, real):\n    if \"jwtSignature\" in stored:\n        header_and_claims = real.rsplit(\".\", 1)[0] if real.count(\".\") == 2 else real\n        return f\"{header_and_claims}.{stored['jwtSignature']}\"\n    return stored[\"value\"]\n\n\ndef _is_fake(stored, candidate):\n    if not stored or not candidate:\n        return False\n    if \"jwtSignature\" in stored:\n        return candidate.count(\".\") == 2 and candidate.rsplit(\".\", 1)[1] == stored[\"jwtSignature\"]\n    return candidate == stored[\"value\"]\n\n\n# --- presenting secrets ----------------------------------------------------------------------------\n\n\ndef _presented(request, present):\n    if \"query\" in present:\n        return request.query.get(present[\"query\"])\n    value = request.headers.get(present[\"header\"])\n    scheme = present.get(\"scheme\")\n    if value and scheme:\n        prefix = f\"{scheme} \"\n        return value[len(prefix):] if value.lower().startswith(prefix.lower()) else None\n    return value\n\n\ndef _present(request, present, secret):\n    if \"query\" in present:\n        request.query[present[\"query\"]] = secret\n    elif present.get(\"scheme\"):\n        request.headers[present[\"header\"]] = f\"{present['scheme']} {secret}\"\n    else:\n        request.headers[present[\"header\"]] = secret\n\n\ndef _force(request, force):\n    if not force:\n        return\n    if force.get(\"target\"):\n        target = urllib.parse.urlsplit(force[\"target\"])\n        request.scheme = target.scheme or request.scheme\n        request.host = target.hostname or request.host\n        request.port = target.port or (443 if request.scheme == \"https\" else 80)\n    if force.get(\"path\"):\n        request.path = force[\"path\"].rstrip(\"/\") + request.path\n    for key, value in force.get(\"query\", {}).items():\n        request.query[key] = value\n    for key, value in force.get(\"headers\", {}).items():\n        request.headers[key] = value\n\n\ndef _swap_oauth(request, name, mode):\n    credential = _load(_oauth_path(name))\n    presented = _presented(request, mode[\"present\"])\n    if credential and _is_fake(credential[\"fakes\"].get(\"access\"), presented):\n        _present(request, mode[\"present\"], credential[\"access\"])\n        return True\n    return False\n\n\ndef _swap_api_key(request, name, mode):\n    stored = _load(_keys_path(name)) or {}\n    presented = _presented(request, mode[\"present\"])\n    for key in stored.get(\"keys\", {}).values():\n        if presented and presented == key[\"fake\"]:\n            _present(request, mode[\"present\"], key[\"key\"])\n            return True\n    return False\n\n\n# --- the token endpoint ----------------------------------------------------------------------------\n\n\ndef _parse_form_or_json(text, content_type):\n    if not text:\n        return {}\n    if \"json\" in (content_type or \"\"):\n        try:\n            value = json.loads(text)\n            return value if isinstance(value, dict) else {}\n        except ValueError:\n            return {}\n    return dict(urllib.parse.parse_qsl(text))\n\n\ndef _refresh_request(flow, fields):\n    \"\"\"How a client refreshes, to replay with the real refresh token: everything but that token.\"\"\"\n    return {\n        \"contentType\": flow.request.headers.get(\"content-type\", \"\"),\n        \"userAgent\": flow.request.headers.get(\"user-agent\"),\n        \"fields\": {k: v for k, v in fields.items() if k != \"refresh_token\"},\n    }\n\n\ndef _store_tokens(name, mode, body, refresh_request):\n    \"\"\"Keeps a token response as the upstream's credential and returns it.\"\"\"\n    path = _oauth_path(name)\n    credential = dict(_load(path) or {})\n    if refresh_request:\n        credential[\"refreshRequest\"] = refresh_request\n    now = _now_ms()\n    for key, field in TOKEN_FIELDS:\n        if body.get(field):\n            credential[key] = body[field]\n    if \"expires_in\" in body:\n        credential[\"expiresAt\"] = now + int(body[\"expires_in\"]) * 1000\n    if \"refresh_token_expires_in\" in body:\n        credential[\"refreshExpiresAt\"] = now + int(body[\"refresh_token_expires_in\"]) * 1000\n    if body.get(\"scope\"):\n        credential[\"scopes\"] = body[\"scope\"].split()\n    credential[\"lastRefresh\"] = now\n    fakes = dict(credential.get(\"fakes\") or {})\n    for key, _ in TOKEN_FIELDS:\n        shape = mode[\"fakes\"].get(key)\n        if shape and credential.get(key) and key not in fakes:\n            fakes[key] = _issue_fake(shape, credential[key])\n    credential[\"fakes\"] = fakes\n    stripped = set((mode.get(\"strip\") or {}).get(\"fields\", []))\n    credential[\"response\"] = {k: v for k, v in body.items() if k not in stripped}\n    _save(path, credential)\n    return credential\n\n\ndef _client_body(mode, credential):\n    \"\"\"The token response the dimension gets: the last real one, with fakes for every token.\"\"\"\n    body = dict(credential.get(\"response\") or {})\n    fakes = credential[\"fakes\"]\n    for key, field in TOKEN_FIELDS:\n        if field not in body:\n            continue\n        if key in fakes and credential.get(key):\n            body[field] = _fake_value(fakes[key], credential[key])\n        else:\n            del body[field]  # a token with no fake shape never reaches a dimension\n    now = _now_ms()\n    if credential.get(\"expiresAt\") and \"expires_in\" in body:\n        body[\"expires_in\"] = max(0, (credential[\"expiresAt\"] - now) // 1000)\n    if credential.get(\"refreshExpiresAt\") and \"refresh_token_expires_in\" in body:\n        body[\"refresh_token_expires_in\"] = max(0, (credential[\"refreshExpiresAt\"] - now) // 1000)\n    return body\n\n\ndef _refresh_upstream(mode, credential):\n    \"\"\"Replays the client's refresh request with the real refresh token; returns (status, parsed body\n    or raw bytes). http.client, unlike urllib, adds no User-Agent of its own: a client that sent none\n    stays that way.\"\"\"\n    template = credential[\"refreshRequest\"]\n    fields = dict(template[\"fields\"], refresh_token=credential[\"refresh\"])\n    content_type = template[\"contentType\"] or \"application/x-www-form-urlencoded\"\n    data = json.dumps(fields) if \"json\" in content_type else urllib.parse.urlencode(fields)\n    headers = {\"content-type\": content_type, \"accept\": \"application/json\"}\n    if template.get(\"userAgent\"):\n        headers[\"user-agent\"] = template[\"userAgent\"]\n    url = urllib.parse.urlsplit(mode[\"tokenUrl\"])\n    connection_type = http_client.HTTPSConnection if url.scheme == \"https\" else http_client.HTTPConnection\n    connection = connection_type(url.hostname, url.port, timeout=30)\n    try:\n        connection.request(\"POST\", url.path + (f\"?{url.query}\" if url.query else \"\"), body=data.encode(), headers=headers)\n        response = connection.getresponse()\n        raw = response.read()\n        try:\n            return response.status, json.loads(raw)\n        except ValueError:\n            return response.status, raw\n    finally:\n        connection.close()\n\n\nasync def _answer_from_credential(flow, name, mode, refresh_request):\n    \"\"\"Answers a sign-in, or a refresh made with a fake (`refresh_request`), with the credential's\n    fakes. A refresh is replayed upstream first unless another one just was; a sign-in only if the\n    credential has expired. Returns False, answering nothing, when a sign-in can't be given valid\n    tokens: expired, and no client refresh to replay yet or upstream refused it. A client's refresh\n    gets upstream's refusal instead.\"\"\"\n    lock = _refresh_locks.setdefault(name, asyncio.Lock())\n    async with lock:\n        credential = _load(_oauth_path(name))\n        if refresh_request and credential.get(\"refreshRequest\") != refresh_request:\n            credential[\"refreshRequest\"] = refresh_request\n            _save(_oauth_path(name), credential)\n        now = _now_ms()\n        expired = not credential.get(\"expiresAt\") or credential[\"expiresAt\"] - now < REFRESH_MARGIN_MS\n        just_refreshed = now - credential.get(\"lastRefresh\", 0) < REFRESH_SHARED_MS\n        if expired or (refresh_request and not just_refreshed):\n            if not (credential.get(\"refresh\") and credential.get(\"refreshRequest\")):\n                if not refresh_request:\n                    return False\n                # A client's refresh with nothing to refresh with: all there is to give it.\n                flow.response = http.Response.make(\n                    200, json.dumps(_client_body(mode, credential)), {\"content-type\": \"application/json\"}\n                )\n                return True\n            status, body = await asyncio.to_thread(_refresh_upstream, mode, credential)\n            if status != 200 or not isinstance(body, dict) or \"access_token\" not in body:\n                if not refresh_request:\n                    return False\n                content = body if isinstance(body, bytes) else json.dumps(body).encode()\n                flow.response = http.Response.make(status, content, {\"content-type\": \"application/json\"})\n                return True\n            credential = _store_tokens(name, mode, body, None)\n    flow.response = http.Response.make(\n        200, json.dumps(_client_body(mode, credential)), {\"content-type\": \"application/json\"}\n    )\n    return True\n\n\nasync def request(flow: http.HTTPFlow):\n    if flow.response:  # already answered, e.g. by a traffic rule\n        return\n    host = flow.request.pretty_host.lower()\n    url = _base(flow.request.pretty_url)\n    for definition in _definitions():\n        name = definition[\"name\"]\n        oauth = definition.get(\"modes\", {}).get(\"oauth\")\n        api_key = definition.get(\"modes\", {}).get(\"apiKey\")\n        forced = definition.get(\"modes\", {}).get(\"forced\")\n        credential = _load(_oauth_path(name)) if oauth else None\n        if oauth and url == _base(oauth[\"tokenUrl\"]):\n            fields = _parse_form_or_json(flow.request.get_text(strict=False), flow.request.headers.get(\"content-type\"))\n            refreshing = fields.get(\"grant_type\") == \"refresh_token\"\n            refresh_request = _refresh_request(flow, fields) if refreshing else None\n            # A refresh token the gateway didn't issue is a client that signed in before the upstream\n            # was registered (or elsewhere): let it through, and keep what it gets.\n            foreign_refresh = refreshing and not _is_fake(\n                (credential or {}).get(\"fakes\", {}).get(\"refresh\"), fields.get(\"refresh_token\")\n            )\n            if credential and not foreign_refresh and await _answer_from_credential(flow, name, oauth, refresh_request):\n                return\n            flow.metadata[\"hyperbrane_token\"] = (name, refresh_request)\n            return\n        if credential:\n            for answer in oauth.get(\"answers\", []):\n                if url == _base(answer[\"url\"]):\n                    flow.response = http.Response.make(\n                        answer.get(\"status\", 200), json.dumps(answer[\"json\"]), {\"content-type\": \"application/json\"}\n                    )\n                    return\n        for mode, swap in ((oauth, _swap_oauth), (api_key, _swap_api_key), (forced, None)):\n            if mode and host in _hosts(mode):\n                if swap:\n                    swap(flow.request, name, mode)\n                _force(flow.request, mode.get(\"force\"))\n\n\ndef responseheaders(flow: http.HTTPFlow):\n    # Only token responses are rewritten; everything else streams through as it arrives. mitmproxy\n    # otherwise holds each response until its body is complete, so a model's streamed reply (SSE)\n    # reached the dimension in one piece at the end of the turn.\n    if not flow.metadata.get(\"hyperbrane_token\"):\n        flow.response.stream = True\n\n\ndef response(flow: http.HTTPFlow):\n    token = flow.metadata.get(\"hyperbrane_token\")\n    if not token:\n        return\n    name, refresh_request = token\n    definition = next((d for d in _definitions() if d[\"name\"] == name), None)\n    oauth = (definition or {}).get(\"modes\", {}).get(\"oauth\")\n    if not oauth:\n        return\n    body = _parse_form_or_json(flow.response.get_text(strict=False), \"json\")\n    if flow.response.status_code == 200 and body.get(\"access_token\"):\n        credential = _store_tokens(name, oauth, body, refresh_request)\n        flow.response.text = json.dumps(_client_body(oauth, credential))\n    for header in (oauth.get(\"strip\") or {}).get(\"headers\", []):\n        flow.response.headers.pop(header, None)\n",
 };

package/dist/portal-gateway/portalClient.d.ts

@@ -0,0 +1,5 @@
+import type { Sandbox } from "../sandbox/api.ts";
+import type { PortalClient } from "./api.ts";
+export declare function createPortalClient(sandbox: Sandbox, options?: {
+    dataDirectory?: string;
+}): PortalClient;

package/dist/portal-gateway/portalClient.js

@@ -0,0 +1,4 @@
+import { createPortalService } from "./portalService.js";
+export function createPortalClient(sandbox, options = {}) {
+    return createPortalService(sandbox, options);
+}

package/dist/portal-gateway/portalService.d.ts

@@ -1,50 +1,8 @@
-import { type DockerClient } from "../lib/docker.ts";
-import type { ManagedImage } from "../lib/image.ts";
 import type { Sandbox } from "../sandbox/api.ts";
-import { type Portal } from "./api.ts";
-import type { GatewayStorage } from "./gatewayStorage.ts";
-import type { PortalStorage } from "./portalStorage.ts";
+import type { PortalClient } from "./api.ts";
 /** @sealed */
-export interface PortalService {
-    createPortal(portal: Portal): Promise<Portal>;
-    destroyPortal(portalName: string): Promise<void>;
-    listPortals(): Promise<Portal[]>;
-    /**
-     * Re-wires the ready portals (optionally only those of one gateway or dimension) whose containers
-     * are running and whose link is gone: a container restart takes its network namespace, and the
-     * portal's veth, with it. Portals still wired cost one look at each gateway's interfaces, so this
-     * is cheap enough to run before every command. Failures are logged per portal rather than thrown.
-     */
-    restorePortals(filter?: {
-        gatewayName?: string;
-        dimensionName?: string;
-    }): Promise<void>;
+export interface PortalService extends PortalClient {
 }
-type PortalLink = {
-    dimensionIf: string;
-    gatewayIf: string;
-    dimensionIp: string;
-    gatewayIp: string;
-    metric: number;
-};
-/**
- * Interface names come from the portal name (so a portal can always be found again) and addresses
- * from its claimed subnet. The gateway only intercepts interfaces named "hvg*".
- */
-export declare function portalLink(portalName: string, subnet: number): PortalLink;
-/** The OS trust bundle; the gateways' CAs end up in it. */
-export declare const OS_CA_BUNDLE = "/etc/ssl/certs/ca-certificates.crt";
-/**
- * `host` is the machine the docker daemon runs on, rooted at "/" (see createGatewayService).
- * `image` is the gateway image: it carries the wiring script run by the privileged helper.
- */
-export declare function createPortalService(storage: PortalStorage, gateways: GatewayStorage, docker: DockerClient, image: ManagedImage, host: Sandbox): PortalService;
-/** @sealed */
-export interface PortalWatcher {
-    /** Restores every portal, then keeps restoring those of each hyperbrane container that starts. */
-    start(): void;
-    stop(): Promise<void>;
-}
-/** Run one per host: the portals of a container that restarts are gone until something re-wires them. */
-export declare function createPortalWatcher(portals: PortalService, docker: DockerClient): PortalWatcher;
-export {};
+export declare function createPortalService(host: Sandbox, { dataDirectory }?: {
+    dataDirectory?: string;
+}): PortalService;

package/dist/portal-gateway/portalService.js

@@ -1,327 +1,29 @@
-import { createHash } from "node:crypto";
-import { containerName as dimensionContainerName } from "../dimension/service.js";
-import { DockerError } from "../lib/docker.js";
-import { sleep } from "../lib/sleep.js";
-import { GatewayConflictError, GatewayNotFoundError } from "./api.js";
-import { assertValidName, gatewayCaCertificatePath, gatewayContainerName } from "./gatewayService.js";
-const RESTORE_RECHECK_MS = 1_000;
-/** Each portal is a point-to-point /30 out of 10.77.0.0/16. */
-const SUBNET_COUNT = 1 << 14;
-/** A dimension's default routes are ordered by subnet: its portal with the lowest one is preferred. */
-const ROUTE_METRIC_BASE = 100;
-/**
- * Interface names come from the portal name (so a portal can always be found again) and addresses
- * from its claimed subnet. The gateway only intercepts interfaces named "hvg*".
- */
-export function portalLink(portalName, subnet) {
-    const hash = createHash("sha256").update(portalName).digest("hex").slice(0, 8);
-    const base = `10.77.${subnet >> 6}`;
-    const offset = (subnet & 0x3f) << 2;
-    return {
-        dimensionIf: `hvd${hash}`,
-        gatewayIf: `hvg${hash}`,
-        gatewayIp: `${base}.${offset + 1}`,
-        dimensionIp: `${base}.${offset + 2}`,
-        metric: ROUTE_METRIC_BASE + subnet,
-    };
-}
-/** The OS trust bundle; the gateways' CAs end up in it. */
-export const OS_CA_BUNDLE = "/etc/ssl/certs/ca-certificates.crt";
-// Run as root in the dimension: installs GATEWAY_CA as the gateway's certificate in the OS trust
-// store, or removes it when empty. Nothing but its gateways' CAs ever signs what a dimension sees
-// (every TCP connection is intercepted), so without the ca-certificates package the bundle can be
-// just those CAs. Node ignores the OS store, so while any gateway is trusted NODE_EXTRA_CA_CERTS
-// points it at the bundle, for login shells (profile.d) and PAM sessions (/etc/environment); a
-// running container's own environment can't be changed. DNS needs no setup: the gateway answers
-// UDP 53 whatever address it was sent to.
-const TRUST_GATEWAY_SCRIPT = `set -eu
-# Portals of one dimension are wired concurrently (e.g. restored together after a restart), and
-# they share the trust store and /etc/environment.
-exec 9>/run/hyperbrane-trust.lock
-flock 9
-bundle=${OS_CA_BUNDLE}
-mkdir -p /usr/local/share/ca-certificates /etc/ssl/certs /etc/profile.d
-cert="/usr/local/share/ca-certificates/hyperbrane-gateway-$GATEWAY.crt"
-if [ -n "$GATEWAY_CA" ]; then printf '%s\\n' "$GATEWAY_CA" > "$cert"; else rm -f "$cert"; fi
-if command -v update-ca-certificates >/dev/null; then
-  update-ca-certificates --fresh >/dev/null
-else
-  cat /usr/local/share/ca-certificates/hyperbrane-gateway-*.crt > "$bundle" 2>/dev/null || : > "$bundle"
-  if command -v openssl >/dev/null; then openssl rehash /etc/ssl/certs; fi
-fi
-{ grep -v '^NODE_EXTRA_CA_CERTS=' /etc/environment 2>/dev/null || true; } > /etc/environment.hyperbrane
-if ls /usr/local/share/ca-certificates/hyperbrane-gateway-*.crt >/dev/null 2>&1; then
-  echo "NODE_EXTRA_CA_CERTS=$bundle" >> /etc/environment.hyperbrane
-  echo "export NODE_EXTRA_CA_CERTS=$bundle" > /etc/profile.d/hyperbrane-gateway.sh
-else
-  rm -f /etc/profile.d/hyperbrane-gateway.sh
-fi
-mv /etc/environment.hyperbrane /etc/environment
-`;
-/**
- * `host` is the machine the docker daemon runs on, rooted at "/" (see createGatewayService).
- * `image` is the gateway image: it carries the wiring script run by the privileged helper.
- */
-export function createPortalService(storage, gateways, docker, image, host) {
-    // Re-running re-wires, so it also repairs a portal whose veth went away.
+import { createDockerClient } from "../lib/docker.js";
+import { gatewayControl } from "./control.js";
+import { assertValidName } from "./gatewayService.js";
+import { createGatewayImage } from "./image.js";
+export function createPortalService(host, { dataDirectory } = {}) {
+    const control = gatewayControl(host, () => createGatewayImage(createDockerClient(host)).ensure(), { dataDirectory });
     async function createPortal(portal) {
         assertValidName("portal", portal.name);
         assertValidName("dimension", portal.dimensionName);
         assertValidName("gateway", portal.gatewayName);
-        const gateway = await gateways.readGateway(portal.gatewayName);
-        if (gateway?.state !== "ready")
-            throw new GatewayNotFoundError(`gateway ${portal.gatewayName} is not ready`);
-        // Checked again when wiring; failing here first leaves no record behind.
-        await runningPid(gatewayContainerName(portal.gatewayName), "hyperbrane.gateway", portal.gatewayName);
-        await runningPid(dimensionContainerName(portal.dimensionName), "hyperbrane.dimension", portal.dimensionName);
-        let record = await storage.insertPortal({ ...portal, state: "creating" });
-        if (record.state === "destroying")
-            throw new GatewayConflictError(`portal ${portal.name} is being destroyed`);
-        if (record.dimensionName !== portal.dimensionName || record.gatewayName !== portal.gatewayName) {
-            throw new GatewayConflictError(`portal ${portal.name} already exists with a different spec`);
-        }
-        if (record.subnet === undefined) {
-            const subnet = await storage.claimSubnet(portal.name);
-            if (subnet >= SUBNET_COUNT) {
-                await storage.releaseSubnet(subnet);
-                throw new GatewayConflictError("no portal subnets left");
-            }
-            record = { ...record, subnet };
-            await storage.savePortal(record);
-        }
-        await connect(record);
-        if (record.state !== "ready")
-            await storage.savePortal({ ...record, state: "ready" });
-        return publicPortal(record);
-    }
-    async function connect(record, pids) {
-        const { gateway, dimension } = pids ?? (await runningPids(record));
-        await wire(record.name, portalLink(record.name, record.subnet), dimension, gateway);
-        await trustGateway(dimensionContainerName(record.dimensionName), record.gatewayName, true);
-    }
-    async function runningPids(record) {
-        return {
-            gateway: await runningPid(gatewayContainerName(record.gatewayName), "hyperbrane.gateway", record.gatewayName),
-            dimension: await runningPid(dimensionContainerName(record.dimensionName), "hyperbrane.dimension", record.dimensionName),
-        };
-    }
-    async function runningPid(name, label, owner) {
-        const kind = label.replace("hyperbrane.", "");
-        const container = await docker.inspectContainer(name);
-        if (!container || container.Config?.Labels?.[label] !== owner) {
-            throw new GatewayNotFoundError(`${kind} ${owner} does not exist`);
-        }
-        if (!container.State?.Running || !container.State.Pid) {
-            throw new GatewayConflictError(`${kind} ${owner} is not running`);
-        }
-        return container.State.Pid;
+        await control.createPortal(portal);
+        return { name: portal.name, dimensionName: portal.dimensionName, gatewayName: portal.gatewayName };
     }
-    // Moving a veth between two containers' namespaces takes the host's view of both, so it runs in a
-    // throwaway privileged container; neither the dimension nor the gateway gets that power.
-    async function wire(portalName, link, dimensionPid, gatewayPid) {
-        const helper = `hybwire-${crypto.randomUUID().slice(0, 8)}`;
-        await docker.createContainer(helper, {
-            Image: await image.ensure(),
-            Cmd: ["sleep", "infinity"],
-            Labels: { hyperbrane: "true", "hyperbrane.helper": "wire-portal" },
-            HostConfig: {
-                NetworkMode: "host",
-                PidMode: "host",
-                Privileged: true,
-                Binds: ["/run:/hostrun"],
-                Init: true,
-            },
-        });
-        try {
-            await docker.startContainer(helper);
-            const result = await docker.exec(helper, {
-                cmd: ["hyb-wire-portal"],
-                user: "root",
-                env: {
-                    PORTAL: portalName,
-                    DIMENSION_PID: String(dimensionPid),
-                    GATEWAY_PID: String(gatewayPid),
-                    DIMENSION_IF: link.dimensionIf,
-                    GATEWAY_IF: link.gatewayIf,
-                    DIMENSION_IP: link.dimensionIp,
-                    GATEWAY_IP: link.gatewayIp,
-                    ROUTE_METRIC: String(link.metric),
-                },
-            });
-            if (result.exitCode !== 0)
-                throw new Error(`wiring portal ${portalName} failed: ${result.stderr.trim()}`);
-        }
-        finally {
-            await docker.removeContainer(helper);
-        }
-    }
-    async function trustGateway(dimensionName, gatewayName, trusted) {
-        let ca = "";
-        if (trusted) {
-            const stateDirectory = await gateways.ensureStateDirectory(gatewayName);
-            ca = new TextDecoder().decode(await host.readFile(gatewayCaCertificatePath(stateDirectory))).trim();
-        }
-        const result = await docker.exec(dimensionName, {
-            cmd: ["sh", "-c", TRUST_GATEWAY_SCRIPT],
-            user: "root",
-            env: { GATEWAY: gatewayName, GATEWAY_CA: ca },
-        });
-        if (result.exitCode !== 0) {
-            throw new Error(`updating gateway ${gatewayName} trust in ${dimensionName} failed (exit ${result.exitCode}): ${result.stderr.trim()}`);
-        }
-    }
-    // Deleting the gateway end removes the pair, and the dimension's route through it.
-    // Marked first: a restore already under way re-checks the record after wiring and undoes its work,
-    // and the link is removed after the mark, so whichever runs last, the link is gone.
     async function destroyPortal(portalName) {
         assertValidName("portal", portalName);
-        const record = await storage.readPortal(portalName);
-        if (!record)
-            return;
-        if (record.state !== "destroying")
-            await storage.savePortal({ ...record, state: "destroying" });
-        if (record.subnet !== undefined) {
-            await ignoringStopped(() => unwire(record.gatewayName, portalLink(portalName, record.subnet)));
-            await storage.releaseSubnet(record.subnet);
-        }
-        const sameGateway = (await storage.listPortals()).some((other) => other.name !== portalName &&
-            other.dimensionName === record.dimensionName &&
-            other.gatewayName === record.gatewayName);
-        if (!sameGateway) {
-            await ignoringStopped(() => trustGateway(dimensionContainerName(record.dimensionName), record.gatewayName, false));
-        }
-        await storage.removePortal(portalName);
-    }
-    async function unwire(gatewayName, link) {
-        const result = await docker.exec(gatewayContainerName(gatewayName), {
-            cmd: ["sh", "-c", `ip link del ${link.gatewayIf} 2>/dev/null || true`],
-            user: "root",
-        });
-        if (result.exitCode !== 0)
-            throw new Error(`removing ${link.gatewayIf} failed: ${result.stderr.trim()}`);
-    }
-    async function listPortals() {
-        return (await storage.listPortals()).map(publicPortal);
-    }
-    async function restorePortals({ gatewayName, dimensionName } = {}) {
-        const records = (await storage.listPortals()).filter((record) => record.state === "ready" &&
-            (gatewayName === undefined || record.gatewayName === gatewayName) &&
-            (dimensionName === undefined || record.dimensionName === dimensionName));
-        // A veth pair only ever goes as a whole, so a gateway end still there means the portal is intact.
-        const interfaces = new Map();
-        const gatewayInterfaces = (name) => {
-            if (!interfaces.has(name))
-                interfaces.set(name, listInterfaces(gatewayContainerName(name)));
-            return interfaces.get(name);
-        };
-        await Promise.all(records.map(async (record) => {
-            if ((await gatewayInterfaces(record.gatewayName))?.has(portalLink(record.name, record.subnet).gatewayIf))
-                return;
-            let pids;
-            try {
-                pids = await runningPids(record);
-            }
-            catch (error) {
-                // A stopped or removed container gets wired when it starts again, or never.
-                if (error instanceof GatewayConflictError || error instanceof GatewayNotFoundError)
-                    return;
-                throw error;
-            }
-            for (let attempt = 1;; attempt++) {
-                try {
-                    await connect(record, pids);
-                    // Destroyed meanwhile: the link just made must not outlive it (see destroyPortal).
-                    if ((await storage.readPortal(record.name))?.state !== "ready") {
-                        await ignoringStopped(() => unwire(record.gatewayName, portalLink(record.name, record.subnet)));
-                    }
-                    return;
-                }
-                catch (error) {
-                    // A container that stopped or restarted meanwhile (killing what ran in it) is restored by
-                    // its own start event. Docker kills a restarting container's processes before its state
-                    // changes, so give it a moment before deciding it's still the same container.
-                    await sleep(RESTORE_RECHECK_MS);
-                    const now = await runningPids(record).catch(() => null);
-                    if (now?.gateway !== pids.gateway || now?.dimension !== pids.dimension)
-                        return;
-                    if (attempt === 2)
-                        return console.warn(`restoring portal ${record.name} failed:`, error);
-                }
-            }
-        }));
+        await control.destroyPortal(portalName);
     }
-    /** Null when the container isn't running. */
-    async function listInterfaces(container) {
-        try {
-            const result = await docker.exec(container, { cmd: ["ip", "-br", "link"] });
-            if (result.exitCode !== 0)
-                return null;
-            return new Set(result.stdout.split("\n").map((line) => line.split(/[@\s]/)[0]).filter(Boolean));
-        }
-        catch (error) {
-            if (error instanceof DockerError && (error.status === 404 || error.status === 409))
-                return null;
-            throw error;
-        }
+    async function restorePortals(filter = {}) {
+        const failed = await control.restorePortals(filter);
+        if (failed)
+            console.warn(failed);
     }
-    return { createPortal, destroyPortal, listPortals, restorePortals };
-}
-/** Runs `action` unless its container isn't running, in which case there's nothing to undo. */
-async function ignoringStopped(action) {
-    try {
-        await action();
-    }
-    catch (error) {
-        if (!(error instanceof DockerError && (error.status === 404 || error.status === 409)))
-            throw error;
-    }
-}
-function publicPortal({ name, dimensionName, gatewayName }) {
-    return { name, dimensionName, gatewayName };
-}
-const WATCH_RETRY_MS = 1_000;
-/** Run one per host: the portals of a container that restarts are gone until something re-wires them. */
-export function createPortalWatcher(portals, docker) {
-    let controller;
-    let watching;
-    function restore(filter) {
-        portals.restorePortals(filter).catch((error) => console.warn("restoring portals failed:", error));
-    }
-    async function watch(signal) {
-        while (!signal.aborted) {
-            try {
-                const events = docker
-                    .events({ type: ["container"], event: ["start"], label: ["hyperbrane"] }, signal)[Symbol.asyncIterator]();
-                // Subscribe first, then restore: a container starting in between is covered either way.
-                let next = events.next();
-                restore();
-                for (let event = await next; !event.done; event = await next) {
-                    next = events.next();
-                    const labels = event.value.Actor?.Attributes ?? {};
-                    if (labels["hyperbrane.gateway"])
-                        restore({ gatewayName: labels["hyperbrane.gateway"] });
-                    if (labels["hyperbrane.dimension"])
-                        restore({ dimensionName: labels["hyperbrane.dimension"] });
-                }
-            }
-            catch (error) {
-                console.warn("watching docker events failed:", error);
-            }
-            if (!signal.aborted)
-                await sleep(WATCH_RETRY_MS);
-        }
-    }
-    function start() {
-        if (controller)
-            return;
-        controller = new AbortController();
-        watching = watch(controller.signal);
-    }
-    async function stop() {
-        controller?.abort();
-        await watching;
-        controller = undefined;
-        watching = undefined;
-    }
-    return { start, stop };
+    return {
+        createPortal: (portal) => createPortal(portal),
+        destroyPortal: (portalName) => destroyPortal(portalName),
+        listPortals: () => control.listPortals(),
+        restorePortals,
+    };
 }

package/dist/portal-gateway/transport-worker.d.ts

@@ -0,0 +1 @@
+export {};