hyperbrane 0.1.1 → 0.1.2

package/dist/cli/agent.js

@@ -101,6 +101,19 @@ export const agentCommands = {
                 await forEach(backends, (backend) => agents.install(agent, backendArg(backend)));
             },
         },
+        backends: {
+            summary: "Show the backends installed in an agent's dimension",
+            args: ["AGENT"],
+            flags: QUIET_FLAG,
+            async run([agent], flags, { agents }) {
+                const rows = (await agents.listBackends(agent)).map(({ backend, version, ready }) => [
+                    backend,
+                    version ?? "-",
+                    ready ? "ready" : version ? "incomplete or outdated: install it" : "not installed",
+                ]);
+                printList(flags, ["BACKEND", "VERSION", "STATUS"], rows);
+            },
+        },
         rm: {
             summary: "Stop one or more agents' workers (their sessions stay in the dimension)",
             args: ["NAME..."],

package/dist/cli/services.js

@@ -21,13 +21,14 @@ export function createServices(host = createLocalSandbox("/")) {
     const portalStorage = createPortalStorage(host);
     const gatewayImage = createGatewayImage(docker);
     const portals = createPortalService(portalStorage, gatewayStorage, docker, gatewayImage, host);
-    const gateways = createGatewayService(gatewayStorage, portalStorage, docker, gatewayImage, host);
-    const dimensions = createDimensionService(createFileStorage(host), docker, createDimensionImage(docker), host);
+    // Each restart they make, and each worker an agent revives, re-wires what it took: the portals.
+    const gateways = createGatewayService(gatewayStorage, portalStorage, docker, gatewayImage, host, portals);
+    const dimensions = createDimensionService(createFileStorage(host), docker, createDimensionImage(docker), host, portals);
     return {
         docker,
         dimensions,
         gateways,
         portals,
-        agents: createAgentService(createAgentStorage(host), docker, host, createGatewaySignIns(portals, gateways)),
+        agents: createAgentService(createAgentStorage(host), docker, host, createGatewaySignIns(portals, gateways), portals),
     };
 }

package/dist/dimension/service.d.ts

@@ -4,6 +4,7 @@ import { type Dimension, type DimensionInfo, type DimensionLimits } from "./api.
 import type { DimensionImage } from "./image.ts";
 import type { Storage } from "./storage.ts";
 import { RUN_PATH, STATE_PATH, WORKSPACE_PATH } from "./paths.ts";
+import type { PortalService } from "../portal-gateway/portalService.ts";
 export { RUN_PATH, STATE_PATH, WORKSPACE_PATH };
 /** @sealed */
 export interface DimensionService {
@@ -20,4 +21,6 @@ export declare function containerName(dimensionName: string): string;
  * `host` is the machine the docker daemon runs on, rooted at "/" so dimension directories
  * (bind-mount sources) resolve the same way docker resolves them.
  */
-export declare function createDimensionService(storage: Storage, docker: DockerClient, image: DimensionImage, host: Sandbox): DimensionService;
+export declare function createDimensionService(storage: Storage, docker: DockerClient, image: DimensionImage, host: Sandbox, 
+/** Re-wires the portals of a dimension whose container restarted (portal-gateway/portalService.ts). */
+restoring?: Pick<PortalService, "restorePortals">): DimensionService;

package/dist/dimension/service.js

@@ -15,7 +15,9 @@ export function containerName(dimensionName) {
  * `host` is the machine the docker daemon runs on, rooted at "/" so dimension directories
  * (bind-mount sources) resolve the same way docker resolves them.
  */
-export function createDimensionService(storage, docker, image, host) {
+export function createDimensionService(storage, docker, image, host, 
+/** Re-wires the portals of a dimension whose container restarted (portal-gateway/portalService.ts). */
+restoring) {
     async function getDimensions() {
         return Promise.all((await storage.getDimensions()).map(describeDimension));
     }
@@ -159,6 +161,8 @@ export function createDimensionService(storage, docker, image, host) {
             throw new DimensionNotFoundError(`dimension ${dimensionName} has no container; create it again`);
         assertOwnedContainer(existing, dimensionName);
         await docker.restartContainer(containerName(dimensionName));
+        // The restart took its end of every portal with its network namespace.
+        await restoring?.restorePortals({ dimensionName });
     }
     // Container first: a limit docker refuses leaves the record as it was.
     async function updateDimension(dimensionName, limits) {

package/dist/index.d.ts

@@ -11,6 +11,7 @@ export { parseAccessibleHost, proxyCredentialsAuth, type SandboxEndpointAuth, ty
 export { exeDevProxyAuth, type ExeDevAuthOptions } from "./sandbox/exeDevAuth.ts";
 export { EndpointLoginRequiredError, endpointRequest, exeDevLoginUrl, openSseConnection, type EndpointAccess } from "./sandbox/sseConnection.ts";
 export * from "./agent/client.ts";
+export { createPortalWatcher, type PortalWatcher } from "./portal-gateway/portalService.ts";
 export { createDockerClient, DockerError, type DockerClient } from "./lib/docker.ts";
 export { BUILTIN_UPSTREAMS } from "./upstreams/index.ts";
 export { HYPERBRANE_VERSION } from "./lib/version.ts";

package/dist/index.js

@@ -19,6 +19,9 @@ export { exeDevProxyAuth } from "./sandbox/exeDevAuth.js";
 export { EndpointLoginRequiredError, endpointRequest, exeDevLoginUrl, openSseConnection } from "./sandbox/sseConnection.js";
 // An agent's worker, reached through its endpoint (AgentService.exposeAgent), from a server or a browser.
 export * from "./agent/client.js";
+// Portals come back by themselves after the services' own restarts; after any other (docker's own, a
+// reboot), restorePortals, or a watcher that runs it for each container that starts.
+export { createPortalWatcher } from "./portal-gateway/portalService.js";
 export { createDockerClient, DockerError } from "./lib/docker.js";
 export { BUILTIN_UPSTREAMS } from "./upstreams/index.js";
 export { HYPERBRANE_VERSION } from "./lib/version.js";

package/dist/portal-gateway/gatewayService.d.ts

@@ -4,6 +4,7 @@ import { type Sandbox } from "../sandbox/api.ts";
 import { type Gateway, type GatewayCredential, type TrafficRule, type Upstream } from "./api.ts";
 import type { GatewayStorage } from "./gatewayStorage.ts";
 import type { PortalStorage } from "./portalStorage.ts";
+import type { PortalService } from "./portalService.ts";
 /** @sealed */
 export interface GatewayService {
     createGateway(gateway: Gateway): Promise<Gateway>;
@@ -35,4 +36,6 @@ export declare function assertValidName(kind: string, name: string): void;
  * `host` is the machine the docker daemon runs on, rooted at "/" so the state directory (a
  * bind-mount source) resolves the same way docker resolves it.
  */
-export declare function createGatewayService(storage: GatewayStorage, portals: PortalStorage, docker: DockerClient, image: ManagedImage, host: Sandbox): GatewayService;
+export declare function createGatewayService(storage: GatewayStorage, portals: PortalStorage, docker: DockerClient, image: ManagedImage, host: Sandbox, 
+/** Re-wires the portals of a gateway whose container restarted or was replaced. */
+restoring?: Pick<PortalService, "restorePortals">): GatewayService;

package/dist/portal-gateway/gatewayService.js

@@ -10,6 +10,7 @@ import { BUILTIN_UPSTREAMS } from "../upstreams/index.js";
 /** Where the gateway's state directory is mounted in its container. */
 export const GATEWAY_STATE_PATH = "/state";
 const PROXY_PORT = 8080;
+const DNS_PORT = 5353;
 const READY_TIMEOUT_MS = 60_000;
 // "hybgw-" can't collide with a dimension's "hyb-<name>": dimension names start with a letter or digit.
 export function gatewayContainerName(gatewayName) {
@@ -33,7 +34,9 @@ export function assertValidName(kind, name) {
  * `host` is the machine the docker daemon runs on, rooted at "/" so the state directory (a
  * bind-mount source) resolves the same way docker resolves it.
  */
-export function createGatewayService(storage, portals, docker, image, host) {
+export function createGatewayService(storage, portals, docker, image, host, 
+/** Re-wires the portals of a gateway whose container restarted or was replaced. */
+restoring) {
     // Keeps no state between calls: the record is written first so a slow or crashed create stays
     // visible, and every later step checks before it acts, so re-running converges.
     async function createGateway(gateway) {
@@ -53,11 +56,12 @@ export function createGatewayService(storage, portals, docker, image, host) {
         await awaitReady(gateway.name, stateDirectory);
         if (record.state !== "ready")
             await storage.saveGateway({ name: gateway.name, state: "ready" });
+        // A replaced container (an older image's) came without them; still wired, this is one look.
+        await restoring?.restorePortals({ gatewayName: gateway.name });
         return { name: gateway.name };
     }
     // A container from an older image is replaced: everything worth keeping (CA, credentials, rules)
-    // is in the mounted state directory. Its portals are re-wired by the portal watcher, or by the
-    // restore every command runs first.
+    // is in the mounted state directory. createGateway re-wires its portals once it's ready.
     async function ensureContainer(gatewayName, stateDirectory, imageTag) {
         const name = gatewayContainerName(gatewayName);
         const existing = await docker.inspectContainer(name);
@@ -104,8 +108,8 @@ export function createGatewayService(storage, portals, docker, image, host) {
         }
         throw conflict;
     }
-    // Ready = the proxy is listening (so the redirect rules are in) and the CA exists (so portals can
-    // install it into dimensions).
+    // Ready = the proxy and its resolver are listening (so the redirect rules are in, and nothing
+    // through a portal is refused) and the CA exists (so portals can install it into dimensions).
     async function awaitReady(gatewayName, stateDirectory) {
         const name = gatewayContainerName(gatewayName);
         const deadline = Date.now() + READY_TIMEOUT_MS;
@@ -119,8 +123,9 @@ export function createGatewayService(storage, portals, docker, image, host) {
     }
     async function proxyListening(name) {
         try {
-            const result = await docker.exec(name, { cmd: ["ss", "-Hltn", `sport = :${PROXY_PORT}`], user: "root" });
-            return result.exitCode === 0 && result.stdout.trim() !== "";
+            const result = await docker.exec(name, { cmd: ["ss", "-Hln", `( sport = :${PROXY_PORT} or sport = :${DNS_PORT} )`], user: "root" });
+            // Both: TCP for the proxy, UDP for the resolver.
+            return result.exitCode === 0 && /^tcp\b/m.test(result.stdout) && /^udp\b/m.test(result.stdout);
         }
         catch (error) {
             // Not running (yet, or restarting after a failed start).
@@ -165,6 +170,8 @@ export function createGatewayService(storage, portals, docker, image, host) {
         assertOwnedContainer(container, gatewayName);
         await docker.restartContainer(name);
         await awaitReady(gatewayName, await storage.ensureStateDirectory(record.name));
+        // The restart took its end of every portal with its network namespace.
+        await restoring?.restorePortals({ gatewayName });
     }
     async function listGateways() {
         return (await storage.listGateways()).map(({ name }) => ({ name }));
@@ -322,14 +329,19 @@ export function createGatewayService(storage, portals, docker, image, host) {
 /** The checks the gateway relies on; the rest of the shape is the SDK client's (see upstream.ts). */
 function validUpstream(upstream) {
     assertValidName("upstream", upstream.name);
-    const { oauth, apiKey } = upstream.modes ?? {};
-    if (!oauth && !apiKey)
-        throw new GatewayInvalidError(`upstream ${upstream.name} needs an oauth or apiKey mode`);
+    const { oauth, apiKey, forced } = upstream.modes ?? {};
+    if (!oauth && !apiKey && !forced)
+        throw new GatewayInvalidError(`upstream ${upstream.name} needs an oauth, apiKey or forced mode`);
+    for (const mode of [oauth, apiKey, forced]) {
+        if (mode && (!Array.isArray(mode.hosts) || mode.hosts.length === 0))
+            throw new GatewayInvalidError(`upstream ${upstream.name}: every mode needs hosts`);
+    }
     for (const mode of [oauth, apiKey]) {
-        if (mode && (!Array.isArray(mode.hosts) || mode.hosts.length === 0 || !mode.present)) {
-            throw new GatewayInvalidError(`upstream ${upstream.name}: every mode needs hosts and present`);
-        }
+        if (mode && !mode.present)
+            throw new GatewayInvalidError(`upstream ${upstream.name}: oauth and apiKey modes need present`);
     }
+    if (forced && !forced.force)
+        throw new GatewayInvalidError(`upstream ${upstream.name}: a forced mode needs force`);
     if (oauth && (!oauth.tokenUrl || !oauth.fakes?.access)) {
         throw new GatewayInvalidError(`upstream ${upstream.name}: oauth needs tokenUrl and fakes.access`);
     }

package/dist/portal-gateway/image.d.ts

@@ -1,6 +1,6 @@
 import type { DockerClient } from "../lib/docker.ts";
 import { type ManagedImage } from "../lib/image.ts";
 /** Bump whenever anything under ./image changes (and run `bun run embed:images`). */
-export declare const GATEWAY_IMAGE_VERSION = "v16";
+export declare const GATEWAY_IMAGE_VERSION = "v17";
 export declare const GATEWAY_IMAGE_REPOSITORY = "hyperbrane-gateway";
 export declare function createGatewayImage(docker: DockerClient): ManagedImage;

package/dist/portal-gateway/image.js

@@ -2,7 +2,7 @@ import { createManagedImage } from "../lib/image.js";
 // The files under ./image, as strings (see dimension/image.ts).
 import { IMAGE_FILES } from "./imageFiles.generated.js";
 /** Bump whenever anything under ./image changes (and run `bun run embed:images`). */
-export const GATEWAY_IMAGE_VERSION = "v16";
+export const GATEWAY_IMAGE_VERSION = "v17";
 export const GATEWAY_IMAGE_REPOSITORY = "hyperbrane-gateway";
 export function createGatewayImage(docker) {
     return createManagedImage(docker, {

package/dist/portal-gateway/imageFiles.generated.js

@@ -4,5 +4,5 @@ export const IMAGE_FILES = {
     "entry.sh": "#!/bin/sh\n# Gateway container entry: redirects everything arriving over a portal into mitmproxy, then becomes\n# mitmproxy. If the redirect can't be installed, or mitmproxy exits, the container exits: a gateway\n# that isn't intercepting must not look healthy.\nset -eu\n\nSTATE=/state\nPROXY_PORT=8080\nDNS_PORT=5353\n\n# Portals end in \"hvg*\" interfaces. All TCP goes to the transparent proxy and DNS to its resolver;\n# nothing is forwarded, so there is no path out that bypasses mitmproxy. mitmproxy's own upstream\n# connections are locally generated and never pass PREROUTING, so it doesn't intercept itself.\nnft -f - <<EOF\ntable ip hyperbrane\ndelete table ip hyperbrane\ntable ip hyperbrane {\n  chain prerouting {\n    type nat hook prerouting priority dstnat; policy accept;\n    iifname \"hvg*\" meta l4proto tcp redirect to :$PROXY_PORT\n    iifname \"hvg*\" udp dport 53 redirect to :$DNS_PORT\n  }\n  chain input {\n    type filter hook input priority filter; policy accept;\n    iifname \"hvg*\" tcp dport $PROXY_PORT accept\n    iifname \"hvg*\" udp dport $DNS_PORT accept\n    iifname \"hvg*\" drop\n  }\n  chain forward {\n    type filter hook forward priority filter; policy accept;\n    iifname \"hvg*\" drop\n  }\n}\nEOF\n\n# An exit code of 0 is not proof: read the rules back.\nrules=\"$(nft list table ip hyperbrane)\"\ncase \"$rules\" in\n  *\"redirect to :$PROXY_PORT\"*\"redirect to :$DNS_PORT\"*) ;;\n  *) echo \"gateway: redirect rules are not installed\" >&2; exit 1 ;;\nesac\n\n# mitmproxy runs as the owner of the state directory, so everything it writes there (the CA) stays\n# readable and removable by the host service.\nowner_uid=\"$(stat -c %u \"$STATE\")\"\nowner_gid=\"$(stat -c %g \"$STATE\")\"\nexport HOME=/tmp\n# Eager: connect upstream as soon as the client connects, or protocols where the server speaks first\n# (SSH, SMTP) wait forever for a client that is waiting for them.\nexec setpriv --reuid=\"$owner_uid\" --regid=\"$owner_gid\" --clear-groups -- \\\n  mitmdump \\\n  --mode \"transparent@$PROXY_PORT\" \\\n  --mode \"dns@$DNS_PORT\" \\\n  --set \"confdir=$STATE/mitmproxy\" \\\n  --set connection_strategy=eager \\\n  -s /opt/hyperbrane/traffic_rules.py \\\n  -s /opt/hyperbrane/upstreams.py\n",
     "wire-portal.sh": "#!/bin/sh\n# Wires a portal: a veth pair with one end in the dimension's network namespace and the other in the\n# gateway's, and a default route down it. A dimension with several portals has one default route per\n# portal; the lowest ROUTE_METRIC wins, and when its veth goes away (its gateway stopped) so does the\n# route, so traffic falls over to the next portal.\n# Runs in a privileged helper container sharing the host's network and pid namespaces, with the\n# host's /run at /hostrun.\n#\n# Inputs: PORTAL, DIMENSION_PID, GATEWAY_PID, DIMENSION_IF, GATEWAY_IF, DIMENSION_IP, GATEWAY_IP,\n# ROUTE_METRIC.\n# Safe to re-run: an already correct wiring is left alone, anything else is torn down and rebuilt.\nset -eu\n\n# Concurrent wirings of one portal would delete each other's veths; the lock lives on the host so it\n# is shared by every helper container.\nexec 9>\"/hostrun/hyperbrane-portal-$PORTAL.lock\"\nflock -w 30 9 || { echo \"timed out waiting for the wiring lock of $PORTAL\" >&2; exit 1; }\n\nmkdir -p /var/run/netns\nln -sf \"/proc/$DIMENSION_PID/ns/net\" /var/run/netns/dimension\nln -sf \"/proc/$GATEWAY_PID/ns/net\" /var/run/netns/gateway\ntrap 'rm -f /var/run/netns/dimension /var/run/netns/gateway' EXIT\n\nwired() {\n  ip -n dimension addr show dev \"$DIMENSION_IF\" 2>/dev/null | grep -q \"inet $DIMENSION_IP/30\" \\\n    && ip -n dimension route show default 2>/dev/null | grep -q \"via $GATEWAY_IP dev $DIMENSION_IF metric $ROUTE_METRIC\" \\\n    && ip -n gateway addr show dev \"$GATEWAY_IF\" 2>/dev/null | grep -q \"inet $GATEWAY_IP/30\"\n}\nwired && exit 0\n\n# A stale end can be in either namespace, or left on the host when a container died mid-wiring.\nip -n dimension link del \"$DIMENSION_IF\" 2>/dev/null || true\nip -n gateway link del \"$GATEWAY_IF\" 2>/dev/null || true\nip link del \"$DIMENSION_IF\" 2>/dev/null || true\nip link del \"$GATEWAY_IF\" 2>/dev/null || true\n\nip link add \"$DIMENSION_IF\" type veth peer name \"$GATEWAY_IF\"\nip link set \"$DIMENSION_IF\" netns dimension\nip link set \"$GATEWAY_IF\" netns gateway\nip -n dimension addr replace \"$DIMENSION_IP/30\" dev \"$DIMENSION_IF\"\nip -n dimension link set \"$DIMENSION_IF\" up\nip -n gateway addr replace \"$GATEWAY_IP/30\" dev \"$GATEWAY_IF\"\nip -n gateway link set \"$GATEWAY_IF\" up\nip -n dimension route replace default via \"$GATEWAY_IP\" dev \"$DIMENSION_IF\" metric \"$ROUTE_METRIC\"\n",
     "traffic_rules.py": "# mitmproxy addon applying the gateway's traffic rules to everything arriving through a portal.\n#\n# Rules are the JSON files the gateway service writes to /state/traffic-rules, re-read every time, so\n# a change takes effect without a reload. A rule's urlMask is a glob, and its scheme says what it\n# applies to:\n#   tcp://<address>:<port>  every TCP connection, by its destination, before it is made upstream.\n#                           address is \"*\", an IPv4 address or a CIDR; port is \"*\" or a number.\n#                           Hostnames aren't known at that point, so rules don't use them.\n#   anything else           every HTTP(S) request, matched as a glob against host:port/path?query.\n#                           The scheme doesn't matter (http, https or none match the same), and a\n#                           mask without a port matches any port, one without a path any path:\n#                           \"github.com\" is the whole host, \"github.com:80/*\" only its plain HTTP.\n# Rules are checked in name order and the first match decides:\n#   BLOCK  refuse: a 403 for a request, a closed connection for TCP\n#   DELAY  hold for delayMs, then carry on\n#   ASK    hold until the service records an answer for the rule in /state/traffic-responses (only\n#          answers given after the hold began count); DENY refuses, ALLOW carries on\n\nimport asyncio\nimport fnmatch\nimport ipaddress\nimport json\nimport os\nimport time\n\nfrom mitmproxy import http\nfrom mitmproxy.proxy import server_hooks\n\nSTATE = \"/state\"\nRULES = os.path.join(STATE, \"traffic-rules\")\nRESPONSES = os.path.join(STATE, \"traffic-responses\")\nASK_POLL_SECONDS = 0.2\nTCP_SCHEME = \"tcp://\"\n\n\ndef _load(path):\n    try:\n        with open(path) as f:\n            return json.load(f)\n    except (OSError, ValueError):\n        return None\n\n\ndef _rules(tcp):\n    try:\n        files = sorted(f for f in os.listdir(RULES) if f.endswith(\".json\"))\n    except OSError:\n        return []\n    rules = (_load(os.path.join(RULES, f)) for f in files)\n    return [r for r in rules if r and r.get(\"urlMask\", \"\").startswith(TCP_SCHEME) == tcp]\n\n\ndef _request_target(request):\n    \"\"\"host:port/path?query: the port always written out, the scheme left out.\"\"\"\n    return f\"{request.pretty_host.lower()}:{request.port}{request.path}\"\n\n\ndef _url_mask(mask):\n    \"\"\"The mask in _request_target's form: no scheme, and \"any\" for a port or path it leaves out.\"\"\"\n    if \"://\" in mask:\n        mask = mask.split(\"://\", 1)[1]\n    host, slash, path = mask.partition(\"/\")\n    if \":\" not in host:\n        host += \":*\"\n    return f\"{host.lower()}/{path if slash else '*'}\"\n\n\ndef _matching_url_rule(request):\n    target = _request_target(request)\n    for rule in _rules(tcp=False):\n        if fnmatch.fnmatchcase(target, _url_mask(rule[\"urlMask\"])):\n            return rule\n    return None\n\n\ndef _tcp_mask_matches(mask, ip, port):\n    address, _, mask_port = mask[len(TCP_SCHEME):].rpartition(\":\")\n    if mask_port != \"*\" and mask_port != str(port):\n        return False\n    if address == \"*\":\n        return True\n    try:\n        return ipaddress.ip_address(ip) in ipaddress.ip_network(address, strict=False)\n    except ValueError:\n        return False\n\n\ndef _matching_tcp_rule(ip, port):\n    for rule in _rules(tcp=True):\n        if _tcp_mask_matches(rule[\"urlMask\"], ip, port):\n            return rule\n    return None\n\n\nasync def _await_answer(name):\n    \"\"\"The ALLOW/DENY given for the rule after now, or None once the rule is no longer an ASK.\"\"\"\n    asked_at = time.time() * 1000\n    while True:\n        answer = _load(os.path.join(RESPONSES, f\"{name}.json\"))\n        if answer and answer.get(\"at\", 0) >= asked_at:\n            return answer.get(\"action\")\n        rule = _load(os.path.join(RULES, f\"{name}.json\"))\n        if not rule or rule.get(\"action\") != \"ASK\":\n            return None\n        # Never time.sleep: this runs on mitmproxy's event loop, shared by every connection.\n        await asyncio.sleep(ASK_POLL_SECONDS)\n\n\nasync def _verdict(rule):\n    \"\"\"Applies the rule's hold, if any; returns the refusal message, or None to carry on.\"\"\"\n    action = rule.get(\"action\")\n    if action == \"BLOCK\":\n        return f\"blocked by gateway rule {rule['name']}\"\n    if action == \"DELAY\":\n        await asyncio.sleep(max(0, rule.get(\"delayMs\", 0)) / 1000)\n    elif action == \"ASK\" and await _await_answer(rule[\"name\"]) == \"DENY\":\n        return f\"denied by gateway rule {rule['name']}\"\n    return None\n\n\nasync def request(flow: http.HTTPFlow):\n    rule = _matching_url_rule(flow.request)\n    refusal = rule and await _verdict(rule)\n    if refusal:\n        flow.response = http.Response.make(403, f\"{refusal}\\n\", {\"content-type\": \"text/plain\"})\n\n\nasync def server_connect(data: server_hooks.ServerConnectionHookData):\n    if not data.server.address:\n        return\n    rule = _matching_tcp_rule(data.server.address[0], data.server.address[1])\n    refusal = rule and await _verdict(rule)\n    if refusal:\n        data.server.error = refusal\n\n",
-    "upstreams.py": "# mitmproxy addon applying the gateway's upstreams (see portal-gateway/upstream.ts) to traffic from\n# portals. Definitions are the JSON files the gateway service writes to /state/upstreams; this addon\n# keeps each upstream's OAuth credential in /state/credentials/<upstream>.oauth.json and reads its\n# API keys from /state/credentials/<upstream>.api-keys.json. Everything is re-read per request, so a\n# change takes effect without a reload.\n#\n# Clients sign in from dimensions, through the proxy:\n#   token endpoint   without a usable credential, requests go through and the response is kept as\n#                    the credential, reaching the dimension with fakes in place of the tokens. With\n#                    one, the gateway answers from it: a sign-in (whatever code it carries) and a\n#                    refresh made with a fake refresh token. A client's refresh goes upstream: the\n#                    gateway replays it (encoding, fields, User-Agent) with the real refresh token,\n#                    one at a time, and refreshes arriving together share its result. Every client\n#                    holds the same fakes, so they all get the new tokens. A sign-in is answered from\n#                    the credential, refreshed first if expired by replaying the last client refresh.\n#                    An expired credential it can't refresh (no client refresh to replay yet, or\n#                    upstream refused) counts as none: a sign-in goes to the upstream again.\n#   answers          other sign-in steps (a device flow's poll) answered as the definition says, once\n#                    there is a credential\n#   other requests   a fake in the mode's `present` location is replaced by the real secret\n\nimport asyncio\nimport http.client as http_client\nimport json\nimport os\nimport secrets\nimport string\nimport time\nimport urllib.parse\n\nfrom mitmproxy import http\n\nSTATE = \"/state\"\nUPSTREAMS = os.path.join(STATE, \"upstreams\")\nCREDENTIALS = os.path.join(STATE, \"credentials\")\n# A token this close to expiry counts as expired.\nREFRESH_MARGIN_MS = 60_000\n# Refreshes arriving within this long of the last one get its tokens instead of going upstream again.\nREFRESH_SHARED_MS = 5_000\nFAKE_ALPHABET = string.ascii_letters + string.digits + \"-_\"\nTOKEN_FIELDS = ((\"access\", \"access_token\"), (\"refresh\", \"refresh_token\"), (\"id\", \"id_token\"))\n\n# One refresh per upstream at a time: concurrent ones would each spend the rotating refresh token.\n_refresh_locks: dict[str, asyncio.Lock] = {}\n\n\ndef _now_ms():\n    return int(time.time() * 1000)\n\n\ndef _load(path):\n    try:\n        with open(path) as f:\n            return json.load(f)\n    except (OSError, ValueError):\n        return None\n\n\ndef _save(path, data):\n    tmp = f\"{path}.{secrets.token_hex(4)}.tmp\"\n    with open(tmp, \"w\") as f:\n        json.dump(data, f, indent=2)\n    os.replace(tmp, path)\n\n\ndef _definitions():\n    try:\n        files = sorted(f for f in os.listdir(UPSTREAMS) if f.endswith(\".json\"))\n    except OSError:\n        return []\n    definitions = (_load(os.path.join(UPSTREAMS, f)) for f in files)\n    return [d for d in definitions if d and d.get(\"name\")]\n\n\ndef _oauth_path(name):\n    return os.path.join(CREDENTIALS, f\"{name}.oauth.json\")\n\n\ndef _keys_path(name):\n    return os.path.join(CREDENTIALS, f\"{name}.api-keys.json\")\n\n\ndef _base(url):\n    return url.split(\"?\", 1)[0].split(\"#\", 1)[0]\n\n\ndef _hosts(mode):\n    return {h.lower() for h in mode.get(\"hosts\", [])}\n\n\n# --- fakes -----------------------------------------------------------------------------------------\n\n\ndef _random(length):\n    return \"\".join(secrets.choice(FAKE_ALPHABET) for _ in range(length))\n\n\ndef _issue_fake(shape, real):\n    if shape.get(\"jwt\"):\n        signature = real.rsplit(\".\", 1)[-1] if real.count(\".\") == 2 else \"\"\n        return {\"jwtSignature\": _random(max(len(signature), 43))}\n    prefix = shape.get(\"prefix\", \"\")\n    return {\"value\": prefix + _random(max(len(real) - len(prefix), 16))}\n\n\ndef _fake_value(stored, real):\n    if \"jwtSignature\" in stored:\n        header_and_claims = real.rsplit(\".\", 1)[0] if real.count(\".\") == 2 else real\n        return f\"{header_and_claims}.{stored['jwtSignature']}\"\n    return stored[\"value\"]\n\n\ndef _is_fake(stored, candidate):\n    if not stored or not candidate:\n        return False\n    if \"jwtSignature\" in stored:\n        return candidate.count(\".\") == 2 and candidate.rsplit(\".\", 1)[1] == stored[\"jwtSignature\"]\n    return candidate == stored[\"value\"]\n\n\n# --- presenting secrets ----------------------------------------------------------------------------\n\n\ndef _presented(request, present):\n    if \"query\" in present:\n        return request.query.get(present[\"query\"])\n    value = request.headers.get(present[\"header\"])\n    scheme = present.get(\"scheme\")\n    if value and scheme:\n        prefix = f\"{scheme} \"\n        return value[len(prefix):] if value.lower().startswith(prefix.lower()) else None\n    return value\n\n\ndef _present(request, present, secret):\n    if \"query\" in present:\n        request.query[present[\"query\"]] = secret\n    elif present.get(\"scheme\"):\n        request.headers[present[\"header\"]] = f\"{present['scheme']} {secret}\"\n    else:\n        request.headers[present[\"header\"]] = secret\n\n\ndef _force(request, force):\n    if not force:\n        return\n    if force.get(\"target\"):\n        target = urllib.parse.urlsplit(force[\"target\"])\n        request.scheme = target.scheme or request.scheme\n        request.host = target.hostname or request.host\n        request.port = target.port or (443 if request.scheme == \"https\" else 80)\n    if force.get(\"path\"):\n        request.path = force[\"path\"].rstrip(\"/\") + request.path\n    for key, value in force.get(\"query\", {}).items():\n        request.query[key] = value\n    for key, value in force.get(\"headers\", {}).items():\n        request.headers[key] = value\n\n\ndef _swap_oauth(request, name, mode):\n    credential = _load(_oauth_path(name))\n    presented = _presented(request, mode[\"present\"])\n    if credential and _is_fake(credential[\"fakes\"].get(\"access\"), presented):\n        _present(request, mode[\"present\"], credential[\"access\"])\n        return True\n    return False\n\n\ndef _swap_api_key(request, name, mode):\n    stored = _load(_keys_path(name)) or {}\n    presented = _presented(request, mode[\"present\"])\n    for key in stored.get(\"keys\", {}).values():\n        if presented and presented == key[\"fake\"]:\n            _present(request, mode[\"present\"], key[\"key\"])\n            return True\n    return False\n\n\n# --- the token endpoint ----------------------------------------------------------------------------\n\n\ndef _parse_form_or_json(text, content_type):\n    if not text:\n        return {}\n    if \"json\" in (content_type or \"\"):\n        try:\n            value = json.loads(text)\n            return value if isinstance(value, dict) else {}\n        except ValueError:\n            return {}\n    return dict(urllib.parse.parse_qsl(text))\n\n\ndef _refresh_request(flow, fields):\n    \"\"\"How a client refreshes, to replay with the real refresh token: everything but that token.\"\"\"\n    return {\n        \"contentType\": flow.request.headers.get(\"content-type\", \"\"),\n        \"userAgent\": flow.request.headers.get(\"user-agent\"),\n        \"fields\": {k: v for k, v in fields.items() if k != \"refresh_token\"},\n    }\n\n\ndef _store_tokens(name, mode, body, refresh_request):\n    \"\"\"Keeps a token response as the upstream's credential and returns it.\"\"\"\n    path = _oauth_path(name)\n    credential = dict(_load(path) or {})\n    if refresh_request:\n        credential[\"refreshRequest\"] = refresh_request\n    now = _now_ms()\n    for key, field in TOKEN_FIELDS:\n        if body.get(field):\n            credential[key] = body[field]\n    if \"expires_in\" in body:\n        credential[\"expiresAt\"] = now + int(body[\"expires_in\"]) * 1000\n    if \"refresh_token_expires_in\" in body:\n        credential[\"refreshExpiresAt\"] = now + int(body[\"refresh_token_expires_in\"]) * 1000\n    if body.get(\"scope\"):\n        credential[\"scopes\"] = body[\"scope\"].split()\n    credential[\"lastRefresh\"] = now\n    fakes = dict(credential.get(\"fakes\") or {})\n    for key, _ in TOKEN_FIELDS:\n        shape = mode[\"fakes\"].get(key)\n        if shape and credential.get(key) and key not in fakes:\n            fakes[key] = _issue_fake(shape, credential[key])\n    credential[\"fakes\"] = fakes\n    stripped = set((mode.get(\"strip\") or {}).get(\"fields\", []))\n    credential[\"response\"] = {k: v for k, v in body.items() if k not in stripped}\n    _save(path, credential)\n    return credential\n\n\ndef _client_body(mode, credential):\n    \"\"\"The token response the dimension gets: the last real one, with fakes for every token.\"\"\"\n    body = dict(credential.get(\"response\") or {})\n    fakes = credential[\"fakes\"]\n    for key, field in TOKEN_FIELDS:\n        if field not in body:\n            continue\n        if key in fakes and credential.get(key):\n            body[field] = _fake_value(fakes[key], credential[key])\n        else:\n            del body[field]  # a token with no fake shape never reaches a dimension\n    now = _now_ms()\n    if credential.get(\"expiresAt\") and \"expires_in\" in body:\n        body[\"expires_in\"] = max(0, (credential[\"expiresAt\"] - now) // 1000)\n    if credential.get(\"refreshExpiresAt\") and \"refresh_token_expires_in\" in body:\n        body[\"refresh_token_expires_in\"] = max(0, (credential[\"refreshExpiresAt\"] - now) // 1000)\n    return body\n\n\ndef _refresh_upstream(mode, credential):\n    \"\"\"Replays the client's refresh request with the real refresh token; returns (status, parsed body\n    or raw bytes). http.client, unlike urllib, adds no User-Agent of its own: a client that sent none\n    stays that way.\"\"\"\n    template = credential[\"refreshRequest\"]\n    fields = dict(template[\"fields\"], refresh_token=credential[\"refresh\"])\n    content_type = template[\"contentType\"] or \"application/x-www-form-urlencoded\"\n    data = json.dumps(fields) if \"json\" in content_type else urllib.parse.urlencode(fields)\n    headers = {\"content-type\": content_type, \"accept\": \"application/json\"}\n    if template.get(\"userAgent\"):\n        headers[\"user-agent\"] = template[\"userAgent\"]\n    url = urllib.parse.urlsplit(mode[\"tokenUrl\"])\n    connection_type = http_client.HTTPSConnection if url.scheme == \"https\" else http_client.HTTPConnection\n    connection = connection_type(url.hostname, url.port, timeout=30)\n    try:\n        connection.request(\"POST\", url.path + (f\"?{url.query}\" if url.query else \"\"), body=data.encode(), headers=headers)\n        response = connection.getresponse()\n        raw = response.read()\n        try:\n            return response.status, json.loads(raw)\n        except ValueError:\n            return response.status, raw\n    finally:\n        connection.close()\n\n\nasync def _answer_from_credential(flow, name, mode, refresh_request):\n    \"\"\"Answers a sign-in, or a refresh made with a fake (`refresh_request`), with the credential's\n    fakes. A refresh is replayed upstream first unless another one just was; a sign-in only if the\n    credential has expired. Returns False, answering nothing, when a sign-in can't be given valid\n    tokens: expired, and no client refresh to replay yet or upstream refused it. A client's refresh\n    gets upstream's refusal instead.\"\"\"\n    lock = _refresh_locks.setdefault(name, asyncio.Lock())\n    async with lock:\n        credential = _load(_oauth_path(name))\n        if refresh_request and credential.get(\"refreshRequest\") != refresh_request:\n            credential[\"refreshRequest\"] = refresh_request\n            _save(_oauth_path(name), credential)\n        now = _now_ms()\n        expired = not credential.get(\"expiresAt\") or credential[\"expiresAt\"] - now < REFRESH_MARGIN_MS\n        just_refreshed = now - credential.get(\"lastRefresh\", 0) < REFRESH_SHARED_MS\n        if expired or (refresh_request and not just_refreshed):\n            if not (credential.get(\"refresh\") and credential.get(\"refreshRequest\")):\n                if not refresh_request:\n                    return False\n                # A client's refresh with nothing to refresh with: all there is to give it.\n                flow.response = http.Response.make(\n                    200, json.dumps(_client_body(mode, credential)), {\"content-type\": \"application/json\"}\n                )\n                return True\n            status, body = await asyncio.to_thread(_refresh_upstream, mode, credential)\n            if status != 200 or not isinstance(body, dict) or \"access_token\" not in body:\n                if not refresh_request:\n                    return False\n                content = body if isinstance(body, bytes) else json.dumps(body).encode()\n                flow.response = http.Response.make(status, content, {\"content-type\": \"application/json\"})\n                return True\n            credential = _store_tokens(name, mode, body, None)\n    flow.response = http.Response.make(\n        200, json.dumps(_client_body(mode, credential)), {\"content-type\": \"application/json\"}\n    )\n    return True\n\n\nasync def request(flow: http.HTTPFlow):\n    if flow.response:  # already answered, e.g. by a traffic rule\n        return\n    host = flow.request.pretty_host.lower()\n    url = _base(flow.request.pretty_url)\n    for definition in _definitions():\n        name = definition[\"name\"]\n        oauth = definition.get(\"modes\", {}).get(\"oauth\")\n        api_key = definition.get(\"modes\", {}).get(\"apiKey\")\n        credential = _load(_oauth_path(name)) if oauth else None\n        if oauth and url == _base(oauth[\"tokenUrl\"]):\n            fields = _parse_form_or_json(flow.request.get_text(strict=False), flow.request.headers.get(\"content-type\"))\n            refreshing = fields.get(\"grant_type\") == \"refresh_token\"\n            refresh_request = _refresh_request(flow, fields) if refreshing else None\n            # A refresh token the gateway didn't issue is a client that signed in before the upstream\n            # was registered (or elsewhere): let it through, and keep what it gets.\n            foreign_refresh = refreshing and not _is_fake(\n                (credential or {}).get(\"fakes\", {}).get(\"refresh\"), fields.get(\"refresh_token\")\n            )\n            if credential and not foreign_refresh and await _answer_from_credential(flow, name, oauth, refresh_request):\n                return\n            flow.metadata[\"hyperbrane_token\"] = (name, refresh_request)\n            return\n        if credential:\n            for answer in oauth.get(\"answers\", []):\n                if url == _base(answer[\"url\"]):\n                    flow.response = http.Response.make(\n                        answer.get(\"status\", 200), json.dumps(answer[\"json\"]), {\"content-type\": \"application/json\"}\n                    )\n                    return\n        for mode, swap in ((oauth, _swap_oauth), (api_key, _swap_api_key)):\n            if mode and host in _hosts(mode):\n                swap(flow.request, name, mode)\n                _force(flow.request, mode.get(\"force\"))\n\n\ndef responseheaders(flow: http.HTTPFlow):\n    # Only token responses are rewritten; everything else streams through as it arrives. mitmproxy\n    # otherwise holds each response until its body is complete, so a model's streamed reply (SSE)\n    # reached the dimension in one piece at the end of the turn.\n    if not flow.metadata.get(\"hyperbrane_token\"):\n        flow.response.stream = True\n\n\ndef response(flow: http.HTTPFlow):\n    token = flow.metadata.get(\"hyperbrane_token\")\n    if not token:\n        return\n    name, refresh_request = token\n    definition = next((d for d in _definitions() if d[\"name\"] == name), None)\n    oauth = (definition or {}).get(\"modes\", {}).get(\"oauth\")\n    if not oauth:\n        return\n    body = _parse_form_or_json(flow.response.get_text(strict=False), \"json\")\n    if flow.response.status_code == 200 and body.get(\"access_token\"):\n        credential = _store_tokens(name, oauth, body, refresh_request)\n        flow.response.text = json.dumps(_client_body(oauth, credential))\n    for header in (oauth.get(\"strip\") or {}).get(\"headers\", []):\n        flow.response.headers.pop(header, None)\n",
+    "upstreams.py": "# mitmproxy addon applying the gateway's upstreams (see portal-gateway/upstream.ts) to traffic from\n# portals. Definitions are the JSON files the gateway service writes to /state/upstreams; this addon\n# keeps each upstream's OAuth credential in /state/credentials/<upstream>.oauth.json and reads its\n# API keys from /state/credentials/<upstream>.api-keys.json. Everything is re-read per request, so a\n# change takes effect without a reload.\n#\n# Clients sign in from dimensions, through the proxy:\n#   token endpoint   without a usable credential, requests go through and the response is kept as\n#                    the credential, reaching the dimension with fakes in place of the tokens. With\n#                    one, the gateway answers from it: a sign-in (whatever code it carries) and a\n#                    refresh made with a fake refresh token. A client's refresh goes upstream: the\n#                    gateway replays it (encoding, fields, User-Agent) with the real refresh token,\n#                    one at a time, and refreshes arriving together share its result. Every client\n#                    holds the same fakes, so they all get the new tokens. A sign-in is answered from\n#                    the credential, refreshed first if expired by replaying the last client refresh.\n#                    An expired credential it can't refresh (no client refresh to replay yet, or\n#                    upstream refused) counts as none: a sign-in goes to the upstream again.\n#   answers          other sign-in steps (a device flow's poll) answered as the definition says, once\n#                    there is a credential\n#   other requests   a fake in the mode's `present` location is replaced by the real secret\n\nimport asyncio\nimport http.client as http_client\nimport json\nimport os\nimport secrets\nimport string\nimport time\nimport urllib.parse\n\nfrom mitmproxy import http\n\nSTATE = \"/state\"\nUPSTREAMS = os.path.join(STATE, \"upstreams\")\nCREDENTIALS = os.path.join(STATE, \"credentials\")\n# A token this close to expiry counts as expired.\nREFRESH_MARGIN_MS = 60_000\n# Refreshes arriving within this long of the last one get its tokens instead of going upstream again.\nREFRESH_SHARED_MS = 5_000\nFAKE_ALPHABET = string.ascii_letters + string.digits + \"-_\"\nTOKEN_FIELDS = ((\"access\", \"access_token\"), (\"refresh\", \"refresh_token\"), (\"id\", \"id_token\"))\n\n# One refresh per upstream at a time: concurrent ones would each spend the rotating refresh token.\n_refresh_locks: dict[str, asyncio.Lock] = {}\n\n\ndef _now_ms():\n    return int(time.time() * 1000)\n\n\ndef _load(path):\n    try:\n        with open(path) as f:\n            return json.load(f)\n    except (OSError, ValueError):\n        return None\n\n\ndef _save(path, data):\n    tmp = f\"{path}.{secrets.token_hex(4)}.tmp\"\n    with open(tmp, \"w\") as f:\n        json.dump(data, f, indent=2)\n    os.replace(tmp, path)\n\n\ndef _definitions():\n    try:\n        files = sorted(f for f in os.listdir(UPSTREAMS) if f.endswith(\".json\"))\n    except OSError:\n        return []\n    definitions = (_load(os.path.join(UPSTREAMS, f)) for f in files)\n    return [d for d in definitions if d and d.get(\"name\")]\n\n\ndef _oauth_path(name):\n    return os.path.join(CREDENTIALS, f\"{name}.oauth.json\")\n\n\ndef _keys_path(name):\n    return os.path.join(CREDENTIALS, f\"{name}.api-keys.json\")\n\n\ndef _base(url):\n    return url.split(\"?\", 1)[0].split(\"#\", 1)[0]\n\n\ndef _hosts(mode):\n    return {h.lower() for h in mode.get(\"hosts\", [])}\n\n\n# --- fakes -----------------------------------------------------------------------------------------\n\n\ndef _random(length):\n    return \"\".join(secrets.choice(FAKE_ALPHABET) for _ in range(length))\n\n\ndef _issue_fake(shape, real):\n    if shape.get(\"jwt\"):\n        signature = real.rsplit(\".\", 1)[-1] if real.count(\".\") == 2 else \"\"\n        return {\"jwtSignature\": _random(max(len(signature), 43))}\n    prefix = shape.get(\"prefix\", \"\")\n    return {\"value\": prefix + _random(max(len(real) - len(prefix), 16))}\n\n\ndef _fake_value(stored, real):\n    if \"jwtSignature\" in stored:\n        header_and_claims = real.rsplit(\".\", 1)[0] if real.count(\".\") == 2 else real\n        return f\"{header_and_claims}.{stored['jwtSignature']}\"\n    return stored[\"value\"]\n\n\ndef _is_fake(stored, candidate):\n    if not stored or not candidate:\n        return False\n    if \"jwtSignature\" in stored:\n        return candidate.count(\".\") == 2 and candidate.rsplit(\".\", 1)[1] == stored[\"jwtSignature\"]\n    return candidate == stored[\"value\"]\n\n\n# --- presenting secrets ----------------------------------------------------------------------------\n\n\ndef _presented(request, present):\n    if \"query\" in present:\n        return request.query.get(present[\"query\"])\n    value = request.headers.get(present[\"header\"])\n    scheme = present.get(\"scheme\")\n    if value and scheme:\n        prefix = f\"{scheme} \"\n        return value[len(prefix):] if value.lower().startswith(prefix.lower()) else None\n    return value\n\n\ndef _present(request, present, secret):\n    if \"query\" in present:\n        request.query[present[\"query\"]] = secret\n    elif present.get(\"scheme\"):\n        request.headers[present[\"header\"]] = f\"{present['scheme']} {secret}\"\n    else:\n        request.headers[present[\"header\"]] = secret\n\n\ndef _force(request, force):\n    if not force:\n        return\n    if force.get(\"target\"):\n        target = urllib.parse.urlsplit(force[\"target\"])\n        request.scheme = target.scheme or request.scheme\n        request.host = target.hostname or request.host\n        request.port = target.port or (443 if request.scheme == \"https\" else 80)\n    if force.get(\"path\"):\n        request.path = force[\"path\"].rstrip(\"/\") + request.path\n    for key, value in force.get(\"query\", {}).items():\n        request.query[key] = value\n    for key, value in force.get(\"headers\", {}).items():\n        request.headers[key] = value\n\n\ndef _swap_oauth(request, name, mode):\n    credential = _load(_oauth_path(name))\n    presented = _presented(request, mode[\"present\"])\n    if credential and _is_fake(credential[\"fakes\"].get(\"access\"), presented):\n        _present(request, mode[\"present\"], credential[\"access\"])\n        return True\n    return False\n\n\ndef _swap_api_key(request, name, mode):\n    stored = _load(_keys_path(name)) or {}\n    presented = _presented(request, mode[\"present\"])\n    for key in stored.get(\"keys\", {}).values():\n        if presented and presented == key[\"fake\"]:\n            _present(request, mode[\"present\"], key[\"key\"])\n            return True\n    return False\n\n\n# --- the token endpoint ----------------------------------------------------------------------------\n\n\ndef _parse_form_or_json(text, content_type):\n    if not text:\n        return {}\n    if \"json\" in (content_type or \"\"):\n        try:\n            value = json.loads(text)\n            return value if isinstance(value, dict) else {}\n        except ValueError:\n            return {}\n    return dict(urllib.parse.parse_qsl(text))\n\n\ndef _refresh_request(flow, fields):\n    \"\"\"How a client refreshes, to replay with the real refresh token: everything but that token.\"\"\"\n    return {\n        \"contentType\": flow.request.headers.get(\"content-type\", \"\"),\n        \"userAgent\": flow.request.headers.get(\"user-agent\"),\n        \"fields\": {k: v for k, v in fields.items() if k != \"refresh_token\"},\n    }\n\n\ndef _store_tokens(name, mode, body, refresh_request):\n    \"\"\"Keeps a token response as the upstream's credential and returns it.\"\"\"\n    path = _oauth_path(name)\n    credential = dict(_load(path) or {})\n    if refresh_request:\n        credential[\"refreshRequest\"] = refresh_request\n    now = _now_ms()\n    for key, field in TOKEN_FIELDS:\n        if body.get(field):\n            credential[key] = body[field]\n    if \"expires_in\" in body:\n        credential[\"expiresAt\"] = now + int(body[\"expires_in\"]) * 1000\n    if \"refresh_token_expires_in\" in body:\n        credential[\"refreshExpiresAt\"] = now + int(body[\"refresh_token_expires_in\"]) * 1000\n    if body.get(\"scope\"):\n        credential[\"scopes\"] = body[\"scope\"].split()\n    credential[\"lastRefresh\"] = now\n    fakes = dict(credential.get(\"fakes\") or {})\n    for key, _ in TOKEN_FIELDS:\n        shape = mode[\"fakes\"].get(key)\n        if shape and credential.get(key) and key not in fakes:\n            fakes[key] = _issue_fake(shape, credential[key])\n    credential[\"fakes\"] = fakes\n    stripped = set((mode.get(\"strip\") or {}).get(\"fields\", []))\n    credential[\"response\"] = {k: v for k, v in body.items() if k not in stripped}\n    _save(path, credential)\n    return credential\n\n\ndef _client_body(mode, credential):\n    \"\"\"The token response the dimension gets: the last real one, with fakes for every token.\"\"\"\n    body = dict(credential.get(\"response\") or {})\n    fakes = credential[\"fakes\"]\n    for key, field in TOKEN_FIELDS:\n        if field not in body:\n            continue\n        if key in fakes and credential.get(key):\n            body[field] = _fake_value(fakes[key], credential[key])\n        else:\n            del body[field]  # a token with no fake shape never reaches a dimension\n    now = _now_ms()\n    if credential.get(\"expiresAt\") and \"expires_in\" in body:\n        body[\"expires_in\"] = max(0, (credential[\"expiresAt\"] - now) // 1000)\n    if credential.get(\"refreshExpiresAt\") and \"refresh_token_expires_in\" in body:\n        body[\"refresh_token_expires_in\"] = max(0, (credential[\"refreshExpiresAt\"] - now) // 1000)\n    return body\n\n\ndef _refresh_upstream(mode, credential):\n    \"\"\"Replays the client's refresh request with the real refresh token; returns (status, parsed body\n    or raw bytes). http.client, unlike urllib, adds no User-Agent of its own: a client that sent none\n    stays that way.\"\"\"\n    template = credential[\"refreshRequest\"]\n    fields = dict(template[\"fields\"], refresh_token=credential[\"refresh\"])\n    content_type = template[\"contentType\"] or \"application/x-www-form-urlencoded\"\n    data = json.dumps(fields) if \"json\" in content_type else urllib.parse.urlencode(fields)\n    headers = {\"content-type\": content_type, \"accept\": \"application/json\"}\n    if template.get(\"userAgent\"):\n        headers[\"user-agent\"] = template[\"userAgent\"]\n    url = urllib.parse.urlsplit(mode[\"tokenUrl\"])\n    connection_type = http_client.HTTPSConnection if url.scheme == \"https\" else http_client.HTTPConnection\n    connection = connection_type(url.hostname, url.port, timeout=30)\n    try:\n        connection.request(\"POST\", url.path + (f\"?{url.query}\" if url.query else \"\"), body=data.encode(), headers=headers)\n        response = connection.getresponse()\n        raw = response.read()\n        try:\n            return response.status, json.loads(raw)\n        except ValueError:\n            return response.status, raw\n    finally:\n        connection.close()\n\n\nasync def _answer_from_credential(flow, name, mode, refresh_request):\n    \"\"\"Answers a sign-in, or a refresh made with a fake (`refresh_request`), with the credential's\n    fakes. A refresh is replayed upstream first unless another one just was; a sign-in only if the\n    credential has expired. Returns False, answering nothing, when a sign-in can't be given valid\n    tokens: expired, and no client refresh to replay yet or upstream refused it. A client's refresh\n    gets upstream's refusal instead.\"\"\"\n    lock = _refresh_locks.setdefault(name, asyncio.Lock())\n    async with lock:\n        credential = _load(_oauth_path(name))\n        if refresh_request and credential.get(\"refreshRequest\") != refresh_request:\n            credential[\"refreshRequest\"] = refresh_request\n            _save(_oauth_path(name), credential)\n        now = _now_ms()\n        expired = not credential.get(\"expiresAt\") or credential[\"expiresAt\"] - now < REFRESH_MARGIN_MS\n        just_refreshed = now - credential.get(\"lastRefresh\", 0) < REFRESH_SHARED_MS\n        if expired or (refresh_request and not just_refreshed):\n            if not (credential.get(\"refresh\") and credential.get(\"refreshRequest\")):\n                if not refresh_request:\n                    return False\n                # A client's refresh with nothing to refresh with: all there is to give it.\n                flow.response = http.Response.make(\n                    200, json.dumps(_client_body(mode, credential)), {\"content-type\": \"application/json\"}\n                )\n                return True\n            status, body = await asyncio.to_thread(_refresh_upstream, mode, credential)\n            if status != 200 or not isinstance(body, dict) or \"access_token\" not in body:\n                if not refresh_request:\n                    return False\n                content = body if isinstance(body, bytes) else json.dumps(body).encode()\n                flow.response = http.Response.make(status, content, {\"content-type\": \"application/json\"})\n                return True\n            credential = _store_tokens(name, mode, body, None)\n    flow.response = http.Response.make(\n        200, json.dumps(_client_body(mode, credential)), {\"content-type\": \"application/json\"}\n    )\n    return True\n\n\nasync def request(flow: http.HTTPFlow):\n    if flow.response:  # already answered, e.g. by a traffic rule\n        return\n    host = flow.request.pretty_host.lower()\n    url = _base(flow.request.pretty_url)\n    for definition in _definitions():\n        name = definition[\"name\"]\n        oauth = definition.get(\"modes\", {}).get(\"oauth\")\n        api_key = definition.get(\"modes\", {}).get(\"apiKey\")\n        forced = definition.get(\"modes\", {}).get(\"forced\")\n        credential = _load(_oauth_path(name)) if oauth else None\n        if oauth and url == _base(oauth[\"tokenUrl\"]):\n            fields = _parse_form_or_json(flow.request.get_text(strict=False), flow.request.headers.get(\"content-type\"))\n            refreshing = fields.get(\"grant_type\") == \"refresh_token\"\n            refresh_request = _refresh_request(flow, fields) if refreshing else None\n            # A refresh token the gateway didn't issue is a client that signed in before the upstream\n            # was registered (or elsewhere): let it through, and keep what it gets.\n            foreign_refresh = refreshing and not _is_fake(\n                (credential or {}).get(\"fakes\", {}).get(\"refresh\"), fields.get(\"refresh_token\")\n            )\n            if credential and not foreign_refresh and await _answer_from_credential(flow, name, oauth, refresh_request):\n                return\n            flow.metadata[\"hyperbrane_token\"] = (name, refresh_request)\n            return\n        if credential:\n            for answer in oauth.get(\"answers\", []):\n                if url == _base(answer[\"url\"]):\n                    flow.response = http.Response.make(\n                        answer.get(\"status\", 200), json.dumps(answer[\"json\"]), {\"content-type\": \"application/json\"}\n                    )\n                    return\n        for mode, swap in ((oauth, _swap_oauth), (api_key, _swap_api_key), (forced, None)):\n            if mode and host in _hosts(mode):\n                if swap:\n                    swap(flow.request, name, mode)\n                _force(flow.request, mode.get(\"force\"))\n\n\ndef responseheaders(flow: http.HTTPFlow):\n    # Only token responses are rewritten; everything else streams through as it arrives. mitmproxy\n    # otherwise holds each response until its body is complete, so a model's streamed reply (SSE)\n    # reached the dimension in one piece at the end of the turn.\n    if not flow.metadata.get(\"hyperbrane_token\"):\n        flow.response.stream = True\n\n\ndef response(flow: http.HTTPFlow):\n    token = flow.metadata.get(\"hyperbrane_token\")\n    if not token:\n        return\n    name, refresh_request = token\n    definition = next((d for d in _definitions() if d[\"name\"] == name), None)\n    oauth = (definition or {}).get(\"modes\", {}).get(\"oauth\")\n    if not oauth:\n        return\n    body = _parse_form_or_json(flow.response.get_text(strict=False), \"json\")\n    if flow.response.status_code == 200 and body.get(\"access_token\"):\n        credential = _store_tokens(name, oauth, body, refresh_request)\n        flow.response.text = json.dumps(_client_body(oauth, credential))\n    for header in (oauth.get(\"strip\") or {}).get(\"headers\", []):\n        flow.response.headers.pop(header, None)\n",
 };

package/dist/portal-gateway/portalService.js

@@ -180,17 +180,25 @@ export function createPortalService(storage, gateways, docker, image, host) {
             return;
         if (record.state !== "destroying")
             await storage.savePortal({ ...record, state: "destroying" });
+        // A restore under way finishes first (seeing the mark, it undoes what it wired), so nothing it
+        // does lands after this returns; one that starts from now on sees the mark and does nothing.
+        while (restoring.has(portalName))
+            await restoring.get(portalName).catch(() => { });
         if (record.subnet !== undefined) {
             await ignoringStopped(() => unwire(record.gatewayName, portalLink(portalName, record.subnet)));
             await storage.releaseSubnet(record.subnet);
         }
-        const sameGateway = (await storage.listPortals()).some((other) => other.name !== portalName &&
+        await untrustUnlessShared(record);
+        await storage.removePortal(portalName);
+    }
+    /** Takes the gateway's CA out of the dimension, unless another of its portals goes to that gateway. */
+    async function untrustUnlessShared(record) {
+        const sameGateway = (await storage.listPortals()).some((other) => other.name !== record.name &&
             other.dimensionName === record.dimensionName &&
             other.gatewayName === record.gatewayName);
         if (!sameGateway) {
             await ignoringStopped(() => trustGateway(dimensionContainerName(record.dimensionName), record.gatewayName, false));
         }
-        await storage.removePortal(portalName);
     }
     async function unwire(gatewayName, link) {
         const result = await docker.exec(gatewayContainerName(gatewayName), {
@@ -217,38 +225,62 @@ export function createPortalService(storage, gateways, docker, image, host) {
         await Promise.all(records.map(async (record) => {
             if ((await gatewayInterfaces(record.gatewayName))?.has(portalLink(record.name, record.subnet).gatewayIf))
                 return;
-            let pids;
+            await restoreOnce(record);
+        }));
+    }
+    // One restore of a portal at a time, however many ask (a restart's own and the watcher's): two
+    // wiring the same link at once fail each other, and the retry could land after a destroy. Each
+    // waits for the one before rather than joining it: that one may have given up on a container that
+    // has since restarted, so once it's done the link is looked at again.
+    const restoring = new Map();
+    async function restoreOnce(record) {
+        while (restoring.has(record.name)) {
+            await restoring.get(record.name).catch(() => { });
+            const link = portalLink(record.name, record.subnet);
+            if ((await listInterfaces(gatewayContainerName(record.gatewayName)))?.has(link.gatewayIf))
+                return;
+        }
+        const restored = restore(record).finally(() => restoring.delete(record.name));
+        restoring.set(record.name, restored);
+        return restored;
+    }
+    async function restore(listed) {
+        // As it is now: one being destroyed is left alone.
+        const record = await storage.readPortal(listed.name);
+        if (record?.state !== "ready")
+            return;
+        let pids;
+        try {
+            pids = await runningPids(record);
+        }
+        catch (error) {
+            // A stopped or removed container gets wired when it starts again, or never.
+            if (error instanceof GatewayConflictError || error instanceof GatewayNotFoundError)
+                return;
+            throw error;
+        }
+        for (let attempt = 1;; attempt++) {
             try {
-                pids = await runningPids(record);
+                await connect(record, pids);
+                // Destroyed meanwhile: neither the link nor the trust just made may outlive it (see destroyPortal).
+                if ((await storage.readPortal(record.name))?.state !== "ready") {
+                    await ignoringStopped(() => unwire(record.gatewayName, portalLink(record.name, record.subnet)));
+                    await untrustUnlessShared(record);
+                }
+                return;
             }
             catch (error) {
-                // A stopped or removed container gets wired when it starts again, or never.
-                if (error instanceof GatewayConflictError || error instanceof GatewayNotFoundError)
+                // A container that stopped or restarted meanwhile (killing what ran in it) is restored by
+                // its own start event. Docker kills a restarting container's processes before its state
+                // changes, so give it a moment before deciding it's still the same container.
+                await sleep(RESTORE_RECHECK_MS);
+                const now = await runningPids(record).catch(() => null);
+                if (now?.gateway !== pids.gateway || now?.dimension !== pids.dimension)
                     return;
-                throw error;
+                if (attempt === 2)
+                    return console.warn(`restoring portal ${record.name} failed:`, error);
             }
-            for (let attempt = 1;; attempt++) {
-                try {
-                    await connect(record, pids);
-                    // Destroyed meanwhile: the link just made must not outlive it (see destroyPortal).
-                    if ((await storage.readPortal(record.name))?.state !== "ready") {
-                        await ignoringStopped(() => unwire(record.gatewayName, portalLink(record.name, record.subnet)));
-                    }
-                    return;
-                }
-                catch (error) {
-                    // A container that stopped or restarted meanwhile (killing what ran in it) is restored by
-                    // its own start event. Docker kills a restarting container's processes before its state
-                    // changes, so give it a moment before deciding it's still the same container.
-                    await sleep(RESTORE_RECHECK_MS);
-                    const now = await runningPids(record).catch(() => null);
-                    if (now?.gateway !== pids.gateway || now?.dimension !== pids.dimension)
-                        return;
-                    if (attempt === 2)
-                        return console.warn(`restoring portal ${record.name} failed:`, error);
-                }
-            }
-        }));
+        }
     }
     /** Null when the container isn't running. */
     async function listInterfaces(container) {

package/dist/portal-gateway/upstream.d.ts

@@ -19,8 +19,20 @@ export type UpstreamDefinition = {
     modes: {
         oauth?: OAuthMode;
         apiKey?: ApiKeyMode;
+        forced?: ForcedMode;
     };
 };
+/** What to force onto every matching request, whatever the client sent. */
+export type UpstreamForce = {
+    /** Retarget the request, e.g. "https://api.internal:8443". */
+    target?: string;
+    /** Prefix prepended to the request path, so dimensions needn't know where the API is mounted. */
+    path?: string;
+    /** Query params the client doesn't get to choose. */
+    query?: Record<string, string>;
+    /** Headers whose values the dimension never sees. */
+    headers?: Record<string, string>;
+};
 type ModeBase = {
     /**
      * Requests to these hosts get the credential. Matched on host alone: v1 also matched the
@@ -34,17 +46,16 @@ type ModeBase = {
     } | {
         query: string;
     };
-    /** What to force onto every matching request, whatever the client sent. */
-    force?: {
-        /** Retarget the request, e.g. "https://api.internal:8443". */
-        target?: string;
-        /** Prefix prepended to the request path, so dimensions needn't know where the API is mounted. */
-        path?: string;
-        /** Query params the client doesn't get to choose. */
-        query?: Record<string, string>;
-        /** Headers the client never sees the values of. */
-        headers?: Record<string, string>;
-    };
+    force?: UpstreamForce;
+};
+/**
+ * No credential of the gateway's own, so nothing to present and no fakes: only what `force` writes
+ * onto every request to `hosts`, e.g. a service's own auth header, set here and never seen by the
+ * dimension. Applies as soon as it's registered.
+ */
+export type ForcedMode = {
+    hosts: string[];
+    force: UpstreamForce;
 };
 export type OAuthMode = ModeBase & {
     /**

package/dist/sandbox/api.d.ts

@@ -97,12 +97,14 @@ export type SandboxEndpoint = {
     readonly id: string;
     /**
      * Where the client reaches the endpoint, directly over the network (not through the backend's own
-     * connection), at the host the sandbox was created with.
+     * connection), at the host the sandbox was created with. It lasts until the endpoint is closed,
+     * but what serves it may stop (a reboot, an archived Boat sandbox): ensureEndpoint starts it again.
+     * Until then a browser reports a request to it as a CORS error, since nothing of the endpoint's answers.
      */
     readonly url: string;
     /** What the client presents; requests without it are refused before reaching the target. */
     readonly credentials: SandboxEndpointCredentials;
-    /** As requested: fixed for the endpoint's life. */
+    /** As requested, or as updateEndpoint changed them since. */
     readonly allowedOrigins: string[];
 };
 /**
@@ -177,6 +179,13 @@ export interface SandboxAccessibleEndpoint {
      * and credentials; false if it was closed, or its port has since been taken (expose another).
      */
     ensureEndpoint(endpointId: string): Promise<boolean>;
+    /**
+     * Changes which web origins may call the endpoint, in place: its URL and credentials stay, and so
+     * do connections already open. False if it was closed.
+     */
+    updateEndpoint(endpointId: string, update: {
+        allowedOrigins: string[];
+    }): Promise<boolean>;
 }
 /**
  * A sandbox that stops itself on a timer (Boat).

package/dist/sandbox/endpointProxy.js

@@ -185,6 +185,21 @@ export function createProxyEndpoints(commands, options, where) {
         if (result.exitCode === 10 && firewall)
             await ufw("delete", "allow", `${port}/tcp`);
     }
+    // The proxy reads the record on every request, so the change applies to the next one.
+    async function updateEndpoint(endpointId, { allowedOrigins }) {
+        const { port, name } = parseEndpointId(endpointId);
+        const read = await sh(READ_SCRIPT, { P: port, N: name });
+        if (read.exitCode !== 0)
+            return false;
+        const record = { ...JSON.parse(read.stdout), origins: allowedOrigins };
+        const written = await sh(WRITE_SCRIPT, { P: port, N: name }, JSON.stringify(record));
+        if (written.exitCode === 44)
+            return false; // its proxy's directory went with its last endpoint
+        if (written.exitCode !== 0) {
+            throw new Error(`updating endpoint ${endpointId} on ${where} failed (exit ${written.exitCode}): ${written.stderr.trim()}`);
+        }
+        return true;
+    }
     // Its record says what to serve; the proxy is started again on the port the endpoint's URL names.
     async function ensureEndpoint(endpointId) {
         const { port, name } = parseEndpointId(endpointId);
@@ -202,7 +217,7 @@ export function createProxyEndpoints(commands, options, where) {
         await closeEndpoint(endpointId);
         return false;
     }
-    return { accessibleHost: options.accessibleHost, exposeEndpoint, exposeEndpointAs, closeEndpoint, ensureEndpoint };
+    return { accessibleHost: options.accessibleHost, exposeEndpoint, exposeEndpointAs, closeEndpoint, ensureEndpoint, updateEndpoint };
 }
 /** The credentials a client presents for a recorded check; `none` when the endpoint checks nothing itself. */
 function credentialsOf(check, none) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyperbrane",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Dimensions, gateways and agents on a machine reached locally, over SSH or on Boat: SDK and CLI.",
   "license": "Apache-2.0",
   "type": "module",