@crvouga/dev-fleet 0.0.0-stage → 0.1.0

package/cliproxy/scripts/modelhealth/scenario.py

@@ -0,0 +1,131 @@
+"""Build protocol-specific request bodies from protocol-neutral turn fixtures."""
+
+import copy
+import json
+
+
+def substitute(value, variables):
+    """Replace {name} placeholders in every string of a fixture."""
+    if isinstance(value, str):
+        for name, text in variables.items():
+            value = value.replace("{" + name + "}", text)
+        return value
+    if isinstance(value, list):
+        return [substitute(item, variables) for item in value]
+    if isinstance(value, dict):
+        return {key: substitute(item, variables) for key, item in value.items()}
+    return value
+
+
+def _merge(base, extra):
+    for key, value in (extra or {}).items():
+        if isinstance(value, dict) and isinstance(base.get(key), dict):
+            _merge(base[key], value)
+        else:
+            base[key] = copy.deepcopy(value)
+    return base
+
+
+def _chat_tool(name, spec):
+    return {"type": "function", "function": {"name": name, "description": spec["description"], "parameters": spec["parameters"]}}
+
+
+def _responses_tool(name, spec):
+    return {"type": "function", "name": name, "description": spec["description"], "parameters": spec["parameters"]}
+
+
+def _chat_messages(turn):
+    messages = []
+    if turn.get("system"):
+        messages.append({"role": "system", "content": turn["system"]})
+    if turn.get("developer"):
+        messages.append({"role": "developer", "content": turn["developer"]})
+    for message in turn.get("messages", []):
+        if message["role"] == "assistant" and message.get("tool_calls"):
+            messages.append({
+                "role": "assistant",
+                "content": message.get("content"),
+                "tool_calls": [
+                    {"id": c["id"], "type": "function", "function": {"name": c["name"], "arguments": c["arguments"]}}
+                    for c in message["tool_calls"]
+                ],
+            })
+        elif message["role"] == "tool":
+            messages.append({"role": "tool", "tool_call_id": message["tool_call_id"], "content": message["content"]})
+        else:
+            messages.append({"role": message["role"], "content": message["content"]})
+    return messages
+
+
+def _responses_input(turn):
+    items = []
+    if turn.get("developer"):
+        items.append({"role": "developer", "content": turn["developer"]})
+    for message in turn.get("messages", []):
+        if message["role"] == "assistant" and message.get("tool_calls"):
+            if message.get("content"):
+                items.append({"role": "assistant", "content": message["content"]})
+            for call in message["tool_calls"]:
+                items.append({"type": "function_call", "call_id": call["id"], "name": call["name"], "arguments": call["arguments"]})
+        elif message["role"] == "tool":
+            items.append({"type": "function_call_output", "call_id": message["tool_call_id"], "output": message["content"]})
+        else:
+            items.append({"role": message["role"], "content": message["content"]})
+    return items
+
+
+def _tool_choice(protocol, choice):
+    if choice is None or isinstance(choice, str):
+        return choice
+    name = choice["name"]
+    if protocol == "chat":
+        return {"type": "function", "function": {"name": name}}
+    return {"type": "function", "name": name}
+
+
+def build(protocol, model_id, turn, config, extra=None):
+    """Return the JSON body for one request on the given wire protocol."""
+    tools = {name: config["tools"][name] for name in turn.get("tools", [])}
+    limit_field = config["protocols"][protocol]["token_limit_field"]
+    body = {"model": model_id}
+    if protocol == "chat":
+        body["messages"] = _chat_messages(turn)
+        if tools:
+            body["tools"] = [_chat_tool(n, s) for n, s in tools.items()]
+        if turn.get("json_schema"):
+            spec = turn["json_schema"]
+            body["response_format"] = {
+                "type": "json_schema",
+                "json_schema": {"name": spec["name"], "schema": spec["schema"], "strict": True},
+            }
+        if turn.get("stream") and turn.get("include_usage"):
+            body["stream_options"] = {"include_usage": True}
+        if tools and "parallel_tool_calls" in turn:
+            body["parallel_tool_calls"] = turn["parallel_tool_calls"]
+    else:
+        if turn.get("system"):
+            body["instructions"] = turn["system"]
+        body["input"] = _responses_input(turn)
+        body["store"] = False
+        if tools:
+            body["tools"] = [_responses_tool(n, s) for n, s in tools.items()]
+        if turn.get("json_schema"):
+            spec = turn["json_schema"]
+            body["text"] = {"format": {"type": "json_schema", "name": spec["name"], "schema": spec["schema"], "strict": True}}
+        if tools and "parallel_tool_calls" in turn:
+            body["parallel_tool_calls"] = turn["parallel_tool_calls"]
+    if tools and turn.get("tool_choice") is not None:
+        body["tool_choice"] = _tool_choice(protocol, turn["tool_choice"])
+    if turn.get("max_tokens"):
+        body[limit_field] = turn["max_tokens"]
+    if turn.get("stream"):
+        body["stream"] = True
+    return _merge(body, extra)
+
+
+def tool_schemas(config, names):
+    return {name: config["tools"][name]["parameters"] for name in names}
+
+
+def dumps(body):
+    return json.dumps(body, separators=(",", ":")).encode()

package/cliproxy/scripts/modelhealth/schema.py

@@ -0,0 +1,63 @@
+"""Minimal JSON Schema subset validator (no third-party dependency).
+
+Supports the keywords used by tool and structured-output fixtures: type, enum,
+const, properties, required, additionalProperties, items, minItems, maxItems,
+minimum, maximum, minLength, maxLength and pattern.
+"""
+
+import re
+
+_TYPES = {
+    "object": lambda v: isinstance(v, dict),
+    "array": lambda v: isinstance(v, list),
+    "string": lambda v: isinstance(v, str),
+    "integer": lambda v: isinstance(v, int) and not isinstance(v, bool),
+    "number": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
+    "boolean": lambda v: isinstance(v, bool),
+    "null": lambda v: v is None,
+}
+
+
+def errors(value, schema, path="$"):
+    """Return a list of 'path: problem' strings; empty means valid."""
+    found = []
+    expected = schema.get("type")
+    if expected is not None:
+        names = expected if isinstance(expected, list) else [expected]
+        if not any(_TYPES[name](value) for name in names):
+            return [path + ": expected " + "|".join(names)]
+    if "const" in schema and value != schema["const"]:
+        found.append(path + ": does not equal the required constant")
+    if "enum" in schema and value not in schema["enum"]:
+        found.append(path + ": not one of the allowed values")
+    if isinstance(value, str):
+        if len(value) < schema.get("minLength", 0):
+            found.append(path + ": shorter than minLength")
+        if len(value) > schema.get("maxLength", 1 << 30):
+            found.append(path + ": longer than maxLength")
+        if "pattern" in schema and not re.search(schema["pattern"], value):
+            found.append(path + ": does not match pattern")
+    if _TYPES["number"](value):
+        if "minimum" in schema and value < schema["minimum"]:
+            found.append(path + ": below minimum")
+        if "maximum" in schema and value > schema["maximum"]:
+            found.append(path + ": above maximum")
+    if isinstance(value, list):
+        if len(value) < schema.get("minItems", 0):
+            found.append(path + ": fewer than minItems")
+        if len(value) > schema.get("maxItems", 1 << 30):
+            found.append(path + ": more than maxItems")
+        if "items" in schema:
+            for index, item in enumerate(value):
+                found += errors(item, schema["items"], "%s[%d]" % (path, index))
+    if isinstance(value, dict):
+        properties = schema.get("properties", {})
+        for name in schema.get("required", []):
+            if name not in value:
+                found.append(path + "." + name + ": required")
+        for name, item in value.items():
+            if name in properties:
+                found += errors(item, properties[name], path + "." + name)
+            elif schema.get("additionalProperties") is False:
+                found.append(path + "." + name + ": unexpected property")
+    return found

package/cliproxy/scripts/modelhealth/simulate.py

@@ -0,0 +1,111 @@
+"""Run declarative routing, rate-limit and fault scenarios against the local emulator.
+
+Evidence from these scenarios is labelled ``simulated``. It proves the harness
+detects and classifies the behavior; it never proves the production proxy's
+accounts, quotas or failover are healthy.
+"""
+
+import random
+import time
+
+from . import checks, protocol
+from .emulator import Account, Emulator
+from .redact import Redactor
+from .transport import Client
+
+DEFAULT_TURN = {"messages": [{"role": "user", "content": "Reply with the single word OK."}], "max_tokens": 64}
+
+
+def _expect(step_expect, outcomes, emulator, sent, elapsed):
+    """Return a list of mismatches for one step's expectations."""
+    problems = []
+    last = outcomes[-1]
+    wanted = step_expect
+    if "status" in wanted and any(o.status != wanted["status"] for o in outcomes):
+        problems.append("HTTP status %s, expected %s" % ([o.status for o in outcomes], wanted["status"]))
+    if "category" in wanted and any(o.category != wanted["category"] for o in outcomes):
+        problems.append("category %s, expected %s" % ([o.category for o in outcomes], wanted["category"]))
+    served = [o.headers.get("x-emulator-account") for o in outcomes]
+    if "served_by_all" in wanted and any(s not in wanted["served_by_all"] for s in served):
+        problems.append("served by %s, expected only %s" % (served, wanted["served_by_all"]))
+    if "served_sequence" in wanted and served != wanted["served_sequence"]:
+        problems.append("served by %s, expected the sequence %s" % (served, wanted["served_sequence"]))
+    if "served_set" in wanted and set(served) != set(wanted["served_set"]):
+        problems.append("served by %s, expected exactly %s" % (sorted(set(served)), wanted["served_set"]))
+    if wanted.get("single_account") and len(set(served)) != 1:
+        problems.append("session was not pinned to one account: %s" % sorted(set(served)))
+    if wanted.get("retry_after") and not last.retry_after:
+        problems.append("Retry-After header was not surfaced")
+    if wanted.get("envelope") and "no error envelope" in (last.detail or ""):
+        problems.append("error response lacks an OpenAI error envelope")
+    if wanted.get("no_retry") and emulator.requests != sent:
+        problems.append("harness sent %d requests where %d were issued (hidden retry)" % (emulator.requests, sent))
+    if "max_elapsed_s" in wanted and elapsed > wanted["max_elapsed_s"]:
+        problems.append("took %.1fs, limit %.1fs" % (elapsed, wanted["max_elapsed_s"]))
+    if wanted.get("single_model") and len({o.norm.model for o in outcomes if o.norm}) != 1:
+        problems.append("response model id differed across accounts")
+    if "violation" in wanted:
+        if not any(wanted["violation"] in v for o in outcomes if o.norm for v in o.norm.violations):
+            problems.append("expected protocol violation %r was not detected" % wanted["violation"])
+    elif wanted.get("status") == 200:
+        for o in outcomes:
+            if o.norm is None or o.norm.violations:
+                problems.append("successful response was not contract-valid")
+                break
+    return problems
+
+
+def run_scenario(config, scenario, seed=0):
+    model_ids = [m["id"] for m in config.models]
+    accounts = []
+    for spec in scenario["accounts"]:
+        account = Account(spec["id"], quota=spec.get("quota"))
+        account.fail_next = spec.get("fail_next", 0)
+        accounts.append(account)
+    emulator = Emulator(model_ids, accounts=accounts, strategy=scenario.get("strategy", config.raw["routing"]["strategy"]))
+    base = emulator.start()
+    redact = Redactor([emulator.key])
+    timeouts = dict(config.raw["timeouts"], **scenario.get("timeouts", {}))
+    budget = checks.Budget(10_000, 10_000, 10_000_000)
+    ctx = checks.Context(config, config.models[0],
+                         lambda model_id: Client(base, emulator.key, redact, timeouts["request_s"],
+                                              lambda kind: budget.take(model_id) if kind in ("inference", "probe") else None),
+                         budget, random.Random(seed), redact, timeouts)
+    details, sent = [], 0
+    try:
+        for number, step in enumerate(scenario["steps"], 1):
+            action = step.get("action")
+            if action == "advance":
+                emulator.advance(step["seconds"])
+            elif action == "account":
+                account = emulator.account(step["id"])
+                account.quota = step.get("quota", account.quota)
+                account.fail_next = step.get("fail_next", account.fail_next)
+            elif action == "fault":
+                emulator.fault(**step["faults"])
+            elif action == "clear_faults":
+                emulator.clear_faults()
+            else:
+                request = step["request"]
+                turn = dict(request.get("turn") or DEFAULT_TURN)
+                outcomes, started = [], time.monotonic()
+                for index in range(step.get("count", 1)):
+                    # Distinct sessions are new clients (cold bindings); "same" is one session (affinity).
+                    session = "same" if request.get("session") == "same" else "s%d-%d" % (number, index)
+                    outcomes.append(checks.call(ctx, request["protocol"], turn, {},
+                                                headers={"X-Session-Id": session if session != "same" else "pinned"}))
+                    sent += 1
+                problems = _expect(step["expect"], outcomes, emulator, sent, time.monotonic() - started)
+                details += ["step %d: %s" % (number, p) for p in problems]
+    except Exception as error:  # a scenario defect must be visible, not swallowed
+        details.append("harness error: %s" % type(error).__name__)
+    finally:
+        log = list(emulator.routing_log)
+        emulator.stop()
+    return {"id": scenario["id"], "capability": scenario["capability"], "title": scenario["title"],
+            "status": "fail" if details else "pass", "evidence": "simulated", "details": details,
+            "routing": [{"tried": entry["tried"], "served_by": entry["served_by"]} for entry in log]}
+
+
+def run_all(config, seed=0):
+    return [run_scenario(config, scenario, seed) for scenario in config.raw.get("simulations", [])]

package/cliproxy/scripts/modelhealth/transport.py

@@ -0,0 +1,213 @@
+"""HTTP/SSE client with strict framing checks, deadlines and no redirects or hidden retries."""
+
+import http.client
+import json
+import ssl
+import time
+import urllib.parse
+
+USER_AGENT = "cliproxy-fleet/1.0 model-health"
+MAX_BODY = 4 * 1024 * 1024
+_SSE_FIELDS = ("data", "event", "id", "retry")
+
+
+class TransportError(Exception):
+    """A request that produced no HTTP response; category feeds failure reporting."""
+
+    def __init__(self, category, detail):
+        super().__init__(detail)
+        self.category = category
+        self.detail = detail
+
+
+class Response:
+    def __init__(self, status, headers, body, elapsed):
+        self.status = status
+        self.headers = {k.lower(): v for k, v in headers}
+        self.body = body
+        self.elapsed = elapsed
+
+    def json(self):
+        try:
+            return json.loads(self.body.decode("utf-8"))
+        except (ValueError, UnicodeDecodeError):
+            return None
+
+
+class Frame:
+    def __init__(self, event, data, at):
+        self.event = event
+        self.data = data
+        self.at = at  # seconds since the request was sent
+
+
+class StreamResult:
+    def __init__(self):
+        self.status = 0
+        self.headers = {}
+        self.frames = []
+        self.violations = []
+        self.error_body = b""
+        self.ttft = None  # first frame carrying data
+        self.elapsed = 0.0
+        self.cancelled = False
+        self.done = False  # saw the literal [DONE] sentinel
+        self.clean_eof = False  # server ended the body after a complete frame
+
+    def json_frames(self):
+        out = []
+        for frame in self.frames:
+            if frame.data == "[DONE]":
+                continue
+            try:
+                out.append((frame.event, json.loads(frame.data)))
+            except ValueError:
+                self.violations.append("non-JSON data frame")
+        return out
+
+
+def _loopback(host):
+    return host in ("127.0.0.1", "localhost", "::1")
+
+
+class Client:
+    """One origin + optional bearer key. Every call is a single attempt."""
+
+    def __init__(self, base_url, key=None, redactor=None, timeout=60.0, on_request=None):
+        parts = urllib.parse.urlsplit(base_url)
+        if parts.scheme not in ("https", "http") or not parts.hostname:
+            raise ValueError("Endpoint must be an absolute URL")
+        if parts.scheme == "http" and not _loopback(parts.hostname):
+            raise ValueError("HTTPS required (plain HTTP is allowed only for loopback emulators)")
+        self.scheme = parts.scheme
+        self.host = parts.hostname
+        self.port = parts.port
+        self.prefix = parts.path.rstrip("/")
+        self.base_url = base_url.rstrip("/")
+        self.key = key
+        self.redact = redactor or (lambda text, limit=300: str(text)[:limit])
+        self.timeout = timeout
+        self.on_request = on_request or (lambda kind: None)
+
+    def _connect(self, timeout):
+        if self.scheme == "https":
+            return http.client.HTTPSConnection(
+                self.host, self.port, timeout=timeout, context=ssl.create_default_context()
+            )
+        return http.client.HTTPConnection(self.host, self.port, timeout=timeout)
+
+    def _headers(self, authenticated, body, extra):
+        headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
+        if authenticated and self.key:
+            headers["Authorization"] = "Bearer " + self.key
+        if body is not None:
+            headers["Content-Type"] = "application/json"
+        headers.update(extra or {})
+        return headers
+
+    def _send(self, method, path, body, authenticated, timeout, extra, kind):
+        self.on_request(kind)
+        connection = self._connect(timeout or self.timeout)
+        payload = None
+        if body is not None:
+            payload = body if isinstance(body, bytes) else json.dumps(body).encode()
+        started = time.monotonic()
+        try:
+            connection.request(
+                method, self.prefix + path, payload, self._headers(authenticated, payload, extra)
+            )
+            return connection, connection.getresponse(), started
+        except Exception as error:
+            connection.close()
+            raise self._classify(error) from None
+
+    def _classify(self, error):
+        text = self.redact(type(error).__name__ + ": " + str(error))
+        if isinstance(error, (TimeoutError,)) or "timed out" in str(error):
+            return TransportError("timeout", text)
+        return TransportError("transport", text)
+
+    def request(self, method, path, body=None, authenticated=True, timeout=None,
+                headers=None, kind="request"):
+        connection, response, started = self._send(
+            method, path, body, authenticated, timeout, headers, kind
+        )
+        try:
+            data = response.read(MAX_BODY + 1)
+        except Exception as error:
+            raise self._classify(error) from None
+        finally:
+            connection.close()
+        if len(data) > MAX_BODY:
+            raise TransportError("protocol", "Response body exceeds the harness limit")
+        return Response(response.status, response.getheaders(), data, time.monotonic() - started)
+
+    def stream(self, path, body, timeout=None, deadline=None, stop_after=None,
+               headers=None, kind="inference"):
+        """POST a streaming request and parse SSE frames.
+
+        timeout is the per-read idle limit; deadline bounds the whole stream.
+        stop_after closes the connection after N data frames (cancellation).
+        """
+        result = StreamResult()
+        extra = {"Accept": "text/event-stream"}
+        extra.update(headers or {})
+        connection, response, started = self._send(
+            "POST", path, body, True, timeout, extra, kind
+        )
+        result.status = response.status
+        result.headers = {k.lower(): v for k, v in response.getheaders()}
+        try:
+            if response.status != 200:
+                result.error_body = response.read(64 * 1024)
+                return result
+            if "text/event-stream" not in result.headers.get("content-type", ""):
+                result.violations.append("content-type is not text/event-stream")
+            event, data_lines, saw_field = None, [], False
+            limit = deadline or (self.timeout * 4)
+            while True:
+                if time.monotonic() - started > limit:
+                    raise TransportError("timeout", "Stream exceeded its overall deadline")
+                try:
+                    raw = response.readline()
+                except Exception as error:
+                    raise self._classify(error) from None
+                if not raw:
+                    result.clean_eof = not saw_field
+                    if saw_field:
+                        result.violations.append("stream ended inside an unterminated frame")
+                    break
+                line = raw.decode("utf-8", "replace").rstrip("\r\n")
+                if line == "":
+                    if saw_field:
+                        at = time.monotonic() - started
+                        frame = Frame(event, "\n".join(data_lines), at)
+                        if data_lines:
+                            result.frames.append(frame)
+                            if result.ttft is None:
+                                result.ttft = at
+                            if frame.data == "[DONE]":
+                                result.done = True
+                        event, data_lines, saw_field = None, [], False
+                        if stop_after and len(result.frames) >= stop_after:
+                            result.cancelled = True
+                            break
+                    continue
+                if line.startswith(":"):
+                    continue  # comment / keep-alive
+                name, sep, value = line.partition(":")
+                if name not in _SSE_FIELDS or (not sep and name != "data"):
+                    result.violations.append("invalid SSE line")
+                    continue
+                value = value[1:] if value.startswith(" ") else value
+                saw_field = True
+                if name == "data":
+                    data_lines.append(value)
+                elif name == "event":
+                    event = value
+                if result.done and name == "data":
+                    result.violations.append("data after [DONE]")
+        finally:
+            connection.close()
+            result.elapsed = time.monotonic() - started
+        return result

package/cliproxy/scripts/monitor

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+import sys
+
+sys.dont_write_bytecode = True
+
+import pathlib as _pathlib
+
+sys.path.insert(0, str(_pathlib.Path(__file__).resolve().parent))
+import cliproxy_host
+
+import datetime, json, pathlib, subprocess, sys, os
+
+r = cliproxy_host.default_root().resolve()
+issues = []
+provenance = subprocess.run([str(r / "scripts/audit.py")], capture_output=True)
+if provenance.returncode:
+    issues.append("Installed code differs from its committed-source manifest.")
+p = subprocess.run([str(r / "scripts/ops"), "health"], capture_output=True)
+if p.returncode:
+    issues.append("Application health/authentication failed; inspect sanitized logs.")
+try:
+    import yaml
+
+    strategy = yaml.safe_load((r / "state/runtime/config.yaml").read_text())["routing"]["strategy"]
+    if strategy != "fill-first":
+        issues.append(
+            "Live routing strategy is not fill-first (drift from config.template.yaml); apply a reviewed config with `cliproxy apply`."
+        )
+except Exception:
+    issues.append("Live routing strategy could not be read.")
+f = r / "state/status/backup-success.json"
+if not f.exists():
+    issues.append("No successful independent backup has been recorded.")
+else:
+    try:
+        t = datetime.datetime.fromisoformat(json.loads(f.read_text())["time"])
+        if (datetime.datetime.now(datetime.timezone.utc) - t).total_seconds() > 45 * 60:
+            issues.append("Cloud checkpoint older than 45 minutes.")
+    except Exception:
+        issues.append("Invalid backup status record.")
+if (r / "state/status/vault-failed").exists():
+    issues.append("Vault login/renewal failed; backups cannot fetch credentials.")
+for marker in [
+    "backup-failed",
+    "backup-check-failed",
+    "backup-operation-failed",
+    "gc-failed",
+]:
+    if (r / "state/status" / marker).exists():
+        issues.append(marker + ": inspect failed operation.")
+gc = r / "state/status/gc-success.json"
+try:
+    age = datetime.datetime.now(
+        datetime.timezone.utc
+    ) - datetime.datetime.fromisoformat(json.loads(gc.read_text())["time"])
+    if age.total_seconds() > 9 * 86400:
+        issues.append("Garbage collection older than nine days.")
+except Exception:
+    issues.append("No valid successful garbage collection record.")
+f = r / "state/status/monitor-failed"
+if issues:
+    f.write_text("\n".join(issues) + "\n")
+    if os.geteuid() == 0:
+        os.chown(f, 1000, 1000)
+    print(f.read_text())
+    sys.exit(1)
+f.unlink(missing_ok=True)
+print("Health and backup freshness passed.")