omnius 1.0.841 → 1.0.843
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/py-embed.js +651 -91
- package/dist/computer-use/models.lock.yaml +495 -0
- package/dist/index.js +2 -2
- package/dist/library.d.ts +1 -0
- package/dist/library.js +176452 -55418
- package/dist/scripts/rapid-computer-use/gpu_lease.py +64 -0
- package/dist/scripts/rapid-computer-use/laya_service.py +483 -0
- package/dist/scripts/rapid-computer-use/model_gate.py +755 -0
- package/dist/scripts/rapid-computer-use/photon_service.py +700 -0
- package/dist/scripts/rapid-computer-use/point_benchmark.py +278 -0
- package/dist/update-worker.js +651 -91
- package/docs/DISCOVERY.json +1727 -828
- package/docs/DISCOVERY.md +23 -0
- package/docs/guides/bring-your-own-inference.md +128 -0
- package/docs/reference/slash-commands.md +138 -22
- package/docs/work-orders/WO-inference-fallback-20261012.md +89 -0
- package/docs/work-orders/WO-primary-checkout-fallback-delivery-20261012.md +94 -0
- package/docs/work-orders/WO-telegram-fallback-20261012.md +83 -0
- package/docs/work-orders/inference-fallback-bundle-20261012.json +75 -0
- package/docs/work-orders/inference-fallback-live-20261012.json +39 -0
- package/docs/work-orders/primary-inference-fallback-bundle-20261012.json +88 -0
- package/docs/work-orders/rapid-computer-use-20261011/README.md +125 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-01-native-save-recovery.md +86 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-02-semantic-control.md +93 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-03-policy-workflows.md +130 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-04-live-model-gates.md +322 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-05-verification-delivery.md +389 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-06-medium-horizon-runtime.md +166 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-07-tool-routing-prompts.md +81 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-08-cross-application-desktop.md +214 -0
- package/docs/work-orders/rapid-computer-use-20261011/architecture-source.md +871 -0
- package/docs/work-orders/telegram-progress-browser-20261011/README.md +79 -0
- package/docs/work-orders/telegram-progress-browser-20261011/WO-01-diagnostic-visibility.md +64 -0
- package/docs/work-orders/telegram-progress-browser-20261011/WO-02-task-narration.md +127 -0
- package/docs/work-orders/telegram-progress-browser-20261011/WO-03-browser-observation.md +51 -0
- package/docs/work-orders/telegram-progress-browser-20261011/WO-04-download-desktop-handoff.md +96 -0
- package/docs/work-orders/telegram-progress-browser-20261011/WO-05-verification-delivery.md +129 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/prompts/agentic/system-common.md +7 -0
|
@@ -0,0 +1,755 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Read-only model preflight and finite-choice evaluation. Never loads a model.
|
|
3
|
+
|
|
4
|
+
Standard library only: importing this module cannot initialize CUDA. A short-lived
|
|
5
|
+
operator statement binds an already resident runner to every live request. The
|
|
6
|
+
operator may be the authorized Omnius/Codex agent; no extra user approval is implied.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import csv
|
|
12
|
+
import hashlib
|
|
13
|
+
import http.client
|
|
14
|
+
import ipaddress
|
|
15
|
+
import json
|
|
16
|
+
import math
|
|
17
|
+
import os
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
import re
|
|
20
|
+
import socket
|
|
21
|
+
import subprocess
|
|
22
|
+
import sys
|
|
23
|
+
import threading
|
|
24
|
+
import time
|
|
25
|
+
import urllib.parse
|
|
26
|
+
|
|
27
|
+
ROOT = Path(__file__).resolve().parents[2]
|
|
28
|
+
DEFAULT_LOCK = ROOT / "computer-use/models.lock.yaml"
|
|
29
|
+
MAX_JSON_BYTES = 1_048_576
|
|
30
|
+
MAX_RESPONSE_BYTES = 65_536
|
|
31
|
+
GPU_ID = re.compile(r"GPU-[a-fA-F0-9-]{36}\Z")
|
|
32
|
+
REVISION = re.compile(r"(?:[a-f0-9]{40}|[a-f0-9]{64})\Z")
|
|
33
|
+
ACTION_ID = re.compile(r"[A-Za-z0-9_.:-]{1,128}\Z")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class GateError(ValueError):
|
|
37
|
+
pass
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def require(condition, reason):
|
|
41
|
+
if not condition:
|
|
42
|
+
raise GateError(reason)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def strict_json(raw):
|
|
46
|
+
def unique(pairs):
|
|
47
|
+
result = {}
|
|
48
|
+
for key, value in pairs:
|
|
49
|
+
require(key not in result, f"duplicate JSON key: {key}")
|
|
50
|
+
result[key] = value
|
|
51
|
+
return result
|
|
52
|
+
|
|
53
|
+
def invalid(value):
|
|
54
|
+
raise GateError(f"non-finite JSON number: {value}")
|
|
55
|
+
|
|
56
|
+
try:
|
|
57
|
+
parsed = json.loads(raw, object_pairs_hook=unique, parse_constant=invalid)
|
|
58
|
+
pending = [(parsed, 0)]
|
|
59
|
+
count = 0
|
|
60
|
+
while pending:
|
|
61
|
+
value, depth = pending.pop()
|
|
62
|
+
count += 1
|
|
63
|
+
require(depth <= 64 and count <= 100_000, "JSON structure exceeds bounded depth or size")
|
|
64
|
+
if isinstance(value, float):
|
|
65
|
+
require(math.isfinite(value), "JSON numeric overflow")
|
|
66
|
+
if isinstance(value, (dict, list)):
|
|
67
|
+
pending.extend((item, depth + 1) for item in
|
|
68
|
+
(value.values() if isinstance(value, dict) else value))
|
|
69
|
+
return parsed
|
|
70
|
+
except (ValueError, TypeError, UnicodeError, RecursionError) as error:
|
|
71
|
+
raise GateError(f"invalid JSON: {error}") from error
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def bounded_bytes(path):
|
|
75
|
+
with Path(path).open("rb") as stream:
|
|
76
|
+
raw = stream.read(MAX_JSON_BYTES + 1)
|
|
77
|
+
require(len(raw) <= MAX_JSON_BYTES, "JSON input exceeds size bound")
|
|
78
|
+
return raw
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def read_json_hashed(path):
|
|
82
|
+
raw = bounded_bytes(path)
|
|
83
|
+
return strict_json(raw), hashlib.sha256(raw).hexdigest()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def read_json(path):
|
|
87
|
+
return read_json_hashed(path)[0]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def digest(value):
|
|
91
|
+
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"),
|
|
92
|
+
allow_nan=False).encode()).hexdigest()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def write_json(path, value):
|
|
96
|
+
"""Atomic owner-only artifact; a failed run cannot leave half a success report."""
|
|
97
|
+
destination = Path(path)
|
|
98
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
99
|
+
temporary = destination.with_name(destination.name + f".{os.getpid()}.tmp")
|
|
100
|
+
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
101
|
+
try:
|
|
102
|
+
with os.fdopen(descriptor, "w") as stream:
|
|
103
|
+
json.dump(value, stream, indent=2, allow_nan=False)
|
|
104
|
+
stream.write("\n")
|
|
105
|
+
stream.flush()
|
|
106
|
+
os.fsync(stream.fileno())
|
|
107
|
+
os.replace(temporary, destination)
|
|
108
|
+
finally:
|
|
109
|
+
temporary.unlink(missing_ok=True)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def local_endpoint(url):
|
|
113
|
+
require(isinstance(url, str) and len(url) <= 512, "an exact endpoint is required")
|
|
114
|
+
parsed = urllib.parse.urlsplit(url)
|
|
115
|
+
require(parsed.scheme in ("http", "https") and parsed.hostname is not None,
|
|
116
|
+
"endpoint must be HTTP(S)")
|
|
117
|
+
require(not parsed.username and not parsed.password and not parsed.query and
|
|
118
|
+
not parsed.fragment, "endpoint credentials, query, and fragment are forbidden")
|
|
119
|
+
try:
|
|
120
|
+
address = ipaddress.ip_address(parsed.hostname)
|
|
121
|
+
port = parsed.port
|
|
122
|
+
except ValueError as error:
|
|
123
|
+
raise GateError("endpoint must use a literal loopback IP address") from error
|
|
124
|
+
require(address.is_loopback and port is not None and 1 <= port <= 65535,
|
|
125
|
+
"endpoint must include a loopback address and explicit port")
|
|
126
|
+
require("//" not in parsed.path and "/../" not in parsed.path and "%" not in parsed.path,
|
|
127
|
+
"endpoint path must be literal")
|
|
128
|
+
return parsed
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def local_json(url, body=None, headers=None, timeout=15):
|
|
132
|
+
parsed = local_endpoint(url)
|
|
133
|
+
require(0 < timeout <= 120, "HTTP timeout outside bounded range")
|
|
134
|
+
payload = None if body is None else json.dumps(body, allow_nan=False).encode()
|
|
135
|
+
deadline = time.monotonic() + timeout
|
|
136
|
+
connection_type = http.client.HTTPSConnection if parsed.scheme == "https" else http.client.HTTPConnection
|
|
137
|
+
connection = connection_type(parsed.hostname, parsed.port, timeout=timeout)
|
|
138
|
+
watchdog = None
|
|
139
|
+
expired = threading.Event()
|
|
140
|
+
try:
|
|
141
|
+
# Direct connection ignores proxy environment and never follows redirects.
|
|
142
|
+
connection.connect()
|
|
143
|
+
remaining = deadline - time.monotonic()
|
|
144
|
+
require(remaining > 0, "local HTTP total deadline exceeded")
|
|
145
|
+
transport = connection.sock
|
|
146
|
+
transport.settimeout(remaining)
|
|
147
|
+
def interrupt():
|
|
148
|
+
expired.set()
|
|
149
|
+
try:
|
|
150
|
+
# HTTPResponse can take ownership of this socket; closing only the
|
|
151
|
+
# connection would not interrupt a slow-drip header/body read.
|
|
152
|
+
transport.shutdown(socket.SHUT_RDWR)
|
|
153
|
+
except OSError:
|
|
154
|
+
pass
|
|
155
|
+
watchdog = threading.Timer(remaining, interrupt)
|
|
156
|
+
watchdog.daemon = True
|
|
157
|
+
watchdog.start()
|
|
158
|
+
connection.request("GET" if body is None else "POST", parsed.path or "/", body=payload,
|
|
159
|
+
headers={"Content-Type": "application/json", **(headers or {})})
|
|
160
|
+
response = connection.getresponse()
|
|
161
|
+
require(response.status == 200, "model endpoint did not return HTTP 200")
|
|
162
|
+
raw = response.read(MAX_RESPONSE_BYTES + 1)
|
|
163
|
+
require(not expired.is_set() and time.monotonic() < deadline, "local HTTP total deadline exceeded")
|
|
164
|
+
require(len(raw) <= MAX_RESPONSE_BYTES, "model response exceeds size bound")
|
|
165
|
+
return strict_json(raw)
|
|
166
|
+
except (OSError, http.client.HTTPException) as error:
|
|
167
|
+
if expired.is_set() or time.monotonic() >= deadline:
|
|
168
|
+
raise GateError("local HTTP total deadline exceeded") from error
|
|
169
|
+
raise GateError(f"local endpoint unavailable: {type(error).__name__}") from error
|
|
170
|
+
finally:
|
|
171
|
+
if watchdog:
|
|
172
|
+
watchdog.cancel()
|
|
173
|
+
connection.close()
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def command(argv):
|
|
177
|
+
try:
|
|
178
|
+
result = subprocess.run(argv, check=False, capture_output=True, text=True, timeout=15)
|
|
179
|
+
except (OSError, subprocess.TimeoutExpired) as error:
|
|
180
|
+
raise GateError(f"preflight command unavailable: {argv[0]}") from error
|
|
181
|
+
require(result.returncode == 0 and not result.stderr.strip(),
|
|
182
|
+
f"preflight command failed or reported an error: {argv[0]}")
|
|
183
|
+
require(len(result.stdout) <= MAX_JSON_BYTES, "preflight command output exceeds bound")
|
|
184
|
+
return result.stdout
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def validate_selection(selection, lock):
|
|
188
|
+
require(isinstance(selection, dict), "selection must be an object")
|
|
189
|
+
require(selection.get("schema") == "omnius.computer-use.model-selection.v1",
|
|
190
|
+
"unsupported selection schema")
|
|
191
|
+
registry = lock.get("models", {})
|
|
192
|
+
require(selection.get("model_id") in registry, "model is absent from source lock")
|
|
193
|
+
model = registry[selection["model_id"]]
|
|
194
|
+
require(selection.get("weights_revision") == model["weights"]["revision"],
|
|
195
|
+
"selected weights do not match source lock")
|
|
196
|
+
require(selection.get("runtime_revision") == model["runtime"]["revision"],
|
|
197
|
+
"selected runtime does not match source lock")
|
|
198
|
+
require(REVISION.fullmatch(selection["weights_revision"]) is not None,
|
|
199
|
+
"model weight revision must be immutable")
|
|
200
|
+
require(isinstance(selection.get("model"), str) and 0 < len(selection["model"]) <= 256,
|
|
201
|
+
"exact served model tag required")
|
|
202
|
+
require(selection.get("protocol") in ("systemone", "ollama-chat", "moondream-point"),
|
|
203
|
+
"unsupported model request protocol")
|
|
204
|
+
require(selection["protocol"] in model["runtime"].get("protocols", []),
|
|
205
|
+
"selected model/runtime protocol has not been resolved in the source lock")
|
|
206
|
+
require(service_mode(selection) in ("broker", "leased-external"), "unsupported service mode")
|
|
207
|
+
require(selection["protocol"] == "ollama-chat" or service_mode(selection) == "leased-external",
|
|
208
|
+
"non-Ollama services require a scoped external lease")
|
|
209
|
+
endpoint = local_endpoint(selection.get("endpoint"))
|
|
210
|
+
expected_path = {"systemone": "/v1/systemone", "ollama-chat": "/api/chat",
|
|
211
|
+
"moondream-point": "/v1/point"}[selection["protocol"]]
|
|
212
|
+
require(endpoint.path.endswith(expected_path), "endpoint does not match selected protocol")
|
|
213
|
+
scope = selection.get("gpu_uuids")
|
|
214
|
+
require(isinstance(scope, list) and 0 < len(scope) <= 3 and len(set(scope)) == len(scope)
|
|
215
|
+
and all(isinstance(gpu, str) and GPU_ID.fullmatch(gpu) for gpu in scope),
|
|
216
|
+
"exact nonempty GPU UUID scope required")
|
|
217
|
+
return model
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def service_mode(selection):
|
|
221
|
+
return selection.get("service_mode", "broker" if selection.get("protocol") == "ollama-chat"
|
|
222
|
+
else "leased-external")
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def validate_broker(discovery, selection):
|
|
226
|
+
require(isinstance(discovery, dict) and
|
|
227
|
+
discovery.get("schema") == "io.ollama-unify.gpu-negotiator.discovery.v1",
|
|
228
|
+
"broker discovery missing or incompatible")
|
|
229
|
+
require(discovery.get("available") is True and discovery.get("backend_available") is True
|
|
230
|
+
and discovery.get("backend") == "cuda", "CUDA broker unavailable")
|
|
231
|
+
health = discovery.get("gpu_health", {})
|
|
232
|
+
require(health.get("supported") is True and health.get("admission_blocked") is False
|
|
233
|
+
and not health.get("error") and not health.get("missing_selected_gpu_ids"),
|
|
234
|
+
"GPU broker blocks admission or cannot verify all selected hardware")
|
|
235
|
+
require(all(value in ("None", "none", "No action", None) for value in
|
|
236
|
+
health.get("recovery_actions", {}).values()), "NVIDIA recovery is required")
|
|
237
|
+
require(not discovery.get("draining"), "GPU broker is draining")
|
|
238
|
+
scope = set(selection["gpu_uuids"])
|
|
239
|
+
require(scope.issubset(set(discovery.get("selected_gpu_ids", []))),
|
|
240
|
+
"selected GPU is outside broker scope")
|
|
241
|
+
require(scope.issubset({row.get("uuid") for row in discovery.get("gpus", [])}),
|
|
242
|
+
"selected GPU missing from live broker inventory")
|
|
243
|
+
quarantined = discovery.get("unregistered_gpu_quarantine", [])
|
|
244
|
+
require(not quarantined, "unregistered CUDA workloads require broker resolution")
|
|
245
|
+
for lease in discovery.get("active_leases", []):
|
|
246
|
+
affected = not lease.get("gpu_uuids") or bool(scope.intersection(lease["gpu_uuids"]))
|
|
247
|
+
require(not (affected and lease.get("state") in ("pending", "revoking")),
|
|
248
|
+
"selected GPU has a pending or revoking lease")
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def process_identity(pid):
|
|
252
|
+
require(isinstance(pid, int) and not isinstance(pid, bool) and pid > 0, "invalid runner PID")
|
|
253
|
+
try:
|
|
254
|
+
stat = Path(f"/proc/{pid}/stat").read_text()
|
|
255
|
+
# comm may contain spaces or parentheses; fields after its last ')' start at state (3).
|
|
256
|
+
start_ticks = int(stat.rsplit(")", 1)[1].split()[19])
|
|
257
|
+
raw = Path(f"/proc/{pid}/cmdline").read_bytes()
|
|
258
|
+
except (OSError, ValueError, IndexError) as error:
|
|
259
|
+
raise GateError(f"runner identity cannot be verified: PID {pid}") from error
|
|
260
|
+
require(bool(raw), "runner has no command identity")
|
|
261
|
+
return {"pid": pid, "start_ticks": start_ticks, "command_sha256": hashlib.sha256(raw).hexdigest()}
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def process_cuda_scope(pid):
|
|
265
|
+
try:
|
|
266
|
+
entries = Path(f"/proc/{pid}/environ").read_bytes().split(b"\0")
|
|
267
|
+
values = [row.split(b"=", 1)[1].decode() for row in entries if row.startswith(b"CUDA_VISIBLE_DEVICES=")]
|
|
268
|
+
except (OSError, UnicodeError) as error:
|
|
269
|
+
raise GateError("cannot inspect actual process CUDA visibility") from error
|
|
270
|
+
require(len(values) == 1, "actual process lacks exact CUDA visibility")
|
|
271
|
+
return values[0].split(",")
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def descends_from(pid, ancestor):
|
|
275
|
+
seen = set()
|
|
276
|
+
for _ in range(64):
|
|
277
|
+
if pid == ancestor:
|
|
278
|
+
return True
|
|
279
|
+
if pid <= 1 or pid in seen:
|
|
280
|
+
return False
|
|
281
|
+
seen.add(pid)
|
|
282
|
+
try:
|
|
283
|
+
pid = int(Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[1])
|
|
284
|
+
except (OSError, ValueError, IndexError) as error:
|
|
285
|
+
raise GateError("runner process ancestry cannot be verified") from error
|
|
286
|
+
raise GateError("runner process ancestry exceeds bound")
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def endpoint_listener_matches(endpoint, pid):
|
|
290
|
+
parsed = local_endpoint(endpoint)
|
|
291
|
+
target = ipaddress.ip_address(parsed.hostname)
|
|
292
|
+
inodes = set()
|
|
293
|
+
try:
|
|
294
|
+
for family, name in ((socket.AF_INET, "tcp"), (socket.AF_INET6, "tcp6")):
|
|
295
|
+
for line in Path(f"/proc/{pid}/net/{name}").read_text().splitlines()[1:]:
|
|
296
|
+
fields = line.split()
|
|
297
|
+
address_hex, port_hex = fields[1].split(":")
|
|
298
|
+
if fields[3] != "0A" or int(port_hex, 16) != parsed.port:
|
|
299
|
+
continue
|
|
300
|
+
raw = bytes.fromhex(address_hex)
|
|
301
|
+
raw = b"".join(raw[i:i + 4][::-1] for i in range(0, len(raw), 4))
|
|
302
|
+
address = ipaddress.ip_address(socket.inet_ntop(family, raw))
|
|
303
|
+
if address == target:
|
|
304
|
+
inodes.add(fields[9])
|
|
305
|
+
require(bool(inodes), "exact loopback endpoint is not listening")
|
|
306
|
+
for descriptor in Path(f"/proc/{pid}/fd").iterdir():
|
|
307
|
+
try:
|
|
308
|
+
link = os.readlink(descriptor)
|
|
309
|
+
except FileNotFoundError:
|
|
310
|
+
continue
|
|
311
|
+
if any(link == f"socket:[{inode}]" for inode in inodes):
|
|
312
|
+
return True
|
|
313
|
+
except (OSError, ValueError, IndexError) as error:
|
|
314
|
+
raise GateError("endpoint listener ownership cannot be verified") from error
|
|
315
|
+
return False
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def collect_service_identity(selection, receipt, runners):
|
|
319
|
+
server = receipt.get("server", {})
|
|
320
|
+
identity = process_identity(server.get("pid"))
|
|
321
|
+
require(endpoint_listener_matches(selection["endpoint"], identity["pid"]),
|
|
322
|
+
"endpoint listener is not the attested server process")
|
|
323
|
+
related = [row["pid"] for row in runners if descends_from(row["pid"], identity["pid"])]
|
|
324
|
+
require(set(related) == {row["pid"] for row in receipt.get("runners", [])},
|
|
325
|
+
"receipt does not cover the endpoint's exact resident runner set")
|
|
326
|
+
return {"server": identity, "listener_verified": True,
|
|
327
|
+
"cuda_visible_devices": process_cuda_scope(identity["pid"]),
|
|
328
|
+
"runner_scopes": {str(pid): process_cuda_scope(pid) for pid in set(related)},
|
|
329
|
+
"runner_ancestry_verified": sorted(set(related))}
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def parse_gpu_rows(raw):
|
|
333
|
+
result = []
|
|
334
|
+
for row in csv.reader(raw.splitlines(), skipinitialspace=True):
|
|
335
|
+
require(len(row) == 4, "unrecognized nvidia-smi GPU inventory")
|
|
336
|
+
try:
|
|
337
|
+
result.append({"uuid": row[0].strip(), "name": row[1].strip(),
|
|
338
|
+
"memory_mib": int(row[2]), "compute_capability": float(row[3])})
|
|
339
|
+
except ValueError as error:
|
|
340
|
+
raise GateError("invalid nvidia-smi GPU inventory") from error
|
|
341
|
+
return result
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def parse_runner_rows(raw):
|
|
345
|
+
result = []
|
|
346
|
+
for row in csv.reader(raw.splitlines(), skipinitialspace=True):
|
|
347
|
+
require(len(row) == 3, "unrecognized nvidia-smi runner inventory")
|
|
348
|
+
try:
|
|
349
|
+
result.append({"pid": int(row[0]), "gpu_uuid": row[1].strip(),
|
|
350
|
+
"used_mib": int(row[2])})
|
|
351
|
+
except ValueError as error:
|
|
352
|
+
raise GateError("unverifiable nvidia-smi runner memory") from error
|
|
353
|
+
return result
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def collect_snapshot(selection, runtime_receipt):
|
|
357
|
+
# Test broker health before inspecting/connecting to any selected model endpoint.
|
|
358
|
+
discovery = strict_json(command(["docker", "gpu", "discover"]))
|
|
359
|
+
validate_broker(discovery, selection)
|
|
360
|
+
gpus = parse_gpu_rows(command(["nvidia-smi", "--query-gpu=uuid,name,memory.total,compute_cap",
|
|
361
|
+
"--format=csv,noheader,nounits"]))
|
|
362
|
+
runners = parse_runner_rows(command(["nvidia-smi", "--query-compute-apps=pid,gpu_uuid,used_gpu_memory",
|
|
363
|
+
"--format=csv,noheader,nounits"]))
|
|
364
|
+
identities = [process_identity(row["pid"]) for row in runtime_receipt.get("runners", [])]
|
|
365
|
+
snapshot = {"observed_at": time.time(), "discovery": discovery, "gpus": gpus,
|
|
366
|
+
"runners": runners, "processes": identities,
|
|
367
|
+
"boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip()}
|
|
368
|
+
if service_mode(selection) == "leased-external":
|
|
369
|
+
snapshot["service_identity"] = collect_service_identity(selection, runtime_receipt, runners)
|
|
370
|
+
if selection["protocol"] == "ollama-chat":
|
|
371
|
+
base = selection["endpoint"].removesuffix("/api/chat")
|
|
372
|
+
if service_mode(selection) == "broker":
|
|
373
|
+
require(base == discovery.get("public_ollama_api", "").rstrip("/"),
|
|
374
|
+
"Ollama must use public broker endpoint")
|
|
375
|
+
snapshot["ollama_ps"] = local_json(base + "/api/ps")
|
|
376
|
+
snapshot["ollama_version"] = local_json(base + "/api/version")
|
|
377
|
+
return snapshot
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def verify_preflight(selection, lock, receipt, snapshot, now=None):
|
|
381
|
+
now = time.time() if now is None else now
|
|
382
|
+
validate_selection(selection, lock)
|
|
383
|
+
validate_broker(snapshot["discovery"], selection)
|
|
384
|
+
require(0 <= now - snapshot.get("observed_at", 0) <= 30, "hardware observation is stale")
|
|
385
|
+
require(receipt.get("schema") == "omnius.computer-use.resident-runtime.v1" and
|
|
386
|
+
receipt.get("loaded") is True, "resident-runtime receipt is required")
|
|
387
|
+
for key in ("endpoint", "protocol", "model", "model_id", "weights_revision", "runtime_revision", "gpu_uuids"):
|
|
388
|
+
require(receipt.get(key) == selection.get(key), f"runtime receipt mismatch: {key}")
|
|
389
|
+
require(service_mode(receipt) == service_mode(selection), "runtime receipt service mode mismatch")
|
|
390
|
+
require(receipt.get("boot_id") == snapshot.get("boot_id") and bool(snapshot.get("boot_id")),
|
|
391
|
+
"runtime receipt belongs to another boot")
|
|
392
|
+
require(receipt.get("cuda_visible_devices") == selection["gpu_uuids"],
|
|
393
|
+
"server GPU visibility differs from reserved UUIDs")
|
|
394
|
+
scope = set(selection["gpu_uuids"])
|
|
395
|
+
inventory = {row["uuid"]: row for row in snapshot["gpus"]}
|
|
396
|
+
for gpu in scope:
|
|
397
|
+
require(gpu in inventory, "requested accelerator is missing")
|
|
398
|
+
hardware = inventory[gpu]
|
|
399
|
+
require("A100" in hardware["name"] and hardware["memory_mib"] >= 80_000
|
|
400
|
+
and math.isfinite(hardware["compute_capability"])
|
|
401
|
+
and hardware["compute_capability"] >= 8.0,
|
|
402
|
+
"requested test requires a capable A100 80 GB; no CPU or small-GPU fallback")
|
|
403
|
+
claimed = receipt.get("runners")
|
|
404
|
+
require(isinstance(claimed, list) and 0 < len(claimed) <= 16, "runner identities are required")
|
|
405
|
+
require(len({row.get("pid") for row in claimed}) == len(claimed), "duplicate runner identity")
|
|
406
|
+
require(sorted(claimed, key=lambda row: row["pid"]) ==
|
|
407
|
+
sorted(snapshot.get("processes", []), key=lambda row: row["pid"]),
|
|
408
|
+
"runner restarted or command identity changed")
|
|
409
|
+
actual_scope = set()
|
|
410
|
+
for identity in claimed:
|
|
411
|
+
allocations = [row for row in snapshot["runners"] if row["pid"] == identity["pid"]]
|
|
412
|
+
require(bool(allocations) and all(row["used_mib"] > 0 for row in allocations),
|
|
413
|
+
"requested model runner is not resident in VRAM")
|
|
414
|
+
placement = {row["gpu_uuid"] for row in allocations}
|
|
415
|
+
require(placement.issubset(scope), "test runner is resident on an unintended GPU")
|
|
416
|
+
actual_scope.update(placement)
|
|
417
|
+
require(actual_scope == scope, "actual runner placement does not cover exact requested scope")
|
|
418
|
+
if service_mode(selection) == "leased-external":
|
|
419
|
+
require(isinstance(receipt.get("lease_owner"), str) and bool(receipt["lease_owner"]),
|
|
420
|
+
"external CUDA service must name its lease owner")
|
|
421
|
+
require(any(lease.get("owner") == receipt["lease_owner"] and lease.get("state") == "active"
|
|
422
|
+
and set(lease.get("gpu_uuids", [])) == scope
|
|
423
|
+
for lease in snapshot["discovery"].get("active_leases", [])),
|
|
424
|
+
"external service lacks its exact active CUDA lease")
|
|
425
|
+
actual = snapshot.get("service_identity", {})
|
|
426
|
+
require(actual.get("listener_verified") is True and actual.get("server") == receipt.get("server")
|
|
427
|
+
and bool(receipt.get("server")), "endpoint server identity is unverified or changed")
|
|
428
|
+
require(actual.get("cuda_visible_devices") == selection["gpu_uuids"],
|
|
429
|
+
"actual server CUDA visibility differs from reserved UUIDs")
|
|
430
|
+
require(actual.get("runner_ancestry_verified") == sorted(row["pid"] for row in claimed),
|
|
431
|
+
"runner is not a descendant of the endpoint server")
|
|
432
|
+
require(actual.get("runner_scopes") == {str(row["pid"]): selection["gpu_uuids"] for row in claimed},
|
|
433
|
+
"actual runner CUDA visibility differs from reserved UUIDs")
|
|
434
|
+
if selection["protocol"] == "ollama-chat":
|
|
435
|
+
if service_mode(selection) == "broker":
|
|
436
|
+
require(selection["endpoint"] == snapshot["discovery"].get("public_ollama_api", "").rstrip("/") + "/api/chat",
|
|
437
|
+
"Ollama request bypasses the public broker")
|
|
438
|
+
require(snapshot.get("ollama_version", {}).get("version") == selection["runtime_revision"],
|
|
439
|
+
"Ollama runtime changed")
|
|
440
|
+
models = [row for row in snapshot.get("ollama_ps", {}).get("models", [])
|
|
441
|
+
if row.get("name", row.get("model")) == selection["model"]
|
|
442
|
+
and row.get("digest") == selection["weights_revision"]]
|
|
443
|
+
require(bool(models), "exact model tag and digest are not resident")
|
|
444
|
+
if service_mode(selection) == "broker":
|
|
445
|
+
require(all(set(row.get("gpu_uuids", [row.get("gpu_uuid")])) == scope for row in models),
|
|
446
|
+
"broker model placement differs from observed runner scope")
|
|
447
|
+
else:
|
|
448
|
+
require(len(snapshot["ollama_ps"]["models"]) == len(models) == 1,
|
|
449
|
+
"leased test endpoint must have exactly the selected resident model")
|
|
450
|
+
require(models[0].get("context_length") == 8192 and models[0].get("size_vram", 0) > 0,
|
|
451
|
+
"leased model must remain resident at the bounded 8192 context")
|
|
452
|
+
else:
|
|
453
|
+
# Non-Ollama weight identity is the trusted launcher's assertion, not a
|
|
454
|
+
# tensor introspection claim. Actual endpoint/PID/GPU identity is checked
|
|
455
|
+
# independently above; an unrelated resident PID cannot satisfy it.
|
|
456
|
+
require(receipt.get("attestation_source") == "host-launcher",
|
|
457
|
+
"external model identity requires a trusted host-launcher receipt")
|
|
458
|
+
identity = {"selection": selection, "boot_id": snapshot["boot_id"], "runners": claimed,
|
|
459
|
+
"receipt_sha256": digest(receipt)}
|
|
460
|
+
return {"schema": "omnius.computer-use.preflight.v1", "status": "ready_for_statement",
|
|
461
|
+
"observed_at": snapshot["observed_at"], "fingerprint": digest(identity),
|
|
462
|
+
**identity, "gpus": [inventory[gpu] for gpu in selection["gpu_uuids"]],
|
|
463
|
+
"resident_vram_mib": sum(row["used_mib"] for row in snapshot["runners"]
|
|
464
|
+
if row["pid"] in {item["pid"] for item in claimed})}
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def approve_preflight(preflight, statement, now=None, ttl=120):
|
|
468
|
+
now = time.time() if now is None else now
|
|
469
|
+
require(preflight.get("status") == "ready_for_statement", "preflight did not pass")
|
|
470
|
+
require(0 <= now - preflight["observed_at"] <= 30, "preflight is stale")
|
|
471
|
+
require(isinstance(statement, str) and 20 <= len(statement) <= 2000,
|
|
472
|
+
"record the exact preflight commentary before approving")
|
|
473
|
+
require(0 < ttl <= 300, "approval lifetime must be at most five minutes")
|
|
474
|
+
return {"schema": "omnius.computer-use.inference-approval.v1",
|
|
475
|
+
"fingerprint": preflight["fingerprint"], "selection": preflight["selection"],
|
|
476
|
+
"issued_at": now, "expires_at": now + ttl, "preflight_statement": statement}
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def verify_approval(approval, preflight, now=None):
|
|
480
|
+
now = time.time() if now is None else now
|
|
481
|
+
require(approval.get("schema") == "omnius.computer-use.inference-approval.v1",
|
|
482
|
+
"explicit preflight statement approval is required")
|
|
483
|
+
issued, expires = approval.get("issued_at", 0), approval.get("expires_at", 0)
|
|
484
|
+
require(isinstance(issued, (int, float)) and isinstance(expires, (int, float)) and
|
|
485
|
+
math.isfinite(issued) and math.isfinite(expires) and
|
|
486
|
+
issued <= now < expires and 0 < expires - issued <= 300,
|
|
487
|
+
"preflight approval is expired or has an invalid lifetime")
|
|
488
|
+
require(approval.get("fingerprint") == preflight["fingerprint"] and
|
|
489
|
+
approval.get("selection") == preflight["selection"],
|
|
490
|
+
"approval does not match the current endpoint, model, and resident runner")
|
|
491
|
+
require(isinstance(approval.get("preflight_statement"), str) and
|
|
492
|
+
len(approval["preflight_statement"]) >= 20, "preflight commentary is missing")
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def finite_request(model, case):
|
|
496
|
+
require(isinstance(case, dict), "decision case must be an object")
|
|
497
|
+
actions = case.get("legal_actions")
|
|
498
|
+
require(isinstance(actions, list) and 2 <= len(actions) <= 50, "require two to fifty legal choices")
|
|
499
|
+
criteria = {}
|
|
500
|
+
kinds = set()
|
|
501
|
+
for action in actions:
|
|
502
|
+
require(isinstance(action, dict) and isinstance(action.get("id"), str)
|
|
503
|
+
and ACTION_ID.fullmatch(action["id"]), "invalid candidate ID")
|
|
504
|
+
require(action["id"] not in criteria, "duplicate candidate ID")
|
|
505
|
+
require(isinstance(action.get("description"), str) and 0 < len(action["description"]) <= 1000,
|
|
506
|
+
"candidate description is missing or too long")
|
|
507
|
+
require(action.get("kind") in ("action", "observe", "abstain"), "invalid candidate kind")
|
|
508
|
+
kinds.add(action["kind"])
|
|
509
|
+
criteria[action["id"]] = action["description"]
|
|
510
|
+
require({"observe", "abstain"}.issubset(kinds), "observe and abstain must remain legal")
|
|
511
|
+
state = case.get("state")
|
|
512
|
+
require(isinstance(state, str) and 0 < len(state) <= 20_000, "compact redacted state required")
|
|
513
|
+
return {"model": model, "state": state, "questions": {"action": {
|
|
514
|
+
"type": "choice", "instructions": "Choose one supplied legal action. Observe or abstain if unsure.",
|
|
515
|
+
"criteria": criteria}}}
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def finite_answer(response, request):
|
|
519
|
+
require(isinstance(response, dict), "decision response must be an object")
|
|
520
|
+
answers = response.get("answers")
|
|
521
|
+
require(isinstance(answers, dict) and isinstance(answers.get("action"), dict),
|
|
522
|
+
"typed action answer missing")
|
|
523
|
+
answer = answers["action"]
|
|
524
|
+
require(answer.get("type", "choice") == "choice", "decision answer has wrong type")
|
|
525
|
+
choices = request["questions"]["action"]["criteria"]
|
|
526
|
+
choice = answer.get("choice")
|
|
527
|
+
require(isinstance(choice, str) and choice in choices, "model returned an unadvertised action")
|
|
528
|
+
probabilities = answer.get("probabilities")
|
|
529
|
+
if probabilities is not None:
|
|
530
|
+
require(isinstance(probabilities, dict) and set(probabilities) == set(choices),
|
|
531
|
+
"probabilities must cover the exact legal action set")
|
|
532
|
+
require(all(isinstance(value, (int, float)) and not isinstance(value, bool)
|
|
533
|
+
and math.isfinite(value) and 0 <= value <= 1 for value in probabilities.values()),
|
|
534
|
+
"probabilities must be finite values in [0,1]")
|
|
535
|
+
require(abs(sum(probabilities.values()) - 1) <= 0.02, "probabilities do not sum to one")
|
|
536
|
+
# Returned tools, code, risk, permissions and confidence never become executable authority.
|
|
537
|
+
return {"choice": choice, "probabilities": probabilities,
|
|
538
|
+
"probability_kind": "native_distribution" if probabilities is not None else "none"}
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def quantiles(values):
|
|
542
|
+
require(all(isinstance(value, (int, float)) and not isinstance(value, bool)
|
|
543
|
+
and math.isfinite(value) and value >= 0 for value in values), "invalid latency sample")
|
|
544
|
+
if not values:
|
|
545
|
+
return {"p50": None, "p95": None, "p99": None}
|
|
546
|
+
ordered = sorted(values)
|
|
547
|
+
def percentile(q):
|
|
548
|
+
index = (len(ordered) - 1) * q
|
|
549
|
+
lower, upper = math.floor(index), math.ceil(index)
|
|
550
|
+
return ordered[lower] + (ordered[upper] - ordered[lower]) * (index - lower)
|
|
551
|
+
return {name: percentile(q) for name, q in (("p50", .5), ("p95", .95), ("p99", .99))}
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def decision_metrics(rows, source, threshold=.9):
|
|
555
|
+
require(source in ("live_model", "synthetic_fixture_replay"), "measurement provenance required")
|
|
556
|
+
require(0 <= threshold <= 1 and math.isfinite(threshold), "invalid confidence threshold")
|
|
557
|
+
correct = sum(row.get("choice") == row.get("label") and row.get("status") == "ok" for row in rows)
|
|
558
|
+
valid = [row for row in rows if row.get("status") == "ok"]
|
|
559
|
+
calibrated = []
|
|
560
|
+
for row in valid:
|
|
561
|
+
request = finite_request("metrics-only", row)
|
|
562
|
+
answer = finite_answer({"answers": {"action": row}}, request)
|
|
563
|
+
require(row.get("label") in request["questions"]["action"]["criteria"], "unknown reference label")
|
|
564
|
+
require(row.get("label_source") in ("fixture_expectation", "verified_ui_receipt"),
|
|
565
|
+
"unverified success cannot be an evaluation label")
|
|
566
|
+
if row["label_source"] == "verified_ui_receipt":
|
|
567
|
+
require(re.fullmatch(r"[a-f0-9]{64}", row.get("verification_sha256", "")) is not None,
|
|
568
|
+
"verified UI label requires its receipt hash")
|
|
569
|
+
if answer["probabilities"] is not None:
|
|
570
|
+
calibrated.append((row, answer["probabilities"]))
|
|
571
|
+
brier = sum(sum((p - int(key == row["label"])) ** 2 for key, p in probs.items())
|
|
572
|
+
for row, probs in calibrated) / len(calibrated) if calibrated else None
|
|
573
|
+
bins = [[] for _ in range(10)]
|
|
574
|
+
accepted = []
|
|
575
|
+
for row, probs in calibrated:
|
|
576
|
+
confidence = probs[row["choice"]]
|
|
577
|
+
success = row["choice"] == row["label"]
|
|
578
|
+
bins[min(int(confidence * 10), 9)].append((confidence, success))
|
|
579
|
+
if confidence >= threshold:
|
|
580
|
+
accepted.append(success)
|
|
581
|
+
ece = sum(abs(sum(conf for conf, _ in group) / len(group) - sum(ok for _, ok in group) / len(group))
|
|
582
|
+
* len(group) / len(calibrated) for group in bins if group) if calibrated else None
|
|
583
|
+
quality = {"accuracy_including_errors": correct / len(rows) if rows else None,
|
|
584
|
+
"native_probability_count": len(calibrated), "multiclass_brier_sum": brier,
|
|
585
|
+
"ece_10_equal_width_bins": ece, "selective_threshold": threshold,
|
|
586
|
+
"selective_coverage": len(accepted) / len(rows) if calibrated else None,
|
|
587
|
+
"selective_risk": 1 - sum(accepted) / len(accepted) if accepted else None}
|
|
588
|
+
return {"measurement_kind": source, "count": len(rows), "valid_count": len(valid),
|
|
589
|
+
"error_count": len(rows) - len(valid),
|
|
590
|
+
"model_quality_metrics": quality if source == "live_model" else None,
|
|
591
|
+
"fixture_contract_metrics": quality if source == "synthetic_fixture_replay" else None,
|
|
592
|
+
"stage_latency_ms": {"rank": quantiles([row["latency_ms"] for row in rows
|
|
593
|
+
if "latency_ms" in row]),
|
|
594
|
+
**{key: None for key in ("observe", "ground", "plan", "queue", "execute", "verify")}},
|
|
595
|
+
"cold_start_measured": False, "peak_vram_mib": None,
|
|
596
|
+
"limitations": ["Choice accuracy is not GUI task completion.",
|
|
597
|
+
"Rank timing includes local HTTP transport; no forward-only timing is claimed.",
|
|
598
|
+
"No cold-start or continuously sampled peak VRAM measurement was performed."]}
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def load_corpus(path):
|
|
602
|
+
raw = bounded_bytes(path)
|
|
603
|
+
rows = [strict_json(line) for line in raw.splitlines() if line.strip()]
|
|
604
|
+
require(0 < len(rows) <= 1000, "corpus must contain one to one thousand cases")
|
|
605
|
+
require(len({row.get("id") for row in rows}) == len(rows), "corpus case IDs must be unique")
|
|
606
|
+
for row in rows:
|
|
607
|
+
request = finite_request("corpus-check", row)
|
|
608
|
+
require(isinstance(row.get("id"), str) and bool(row["id"]), "corpus case ID required")
|
|
609
|
+
require(row.get("label") in request["questions"]["action"]["criteria"], "reference label is not legal")
|
|
610
|
+
require(row.get("label_source") in ("fixture_expectation", "verified_ui_receipt"),
|
|
611
|
+
"unverified success cannot label a case")
|
|
612
|
+
return rows, hashlib.sha256(raw).hexdigest()
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def verify_corpus_lock(lock, sha, point=False):
|
|
616
|
+
key = "point_corpus_sha256" if point else "corpus_sha256"
|
|
617
|
+
require(lock.get("benchmark", {}).get(key) == sha,
|
|
618
|
+
"validated corpus bytes do not match the frozen source lock")
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def ollama_finite_request(selection, finite):
|
|
622
|
+
choices = finite["questions"]["action"]["criteria"]
|
|
623
|
+
return {"model": selection["model"], "stream": False, "think": False,
|
|
624
|
+
"keep_alive": "10m", "options": {"num_ctx": 8192, "num_predict": 256, "temperature": 0},
|
|
625
|
+
"format": {"type": "object", "properties": {"choice": {"type": "string", "enum": list(choices)}},
|
|
626
|
+
"required": ["choice"], "additionalProperties": False},
|
|
627
|
+
"messages": [{"role": "system", "content": "Choose exactly one supplied legal action. Observe or abstain if unsure. Return only the JSON object with choice. Do not invent probabilities."},
|
|
628
|
+
{"role": "user", "content": json.dumps({"state": finite["state"], "choices": choices})}]}
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def ollama_finite_answer(response, selection, request):
|
|
632
|
+
require(response.get("done") is True and response.get("model") == selection["model"],
|
|
633
|
+
"bounded Ollama decision did not complete for selected model")
|
|
634
|
+
value = strict_json(response.get("message", {}).get("content", ""))
|
|
635
|
+
require(isinstance(value, dict) and set(value) == {"choice"}, "Ollama decision must contain only choice")
|
|
636
|
+
return finite_answer({"answers": {"action": {"choice": value["choice"]}}}, request)
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def run_policy_benchmark(selection, lock, receipt, approval, cases, collector=collect_snapshot,
|
|
640
|
+
sender=local_json, clock=time.time, timer=time.perf_counter, checkpoint=None):
|
|
641
|
+
require(selection.get("protocol") in ("systemone", "ollama-chat"), "policy protocol unsupported")
|
|
642
|
+
rows = []
|
|
643
|
+
for case in cases:
|
|
644
|
+
request = finite_request(selection["model"], case)
|
|
645
|
+
# Recheck for every request, not just before the first warm-up.
|
|
646
|
+
try:
|
|
647
|
+
preflight = verify_preflight(selection, lock, receipt, collector(selection, receipt), clock())
|
|
648
|
+
verify_approval(approval, preflight, clock())
|
|
649
|
+
except GateError as error:
|
|
650
|
+
error.partial_rows = rows
|
|
651
|
+
raise
|
|
652
|
+
started = timer()
|
|
653
|
+
try:
|
|
654
|
+
if selection["protocol"] == "ollama-chat":
|
|
655
|
+
response = sender(selection["endpoint"], ollama_finite_request(selection, request),
|
|
656
|
+
headers={"X-Ollama-Unify-GPU-UUIDs": ",".join(selection["gpu_uuids"])}, timeout=60)
|
|
657
|
+
answer = ollama_finite_answer(response, selection, request)
|
|
658
|
+
else:
|
|
659
|
+
answer = finite_answer(sender(selection["endpoint"], request, timeout=30), request)
|
|
660
|
+
rows.append({**case, **answer, "status": "ok", "latency_ms": (timer() - started) * 1000})
|
|
661
|
+
except GateError as error:
|
|
662
|
+
rows.append({**case, "status": "error", "error": str(error),
|
|
663
|
+
"latency_ms": (timer() - started) * 1000})
|
|
664
|
+
if checkpoint:
|
|
665
|
+
checkpoint(rows)
|
|
666
|
+
return rows
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def main(argv=None):
|
|
670
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
671
|
+
parser.add_argument("--lock", type=Path, default=DEFAULT_LOCK)
|
|
672
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
673
|
+
for name in ("preflight", "approve", "benchmark-policy", "smoke-planner"):
|
|
674
|
+
item = commands.add_parser(name)
|
|
675
|
+
item.add_argument("--selection", required=True, type=Path)
|
|
676
|
+
item.add_argument("--runtime-receipt", required=True, type=Path)
|
|
677
|
+
item.add_argument("--output", required=True, type=Path)
|
|
678
|
+
if name == "approve":
|
|
679
|
+
item.add_argument("--statement", required=True)
|
|
680
|
+
if name in ("benchmark-policy", "smoke-planner"):
|
|
681
|
+
item.add_argument("--approval", required=True, type=Path)
|
|
682
|
+
if name == "benchmark-policy":
|
|
683
|
+
item.add_argument("--corpus", required=True, type=Path)
|
|
684
|
+
check = commands.add_parser("check-corpus")
|
|
685
|
+
check.add_argument("--corpus", required=True, type=Path)
|
|
686
|
+
args = parser.parse_args(argv)
|
|
687
|
+
inference_attempted = False
|
|
688
|
+
try:
|
|
689
|
+
if args.command == "check-corpus":
|
|
690
|
+
cases, sha = load_corpus(args.corpus)
|
|
691
|
+
print(json.dumps({"status": "valid", "cases": len(cases), "sha256": sha,
|
|
692
|
+
"model_inference_performed": False}))
|
|
693
|
+
return 0
|
|
694
|
+
selection, receipt = read_json(args.selection), read_json(args.runtime_receipt)
|
|
695
|
+
lock, lock_sha = read_json_hashed(args.lock)
|
|
696
|
+
validate_selection(selection, lock)
|
|
697
|
+
preflight = verify_preflight(selection, lock, receipt, collect_snapshot(selection, receipt))
|
|
698
|
+
if args.command == "preflight":
|
|
699
|
+
result = preflight
|
|
700
|
+
elif args.command == "approve":
|
|
701
|
+
result = approve_preflight(preflight, args.statement)
|
|
702
|
+
else:
|
|
703
|
+
approval = read_json(args.approval)
|
|
704
|
+
verify_approval(approval, preflight)
|
|
705
|
+
write_json(args.output, {"schema": "omnius.computer-use.run-start.v1", "status": "in_progress",
|
|
706
|
+
"selection": selection, "preflight_fingerprint": preflight["fingerprint"],
|
|
707
|
+
"models_lock_sha256": lock_sha})
|
|
708
|
+
if args.command == "benchmark-policy":
|
|
709
|
+
cases, sha = load_corpus(args.corpus)
|
|
710
|
+
verify_corpus_lock(lock, sha)
|
|
711
|
+
def checkpoint(rows):
|
|
712
|
+
write_json(args.output, {"schema": "omnius.computer-use.policy-report.v1",
|
|
713
|
+
"status": "in_progress", "selection": selection,
|
|
714
|
+
"corpus_sha256": sha, "rows": rows})
|
|
715
|
+
inference_attempted = True
|
|
716
|
+
rows = run_policy_benchmark(selection, lock, receipt, approval, cases, checkpoint=checkpoint)
|
|
717
|
+
result = {"schema": "omnius.computer-use.policy-report.v1", "selection": selection,
|
|
718
|
+
"status": "completed" if all(row["status"] == "ok" for row in rows)
|
|
719
|
+
else "completed_with_errors", "corpus_sha256": sha, "rows": rows,
|
|
720
|
+
"models_lock_sha256": lock_sha,
|
|
721
|
+
"preflight_fingerprint": preflight["fingerprint"],
|
|
722
|
+
"corpus_purpose": "finite action selection; no UI input or task completion",
|
|
723
|
+
**decision_metrics(rows, "live_model")}
|
|
724
|
+
else:
|
|
725
|
+
require(selection["protocol"] == "ollama-chat", "planner smoke requires Ollama chat")
|
|
726
|
+
body = {"model": selection["model"], "stream": False, "think": False,
|
|
727
|
+
"messages": [{"role": "user", "content": "Return only the JSON object {\"ready\":true}."}],
|
|
728
|
+
"format": "json", "keep_alive": "10m",
|
|
729
|
+
"options": {"num_ctx": 8192, "num_predict": 32, "temperature": 0}}
|
|
730
|
+
started = time.perf_counter()
|
|
731
|
+
inference_attempted = True
|
|
732
|
+
response = local_json(selection["endpoint"], body, headers={
|
|
733
|
+
"X-Ollama-Unify-GPU-UUIDs": ",".join(selection["gpu_uuids"])}, timeout=60)
|
|
734
|
+
require(response.get("done") is True and response.get("model") == selection["model"],
|
|
735
|
+
"planner smoke did not complete for the selected tag")
|
|
736
|
+
require(strict_json(response.get("message", {}).get("content", "")) == {"ready": True},
|
|
737
|
+
"planner smoke did not return the requested structured output")
|
|
738
|
+
result = {"schema": "omnius.computer-use.planner-smoke.v1", "status": "passed",
|
|
739
|
+
"selection": selection, "latency_ms": (time.perf_counter() - started) * 1000,
|
|
740
|
+
"task_completion_claimed": False}
|
|
741
|
+
write_json(args.output, result)
|
|
742
|
+
print(json.dumps({"status": result.get("status", "recorded"), "artifact": str(args.output)}))
|
|
743
|
+
return 3 if result.get("status") == "completed_with_errors" else 0
|
|
744
|
+
except (GateError, OSError, KeyError, TypeError) as error:
|
|
745
|
+
failure = {"schema": "omnius.computer-use.gate-failure.v1", "status": "blocked",
|
|
746
|
+
"reason": str(error), "inference_status": "possibly_started" if inference_attempted
|
|
747
|
+
else "not_attempted", "partial_rows": getattr(error, "partial_rows", [])}
|
|
748
|
+
if hasattr(args, "output"):
|
|
749
|
+
write_json(args.output, failure)
|
|
750
|
+
print(json.dumps(failure), file=sys.stderr)
|
|
751
|
+
return 2
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
if __name__ == "__main__":
|
|
755
|
+
raise SystemExit(main())
|