omnius 1.0.842 → 1.0.845
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/py-embed.js +651 -91
- package/dist/computer-use/models.lock.yaml +495 -0
- package/dist/index.js +2 -2
- package/dist/library.d.ts +1 -0
- package/dist/library.js +176498 -55325
- package/dist/scripts/rapid-computer-use/gpu_lease.py +64 -0
- package/dist/scripts/rapid-computer-use/laya_service.py +483 -0
- package/dist/scripts/rapid-computer-use/model_gate.py +755 -0
- package/dist/scripts/rapid-computer-use/photon_service.py +700 -0
- package/dist/scripts/rapid-computer-use/point_benchmark.py +278 -0
- package/dist/update-worker.js +651 -91
- package/docs/DISCOVERY.json +1781 -808
- package/docs/DISCOVERY.md +25 -0
- package/docs/guides/bring-your-own-inference.md +128 -0
- package/docs/reference/slash-commands.md +138 -22
- package/docs/work-orders/WO-active-inference-route-and-telegram-review-20261012.md +94 -0
- package/docs/work-orders/WO-inference-fallback-20261012.md +89 -0
- package/docs/work-orders/WO-primary-checkout-fallback-delivery-20261012.md +94 -0
- package/docs/work-orders/WO-telegram-fallback-20261012.md +83 -0
- package/docs/work-orders/active-inference-route-remote-probes-20261012.json +34 -0
- package/docs/work-orders/inference-fallback-bundle-20261012.json +75 -0
- package/docs/work-orders/inference-fallback-live-20261012.json +39 -0
- package/docs/work-orders/primary-inference-fallback-bundle-20261012.json +88 -0
- package/docs/work-orders/rapid-computer-use-20261011/README.md +125 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-01-native-save-recovery.md +86 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-02-semantic-control.md +93 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-03-policy-workflows.md +130 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-04-live-model-gates.md +322 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-05-verification-delivery.md +389 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-06-medium-horizon-runtime.md +166 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-07-tool-routing-prompts.md +81 -0
- package/docs/work-orders/rapid-computer-use-20261011/WO-08-cross-application-desktop.md +214 -0
- package/docs/work-orders/rapid-computer-use-20261011/architecture-source.md +871 -0
- package/docs/work-orders/telegram-progress-browser-20261011/README.md +79 -0
- package/docs/work-orders/telegram-progress-browser-20261011/WO-01-diagnostic-visibility.md +64 -0
- package/docs/work-orders/telegram-progress-browser-20261011/WO-02-task-narration.md +127 -0
- package/docs/work-orders/telegram-progress-browser-20261011/WO-03-browser-observation.md +51 -0
- package/docs/work-orders/telegram-progress-browser-20261011/WO-04-download-desktop-handoff.md +96 -0
- package/docs/work-orders/telegram-progress-browser-20261011/WO-05-verification-delivery.md +129 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/prompts/agentic/system-common.md +7 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Bounded scoped broker transitions, without putting lease secrets in argv/logs."""
|
|
2
|
+
import hmac
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import socket
|
|
6
|
+
import time
|
|
7
|
+
|
|
8
|
+
import model_gate as gate
|
|
9
|
+
|
|
10
|
+
CONTROL_SOCKET = "/run/ollama-unify/gpu-negotiator.sock"
|
|
11
|
+
MAX_REPLY_BYTES = 128 * 1024
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ScopedGpuLease:
|
|
15
|
+
def __init__(self, owner, gpu_uuids, control_socket=CONTROL_SOCKET, environ=None,
|
|
16
|
+
socket_factory=socket.socket, timeout=10):
|
|
17
|
+
environment = os.environ if environ is None else environ
|
|
18
|
+
token = environment.get("OLLAMA_UNIFY_GPU_LEASE", "")
|
|
19
|
+
gate.require(isinstance(token, str) and 16 <= len(token) <= 1024,
|
|
20
|
+
"supervisor-owned GPU lease environment is required")
|
|
21
|
+
gate.require(isinstance(owner, str) and bool(owner) and isinstance(gpu_uuids, list)
|
|
22
|
+
and 0 < len(gpu_uuids) <= 3 and len(set(gpu_uuids)) == len(gpu_uuids)
|
|
23
|
+
and all(isinstance(value, str) and gate.GPU_ID.fullmatch(value) for value in gpu_uuids),
|
|
24
|
+
"exact lease owner and GPU scope required")
|
|
25
|
+
gate.require(0 < timeout <= 30, "lease transition timeout is outside bounds")
|
|
26
|
+
self._token, self.owner, self.gpu_uuids = token, owner, list(gpu_uuids)
|
|
27
|
+
self.control_socket, self.socket_factory, self.timeout = control_socket, socket_factory, timeout
|
|
28
|
+
|
|
29
|
+
def _transition(self, action, expected_state):
|
|
30
|
+
# Do not return or log the broker's raw response: it includes the token.
|
|
31
|
+
try:
|
|
32
|
+
deadline = time.monotonic() + self.timeout
|
|
33
|
+
with self.socket_factory(socket.AF_UNIX, socket.SOCK_STREAM) as stream:
|
|
34
|
+
stream.settimeout(self.timeout)
|
|
35
|
+
stream.connect(self.control_socket)
|
|
36
|
+
stream.sendall(json.dumps({"action": action, "token": self._token}).encode() + b"\n")
|
|
37
|
+
raw = bytearray()
|
|
38
|
+
while b"\n" not in raw:
|
|
39
|
+
remaining = deadline - time.monotonic()
|
|
40
|
+
gate.require(remaining > 0, "lease transition timed out")
|
|
41
|
+
stream.settimeout(remaining)
|
|
42
|
+
block = stream.recv(min(4096, MAX_REPLY_BYTES + 1 - len(raw)))
|
|
43
|
+
gate.require(bool(block), "broker closed before transition confirmation")
|
|
44
|
+
raw.extend(block)
|
|
45
|
+
gate.require(len(raw) <= MAX_REPLY_BYTES, "broker reply exceeds bound")
|
|
46
|
+
response = gate.strict_json(bytes(raw).split(b"\n", 1)[0])
|
|
47
|
+
lease = response.get("lease", {})
|
|
48
|
+
gate.require(response.get("ok") is True and isinstance(lease, dict), "broker denied transition")
|
|
49
|
+
token = lease.get("token")
|
|
50
|
+
gate.require(isinstance(token, str) and hmac.compare_digest(token, self._token)
|
|
51
|
+
and lease.get("owner") == self.owner
|
|
52
|
+
and lease.get("gpu_uuids") == self.gpu_uuids
|
|
53
|
+
and lease.get("state") == expected_state, "broker transition identity mismatch")
|
|
54
|
+
except Exception:
|
|
55
|
+
# Even an unexpected broker exception/JSON field must not leak a secret.
|
|
56
|
+
raise gate.GateError(f"GPU lease {action} was not confirmed; no further CUDA work is permitted") from None
|
|
57
|
+
return {"action": action, "owner": self.owner, "gpu_uuids": self.gpu_uuids,
|
|
58
|
+
"state": expected_state}
|
|
59
|
+
|
|
60
|
+
def prepare(self):
|
|
61
|
+
return self._transition("prepare", "pending")
|
|
62
|
+
|
|
63
|
+
def ready(self):
|
|
64
|
+
return self._transition("ready", "active")
|
|
@@ -0,0 +1,483 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Host-staged, local-only Laya evaluation service; never permits CPU inference.
|
|
3
|
+
|
|
4
|
+
Start with ``serve --selection ... --manifest ... --state-dir ... --lease-owner ...``.
|
|
5
|
+
Inspect idle.json and the actual process environment, then use authorize-load.
|
|
6
|
+
After resident-runtime.json exists, independently preflight and state the GPU
|
|
7
|
+
result before writing model_gate's approval to inference-approval.json. Neither
|
|
8
|
+
HTTP requests nor the service itself can authorize loading or inference.
|
|
9
|
+
"""
|
|
10
|
+
import argparse
|
|
11
|
+
import hashlib
|
|
12
|
+
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
13
|
+
import importlib
|
|
14
|
+
import importlib.metadata
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
import secrets
|
|
19
|
+
import socket
|
|
20
|
+
import stat
|
|
21
|
+
import sys
|
|
22
|
+
import threading
|
|
23
|
+
import time
|
|
24
|
+
|
|
25
|
+
import model_gate as gate
|
|
26
|
+
import gpu_lease
|
|
27
|
+
|
|
28
|
+
CODE_REVISION = "c4a805b5160621bbfc53417126990a67d3163bc5"
|
|
29
|
+
WEIGHTS_REVISION = "7b928d828b7b0e022f929d9bd2e44165aa270148"
|
|
30
|
+
ENCODER_REVISION = "45bb4654a4d5aaff24dd11d4781fa46d39bf8c13"
|
|
31
|
+
MAX_REQUEST_BYTES = 96 * 1024
|
|
32
|
+
INFERENCE_DEADLINE_SECONDS = 30
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class StrictCudaError(gate.GateError):
|
|
36
|
+
"""Deliberately not RuntimeError, which upstream catches to retry on CPU."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def file_sha256(path):
|
|
40
|
+
digest = hashlib.sha256()
|
|
41
|
+
with Path(path).open("rb") as stream:
|
|
42
|
+
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
|
43
|
+
digest.update(block)
|
|
44
|
+
return digest.hexdigest()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def private_json(path):
|
|
48
|
+
path = Path(path)
|
|
49
|
+
info = path.lstat()
|
|
50
|
+
gate.require(stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid()
|
|
51
|
+
and stat.S_IMODE(info.st_mode) & 0o077 == 0,
|
|
52
|
+
"host marker must be an owner-only regular file")
|
|
53
|
+
return gate.read_json(path)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def verify_files(directory, entries, suffix=None):
|
|
57
|
+
directory = Path(directory).resolve(strict=True)
|
|
58
|
+
gate.require(isinstance(entries, dict) and 0 < len(entries) <= 1000,
|
|
59
|
+
"bounded checkpoint/source file manifest required")
|
|
60
|
+
found = {str(path.relative_to(directory)) for path in directory.rglob("*")
|
|
61
|
+
if path.is_file() and (suffix is None or path.suffix == suffix)}
|
|
62
|
+
gate.require(found == set(entries), "checkpoint/source file inventory changed")
|
|
63
|
+
for name, expected in entries.items():
|
|
64
|
+
path = directory / name
|
|
65
|
+
gate.require(not Path(name).is_absolute() and ".." not in Path(name).parts
|
|
66
|
+
and path.resolve().is_relative_to(directory), "manifest path escapes its directory")
|
|
67
|
+
gate.require(isinstance(expected, str) and len(expected) == 64
|
|
68
|
+
and file_sha256(path) == expected, f"checkpoint/source hash mismatch: {name}")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def verify_manifest(manifest):
|
|
72
|
+
gate.require(manifest.get("schema") == "omnius.computer-use.laya-checkpoint.v1"
|
|
73
|
+
and manifest.get("code_revision") == CODE_REVISION
|
|
74
|
+
and manifest.get("weights_revision") == WEIGHTS_REVISION,
|
|
75
|
+
"Laya checkpoint/runtime pins differ from reviewed source")
|
|
76
|
+
encoder = manifest.get("encoder", {})
|
|
77
|
+
gate.require(encoder.get("repository") == "answerdotai/ModernBERT-large"
|
|
78
|
+
and encoder.get("revision") == ENCODER_REVISION,
|
|
79
|
+
"encoder config must use the pinned ModernBERT revision")
|
|
80
|
+
files = manifest.get("files", {})
|
|
81
|
+
gate.require({"model.safetensors", "rl_agent_config.json", "encoder/config.json",
|
|
82
|
+
"tokenizer/tokenizer.json", "tokenizer/tokenizer_config.json"}.issubset(files),
|
|
83
|
+
"resolved local checkpoint is incomplete")
|
|
84
|
+
gate.require(files["encoder/config.json"] == encoder.get("config_sha256"),
|
|
85
|
+
"encoder config digest differs from source receipt")
|
|
86
|
+
verify_files(manifest["checkpoint"], files)
|
|
87
|
+
package = Path(importlib.metadata.distribution("laya").locate_file("laya")).resolve()
|
|
88
|
+
gate.require(package == Path(manifest["laya_package_root"]).resolve(),
|
|
89
|
+
"installed Laya package differs from the prepared runtime")
|
|
90
|
+
verify_files(package, manifest["laya_code_files"], suffix=".py")
|
|
91
|
+
for name, version in manifest["versions"].items():
|
|
92
|
+
gate.require(importlib.metadata.version(name) == version, f"runtime version changed: {name}")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def validate_request(value, model):
|
|
96
|
+
gate.require(isinstance(value, dict) and set(value) == {"model", "state", "questions"},
|
|
97
|
+
"request must contain only model, state and questions")
|
|
98
|
+
gate.require(value["model"] == model, "request model is not the exact loaded model")
|
|
99
|
+
gate.require(isinstance(value["state"], str) and 0 < len(value["state"]) <= 20_000,
|
|
100
|
+
"state must be a bounded redacted string")
|
|
101
|
+
questions = value["questions"]
|
|
102
|
+
gate.require(isinstance(questions, dict) and set(questions) == {"action"},
|
|
103
|
+
"only one finite action question is supported")
|
|
104
|
+
action = questions["action"]
|
|
105
|
+
gate.require(isinstance(action, dict) and set(action) == {"type", "instructions", "criteria"}
|
|
106
|
+
and action["type"] == "choice", "finite choice question required")
|
|
107
|
+
gate.require(isinstance(action["instructions"], str) and 0 < len(action["instructions"]) <= 2000,
|
|
108
|
+
"bounded action instructions required")
|
|
109
|
+
criteria = action["criteria"]
|
|
110
|
+
gate.require(isinstance(criteria, dict) and 2 <= len(criteria) <= 50,
|
|
111
|
+
"require two to fifty finite choices")
|
|
112
|
+
for key, description in criteria.items():
|
|
113
|
+
gate.require(isinstance(key, str) and gate.ACTION_ID.fullmatch(key)
|
|
114
|
+
and isinstance(description, str) and 0 < len(description) <= 1000,
|
|
115
|
+
"invalid candidate ID or description")
|
|
116
|
+
return value
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class StrictCudaGuard:
|
|
120
|
+
"""Install before Agent's first model.to and before any possible forward."""
|
|
121
|
+
def __init__(self, authorized, is_tensor):
|
|
122
|
+
self.authorized = authorized
|
|
123
|
+
self.is_tensor = is_tensor
|
|
124
|
+
|
|
125
|
+
@staticmethod
|
|
126
|
+
def require_device(device):
|
|
127
|
+
if str(device) != "cuda:0":
|
|
128
|
+
raise StrictCudaError("strict CUDA forbids CPU fallback or another device")
|
|
129
|
+
|
|
130
|
+
def assert_resident(self, model):
|
|
131
|
+
parameters = list(model.parameters())
|
|
132
|
+
if not parameters:
|
|
133
|
+
raise StrictCudaError("loaded model has no resident parameters")
|
|
134
|
+
for tensor in parameters + list(model.buffers()):
|
|
135
|
+
self.require_device(tensor.device)
|
|
136
|
+
|
|
137
|
+
def before_forward(self, model, args, kwargs):
|
|
138
|
+
if not self.authorized():
|
|
139
|
+
raise StrictCudaError("forward pass is forbidden before host inference approval")
|
|
140
|
+
self.assert_resident(model)
|
|
141
|
+
pending = [args, kwargs]
|
|
142
|
+
while pending:
|
|
143
|
+
value = pending.pop()
|
|
144
|
+
if self.is_tensor(value):
|
|
145
|
+
self.require_device(value.device)
|
|
146
|
+
elif isinstance(value, (tuple, list)):
|
|
147
|
+
pending.extend(value)
|
|
148
|
+
elif isinstance(value, dict):
|
|
149
|
+
pending.extend(value.values())
|
|
150
|
+
|
|
151
|
+
def install(self, model):
|
|
152
|
+
original_to = model.to
|
|
153
|
+
|
|
154
|
+
def guarded_to(*args, **kwargs):
|
|
155
|
+
device = kwargs.get("device", args[0] if args else None)
|
|
156
|
+
self.require_device(device)
|
|
157
|
+
return original_to(*args, **kwargs)
|
|
158
|
+
|
|
159
|
+
model.to = guarded_to
|
|
160
|
+
model.register_forward_pre_hook(self.before_forward, with_kwargs=True)
|
|
161
|
+
return model
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def deny_outbound_runtime_connections():
|
|
165
|
+
"""Dedicated service process: preserve incoming HTTP, deny network clients."""
|
|
166
|
+
original_connect, original_connect_ex = socket.socket.connect, socket.socket.connect_ex
|
|
167
|
+
|
|
168
|
+
def guarded(original):
|
|
169
|
+
def connect(stream, address):
|
|
170
|
+
if stream.family in (socket.AF_INET, socket.AF_INET6):
|
|
171
|
+
raise StrictCudaError("outbound runtime networking is disabled")
|
|
172
|
+
return original(stream, address)
|
|
173
|
+
return connect
|
|
174
|
+
|
|
175
|
+
socket.socket.connect = guarded(original_connect)
|
|
176
|
+
socket.socket.connect_ex = guarded(original_connect_ex)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class LocalLayaAdapter:
|
|
180
|
+
def __init__(self, manifest, authorized):
|
|
181
|
+
# No torch/laya import occurs until the host load marker was verified.
|
|
182
|
+
verify_manifest(manifest)
|
|
183
|
+
os.environ.update(HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1",
|
|
184
|
+
HF_HUB_DISABLE_TELEMETRY="1", TOKENIZERS_PARALLELISM="false")
|
|
185
|
+
deny_outbound_runtime_connections()
|
|
186
|
+
import torch
|
|
187
|
+
gate.require(torch.cuda.is_available() and torch.cuda.device_count() == 1,
|
|
188
|
+
"strict CUDA requires exactly the reserved visible GPU")
|
|
189
|
+
properties = torch.cuda.get_device_properties(0)
|
|
190
|
+
gate.require("A100" in properties.name and properties.total_memory >= 80_000 * 1024**2
|
|
191
|
+
and properties.major >= 8, "load probe requires the reserved capable A100 80 GB")
|
|
192
|
+
module = importlib.import_module("laya.agent")
|
|
193
|
+
self.guard = StrictCudaGuard(authorized, torch.is_tensor)
|
|
194
|
+
original_factory = module.build_model
|
|
195
|
+
|
|
196
|
+
def guarded_factory(*args, **kwargs):
|
|
197
|
+
return self.guard.install(original_factory(*args, **kwargs))
|
|
198
|
+
|
|
199
|
+
module.build_model = guarded_factory
|
|
200
|
+
try:
|
|
201
|
+
self.agent = module.Agent(manifest["checkpoint"], device="cuda:0", fast=False,
|
|
202
|
+
compile=False, compile_warmup=False,
|
|
203
|
+
expected_sha256=manifest["files"])
|
|
204
|
+
finally:
|
|
205
|
+
module.build_model = original_factory
|
|
206
|
+
self.assert_resident()
|
|
207
|
+
gate.require(self.agent.backend == "eager", "only the reviewed eager backend is permitted")
|
|
208
|
+
self.synchronize()
|
|
209
|
+
|
|
210
|
+
def assert_resident(self):
|
|
211
|
+
self.guard.require_device(self.agent.device)
|
|
212
|
+
self.guard.assert_resident(self.agent.model)
|
|
213
|
+
|
|
214
|
+
def predict(self, request):
|
|
215
|
+
self.assert_resident()
|
|
216
|
+
result = self.agent.system_one(request["state"], request["questions"],
|
|
217
|
+
max_len=512, head_max_len=192)
|
|
218
|
+
self.assert_resident()
|
|
219
|
+
return result
|
|
220
|
+
|
|
221
|
+
def synchronize(self):
|
|
222
|
+
import torch
|
|
223
|
+
self.assert_resident()
|
|
224
|
+
torch.cuda.synchronize(0)
|
|
225
|
+
self.assert_resident()
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
class StagedService:
|
|
229
|
+
def __init__(self, selection, lock, manifest, manifest_sha, state_dir, lease_owner,
|
|
230
|
+
adapter_factory=LocalLayaAdapter, identity=None, cuda_scope=None,
|
|
231
|
+
boot_id=None, clock=time.time, preflight=None, timer_factory=threading.Timer,
|
|
232
|
+
terminate=os._exit, lease=None):
|
|
233
|
+
gate.validate_selection(selection, lock)
|
|
234
|
+
endpoint = gate.local_endpoint(selection["endpoint"])
|
|
235
|
+
gate.require(selection["model_id"] == "laya" and selection["protocol"] == "systemone"
|
|
236
|
+
and selection["weights_revision"] == WEIGHTS_REVISION
|
|
237
|
+
and selection["runtime_revision"] == CODE_REVISION,
|
|
238
|
+
"service requires the exact pinned Laya selection")
|
|
239
|
+
gate.require(endpoint.hostname == "127.0.0.1" and endpoint.path == "/v1/systemone"
|
|
240
|
+
and endpoint.port and len(selection["gpu_uuids"]) == 1,
|
|
241
|
+
"bind an exact literal loopback port and one leased GPU")
|
|
242
|
+
gate.require(isinstance(lease_owner, str) and 1 <= len(lease_owner) <= 256,
|
|
243
|
+
"explicit lease owner required")
|
|
244
|
+
self.identity = identity or gate.process_identity(os.getpid())
|
|
245
|
+
self.cuda_scope = cuda_scope or gate.process_cuda_scope(os.getpid())
|
|
246
|
+
gate.require(self.cuda_scope == selection["gpu_uuids"], "actual process CUDA visibility differs from selection")
|
|
247
|
+
self.boot_id = boot_id or Path("/proc/sys/kernel/random/boot_id").read_text().strip()
|
|
248
|
+
self.selection, self.lock, self.manifest, self.manifest_sha = selection, lock, manifest, manifest_sha
|
|
249
|
+
self.state_dir = Path(state_dir)
|
|
250
|
+
self.state_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
251
|
+
info = self.state_dir.stat()
|
|
252
|
+
gate.require(info.st_uid == os.getuid() and stat.S_IMODE(info.st_mode) & 0o077 == 0,
|
|
253
|
+
"state directory must be owner-only")
|
|
254
|
+
self.lease_owner, self.clock, self.adapter_factory = lease_owner, clock, adapter_factory
|
|
255
|
+
self.preflight = preflight or (lambda receipt: gate.verify_preflight(
|
|
256
|
+
selection, lock, receipt, gate.collect_snapshot(selection, receipt)))
|
|
257
|
+
self.timer_factory, self.terminate = timer_factory, terminate
|
|
258
|
+
self.lease = lease or gpu_lease.ScopedGpuLease(lease_owner, self.cuda_scope)
|
|
259
|
+
self.stage, self.adapter, self.receipt = "idle", None, None
|
|
260
|
+
self.inference_active = False
|
|
261
|
+
self.lease_transition = None
|
|
262
|
+
self.mutex = threading.Lock()
|
|
263
|
+
self.nonce = secrets.token_hex(24)
|
|
264
|
+
self.service_code_hashes = self.current_code_hashes()
|
|
265
|
+
self.idle = {"schema": "omnius.computer-use.laya-idle.v1", "stage": self.stage,
|
|
266
|
+
"loaded": False, "server": self.identity, "startup_nonce": self.nonce,
|
|
267
|
+
"manifest_sha256": manifest_sha, "selection": selection,
|
|
268
|
+
"cuda_visible_devices": self.cuda_scope, "strict_cuda": True,
|
|
269
|
+
"forward_calls": 0, "network_runtime": "disabled", "lease_owner": lease_owner,
|
|
270
|
+
"service_code_hashes": self.service_code_hashes}
|
|
271
|
+
gate.write_json(self.state_dir / "idle.json", self.idle)
|
|
272
|
+
self.write_state()
|
|
273
|
+
|
|
274
|
+
@staticmethod
|
|
275
|
+
def current_code_hashes():
|
|
276
|
+
return {"laya_service.py": file_sha256(__file__), "model_gate.py": file_sha256(gate.__file__),
|
|
277
|
+
"gpu_lease.py": file_sha256(gpu_lease.__file__)}
|
|
278
|
+
|
|
279
|
+
def verify_service_code(self):
|
|
280
|
+
gate.require(self.current_code_hashes() == self.service_code_hashes,
|
|
281
|
+
"reviewed service or model gate source changed after startup")
|
|
282
|
+
|
|
283
|
+
def write_state(self, error=None):
|
|
284
|
+
gate.write_json(self.state_dir / "state.json", {
|
|
285
|
+
"schema": "omnius.computer-use.laya-service-state.v1", "stage": self.stage,
|
|
286
|
+
"server": self.identity, "startup_nonce": self.nonce,
|
|
287
|
+
"loaded": self.receipt is not None, "strict_cuda": True,
|
|
288
|
+
"lease_transition": self.lease_transition,
|
|
289
|
+
**({"error": str(error)[:2000]} if error else {})})
|
|
290
|
+
|
|
291
|
+
def fail(self, error):
|
|
292
|
+
self.stage = "failed"
|
|
293
|
+
self.inference_active = False
|
|
294
|
+
self.write_state(error)
|
|
295
|
+
|
|
296
|
+
def verify_load_marker(self, marker):
|
|
297
|
+
gate.require(marker.get("schema") == "omnius.computer-use.load-authorization.v1",
|
|
298
|
+
"host load authorization required")
|
|
299
|
+
for key in ("server", "startup_nonce", "manifest_sha256", "selection", "cuda_visible_devices", "service_code_hashes"):
|
|
300
|
+
gate.require(marker.get(key) == self.idle[key], f"load authorization mismatch: {key}")
|
|
301
|
+
issued, expires = marker.get("issued_at", 0), marker.get("expires_at", 0)
|
|
302
|
+
gate.require(isinstance(issued, (int, float)) and isinstance(expires, (int, float))
|
|
303
|
+
and issued <= self.clock() < expires and 0 < expires - issued <= 120,
|
|
304
|
+
"load authorization is stale")
|
|
305
|
+
gate.require(isinstance(marker.get("preload_statement"), str)
|
|
306
|
+
and 20 <= len(marker["preload_statement"]) <= 2000,
|
|
307
|
+
"record actual process CUDA environment inspection before load")
|
|
308
|
+
|
|
309
|
+
def advance(self):
|
|
310
|
+
marker = self.state_dir / "load-authorization.json"
|
|
311
|
+
if self.stage != "idle" or not marker.exists():
|
|
312
|
+
return
|
|
313
|
+
with self.mutex:
|
|
314
|
+
try:
|
|
315
|
+
self.verify_service_code()
|
|
316
|
+
self.verify_load_marker(private_json(marker))
|
|
317
|
+
self.stage = "loading"
|
|
318
|
+
self.write_state()
|
|
319
|
+
self.adapter = self.adapter_factory(self.manifest, lambda: self.inference_active)
|
|
320
|
+
self.adapter.assert_resident()
|
|
321
|
+
self.receipt = {**self.selection, "schema": "omnius.computer-use.resident-runtime.v1",
|
|
322
|
+
"service_mode": "leased-external", "loaded": True,
|
|
323
|
+
"attestation_source": "host-launcher", "server": self.identity,
|
|
324
|
+
"runners": [self.identity], "boot_id": self.boot_id,
|
|
325
|
+
"cuda_visible_devices": self.cuda_scope, "lease_owner": self.lease_owner,
|
|
326
|
+
"manifest_sha256": self.manifest_sha, "strict_cuda": True,
|
|
327
|
+
"backend": "eager", "warmup_performed": False,
|
|
328
|
+
"service_code_hashes": self.service_code_hashes}
|
|
329
|
+
gate.write_json(self.state_dir / "resident-runtime.json", self.receipt)
|
|
330
|
+
self.stage = "resident"
|
|
331
|
+
self.write_state()
|
|
332
|
+
except Exception as error:
|
|
333
|
+
self.fail(error)
|
|
334
|
+
|
|
335
|
+
def infer(self, request):
|
|
336
|
+
request = validate_request(request, self.selection["model"])
|
|
337
|
+
gate.require(self.mutex.acquire(blocking=False), "another request is in progress")
|
|
338
|
+
try:
|
|
339
|
+
gate.require(self.stage in ("resident", "ready") and self.adapter is not None,
|
|
340
|
+
"service is not resident and authorized for inference")
|
|
341
|
+
self.verify_service_code()
|
|
342
|
+
approval = private_json(self.state_dir / "inference-approval.json")
|
|
343
|
+
preflight = self.preflight(self.receipt)
|
|
344
|
+
gate.verify_approval(approval, preflight, self.clock())
|
|
345
|
+
self.adapter.assert_resident()
|
|
346
|
+
|
|
347
|
+
def timeout():
|
|
348
|
+
self.fail("inference exceeded its hard 30 second deadline; process terminated")
|
|
349
|
+
self.terminate(124)
|
|
350
|
+
|
|
351
|
+
watchdog = self.timer_factory(INFERENCE_DEADLINE_SECONDS, timeout)
|
|
352
|
+
watchdog.daemon = True
|
|
353
|
+
watchdog.start()
|
|
354
|
+
try:
|
|
355
|
+
# Preflight above required an active exact lease. Prepare then
|
|
356
|
+
# owns a pending growth interval; do not re-run the active-state
|
|
357
|
+
# gate inside that interval or release it before CUDA settles.
|
|
358
|
+
self.lease.prepare()
|
|
359
|
+
self.lease_transition = {"action": "prepare", "state": "pending", "confirmed_at": self.clock()}
|
|
360
|
+
self.write_state()
|
|
361
|
+
self.inference_active = True
|
|
362
|
+
answer = gate.finite_answer(self.adapter.predict(request), request)
|
|
363
|
+
self.adapter.assert_resident()
|
|
364
|
+
self.adapter.synchronize()
|
|
365
|
+
self.inference_active = False
|
|
366
|
+
self.lease.ready()
|
|
367
|
+
self.lease_transition = {"action": "ready", "state": "active", "confirmed_at": self.clock()}
|
|
368
|
+
except Exception as error:
|
|
369
|
+
self.fail(error)
|
|
370
|
+
# Never mark an unknown/failed CUDA interval ready. The scoped
|
|
371
|
+
# supervisor waits for process exit and freed VRAM, then releases.
|
|
372
|
+
self.terminate(125)
|
|
373
|
+
raise
|
|
374
|
+
finally:
|
|
375
|
+
self.inference_active = False
|
|
376
|
+
watchdog.cancel()
|
|
377
|
+
self.stage = "ready"
|
|
378
|
+
self.write_state()
|
|
379
|
+
return {"model": self.selection["model"], "revision": WEIGHTS_REVISION,
|
|
380
|
+
"answers": {"action": {"type": "choice", "choice": answer["choice"],
|
|
381
|
+
**({"probabilities": answer["probabilities"]} if answer["probabilities"] is not None else {})}}}
|
|
382
|
+
finally:
|
|
383
|
+
self.mutex.release()
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
class LocalServer(HTTPServer):
|
|
387
|
+
allow_reuse_address = False
|
|
388
|
+
request_queue_size = 4
|
|
389
|
+
|
|
390
|
+
def service_actions(self):
|
|
391
|
+
self.service.advance()
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
class Handler(BaseHTTPRequestHandler):
|
|
395
|
+
def setup(self):
|
|
396
|
+
super().setup()
|
|
397
|
+
self.connection.settimeout(5)
|
|
398
|
+
|
|
399
|
+
def log_message(self, *_args):
|
|
400
|
+
pass # Never log states, criteria, or request bodies.
|
|
401
|
+
|
|
402
|
+
def reply(self, status, value):
|
|
403
|
+
raw = json.dumps(value, allow_nan=False, separators=(",", ":")).encode()
|
|
404
|
+
self.send_response(status)
|
|
405
|
+
self.send_header("Content-Type", "application/json")
|
|
406
|
+
self.send_header("Content-Length", str(len(raw)))
|
|
407
|
+
self.send_header("Connection", "close")
|
|
408
|
+
self.end_headers()
|
|
409
|
+
self.wfile.write(raw)
|
|
410
|
+
|
|
411
|
+
def do_GET(self):
|
|
412
|
+
if self.path != "/health":
|
|
413
|
+
return self.reply(404, {"error": "unknown route"})
|
|
414
|
+
self.reply(200, {"stage": self.server.service.stage, "strict_cuda": True,
|
|
415
|
+
"pid": os.getpid()})
|
|
416
|
+
|
|
417
|
+
def do_POST(self):
|
|
418
|
+
if self.path != "/v1/systemone":
|
|
419
|
+
return self.reply(404, {"error": "unknown route"})
|
|
420
|
+
try:
|
|
421
|
+
gate.require(not self.headers.get("Transfer-Encoding"), "chunked requests are unsupported")
|
|
422
|
+
lengths = self.headers.get_all("Content-Length", [])
|
|
423
|
+
gate.require(len(lengths) == 1 and lengths[0].isascii() and lengths[0].isdigit(),
|
|
424
|
+
"one bounded Content-Length is required")
|
|
425
|
+
length = int(lengths[0])
|
|
426
|
+
gate.require(0 < length <= MAX_REQUEST_BYTES, "request body exceeds bound")
|
|
427
|
+
gate.require(self.headers.get_content_type() == "application/json", "JSON content type required")
|
|
428
|
+
raw = self.rfile.read(length)
|
|
429
|
+
gate.require(len(raw) == length, "request body ended early")
|
|
430
|
+
request = validate_request(gate.strict_json(raw), self.server.service.selection["model"])
|
|
431
|
+
except (gate.GateError, OSError, ValueError) as error:
|
|
432
|
+
return self.reply(400, {"error": str(error)[:1000]})
|
|
433
|
+
try:
|
|
434
|
+
self.reply(200, self.server.service.infer(request))
|
|
435
|
+
except Exception as error:
|
|
436
|
+
self.reply(503, {"error": str(error)[:1000], "stage": self.server.service.stage})
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def authorize_load(state_dir, statement, now=None):
|
|
440
|
+
now = time.time() if now is None else now
|
|
441
|
+
idle = private_json(Path(state_dir) / "idle.json")
|
|
442
|
+
gate.require(idle.get("schema") == "omnius.computer-use.laya-idle.v1" and idle.get("loaded") is False,
|
|
443
|
+
"fresh idle process receipt required")
|
|
444
|
+
gate.require(isinstance(statement, str) and 20 <= len(statement) <= 2000,
|
|
445
|
+
"record the actual process CUDA environment inspection")
|
|
446
|
+
marker = {**{key: idle[key] for key in ("server", "startup_nonce", "manifest_sha256", "selection", "cuda_visible_devices", "service_code_hashes")},
|
|
447
|
+
"schema": "omnius.computer-use.load-authorization.v1", "issued_at": now,
|
|
448
|
+
"expires_at": now + 120, "preload_statement": statement}
|
|
449
|
+
gate.write_json(Path(state_dir) / "load-authorization.json", marker)
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def main(argv=None):
|
|
453
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
454
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
455
|
+
serve = commands.add_parser("serve")
|
|
456
|
+
serve.add_argument("--selection", required=True, type=Path)
|
|
457
|
+
serve.add_argument("--manifest", required=True, type=Path)
|
|
458
|
+
serve.add_argument("--state-dir", required=True, type=Path)
|
|
459
|
+
serve.add_argument("--lease-owner", required=True)
|
|
460
|
+
serve.add_argument("--lock", type=Path, default=gate.DEFAULT_LOCK)
|
|
461
|
+
authorize = commands.add_parser("authorize-load")
|
|
462
|
+
authorize.add_argument("--state-dir", required=True, type=Path)
|
|
463
|
+
authorize.add_argument("--statement", required=True)
|
|
464
|
+
args = parser.parse_args(argv)
|
|
465
|
+
try:
|
|
466
|
+
if args.command == "authorize-load":
|
|
467
|
+
authorize_load(args.state_dir, args.statement)
|
|
468
|
+
return 0
|
|
469
|
+
selection, lock = gate.read_json(args.selection), gate.read_json(args.lock)
|
|
470
|
+
manifest, manifest_sha = gate.read_json_hashed(args.manifest)
|
|
471
|
+
service = StagedService(selection, lock, manifest, manifest_sha, args.state_dir, args.lease_owner)
|
|
472
|
+
endpoint = gate.local_endpoint(selection["endpoint"])
|
|
473
|
+
with LocalServer(("127.0.0.1", endpoint.port), Handler) as server:
|
|
474
|
+
server.service = service
|
|
475
|
+
server.serve_forever(poll_interval=0.25)
|
|
476
|
+
return 0
|
|
477
|
+
except (gate.GateError, OSError, ValueError) as error:
|
|
478
|
+
print(json.dumps({"status": "blocked", "error": str(error)}), file=sys.stderr)
|
|
479
|
+
return 2
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
if __name__ == "__main__":
|
|
483
|
+
raise SystemExit(main())
|