@infernetprotocol/liquid-agent 0.0.0-stage → 0.1.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +15 -2
- package/fixtures/contract.json +6 -0
- package/fixtures/invalid/episode-extra-field.json +275 -0
- package/fixtures/invalid/episode-no-capture.json +265 -0
- package/fixtures/invalid/episode-unknown-action.json +274 -0
- package/fixtures/invalid/model-manifest-bad-hash.json +92 -0
- package/fixtures/invalid/predict-negative-seq.json +48 -0
- package/fixtures/invalid/proposal-bad-abstain.json +31 -0
- package/fixtures/invalid/proposal-confidence-range.json +31 -0
- package/fixtures/invalid/session-bad-seq.json +17 -0
- package/fixtures/invalid/train-job-bad-family.json +44 -0
- package/fixtures/invalid/train-job-zero-lr.json +44 -0
- package/fixtures/valid/action-proposal.json +27 -0
- package/fixtures/valid/dataset-bundle.json +3747 -0
- package/fixtures/valid/dataset-manifest.json +126 -0
- package/fixtures/valid/episode.json +270 -0
- package/fixtures/valid/inference-session.json +13 -0
- package/fixtures/valid/model-manifest.json +88 -0
- package/fixtures/valid/predict-request.json +44 -0
- package/fixtures/valid/train-job.json +40 -0
- package/package.json +28 -3
- package/runtime/infernet_liquid/__init__.py +6 -0
- package/runtime/infernet_liquid/__main__.py +95 -0
- package/runtime/infernet_liquid/bench.py +125 -0
- package/runtime/infernet_liquid/bundle.py +63 -0
- package/runtime/infernet_liquid/canonical.py +75 -0
- package/runtime/infernet_liquid/data.py +121 -0
- package/runtime/infernet_liquid/errors.py +13 -0
- package/runtime/infernet_liquid/evaluate.py +145 -0
- package/runtime/infernet_liquid/export.py +136 -0
- package/runtime/infernet_liquid/features.py +105 -0
- package/runtime/infernet_liquid/models.py +63 -0
- package/runtime/infernet_liquid/schema.py +155 -0
- package/runtime/infernet_liquid/secretscan.py +64 -0
- package/runtime/infernet_liquid/server.py +251 -0
- package/runtime/infernet_liquid/sessions.py +279 -0
- package/runtime/infernet_liquid/synth.py +194 -0
- package/runtime/infernet_liquid/train.py +136 -0
- package/runtime/pyproject.toml +14 -0
- package/runtime/requirements-cuda.lock +461 -0
- package/runtime/requirements.lock +344 -0
- package/schemas/action-proposal.schema.json +40 -0
- package/schemas/common.schema.json +88 -0
- package/schemas/dataset-bundle.schema.json +15 -0
- package/schemas/dataset-manifest.schema.json +100 -0
- package/schemas/episode.schema.json +71 -0
- package/schemas/inference-session.schema.json +23 -0
- package/schemas/model-manifest.schema.json +103 -0
- package/schemas/predict-request.schema.json +20 -0
- package/schemas/train-job.schema.json +75 -0
- package/src/canonical.js +49 -0
- package/src/contract.js +62 -0
- package/src/dataset.js +133 -0
- package/src/index.js +13 -0
- package/src/runtime.js +206 -0
- package/src/secrets.js +79 -0
- package/src/validate.js +161 -0
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""stdio NDJSON server: the subprocess contract Infernet speaks to this runtime.
|
|
2
|
+
|
|
3
|
+
Request: {"v":1, "id":"r1", "op":"predict", ...params}
|
|
4
|
+
Response: {"v":1, "id":"r1", "ok":true, "result":{...}}
|
|
5
|
+
{"v":1, "id":"r1", "ok":false, "error":{"code","message","retriable"}}
|
|
6
|
+
Events: {"v":1, "id":"r1", "event":"epoch", ...} (train/evaluate progress)
|
|
7
|
+
|
|
8
|
+
train and evaluate run on worker threads so health, cancel and predict stay
|
|
9
|
+
responsive. Nothing from a dataset or observation is written to stderr.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
import traceback
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from . import PROTOCOL_VERSION, SCHEMA_VERSION, __version__
|
|
21
|
+
from .data import load_json, split_episodes, verify_bundle
|
|
22
|
+
from .errors import LiquidError
|
|
23
|
+
from .export import load_model, read_manifest, runtime_ref, save_model
|
|
24
|
+
from .schema import validate
|
|
25
|
+
from .sessions import LoadedModel, SessionManager
|
|
26
|
+
|
|
27
|
+
LONG_OPS = {"train", "evaluate"}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def devices():
|
|
31
|
+
import torch
|
|
32
|
+
|
|
33
|
+
mps = bool(getattr(torch.backends, "mps", None) and torch.backends.mps.is_available())
|
|
34
|
+
return {"cpu": True, "cuda": torch.cuda.is_available(), "cuda_devices": torch.cuda.device_count() if torch.cuda.is_available() else 0, "mps": mps}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def ram_mb():
|
|
38
|
+
try:
|
|
39
|
+
return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") // (1024 * 1024)
|
|
40
|
+
except (ValueError, OSError, AttributeError):
|
|
41
|
+
return 0
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _bundle(params):
|
|
45
|
+
if "bundle" in params:
|
|
46
|
+
return params["bundle"]
|
|
47
|
+
if "bundle_path" in params:
|
|
48
|
+
return load_json(params["bundle_path"])
|
|
49
|
+
raise LiquidError("E_BAD_REQUEST", "bundle_path is required")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def run_train(params, on_event=None, cancelled=None):
|
|
53
|
+
from .evaluate import evaluate_encoded
|
|
54
|
+
from .train import train
|
|
55
|
+
|
|
56
|
+
bundle = _bundle(params)
|
|
57
|
+
dhash = verify_bundle(bundle)
|
|
58
|
+
job = params.get("job")
|
|
59
|
+
family = params.get("family", "cfc")
|
|
60
|
+
hp = dict(params.get("hyperparameters") or {})
|
|
61
|
+
device = params.get("device", "auto")
|
|
62
|
+
threads = params.get("max_threads")
|
|
63
|
+
timeout_s = params.get("timeout_s")
|
|
64
|
+
job_id = None
|
|
65
|
+
if job:
|
|
66
|
+
errors = validate(job, "train_job")
|
|
67
|
+
if errors:
|
|
68
|
+
raise LiquidError("E_SCHEMA", "train job failed validation: " + "; ".join(f"{a or '/'} {b}" for a, b in errors[:5]))
|
|
69
|
+
if job["dataset_hash"] != dhash:
|
|
70
|
+
raise LiquidError("E_HASH_MISMATCH", f"job names dataset {job['dataset_hash']} but the bundle is {dhash}")
|
|
71
|
+
if job["privacy"]["mode"] == "trusted_worker" and not job["privacy"]["upload_approved"]:
|
|
72
|
+
raise LiquidError("E_CONSENT", "remote training needs upload_approved in the job's privacy block")
|
|
73
|
+
family = job["model_type"]
|
|
74
|
+
hp.update(job["hyperparameters"])
|
|
75
|
+
device = job["resources"]["device"]
|
|
76
|
+
threads = job["resources"].get("max_threads", threads)
|
|
77
|
+
timeout_s = job["limits"]["timeout_s"]
|
|
78
|
+
job_id = job["job_id"]
|
|
79
|
+
if family not in ("cfc", "ltc", "gru"):
|
|
80
|
+
raise LiquidError("E_BAD_REQUEST", f"unknown family {family}")
|
|
81
|
+
if "consent" in bundle["manifest"] and "model_training" not in bundle["manifest"]["consent"]["purposes"]:
|
|
82
|
+
raise LiquidError("E_CONSENT", "dataset consent does not cover model_training")
|
|
83
|
+
deadline = time.monotonic() + float(timeout_s) if timeout_s else None
|
|
84
|
+
model, info = train(bundle, family, hp, device, threads, on_event, cancelled, deadline)
|
|
85
|
+
enc = info["encoded"]
|
|
86
|
+
split = "test" if enc["test"] else "validation"
|
|
87
|
+
metrics = evaluate_encoded(model, enc[split] or enc["train"])
|
|
88
|
+
tasks = sorted({e["task"] for e in bundle["episodes"]})
|
|
89
|
+
out = params.get("output_dir")
|
|
90
|
+
if not out:
|
|
91
|
+
raise LiquidError("E_BAD_REQUEST", "output_dir is required")
|
|
92
|
+
manifest = save_model(out, model, info, dhash, metrics, split, len(enc[split]), params.get("name"), job_id, params.get("license", "UNLICENSED-private"), tasks)
|
|
93
|
+
report = {"model_hash": manifest["weights"]["weights_hash"], "dataset_hash": dhash, "split": split, "metrics": metrics}
|
|
94
|
+
Path(out, "eval.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", "utf-8")
|
|
95
|
+
return {"model_dir": str(out), "manifest": manifest, "eval": report}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def run_evaluate(params, on_event=None, cancelled=None):
|
|
99
|
+
from .data import encode_episode
|
|
100
|
+
from .evaluate import evaluate_encoded
|
|
101
|
+
|
|
102
|
+
model, manifest = load_model(params["model_dir"], params.get("expected_hash"))
|
|
103
|
+
bundle = _bundle(params)
|
|
104
|
+
dhash = verify_bundle(bundle)
|
|
105
|
+
split = params.get("split", "test")
|
|
106
|
+
eps = split_episodes(bundle)[split] if split != "all" else bundle["episodes"]
|
|
107
|
+
if not eps:
|
|
108
|
+
raise LiquidError("E_BAD_REQUEST", f"dataset has no {split} episodes")
|
|
109
|
+
enc = [encode_episode(e, manifest["architecture"]["feature_dim"]) for e in eps]
|
|
110
|
+
metrics = evaluate_encoded(model, enc)
|
|
111
|
+
return {"model_hash": manifest["weights"]["weights_hash"], "dataset_hash": dhash, "split": split, "metrics": metrics}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class Server:
|
|
115
|
+
def __init__(self, inp=sys.stdin, out=sys.stdout, max_sessions=64, ttl_s=600, worker="local"):
|
|
116
|
+
self.inp = inp
|
|
117
|
+
self.out = out
|
|
118
|
+
self.out_lock = threading.Lock()
|
|
119
|
+
self.sm = SessionManager(max_sessions=max_sessions, default_ttl_s=ttl_s, worker=worker)
|
|
120
|
+
self.jobs = {}
|
|
121
|
+
self.kill_switch = False
|
|
122
|
+
self.running = True
|
|
123
|
+
|
|
124
|
+
def send(self, obj):
|
|
125
|
+
line = json.dumps({"v": PROTOCOL_VERSION, **obj}, separators=(",", ":"))
|
|
126
|
+
with self.out_lock:
|
|
127
|
+
self.out.write(line + "\n")
|
|
128
|
+
self.out.flush()
|
|
129
|
+
|
|
130
|
+
def ok(self, rid, result):
|
|
131
|
+
self.send({"id": rid, "ok": True, "result": result})
|
|
132
|
+
|
|
133
|
+
def fail(self, rid, err):
|
|
134
|
+
self.send({"id": rid, "ok": False, "error": err.to_json() if isinstance(err, LiquidError) else {"code": "E_RUNTIME", "message": str(err), "retriable": True}})
|
|
135
|
+
|
|
136
|
+
def serve(self):
|
|
137
|
+
for line in self.inp:
|
|
138
|
+
line = line.strip()
|
|
139
|
+
if not line:
|
|
140
|
+
continue
|
|
141
|
+
try:
|
|
142
|
+
req = json.loads(line)
|
|
143
|
+
except ValueError:
|
|
144
|
+
self.fail(None, LiquidError("E_BAD_REQUEST", "request is not JSON"))
|
|
145
|
+
continue
|
|
146
|
+
self.handle(req)
|
|
147
|
+
if not self.running:
|
|
148
|
+
break
|
|
149
|
+
for ev in self.jobs.values():
|
|
150
|
+
ev.set()
|
|
151
|
+
|
|
152
|
+
def handle(self, req):
|
|
153
|
+
rid = req.get("id")
|
|
154
|
+
if req.get("v") != PROTOCOL_VERSION:
|
|
155
|
+
return self.fail(rid, LiquidError("E_BAD_REQUEST", f"protocol version must be {PROTOCOL_VERSION}"))
|
|
156
|
+
op = req.get("op")
|
|
157
|
+
if op in LONG_OPS:
|
|
158
|
+
ev = threading.Event()
|
|
159
|
+
self.jobs[rid] = ev
|
|
160
|
+
threading.Thread(target=self._long, args=(rid, op, req, ev), daemon=True).start()
|
|
161
|
+
return
|
|
162
|
+
try:
|
|
163
|
+
self.ok(rid, self.dispatch(op, req))
|
|
164
|
+
except LiquidError as e:
|
|
165
|
+
self.fail(rid, e)
|
|
166
|
+
except Exception as e: # noqa: BLE001 - reported to the caller, never crash the loop
|
|
167
|
+
traceback.print_exc(file=sys.stderr)
|
|
168
|
+
self.fail(rid, e)
|
|
169
|
+
|
|
170
|
+
def _long(self, rid, op, req, ev):
|
|
171
|
+
fn = run_train if op == "train" else run_evaluate
|
|
172
|
+
try:
|
|
173
|
+
res = fn(req, on_event=lambda e: self.send({"id": rid, **e}), cancelled=ev.is_set)
|
|
174
|
+
self.ok(rid, res)
|
|
175
|
+
except LiquidError as e:
|
|
176
|
+
self.fail(rid, e)
|
|
177
|
+
except Exception as e: # noqa: BLE001
|
|
178
|
+
traceback.print_exc(file=sys.stderr)
|
|
179
|
+
self.fail(rid, e)
|
|
180
|
+
finally:
|
|
181
|
+
self.jobs.pop(rid, None)
|
|
182
|
+
|
|
183
|
+
def dispatch(self, op, p):
|
|
184
|
+
sm = self.sm
|
|
185
|
+
scope = p.get("account_scope")
|
|
186
|
+
if op == "health":
|
|
187
|
+
return {
|
|
188
|
+
"status": "ok",
|
|
189
|
+
"schema_version": SCHEMA_VERSION,
|
|
190
|
+
"protocol": PROTOCOL_VERSION,
|
|
191
|
+
"runtime": runtime_ref(),
|
|
192
|
+
"devices": devices(),
|
|
193
|
+
"loaded_models": sorted(sm.models),
|
|
194
|
+
"sessions": len(sm.sessions),
|
|
195
|
+
"kill_switch": self.kill_switch,
|
|
196
|
+
}
|
|
197
|
+
if op == "capabilities":
|
|
198
|
+
d = devices()
|
|
199
|
+
return {
|
|
200
|
+
"workloads": ["infernet.temporal.train", "infernet.temporal.infer"],
|
|
201
|
+
"families": ["cfc", "ltc", "gru"],
|
|
202
|
+
"research_families": ["ltc"],
|
|
203
|
+
"devices": d,
|
|
204
|
+
"ram_mb": ram_mb(),
|
|
205
|
+
"threads": os.cpu_count() or 1,
|
|
206
|
+
"max_sessions": sm.max_sessions,
|
|
207
|
+
"runtime": runtime_ref(),
|
|
208
|
+
}
|
|
209
|
+
if op == "validate":
|
|
210
|
+
bundle = _bundle(p)
|
|
211
|
+
dhash = verify_bundle(bundle)
|
|
212
|
+
return {
|
|
213
|
+
"dataset_hash": dhash,
|
|
214
|
+
"episodes": len(bundle["episodes"]),
|
|
215
|
+
"counts": bundle["manifest"]["splits"]["counts"],
|
|
216
|
+
"tasks": sorted({e["task"] for e in bundle["episodes"]}),
|
|
217
|
+
}
|
|
218
|
+
if op == "load":
|
|
219
|
+
model, manifest = load_model(p["model_dir"], p.get("expected_hash"))
|
|
220
|
+
lm = LoadedModel(model, manifest, float(p.get("abstain_threshold", 0.5)))
|
|
221
|
+
sm.add_model(lm)
|
|
222
|
+
return {"model_hash": lm.hash, "manifest": manifest}
|
|
223
|
+
if op == "manifest":
|
|
224
|
+
return read_manifest(p["model_dir"])
|
|
225
|
+
if op == "unload":
|
|
226
|
+
sm.remove_model(p["model_hash"])
|
|
227
|
+
return {"model_hash": p["model_hash"], "unloaded": True}
|
|
228
|
+
if op == "open_session":
|
|
229
|
+
return sm.open(p["model_hash"], p["session_id"], p.get("ttl_s"), scope)
|
|
230
|
+
if op == "predict":
|
|
231
|
+
return sm.predict(p["request"], scope, self.kill_switch)
|
|
232
|
+
if op == "reset":
|
|
233
|
+
return sm.reset(p["session_id"], scope)
|
|
234
|
+
if op == "close_session":
|
|
235
|
+
return sm.close(p["session_id"], scope)
|
|
236
|
+
if op == "checkpoint":
|
|
237
|
+
return sm.checkpoint(p["session_id"], p["dir"], scope)
|
|
238
|
+
if op == "restore":
|
|
239
|
+
return sm.restore(p["path"], p.get("session_id"), scope)
|
|
240
|
+
if op == "kill_switch":
|
|
241
|
+
self.kill_switch = bool(p.get("on", True))
|
|
242
|
+
return {"kill_switch": self.kill_switch}
|
|
243
|
+
if op == "cancel":
|
|
244
|
+
ev = self.jobs.get(p.get("target"))
|
|
245
|
+
if ev:
|
|
246
|
+
ev.set()
|
|
247
|
+
return {"cancelled": bool(ev)}
|
|
248
|
+
if op == "shutdown":
|
|
249
|
+
self.running = False
|
|
250
|
+
return {"bye": True}
|
|
251
|
+
raise LiquidError("E_BAD_REQUEST", f"unknown op {op}")
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
"""Stateful inference sessions: one hidden state per session, strict sequence order, TTL, checkpoints."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import math
|
|
5
|
+
import re
|
|
6
|
+
import threading
|
|
7
|
+
import time
|
|
8
|
+
from datetime import datetime, timezone
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import torch
|
|
12
|
+
from safetensors.torch import load_file, save_file
|
|
13
|
+
|
|
14
|
+
from . import SCHEMA_VERSION
|
|
15
|
+
from .canonical import file_sha256, hash_canonical
|
|
16
|
+
from .data import collate
|
|
17
|
+
from .errors import LiquidError
|
|
18
|
+
from .features import ACTION_TYPES, EXTERNAL_WRITE, MAX_ELEMENTS, TARGETED, encode_element, encode_step
|
|
19
|
+
from .schema import validate
|
|
20
|
+
|
|
21
|
+
SESSION_ID = re.compile(r"^[A-Za-z0-9_.:-]{1,128}$")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def iso(ts):
|
|
25
|
+
return datetime.fromtimestamp(ts, timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class LoadedModel:
|
|
29
|
+
def __init__(self, model, manifest, abstain_threshold=0.5):
|
|
30
|
+
self.model = model
|
|
31
|
+
self.manifest = manifest
|
|
32
|
+
self.hash = manifest["weights"]["weights_hash"]
|
|
33
|
+
self.abstain_threshold = abstain_threshold
|
|
34
|
+
self.lock = threading.Lock()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Session:
|
|
38
|
+
def __init__(self, sid, lm, ttl_s, account_scope, worker, now):
|
|
39
|
+
self.id = sid
|
|
40
|
+
self.lm = lm
|
|
41
|
+
self.ttl_s = ttl_s
|
|
42
|
+
self.account_scope = account_scope
|
|
43
|
+
self.worker = worker
|
|
44
|
+
self.created = now
|
|
45
|
+
self.touched = now
|
|
46
|
+
self.seq = -1
|
|
47
|
+
self.h = lm.model.init_state(1)
|
|
48
|
+
self.state = "active"
|
|
49
|
+
self.checkpoint_ref = None
|
|
50
|
+
|
|
51
|
+
def expires(self):
|
|
52
|
+
return self.touched + self.ttl_s
|
|
53
|
+
|
|
54
|
+
def to_json(self):
|
|
55
|
+
d = {
|
|
56
|
+
"schema_version": SCHEMA_VERSION,
|
|
57
|
+
"kind": "inference_session",
|
|
58
|
+
"session_id": self.id,
|
|
59
|
+
"model_hash": self.lm.hash,
|
|
60
|
+
"created_at": iso(self.created),
|
|
61
|
+
"expires_at": iso(self.expires()),
|
|
62
|
+
"ttl_s": self.ttl_s,
|
|
63
|
+
"sequence_index": self.seq,
|
|
64
|
+
"state": self.state,
|
|
65
|
+
"checkpoint_ref": self.checkpoint_ref,
|
|
66
|
+
"worker": self.worker,
|
|
67
|
+
}
|
|
68
|
+
if self.account_scope:
|
|
69
|
+
d["account_scope"] = self.account_scope
|
|
70
|
+
return d
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class SessionManager:
|
|
74
|
+
def __init__(self, max_sessions=64, default_ttl_s=600, worker="local", clock=time.time):
|
|
75
|
+
self.max_sessions = max_sessions
|
|
76
|
+
self.default_ttl_s = default_ttl_s
|
|
77
|
+
self.worker = worker
|
|
78
|
+
self.clock = clock
|
|
79
|
+
self.models = {}
|
|
80
|
+
self.sessions = {}
|
|
81
|
+
self.lock = threading.RLock()
|
|
82
|
+
|
|
83
|
+
# models ---------------------------------------------------------------
|
|
84
|
+
def add_model(self, lm):
|
|
85
|
+
with self.lock:
|
|
86
|
+
self.models[lm.hash] = lm
|
|
87
|
+
|
|
88
|
+
def remove_model(self, model_hash):
|
|
89
|
+
with self.lock:
|
|
90
|
+
if model_hash not in self.models:
|
|
91
|
+
raise LiquidError("E_NOT_LOADED", f"model {model_hash} is not loaded")
|
|
92
|
+
for sid in [s for s, v in self.sessions.items() if v.lm.hash == model_hash]:
|
|
93
|
+
self.sessions.pop(sid).state = "closed"
|
|
94
|
+
del self.models[model_hash]
|
|
95
|
+
|
|
96
|
+
def model(self, model_hash):
|
|
97
|
+
lm = self.models.get(model_hash)
|
|
98
|
+
if lm is None:
|
|
99
|
+
raise LiquidError("E_NOT_LOADED", f"model {model_hash} is not loaded")
|
|
100
|
+
return lm
|
|
101
|
+
|
|
102
|
+
# sessions -------------------------------------------------------------
|
|
103
|
+
def sweep(self):
|
|
104
|
+
now = self.clock()
|
|
105
|
+
for sid in [s for s, v in self.sessions.items() if v.expires() <= now]:
|
|
106
|
+
self.sessions.pop(sid).state = "expired"
|
|
107
|
+
|
|
108
|
+
def open(self, model_hash, session_id, ttl_s=None, account_scope=None):
|
|
109
|
+
if not isinstance(session_id, str) or not SESSION_ID.match(session_id):
|
|
110
|
+
raise LiquidError("E_BAD_REQUEST", "session_id must match ^[A-Za-z0-9_.:-]{1,128}$")
|
|
111
|
+
ttl = int(ttl_s or self.default_ttl_s)
|
|
112
|
+
if not 1 <= ttl <= 86400:
|
|
113
|
+
raise LiquidError("E_BAD_REQUEST", "ttl_s must be 1..86400")
|
|
114
|
+
with self.lock:
|
|
115
|
+
self.sweep()
|
|
116
|
+
lm = self.model(model_hash)
|
|
117
|
+
if session_id in self.sessions:
|
|
118
|
+
raise LiquidError("E_BAD_REQUEST", f"session {session_id} already exists; reset or close it")
|
|
119
|
+
if len(self.sessions) >= self.max_sessions:
|
|
120
|
+
raise LiquidError("E_CAPACITY", f"session limit {self.max_sessions} reached")
|
|
121
|
+
s = Session(session_id, lm, ttl, account_scope, self.worker, self.clock())
|
|
122
|
+
self.sessions[session_id] = s
|
|
123
|
+
return s.to_json()
|
|
124
|
+
|
|
125
|
+
def get(self, session_id, account_scope=None):
|
|
126
|
+
with self.lock:
|
|
127
|
+
s = self.sessions.get(session_id)
|
|
128
|
+
if s is not None and s.expires() <= self.clock():
|
|
129
|
+
self.sessions.pop(session_id)
|
|
130
|
+
raise LiquidError("E_SESSION_EXPIRED", f"session {session_id} expired; open a new one or restore a checkpoint")
|
|
131
|
+
if s is None:
|
|
132
|
+
raise LiquidError("E_SESSION_UNKNOWN", f"no session {session_id}")
|
|
133
|
+
if s.account_scope and account_scope and s.account_scope != account_scope:
|
|
134
|
+
raise LiquidError("E_SESSION_UNKNOWN", f"no session {session_id} in this account scope")
|
|
135
|
+
return s
|
|
136
|
+
|
|
137
|
+
def reset(self, session_id, account_scope=None):
|
|
138
|
+
s = self.get(session_id, account_scope)
|
|
139
|
+
with s.lm.lock:
|
|
140
|
+
s.h = s.lm.model.init_state(1)
|
|
141
|
+
s.seq = -1
|
|
142
|
+
s.touched = self.clock()
|
|
143
|
+
s.checkpoint_ref = None
|
|
144
|
+
return s.to_json()
|
|
145
|
+
|
|
146
|
+
def close(self, session_id, account_scope=None):
|
|
147
|
+
s = self.get(session_id, account_scope)
|
|
148
|
+
with self.lock:
|
|
149
|
+
self.sessions.pop(session_id, None)
|
|
150
|
+
s.state = "closed"
|
|
151
|
+
return s.to_json()
|
|
152
|
+
|
|
153
|
+
def predict(self, req, account_scope=None, kill_switch=False):
|
|
154
|
+
errors = validate(req, "predict_request")
|
|
155
|
+
if errors:
|
|
156
|
+
raise LiquidError("E_SCHEMA", "predict request failed validation: " + "; ".join(f"{a or '/'} {b}" for a, b in errors[:5]))
|
|
157
|
+
s = self.get(req["session_id"], account_scope)
|
|
158
|
+
lm = s.lm
|
|
159
|
+
if req.get("model_hash") and req["model_hash"] != lm.hash:
|
|
160
|
+
raise LiquidError("E_MODEL_HASH", f"session {s.id} serves {lm.hash}, request names {req['model_hash']}")
|
|
161
|
+
idx = req["sequence_index"]
|
|
162
|
+
with lm.lock:
|
|
163
|
+
if idx == s.seq:
|
|
164
|
+
raise LiquidError("E_SEQ_DUPLICATE", f"sequence_index {idx} already processed", expected=s.seq + 1)
|
|
165
|
+
if idx < s.seq:
|
|
166
|
+
raise LiquidError("E_SEQ_LATE", f"sequence_index {idx} is older than the last accepted {s.seq}", expected=s.seq + 1)
|
|
167
|
+
if idx > s.seq + 1:
|
|
168
|
+
raise LiquidError("E_SEQ_GAP", f"sequence_index {idx} skips {s.seq + 1}; the state cannot skip steps", expected=s.seq + 1)
|
|
169
|
+
t0 = time.perf_counter()
|
|
170
|
+
proposal = self._step(s, req)
|
|
171
|
+
s.seq = idx
|
|
172
|
+
s.touched = self.clock()
|
|
173
|
+
proposal["latency_ms"] = round((time.perf_counter() - t0) * 1000.0, 4)
|
|
174
|
+
if kill_switch:
|
|
175
|
+
proposal.update({"action": None, "alternatives": [], "requires_approval": False, "abstain_reason": "kill_switch"})
|
|
176
|
+
return proposal
|
|
177
|
+
|
|
178
|
+
@torch.no_grad()
|
|
179
|
+
def _step(self, s, req):
|
|
180
|
+
m = s.lm.model
|
|
181
|
+
F = m.feature_dim
|
|
182
|
+
obs = req["observation"]
|
|
183
|
+
x = torch.tensor([encode_step(req.get("task"), obs, req.get("prev_action"), req["dt_ms"], F)])
|
|
184
|
+
dt = torch.tensor([[min(req["dt_ms"] / 1000.0, 60.0)]])
|
|
185
|
+
els = (obs.get("elements") or [])[:MAX_ELEMENTS]
|
|
186
|
+
enc = collate([{
|
|
187
|
+
"x": [[0.0] * F], "dt": [[0.0]], "elems": [[encode_element(e, i) for i, e in enumerate(els)]],
|
|
188
|
+
"n_elems": [len(els)], "y_type": [0], "y_ptr": [-100], "needs_target": [False],
|
|
189
|
+
}])
|
|
190
|
+
out, s.h = m.step(x, s.h, dt)
|
|
191
|
+
tl, pl = m.heads(out, enc["elems"][:, 0], enc["emask"][:, 0])
|
|
192
|
+
p_type = torch.softmax(tl[0], -1)
|
|
193
|
+
p_ptr = torch.softmax(pl[0], -1) if els else None
|
|
194
|
+
ent = float(-(p_type * torch.log(p_type.clamp_min(1e-12))).sum()) / math.log(len(ACTION_TYPES))
|
|
195
|
+
cands = []
|
|
196
|
+
for ti in torch.topk(p_type, k=3).indices.tolist():
|
|
197
|
+
t = ACTION_TYPES[ti]
|
|
198
|
+
if t in TARGETED:
|
|
199
|
+
if p_ptr is None:
|
|
200
|
+
continue
|
|
201
|
+
for ei in torch.topk(p_ptr, k=min(2, len(els))).indices.tolist():
|
|
202
|
+
el = els[ei]
|
|
203
|
+
target = {k: el[k] for k in ("role", "name", "id") if k in el}
|
|
204
|
+
cands.append(({"type": t, "target": target}, float(p_type[ti] * p_ptr[ei])))
|
|
205
|
+
else:
|
|
206
|
+
cands.append(({"type": t}, float(p_type[ti])))
|
|
207
|
+
cands.sort(key=lambda c: -c[1])
|
|
208
|
+
best_type = ACTION_TYPES[int(p_type.argmax())]
|
|
209
|
+
action, conf, reason = None, 0.0, None
|
|
210
|
+
if best_type in TARGETED and p_ptr is None:
|
|
211
|
+
reason = "unknown_target"
|
|
212
|
+
conf = float(p_type.max())
|
|
213
|
+
elif cands:
|
|
214
|
+
action, conf = cands[0]
|
|
215
|
+
if conf < s.lm.abstain_threshold:
|
|
216
|
+
reason = "low_confidence"
|
|
217
|
+
if reason:
|
|
218
|
+
action = None
|
|
219
|
+
return {
|
|
220
|
+
"schema_version": SCHEMA_VERSION,
|
|
221
|
+
"kind": "action_proposal",
|
|
222
|
+
"session_id": s.id,
|
|
223
|
+
"sequence_index": req["sequence_index"],
|
|
224
|
+
"model_hash": s.lm.hash,
|
|
225
|
+
"action": action,
|
|
226
|
+
"confidence": round(min(1.0, max(0.0, conf)), 6),
|
|
227
|
+
"uncertainty": round(min(1.0, max(0.0, ent)), 6),
|
|
228
|
+
"alternatives": [{"action": a, "p": round(min(1.0, p), 6)} for a, p in cands[1:4]],
|
|
229
|
+
"requires_approval": bool(action and action["type"] in EXTERNAL_WRITE),
|
|
230
|
+
"abstain_reason": reason,
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
# checkpoints ----------------------------------------------------------
|
|
234
|
+
def checkpoint(self, session_id, directory, account_scope=None):
|
|
235
|
+
s = self.get(session_id, account_scope)
|
|
236
|
+
d = Path(directory)
|
|
237
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
238
|
+
with s.lm.lock:
|
|
239
|
+
state_path = d / f"{session_id}.state.safetensors"
|
|
240
|
+
save_file({"h": s.h.contiguous().clone()}, str(state_path))
|
|
241
|
+
meta = {
|
|
242
|
+
"schema_version": SCHEMA_VERSION,
|
|
243
|
+
"session_id": s.id,
|
|
244
|
+
"model_hash": s.lm.hash,
|
|
245
|
+
"sequence_index": s.seq,
|
|
246
|
+
"ttl_s": s.ttl_s,
|
|
247
|
+
"account_scope": s.account_scope,
|
|
248
|
+
"state_hash": file_sha256(state_path),
|
|
249
|
+
}
|
|
250
|
+
ref = hash_canonical(meta)
|
|
251
|
+
(d / f"{session_id}.checkpoint.json").write_text(json.dumps({**meta, "checkpoint_ref": ref}, indent=2, sort_keys=True), "utf-8")
|
|
252
|
+
s.checkpoint_ref = ref
|
|
253
|
+
return {"checkpoint_ref": ref, "path": str(d / f"{session_id}.checkpoint.json"), "session": s.to_json()}
|
|
254
|
+
|
|
255
|
+
def restore(self, path, session_id=None, account_scope=None):
|
|
256
|
+
p = Path(path)
|
|
257
|
+
try:
|
|
258
|
+
meta = json.loads(p.read_text("utf-8"))
|
|
259
|
+
except (OSError, ValueError) as e:
|
|
260
|
+
raise LiquidError("E_BAD_REQUEST", f"cannot read checkpoint {path}: {e}")
|
|
261
|
+
ref = meta.pop("checkpoint_ref", None)
|
|
262
|
+
if hash_canonical(meta) != ref:
|
|
263
|
+
raise LiquidError("E_HASH_MISMATCH", "checkpoint_ref does not match the checkpoint")
|
|
264
|
+
state_path = p.with_name(f"{meta['session_id']}.state.safetensors")
|
|
265
|
+
if file_sha256(state_path) != meta["state_hash"]:
|
|
266
|
+
raise LiquidError("E_HASH_MISMATCH", "checkpoint state does not match its hash")
|
|
267
|
+
if meta.get("account_scope") and account_scope and meta["account_scope"] != account_scope:
|
|
268
|
+
raise LiquidError("E_SESSION_UNKNOWN", "checkpoint belongs to another account scope")
|
|
269
|
+
sid = session_id or meta["session_id"]
|
|
270
|
+
with self.lock:
|
|
271
|
+
lm = self.model(meta["model_hash"])
|
|
272
|
+
if sid in self.sessions:
|
|
273
|
+
self.sessions.pop(sid)
|
|
274
|
+
self.open(lm.hash, sid, meta["ttl_s"], meta.get("account_scope"))
|
|
275
|
+
s = self.sessions[sid]
|
|
276
|
+
s.h = load_file(str(state_path))["h"]
|
|
277
|
+
s.seq = meta["sequence_index"]
|
|
278
|
+
s.checkpoint_ref = ref
|
|
279
|
+
return s.to_json()
|