@infernetprotocol/liquid-agent 0.0.0-stage → 0.1.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +15 -2
- package/fixtures/contract.json +6 -0
- package/fixtures/invalid/episode-extra-field.json +275 -0
- package/fixtures/invalid/episode-no-capture.json +265 -0
- package/fixtures/invalid/episode-unknown-action.json +274 -0
- package/fixtures/invalid/model-manifest-bad-hash.json +92 -0
- package/fixtures/invalid/predict-negative-seq.json +48 -0
- package/fixtures/invalid/proposal-bad-abstain.json +31 -0
- package/fixtures/invalid/proposal-confidence-range.json +31 -0
- package/fixtures/invalid/session-bad-seq.json +17 -0
- package/fixtures/invalid/train-job-bad-family.json +44 -0
- package/fixtures/invalid/train-job-zero-lr.json +44 -0
- package/fixtures/valid/action-proposal.json +27 -0
- package/fixtures/valid/dataset-bundle.json +3747 -0
- package/fixtures/valid/dataset-manifest.json +126 -0
- package/fixtures/valid/episode.json +270 -0
- package/fixtures/valid/inference-session.json +13 -0
- package/fixtures/valid/model-manifest.json +88 -0
- package/fixtures/valid/predict-request.json +44 -0
- package/fixtures/valid/train-job.json +40 -0
- package/package.json +28 -3
- package/runtime/infernet_liquid/__init__.py +6 -0
- package/runtime/infernet_liquid/__main__.py +95 -0
- package/runtime/infernet_liquid/bench.py +125 -0
- package/runtime/infernet_liquid/bundle.py +63 -0
- package/runtime/infernet_liquid/canonical.py +75 -0
- package/runtime/infernet_liquid/data.py +121 -0
- package/runtime/infernet_liquid/errors.py +13 -0
- package/runtime/infernet_liquid/evaluate.py +145 -0
- package/runtime/infernet_liquid/export.py +136 -0
- package/runtime/infernet_liquid/features.py +105 -0
- package/runtime/infernet_liquid/models.py +63 -0
- package/runtime/infernet_liquid/schema.py +155 -0
- package/runtime/infernet_liquid/secretscan.py +64 -0
- package/runtime/infernet_liquid/server.py +251 -0
- package/runtime/infernet_liquid/sessions.py +279 -0
- package/runtime/infernet_liquid/synth.py +194 -0
- package/runtime/infernet_liquid/train.py +136 -0
- package/runtime/pyproject.toml +14 -0
- package/runtime/requirements-cuda.lock +461 -0
- package/runtime/requirements.lock +344 -0
- package/schemas/action-proposal.schema.json +40 -0
- package/schemas/common.schema.json +88 -0
- package/schemas/dataset-bundle.schema.json +15 -0
- package/schemas/dataset-manifest.schema.json +100 -0
- package/schemas/episode.schema.json +71 -0
- package/schemas/inference-session.schema.json +23 -0
- package/schemas/model-manifest.schema.json +103 -0
- package/schemas/predict-request.schema.json +20 -0
- package/schemas/train-job.schema.json +75 -0
- package/src/canonical.js +49 -0
- package/src/contract.js +62 -0
- package/src/dataset.js +133 -0
- package/src/index.js +13 -0
- package/src/runtime.js +206 -0
- package/src/secrets.js +79 -0
- package/src/validate.js +161 -0
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""python -m infernet_liquid <serve|train|evaluate|synth|bench|health>"""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
from .errors import LiquidError
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def main(argv=None):
|
|
11
|
+
ap = argparse.ArgumentParser(prog="infernet-liquid")
|
|
12
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
13
|
+
s = sub.add_parser("serve", help="stdio NDJSON server (the Infernet contract)")
|
|
14
|
+
s.add_argument("--max-sessions", type=int, default=64)
|
|
15
|
+
s.add_argument("--ttl", type=int, default=600)
|
|
16
|
+
s.add_argument("--worker", default="local")
|
|
17
|
+
t = sub.add_parser("train")
|
|
18
|
+
t.add_argument("--dataset", required=True)
|
|
19
|
+
t.add_argument("--family", default="cfc", choices=["cfc", "ltc", "gru"])
|
|
20
|
+
t.add_argument("--output", required=True)
|
|
21
|
+
t.add_argument("--epochs", type=int)
|
|
22
|
+
t.add_argument("--seed", type=int)
|
|
23
|
+
t.add_argument("--hidden", type=int)
|
|
24
|
+
t.add_argument("--threads", type=int)
|
|
25
|
+
e = sub.add_parser("evaluate")
|
|
26
|
+
e.add_argument("--model", required=True)
|
|
27
|
+
e.add_argument("--dataset", required=True)
|
|
28
|
+
e.add_argument("--split", default="test")
|
|
29
|
+
y = sub.add_parser("synth")
|
|
30
|
+
y.add_argument("--out", required=True)
|
|
31
|
+
y.add_argument("--per-task", type=int, default=160)
|
|
32
|
+
y.add_argument("--seed", type=int, default=7)
|
|
33
|
+
y.add_argument("--shifted", action="store_true")
|
|
34
|
+
b = sub.add_parser("bench")
|
|
35
|
+
b.add_argument("--out-dir", required=True)
|
|
36
|
+
b.add_argument("--per-task", type=int, default=160)
|
|
37
|
+
b.add_argument("--families", default="cfc,gru,ltc")
|
|
38
|
+
b.add_argument("--seeds", default="0,1,2")
|
|
39
|
+
b.add_argument("--epochs", type=int)
|
|
40
|
+
b.add_argument("--threads", type=int, default=4)
|
|
41
|
+
b.add_argument("--report")
|
|
42
|
+
sub.add_parser("health")
|
|
43
|
+
a = ap.parse_args(argv)
|
|
44
|
+
|
|
45
|
+
try:
|
|
46
|
+
if a.cmd == "serve":
|
|
47
|
+
from .server import Server
|
|
48
|
+
|
|
49
|
+
Server(max_sessions=a.max_sessions, ttl_s=a.ttl, worker=a.worker).serve()
|
|
50
|
+
return 0
|
|
51
|
+
if a.cmd == "health":
|
|
52
|
+
from .server import Server
|
|
53
|
+
|
|
54
|
+
print(json.dumps(Server().dispatch("health", {}), indent=2))
|
|
55
|
+
return 0
|
|
56
|
+
if a.cmd == "train":
|
|
57
|
+
from .server import run_train
|
|
58
|
+
|
|
59
|
+
hp = {k: v for k, v in {"epochs": a.epochs, "seed": a.seed, "hidden_units": a.hidden}.items() if v is not None}
|
|
60
|
+
res = run_train({"bundle_path": a.dataset, "family": a.family, "output_dir": a.output, "hyperparameters": hp, "max_threads": a.threads},
|
|
61
|
+
on_event=lambda ev: print(json.dumps(ev), file=sys.stderr))
|
|
62
|
+
print(json.dumps({"model_dir": res["model_dir"], "model_hash": res["manifest"]["weights"]["weights_hash"], "metrics": res["eval"]["metrics"]}, indent=2))
|
|
63
|
+
return 0
|
|
64
|
+
if a.cmd == "evaluate":
|
|
65
|
+
from .server import run_evaluate
|
|
66
|
+
|
|
67
|
+
print(json.dumps(run_evaluate({"model_dir": a.model, "bundle_path": a.dataset, "split": a.split}), indent=2))
|
|
68
|
+
return 0
|
|
69
|
+
if a.cmd == "synth":
|
|
70
|
+
from .synth import make_bundle
|
|
71
|
+
|
|
72
|
+
bundle = make_bundle(per_task=a.per_task, seed=a.seed, shifted=a.shifted)
|
|
73
|
+
with open(a.out, "w", encoding="utf-8") as f:
|
|
74
|
+
json.dump(bundle, f, separators=(",", ":"))
|
|
75
|
+
print(json.dumps({"out": a.out, "dataset_hash": bundle["manifest"]["dataset_hash"], "counts": bundle["manifest"]["splits"]["counts"]}))
|
|
76
|
+
return 0
|
|
77
|
+
if a.cmd == "bench":
|
|
78
|
+
from .bench import render_markdown, run
|
|
79
|
+
|
|
80
|
+
s = run(a.out_dir, a.per_task, tuple(a.families.split(",")), tuple(int(x) for x in a.seeds.split(",")), a.epochs,
|
|
81
|
+
on_event=lambda ev: print(json.dumps(ev), file=sys.stderr), threads=a.threads)
|
|
82
|
+
md = render_markdown(s)
|
|
83
|
+
if a.report:
|
|
84
|
+
with open(a.report, "w", encoding="utf-8") as f:
|
|
85
|
+
f.write(md)
|
|
86
|
+
print(md)
|
|
87
|
+
return 0
|
|
88
|
+
except LiquidError as err:
|
|
89
|
+
print(json.dumps({"error": err.to_json()}), file=sys.stderr)
|
|
90
|
+
return 1
|
|
91
|
+
return 2
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
if __name__ == "__main__":
|
|
95
|
+
sys.exit(main())
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""CfC vs GRU (vs LTC research, vs a script baseline) on identical synthetic splits."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import statistics
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from .data import encode_episode, split_episodes, verify_bundle
|
|
8
|
+
from .evaluate import evaluate_encoded, evaluate_markov
|
|
9
|
+
from .export import runtime_ref
|
|
10
|
+
from .synth import make_bundle
|
|
11
|
+
from .train import DEFAULTS, train
|
|
12
|
+
|
|
13
|
+
KEYS = ["action_accuracy", "target_accuracy", "step_accuracy", "episode_success", "external_write_recall", "latency_ms_p50", "latency_ms_p95"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def run(out_dir, per_task=160, families=("cfc", "gru", "ltc"), seeds=(0, 1, 2), epochs=None, shift_per_task=40, on_event=None, threads=4):
|
|
17
|
+
out = Path(out_dir)
|
|
18
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
19
|
+
main = make_bundle(per_task=per_task, seed=7)
|
|
20
|
+
shift = make_bundle(per_task=shift_per_task, seed=11, shifted=True)
|
|
21
|
+
verify_bundle(main)
|
|
22
|
+
verify_bundle(shift)
|
|
23
|
+
(out / "bench-bundle.json").write_text(json.dumps(main, separators=(",", ":")), "utf-8")
|
|
24
|
+
(out / "bench-shift-bundle.json").write_text(json.dumps(shift, separators=(",", ":")), "utf-8")
|
|
25
|
+
splits = split_episodes(main)
|
|
26
|
+
held = {}
|
|
27
|
+
for e in splits["test"]:
|
|
28
|
+
held[e["task"]] = held.get(e["task"], 0) + 1
|
|
29
|
+
results = {}
|
|
30
|
+
hp_base = dict(DEFAULTS)
|
|
31
|
+
if epochs:
|
|
32
|
+
hp_base["epochs"] = epochs
|
|
33
|
+
for fam in families:
|
|
34
|
+
runs = []
|
|
35
|
+
for seed in seeds:
|
|
36
|
+
hp = {**hp_base, "seed": seed}
|
|
37
|
+
model, info = train(main, fam, hp, "cpu", threads, on_event)
|
|
38
|
+
test = evaluate_encoded(model, info["encoded"]["test"])
|
|
39
|
+
enc_shift = [encode_episode(e, hp["feature_dim"]) for e in shift["episodes"]]
|
|
40
|
+
sh = evaluate_encoded(model, enc_shift, timing=False)
|
|
41
|
+
runs.append({"seed": seed, "train_seconds": info["train_seconds"], "epochs_run": info["epochs_run"], "params": model.param_count(), "test": test, "shift": sh})
|
|
42
|
+
if on_event:
|
|
43
|
+
on_event({"event": "bench_run", "family": fam, "seed": seed, "step_accuracy": test["step_accuracy"], "episode_success": test["episode_success"], "shift_episode_success": sh["episode_success"]})
|
|
44
|
+
results[fam] = runs
|
|
45
|
+
markov_test = evaluate_markov(splits["train"], splits["test"])
|
|
46
|
+
markov_shift = evaluate_markov(splits["train"], shift["episodes"])
|
|
47
|
+
summary = {
|
|
48
|
+
"runtime": runtime_ref(),
|
|
49
|
+
"dataset_hash": main["manifest"]["dataset_hash"],
|
|
50
|
+
"shift_dataset_hash": shift["manifest"]["dataset_hash"],
|
|
51
|
+
"split_counts": main["manifest"]["splits"]["counts"],
|
|
52
|
+
"held_out_per_task": held,
|
|
53
|
+
"shift_episodes": len(shift["episodes"]),
|
|
54
|
+
"hyperparameters": hp_base,
|
|
55
|
+
"seeds": list(seeds),
|
|
56
|
+
"families": {},
|
|
57
|
+
"script_baseline": {"test": markov_test, "shift": markov_shift},
|
|
58
|
+
}
|
|
59
|
+
for fam, runs in results.items():
|
|
60
|
+
agg = {}
|
|
61
|
+
for k in KEYS:
|
|
62
|
+
vals = [r["test"].get(k, 0.0) for r in runs]
|
|
63
|
+
agg[k] = {"mean": round(statistics.mean(vals), 4), "min": round(min(vals), 4), "max": round(max(vals), 4)}
|
|
64
|
+
agg["shift_episode_success"] = {"mean": round(statistics.mean(r["shift"]["episode_success"] for r in runs), 4)}
|
|
65
|
+
agg["shift_step_accuracy"] = {"mean": round(statistics.mean(r["shift"]["step_accuracy"] for r in runs), 4)}
|
|
66
|
+
agg["train_seconds"] = {"mean": round(statistics.mean(r["train_seconds"] for r in runs), 2)}
|
|
67
|
+
agg["params"] = runs[0]["params"]
|
|
68
|
+
agg["by_task_episode_success"] = {
|
|
69
|
+
t: round(statistics.mean(r["test"]["by_task"][t]["episode_success"] for r in runs), 4) for t in runs[0]["test"]["by_task"]
|
|
70
|
+
}
|
|
71
|
+
summary["families"][fam] = {"aggregate": agg, "runs": runs}
|
|
72
|
+
(out / "benchmark.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", "utf-8")
|
|
73
|
+
return summary
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def render_markdown(s):
|
|
77
|
+
fams = list(s["families"])
|
|
78
|
+
lines = [
|
|
79
|
+
"# Temporal policy benchmark: CfC vs GRU",
|
|
80
|
+
"",
|
|
81
|
+
"Generated by `python -m infernet_liquid bench` on synthetic mock desktop traces",
|
|
82
|
+
f"(runtime {s['runtime']['name']} {s['runtime']['version']}, torch {s['runtime']['torch']}, ncps {s['runtime']['ncps']}, CPU).",
|
|
83
|
+
"",
|
|
84
|
+
f"- Dataset `{s['dataset_hash']}`: splits {s['split_counts']}, split by recording session.",
|
|
85
|
+
f"- Held-out test episodes per workflow: {s['held_out_per_task']}.",
|
|
86
|
+
f"- Layout-shift holdout: {s['shift_episodes']} episodes on an unseen layout (renamed controls, reordered) `{s['shift_dataset_hash']}`.",
|
|
87
|
+
f"- Same splits, features, hidden size ({s['hyperparameters']['hidden_units']}), optimiser and epoch budget for every family; seeds {s['seeds']}; mean (min to max) over seeds.",
|
|
88
|
+
"",
|
|
89
|
+
"Predeclared metric: **episode_success** on held-out test episodes (every step's action type and target right under teacher forcing).",
|
|
90
|
+
"Secondary: step_accuracy, layout-shift episode success, external_write_recall (an external write predicted as one, so approval gating fires), batch-1 step latency.",
|
|
91
|
+
"",
|
|
92
|
+
"| model | params | episode success | step acc | target acc | ext-write recall | shift episode success | latency p50 / p95 ms | train s |",
|
|
93
|
+
"|---|---|---|---|---|---|---|---|---|",
|
|
94
|
+
]
|
|
95
|
+
for f in fams:
|
|
96
|
+
a = s["families"][f]["aggregate"]
|
|
97
|
+
r = lambda k: f"{a[k]['mean']:.3f} ({a[k]['min']:.3f} to {a[k]['max']:.3f})"
|
|
98
|
+
label = f + (" (research)" if f == "ltc" else " (baseline)" if f == "gru" else "")
|
|
99
|
+
lines.append(f"| {label} | {a['params']} | {r('episode_success')} | {r('step_accuracy')} | {a['target_accuracy']['mean']:.3f} | {a['external_write_recall']['mean']:.3f} | {a['shift_episode_success']['mean']:.3f} | {a['latency_ms_p50']['mean']:.3f} / {a['latency_ms_p95']['mean']:.3f} | {a['train_seconds']['mean']:.1f} |")
|
|
100
|
+
m = s["script_baseline"]
|
|
101
|
+
lines.append(f"| script (majority next action) | 0 | {m['test']['episode_success']:.3f} | {m['test']['step_accuracy']:.3f} | {m['test']['target_accuracy']:.3f} | {m['test']['external_write_recall']:.3f} | {m['shift']['episode_success']:.3f} | n/a | 0 |")
|
|
102
|
+
lines += ["", "Per-workflow episode success (mean over seeds):", "", "| model | " + " | ".join(s["families"][fams[0]]["aggregate"]["by_task_episode_success"]) + " |", "|---|" + "---|" * len(s["families"][fams[0]]["aggregate"]["by_task_episode_success"])]
|
|
103
|
+
for f in fams:
|
|
104
|
+
bt = s["families"][f]["aggregate"]["by_task_episode_success"]
|
|
105
|
+
lines.append(f"| {f} | " + " | ".join(f"{v:.3f}" for v in bt.values()) + " |")
|
|
106
|
+
if "cfc" in s["families"] and "gru" in s["families"]:
|
|
107
|
+
c = s["families"]["cfc"]["aggregate"]
|
|
108
|
+
g = s["families"]["gru"]["aggregate"]
|
|
109
|
+
d = c["episode_success"]["mean"] - g["episode_success"]["mean"]
|
|
110
|
+
if c["episode_success"]["min"] > g["episode_success"]["max"]:
|
|
111
|
+
verdict = "CfC beats GRU on every seed: CfC may be selected for these tasks."
|
|
112
|
+
elif g["episode_success"]["min"] > c["episode_success"]["max"]:
|
|
113
|
+
verdict = "GRU beats CfC on every seed: keep GRU."
|
|
114
|
+
else:
|
|
115
|
+
verdict = "Neither beats the other on every seed: keep GRU (cheaper, simpler)."
|
|
116
|
+
lines += [
|
|
117
|
+
"",
|
|
118
|
+
"## Verdict",
|
|
119
|
+
"",
|
|
120
|
+
f"{verdict} CfC minus GRU mean episode success: {d:+.3f}.",
|
|
121
|
+
"Rule (pipeline PRD): a liquid model is chosen only when it beats the baseline on the predeclared metric on every seed.",
|
|
122
|
+
"",
|
|
123
|
+
"Limits: synthetic traces from a generator with three layouts. They exercise the pipeline and the comparison method; they say nothing about real desktops. Rerun on reviewed W3BS episodes before choosing a family for a real task.",
|
|
124
|
+
]
|
|
125
|
+
return "\n".join(lines) + "\n"
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Build sealed episodes and DatasetBundleV1 files; mirrors src/dataset.js byte for byte."""
|
|
2
|
+
|
|
3
|
+
from .canonical import dataset_hash, episode_hash, sha256_hex
|
|
4
|
+
from .secretscan import SCANNER_VERSION
|
|
5
|
+
from . import SCHEMA_VERSION
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def seal_episode(ep):
|
|
9
|
+
e = {k: v for k, v in ep.items() if k != "content_hash"}
|
|
10
|
+
e["schema_version"] = SCHEMA_VERSION
|
|
11
|
+
e["kind"] = "episode"
|
|
12
|
+
e["content_hash"] = episode_hash(e)
|
|
13
|
+
return e
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _group(ep, strategy):
|
|
17
|
+
if strategy == "by_task":
|
|
18
|
+
return ep["task"]
|
|
19
|
+
if strategy == "by_variant":
|
|
20
|
+
return ep.get("variant") or ep.get("split_group") or ep["episode_id"]
|
|
21
|
+
return ep.get("split_group") or ep["episode_id"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _bucket(seed, group):
|
|
25
|
+
return int(sha256_hex(f"{seed}:{group}")[:8], 16) / 0x100000000
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def assign_splits(episodes, strategy="by_session", seed=0, validation=0.15, test=0.2):
|
|
29
|
+
out = []
|
|
30
|
+
for ep in episodes:
|
|
31
|
+
g = _group(ep, strategy)
|
|
32
|
+
b = _bucket(seed, g)
|
|
33
|
+
split = "test" if b < test else "validation" if b < test + validation else "train"
|
|
34
|
+
out.append((ep, split, g))
|
|
35
|
+
return out
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def build_bundle(episodes, dataset_id, created_at, consent, name=None, strategy="by_session", seed=0, validation=0.15, test=0.2, provenance=None, license="CC0-1.0"):
|
|
39
|
+
assigned = assign_splits(episodes, strategy, seed, validation, test)
|
|
40
|
+
counts = {"train": 0, "validation": 0, "test": 0}
|
|
41
|
+
for _, split, _ in assigned:
|
|
42
|
+
counts[split] += 1
|
|
43
|
+
manifest = {
|
|
44
|
+
"schema_version": SCHEMA_VERSION,
|
|
45
|
+
"kind": "dataset_manifest",
|
|
46
|
+
"dataset_id": dataset_id,
|
|
47
|
+
"version": 1,
|
|
48
|
+
"created_at": created_at,
|
|
49
|
+
"episodes": [
|
|
50
|
+
{"episode_id": ep["episode_id"], "content_hash": ep["content_hash"], "task": ep["task"], "split": split, "split_group": str(g)[:128]}
|
|
51
|
+
for ep, split, g in assigned
|
|
52
|
+
],
|
|
53
|
+
"splits": {"strategy": strategy, "seed": seed, "counts": counts},
|
|
54
|
+
"license": license,
|
|
55
|
+
"provenance": provenance or {"source": "synthetic", "generator": "infernet-liquid synth"},
|
|
56
|
+
"consent": consent,
|
|
57
|
+
"retention": {"days": None, "delete_on_revoke": True},
|
|
58
|
+
"redaction": {"profile": "strict", "secret_scan": "passed", "scanner_version": SCANNER_VERSION},
|
|
59
|
+
}
|
|
60
|
+
if name:
|
|
61
|
+
manifest["name"] = name
|
|
62
|
+
manifest["dataset_hash"] = dataset_hash(manifest)
|
|
63
|
+
return {"schema_version": SCHEMA_VERSION, "kind": "dataset_bundle", "manifest": manifest, "episodes": episodes}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Canonical JSON identical to src/canonical.js (sorted keys, no spaces, JS number format)."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import math
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _num(v):
|
|
9
|
+
if isinstance(v, bool):
|
|
10
|
+
raise TypeError("bool is not a number")
|
|
11
|
+
if isinstance(v, int):
|
|
12
|
+
return str(v)
|
|
13
|
+
if not math.isfinite(v):
|
|
14
|
+
raise TypeError("canonical JSON: non-finite number")
|
|
15
|
+
if v == int(v) and abs(v) < 1e21:
|
|
16
|
+
return str(int(v))
|
|
17
|
+
r = repr(v)
|
|
18
|
+
if "e" in r:
|
|
19
|
+
mant, exp = r.split("e")
|
|
20
|
+
sign = "-" if exp.startswith("-") else "+"
|
|
21
|
+
r = f"{mant}e{sign}{int(exp.lstrip('+-'))}"
|
|
22
|
+
return r
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def canonicalize(value):
|
|
26
|
+
if value is None:
|
|
27
|
+
return "null"
|
|
28
|
+
if value is True:
|
|
29
|
+
return "true"
|
|
30
|
+
if value is False:
|
|
31
|
+
return "false"
|
|
32
|
+
if isinstance(value, (int, float)):
|
|
33
|
+
return _num(value)
|
|
34
|
+
if isinstance(value, str):
|
|
35
|
+
return json.dumps(value, ensure_ascii=False)
|
|
36
|
+
if isinstance(value, (list, tuple)):
|
|
37
|
+
return "[" + ",".join(canonicalize(v) for v in value) + "]"
|
|
38
|
+
if isinstance(value, dict):
|
|
39
|
+
keys = sorted(k for k, v in value.items())
|
|
40
|
+
return "{" + ",".join(json.dumps(k, ensure_ascii=False) + ":" + canonicalize(value[k]) for k in keys) + "}"
|
|
41
|
+
raise TypeError(f"canonical JSON: unsupported type {type(value).__name__}")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def sha256_hex(data):
|
|
45
|
+
if isinstance(data, str):
|
|
46
|
+
data = data.encode("utf-8")
|
|
47
|
+
return hashlib.sha256(data).hexdigest()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def hash_canonical(value):
|
|
51
|
+
return "sha256:" + sha256_hex(canonicalize(value))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def hash_without(value, fields):
|
|
55
|
+
return hash_canonical({k: v for k, v in value.items() if k not in fields})
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def episode_hash(ep):
|
|
59
|
+
return hash_without(ep, ("content_hash",))
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def dataset_hash(manifest):
|
|
63
|
+
return hash_canonical({"episodes": manifest["episodes"], "splits": manifest["splits"]})
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def manifest_hash(manifest):
|
|
67
|
+
return hash_without(manifest, ("manifest_hash", "signature"))
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def file_sha256(path):
|
|
71
|
+
h = hashlib.sha256()
|
|
72
|
+
with open(path, "rb") as f:
|
|
73
|
+
for chunk in iter(lambda: f.read(1 << 20), b""):
|
|
74
|
+
h.update(chunk)
|
|
75
|
+
return "sha256:" + h.hexdigest()
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Load, verify and tensorise DatasetBundleV1 files."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
|
|
7
|
+
from .canonical import dataset_hash, episode_hash
|
|
8
|
+
from .errors import LiquidError
|
|
9
|
+
from .features import ACTION_TYPES, ELEMENT_DIM, MAX_ELEMENTS, encode_element, encode_step, target_index
|
|
10
|
+
from .schema import validate
|
|
11
|
+
from .secretscan import scan_secrets
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def load_json(path):
|
|
15
|
+
try:
|
|
16
|
+
with open(path, encoding="utf-8") as f:
|
|
17
|
+
return json.load(f)
|
|
18
|
+
except FileNotFoundError:
|
|
19
|
+
raise LiquidError("E_BAD_REQUEST", f"file not found: {path}")
|
|
20
|
+
except json.JSONDecodeError as e:
|
|
21
|
+
raise LiquidError("E_SCHEMA", f"{path} is not JSON: {e}")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def verify_bundle(bundle):
|
|
25
|
+
"""Schema, every hash, manifest/episode agreement and the secret scan. Returns the dataset hash."""
|
|
26
|
+
errors = validate(bundle, "dataset_bundle")
|
|
27
|
+
if errors:
|
|
28
|
+
raise LiquidError("E_SCHEMA", "dataset bundle failed liquid-agent/v1 validation: " + "; ".join(f"{p or '/'} {m}" for p, m in errors[:5]))
|
|
29
|
+
eps = {e["episode_id"]: e for e in bundle["episodes"]}
|
|
30
|
+
if len(eps) != len(bundle["episodes"]):
|
|
31
|
+
raise LiquidError("E_SCHEMA", "duplicate episode_id in bundle")
|
|
32
|
+
manifest = bundle["manifest"]
|
|
33
|
+
if len(manifest["episodes"]) != len(eps):
|
|
34
|
+
raise LiquidError("E_HASH_MISMATCH", "bundle carries episodes the manifest does not name")
|
|
35
|
+
for entry in manifest["episodes"]:
|
|
36
|
+
ep = eps.get(entry["episode_id"])
|
|
37
|
+
if ep is None:
|
|
38
|
+
raise LiquidError("E_HASH_MISMATCH", f"manifest names {entry['episode_id']} but the bundle lacks it")
|
|
39
|
+
if ep["content_hash"] != entry["content_hash"] or episode_hash(ep) != ep["content_hash"]:
|
|
40
|
+
raise LiquidError("E_HASH_MISMATCH", f"episode {entry['episode_id']} does not match its hash")
|
|
41
|
+
if dataset_hash(manifest) != manifest["dataset_hash"]:
|
|
42
|
+
raise LiquidError("E_HASH_MISMATCH", "dataset_hash does not match the manifest")
|
|
43
|
+
hits = scan_secrets(bundle["episodes"])
|
|
44
|
+
if hits:
|
|
45
|
+
raise LiquidError("E_SECRET_DETECTED", f"dataset contains what looks like a secret at {hits[0]['path']} ({','.join(hits[0]['kinds'])})")
|
|
46
|
+
return manifest["dataset_hash"]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def split_episodes(bundle):
|
|
50
|
+
eps = {e["episode_id"]: e for e in bundle["episodes"]}
|
|
51
|
+
out = {"train": [], "validation": [], "test": []}
|
|
52
|
+
for entry in bundle["manifest"]["episodes"]:
|
|
53
|
+
out[entry["split"]].append(eps[entry["episode_id"]])
|
|
54
|
+
return out
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def encode_episode(ep, feature_dim):
|
|
58
|
+
xs, dts, elems, masks, y_type, y_ptr, needs = [], [], [], [], [], [], []
|
|
59
|
+
hidden_targets = 0
|
|
60
|
+
prev, prev_t = None, None
|
|
61
|
+
for step in ep["steps"]:
|
|
62
|
+
dt_ms = 0 if prev_t is None else max(0, step["t_ms"] - prev_t)
|
|
63
|
+
obs = step["observation"]
|
|
64
|
+
xs.append(encode_step(ep["task"], obs, prev, dt_ms, feature_dim))
|
|
65
|
+
dts.append([min(dt_ms / 1000.0, 60.0)])
|
|
66
|
+
els = (obs.get("elements") or [])[:MAX_ELEMENTS]
|
|
67
|
+
elems.append([encode_element(el, i) for i, el in enumerate(els)])
|
|
68
|
+
masks.append(len(els))
|
|
69
|
+
y_type.append(ACTION_TYPES.index(step["action"]["type"]))
|
|
70
|
+
needs.append(bool(step["action"].get("target")))
|
|
71
|
+
ti = target_index(obs, step["action"])
|
|
72
|
+
if ti is None:
|
|
73
|
+
hidden_targets += 1
|
|
74
|
+
ti = -100
|
|
75
|
+
elif ti == -1:
|
|
76
|
+
ti = -100
|
|
77
|
+
y_ptr.append(ti)
|
|
78
|
+
prev, prev_t = step["action"], step["t_ms"]
|
|
79
|
+
return {
|
|
80
|
+
"episode_id": ep["episode_id"],
|
|
81
|
+
"task": ep["task"],
|
|
82
|
+
"variant": ep.get("variant"),
|
|
83
|
+
"x": xs,
|
|
84
|
+
"dt": dts,
|
|
85
|
+
"elems": elems,
|
|
86
|
+
"n_elems": masks,
|
|
87
|
+
"y_type": y_type,
|
|
88
|
+
"y_ptr": y_ptr,
|
|
89
|
+
"needs_target": needs,
|
|
90
|
+
"hidden_targets": hidden_targets,
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def collate(encoded):
|
|
95
|
+
"""Pad a list of encoded episodes into batch tensors."""
|
|
96
|
+
B = len(encoded)
|
|
97
|
+
T = max(len(e["x"]) for e in encoded)
|
|
98
|
+
F = len(encoded[0]["x"][0])
|
|
99
|
+
E = max(1, max(max(e["n_elems"]) if e["n_elems"] else 0 for e in encoded))
|
|
100
|
+
x = torch.zeros(B, T, F)
|
|
101
|
+
dt = torch.zeros(B, T, 1)
|
|
102
|
+
elems = torch.zeros(B, T, E, ELEMENT_DIM)
|
|
103
|
+
emask = torch.zeros(B, T, E, dtype=torch.bool)
|
|
104
|
+
smask = torch.zeros(B, T, dtype=torch.bool)
|
|
105
|
+
y_type = torch.full((B, T), -100, dtype=torch.long)
|
|
106
|
+
y_ptr = torch.full((B, T), -100, dtype=torch.long)
|
|
107
|
+
for b, e in enumerate(encoded):
|
|
108
|
+
t = len(e["x"])
|
|
109
|
+
x[b, :t] = torch.tensor(e["x"])
|
|
110
|
+
dt[b, :t] = torch.tensor(e["dt"])
|
|
111
|
+
smask[b, :t] = True
|
|
112
|
+
y_type[b, :t] = torch.tensor(e["y_type"])
|
|
113
|
+
y_ptr[b, :t] = torch.tensor(e["y_ptr"])
|
|
114
|
+
for s, (vecs, n) in enumerate(zip(e["elems"], e["n_elems"])):
|
|
115
|
+
if n:
|
|
116
|
+
elems[b, s, :n] = torch.tensor(vecs)
|
|
117
|
+
emask[b, s, :n] = True
|
|
118
|
+
# A step with no visible elements still needs one finite pointer slot.
|
|
119
|
+
empty = ~emask.any(-1)
|
|
120
|
+
emask[..., 0] |= empty
|
|
121
|
+
return {"x": x, "dt": dt, "elems": elems, "emask": emask, "smask": smask, "y_type": y_type, "y_ptr": y_ptr}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Failure taxonomy shared with src/contract.js ERROR_CODES."""
|
|
2
|
+
|
|
3
|
+
RETRIABLE = {"E_CAPACITY", "E_TIMEOUT", "E_RUNTIME"}
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class LiquidError(Exception):
|
|
7
|
+
def __init__(self, code, message, **extra):
|
|
8
|
+
super().__init__(message)
|
|
9
|
+
self.code = code
|
|
10
|
+
self.extra = extra
|
|
11
|
+
|
|
12
|
+
def to_json(self):
|
|
13
|
+
return {"code": self.code, "message": str(self), "retriable": self.code in RETRIABLE, **self.extra}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""Metrics: per-step and per-episode accuracy, external-write recall, stateful latency."""
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from collections import Counter, defaultdict
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
|
|
8
|
+
from .data import collate
|
|
9
|
+
from .features import ACTION_TYPES, EXTERNAL_WRITE, MAX_ELEMENTS, words
|
|
10
|
+
from .train import forward_batch
|
|
11
|
+
|
|
12
|
+
EXT = {ACTION_TYPES.index(t) for t in EXTERNAL_WRITE}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _pct(xs, q):
|
|
16
|
+
if not xs:
|
|
17
|
+
return 0.0
|
|
18
|
+
s = sorted(xs)
|
|
19
|
+
k = min(len(s) - 1, max(0, int(round(q / 100.0 * (len(s) - 1)))))
|
|
20
|
+
return s[k]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Tally:
|
|
24
|
+
def __init__(self):
|
|
25
|
+
self.steps = self.type_ok = self.ptr_total = self.ptr_ok = self.step_ok = 0
|
|
26
|
+
self.episodes = self.ep_ok = 0
|
|
27
|
+
self.ext_total = self.ext_caught = 0
|
|
28
|
+
self.by_task = defaultdict(lambda: [0, 0, 0, 0]) # steps, step_ok, episodes, ep_ok
|
|
29
|
+
|
|
30
|
+
def episode(self, task, preds_type, preds_ptr, y_type, y_ptr, needs):
|
|
31
|
+
all_ok = True
|
|
32
|
+
bt = self.by_task[task]
|
|
33
|
+
for pt, pp, yt, yp, nd in zip(preds_type, preds_ptr, y_type, y_ptr, needs):
|
|
34
|
+
self.steps += 1
|
|
35
|
+
bt[0] += 1
|
|
36
|
+
tok = pt == yt
|
|
37
|
+
self.type_ok += tok
|
|
38
|
+
pok = True
|
|
39
|
+
if nd:
|
|
40
|
+
self.ptr_total += 1
|
|
41
|
+
pok = yp != -100 and pp == yp
|
|
42
|
+
self.ptr_ok += pok
|
|
43
|
+
ok = tok and pok
|
|
44
|
+
self.step_ok += ok
|
|
45
|
+
bt[1] += ok
|
|
46
|
+
all_ok &= ok
|
|
47
|
+
if yt in EXT:
|
|
48
|
+
self.ext_total += 1
|
|
49
|
+
self.ext_caught += pt in EXT
|
|
50
|
+
self.episodes += 1
|
|
51
|
+
bt[2] += 1
|
|
52
|
+
self.ep_ok += all_ok
|
|
53
|
+
bt[3] += all_ok
|
|
54
|
+
|
|
55
|
+
def metrics(self):
|
|
56
|
+
r = lambda a, b: round(a / b, 4) if b else 0.0
|
|
57
|
+
return {
|
|
58
|
+
"steps": self.steps,
|
|
59
|
+
"episodes": self.episodes,
|
|
60
|
+
"action_accuracy": r(self.type_ok, self.steps),
|
|
61
|
+
"target_accuracy": r(self.ptr_ok, self.ptr_total),
|
|
62
|
+
"step_accuracy": r(self.step_ok, self.steps),
|
|
63
|
+
"episode_success": r(self.ep_ok, self.episodes),
|
|
64
|
+
"external_write_recall": r(self.ext_caught, self.ext_total) if self.ext_total else 1.0,
|
|
65
|
+
"by_task": {t: {"step_accuracy": r(v[1], v[0]), "episode_success": r(v[3], v[2]), "episodes": v[2]} for t, v in sorted(self.by_task.items())},
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@torch.no_grad()
|
|
70
|
+
def evaluate_encoded(model, encoded, device="cpu", timing=True, batch_size=64):
|
|
71
|
+
was_training = model.training
|
|
72
|
+
model.eval()
|
|
73
|
+
tally = Tally()
|
|
74
|
+
for i in range(0, len(encoded), batch_size):
|
|
75
|
+
chunk = encoded[i:i + batch_size]
|
|
76
|
+
batch = {k: v.to(device) for k, v in collate(chunk).items()}
|
|
77
|
+
tl, pl = forward_batch(model, batch)
|
|
78
|
+
pt = tl.argmax(-1).cpu().tolist()
|
|
79
|
+
pp = pl.argmax(-1).cpu().tolist()
|
|
80
|
+
for b, e in enumerate(chunk):
|
|
81
|
+
n = len(e["x"])
|
|
82
|
+
tally.episode(e["task"], pt[b][:n], pp[b][:n], e["y_type"], e["y_ptr"], e["needs_target"])
|
|
83
|
+
m = tally.metrics()
|
|
84
|
+
if timing and encoded:
|
|
85
|
+
m.update(step_latency(model, encoded[: min(20, len(encoded))]))
|
|
86
|
+
if was_training:
|
|
87
|
+
model.train()
|
|
88
|
+
return m
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@torch.no_grad()
|
|
92
|
+
def step_latency(model, encoded):
|
|
93
|
+
"""Batch-1 stateful step latency, the serving path: one forward per observed step."""
|
|
94
|
+
lat = []
|
|
95
|
+
for e in encoded:
|
|
96
|
+
batch = collate([e])
|
|
97
|
+
h = model.init_state(1)
|
|
98
|
+
for t in range(len(e["x"])):
|
|
99
|
+
t0 = time.perf_counter()
|
|
100
|
+
out, h = model.step(batch["x"][:, t], h, batch["dt"][:, t])
|
|
101
|
+
model.heads(out, batch["elems"][:, t], batch["emask"][:, t])
|
|
102
|
+
lat.append((time.perf_counter() - t0) * 1000.0)
|
|
103
|
+
return {"latency_ms_p50": round(_pct(lat, 50), 4), "latency_ms_p95": round(_pct(lat, 95), 4)}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def evaluate_markov(train_eps, test_eps):
|
|
107
|
+
"""Script baseline: majority next action given (task, previous action type); target by word overlap."""
|
|
108
|
+
nxt = defaultdict(Counter)
|
|
109
|
+
tw = defaultdict(Counter)
|
|
110
|
+
for ep in train_eps:
|
|
111
|
+
prev = "<start>"
|
|
112
|
+
for st in ep["steps"]:
|
|
113
|
+
a = st["action"]
|
|
114
|
+
nxt[(ep["task"], prev)][a["type"]] += 1
|
|
115
|
+
if a.get("target"):
|
|
116
|
+
tw[(ep["task"], a["type"])].update(words(a["target"].get("name")))
|
|
117
|
+
prev = a["type"]
|
|
118
|
+
tally = Tally()
|
|
119
|
+
for ep in test_eps:
|
|
120
|
+
prev = "<start>"
|
|
121
|
+
pts, pps, yts, yps, nds = [], [], [], [], []
|
|
122
|
+
for st in ep["steps"]:
|
|
123
|
+
a = st["action"]
|
|
124
|
+
c = nxt.get((ep["task"], prev))
|
|
125
|
+
ptype = c.most_common(1)[0][0] if c else "wait"
|
|
126
|
+
els = (st["observation"].get("elements") or [])[:MAX_ELEMENTS]
|
|
127
|
+
best, best_s = 0, -1.0
|
|
128
|
+
for i, el in enumerate(els):
|
|
129
|
+
ws = words(el.get("name"))
|
|
130
|
+
s = sum(tw[(ep["task"], ptype)][w] for w in ws) / (len(ws) or 1)
|
|
131
|
+
if s > best_s:
|
|
132
|
+
best, best_s = i, s
|
|
133
|
+
from .features import target_index
|
|
134
|
+
|
|
135
|
+
ti = target_index(st["observation"], a)
|
|
136
|
+
pts.append(ACTION_TYPES.index(ptype))
|
|
137
|
+
pps.append(best)
|
|
138
|
+
yts.append(ACTION_TYPES.index(a["type"]))
|
|
139
|
+
yps.append(-100 if ti in (None, -1) else ti)
|
|
140
|
+
nds.append(bool(a.get("target")))
|
|
141
|
+
prev = a["type"]
|
|
142
|
+
tally.episode(ep["task"], pts, pps, yts, yps, nds)
|
|
143
|
+
m = tally.metrics()
|
|
144
|
+
m.update({"latency_ms_p50": 0.0, "latency_ms_p95": 0.0})
|
|
145
|
+
return m
|