@infernetprotocol/liquid-agent 0.0.0-stage → 0.1.67

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/LICENSE +15 -0
  2. package/README.md +15 -2
  3. package/fixtures/contract.json +6 -0
  4. package/fixtures/invalid/episode-extra-field.json +275 -0
  5. package/fixtures/invalid/episode-no-capture.json +265 -0
  6. package/fixtures/invalid/episode-unknown-action.json +274 -0
  7. package/fixtures/invalid/model-manifest-bad-hash.json +92 -0
  8. package/fixtures/invalid/predict-negative-seq.json +48 -0
  9. package/fixtures/invalid/proposal-bad-abstain.json +31 -0
  10. package/fixtures/invalid/proposal-confidence-range.json +31 -0
  11. package/fixtures/invalid/session-bad-seq.json +17 -0
  12. package/fixtures/invalid/train-job-bad-family.json +44 -0
  13. package/fixtures/invalid/train-job-zero-lr.json +44 -0
  14. package/fixtures/valid/action-proposal.json +27 -0
  15. package/fixtures/valid/dataset-bundle.json +3747 -0
  16. package/fixtures/valid/dataset-manifest.json +126 -0
  17. package/fixtures/valid/episode.json +270 -0
  18. package/fixtures/valid/inference-session.json +13 -0
  19. package/fixtures/valid/model-manifest.json +88 -0
  20. package/fixtures/valid/predict-request.json +44 -0
  21. package/fixtures/valid/train-job.json +40 -0
  22. package/package.json +28 -3
  23. package/runtime/infernet_liquid/__init__.py +6 -0
  24. package/runtime/infernet_liquid/__main__.py +95 -0
  25. package/runtime/infernet_liquid/bench.py +125 -0
  26. package/runtime/infernet_liquid/bundle.py +63 -0
  27. package/runtime/infernet_liquid/canonical.py +75 -0
  28. package/runtime/infernet_liquid/data.py +121 -0
  29. package/runtime/infernet_liquid/errors.py +13 -0
  30. package/runtime/infernet_liquid/evaluate.py +145 -0
  31. package/runtime/infernet_liquid/export.py +136 -0
  32. package/runtime/infernet_liquid/features.py +105 -0
  33. package/runtime/infernet_liquid/models.py +63 -0
  34. package/runtime/infernet_liquid/schema.py +155 -0
  35. package/runtime/infernet_liquid/secretscan.py +64 -0
  36. package/runtime/infernet_liquid/server.py +251 -0
  37. package/runtime/infernet_liquid/sessions.py +279 -0
  38. package/runtime/infernet_liquid/synth.py +194 -0
  39. package/runtime/infernet_liquid/train.py +136 -0
  40. package/runtime/pyproject.toml +14 -0
  41. package/runtime/requirements-cuda.lock +461 -0
  42. package/runtime/requirements.lock +344 -0
  43. package/schemas/action-proposal.schema.json +40 -0
  44. package/schemas/common.schema.json +88 -0
  45. package/schemas/dataset-bundle.schema.json +15 -0
  46. package/schemas/dataset-manifest.schema.json +100 -0
  47. package/schemas/episode.schema.json +71 -0
  48. package/schemas/inference-session.schema.json +23 -0
  49. package/schemas/model-manifest.schema.json +103 -0
  50. package/schemas/predict-request.schema.json +20 -0
  51. package/schemas/train-job.schema.json +75 -0
  52. package/src/canonical.js +49 -0
  53. package/src/contract.js +62 -0
  54. package/src/dataset.js +133 -0
  55. package/src/index.js +13 -0
  56. package/src/runtime.js +206 -0
  57. package/src/secrets.js +79 -0
  58. package/src/validate.js +161 -0
@@ -0,0 +1,95 @@
1
+ """python -m infernet_liquid <serve|train|evaluate|synth|bench|health>"""
2
+
3
+ import argparse
4
+ import json
5
+ import sys
6
+
7
+ from .errors import LiquidError
8
+
9
+
10
+ def main(argv=None):
11
+ ap = argparse.ArgumentParser(prog="infernet-liquid")
12
+ sub = ap.add_subparsers(dest="cmd", required=True)
13
+ s = sub.add_parser("serve", help="stdio NDJSON server (the Infernet contract)")
14
+ s.add_argument("--max-sessions", type=int, default=64)
15
+ s.add_argument("--ttl", type=int, default=600)
16
+ s.add_argument("--worker", default="local")
17
+ t = sub.add_parser("train")
18
+ t.add_argument("--dataset", required=True)
19
+ t.add_argument("--family", default="cfc", choices=["cfc", "ltc", "gru"])
20
+ t.add_argument("--output", required=True)
21
+ t.add_argument("--epochs", type=int)
22
+ t.add_argument("--seed", type=int)
23
+ t.add_argument("--hidden", type=int)
24
+ t.add_argument("--threads", type=int)
25
+ e = sub.add_parser("evaluate")
26
+ e.add_argument("--model", required=True)
27
+ e.add_argument("--dataset", required=True)
28
+ e.add_argument("--split", default="test")
29
+ y = sub.add_parser("synth")
30
+ y.add_argument("--out", required=True)
31
+ y.add_argument("--per-task", type=int, default=160)
32
+ y.add_argument("--seed", type=int, default=7)
33
+ y.add_argument("--shifted", action="store_true")
34
+ b = sub.add_parser("bench")
35
+ b.add_argument("--out-dir", required=True)
36
+ b.add_argument("--per-task", type=int, default=160)
37
+ b.add_argument("--families", default="cfc,gru,ltc")
38
+ b.add_argument("--seeds", default="0,1,2")
39
+ b.add_argument("--epochs", type=int)
40
+ b.add_argument("--threads", type=int, default=4)
41
+ b.add_argument("--report")
42
+ sub.add_parser("health")
43
+ a = ap.parse_args(argv)
44
+
45
+ try:
46
+ if a.cmd == "serve":
47
+ from .server import Server
48
+
49
+ Server(max_sessions=a.max_sessions, ttl_s=a.ttl, worker=a.worker).serve()
50
+ return 0
51
+ if a.cmd == "health":
52
+ from .server import Server
53
+
54
+ print(json.dumps(Server().dispatch("health", {}), indent=2))
55
+ return 0
56
+ if a.cmd == "train":
57
+ from .server import run_train
58
+
59
+ hp = {k: v for k, v in {"epochs": a.epochs, "seed": a.seed, "hidden_units": a.hidden}.items() if v is not None}
60
+ res = run_train({"bundle_path": a.dataset, "family": a.family, "output_dir": a.output, "hyperparameters": hp, "max_threads": a.threads},
61
+ on_event=lambda ev: print(json.dumps(ev), file=sys.stderr))
62
+ print(json.dumps({"model_dir": res["model_dir"], "model_hash": res["manifest"]["weights"]["weights_hash"], "metrics": res["eval"]["metrics"]}, indent=2))
63
+ return 0
64
+ if a.cmd == "evaluate":
65
+ from .server import run_evaluate
66
+
67
+ print(json.dumps(run_evaluate({"model_dir": a.model, "bundle_path": a.dataset, "split": a.split}), indent=2))
68
+ return 0
69
+ if a.cmd == "synth":
70
+ from .synth import make_bundle
71
+
72
+ bundle = make_bundle(per_task=a.per_task, seed=a.seed, shifted=a.shifted)
73
+ with open(a.out, "w", encoding="utf-8") as f:
74
+ json.dump(bundle, f, separators=(",", ":"))
75
+ print(json.dumps({"out": a.out, "dataset_hash": bundle["manifest"]["dataset_hash"], "counts": bundle["manifest"]["splits"]["counts"]}))
76
+ return 0
77
+ if a.cmd == "bench":
78
+ from .bench import render_markdown, run
79
+
80
+ s = run(a.out_dir, a.per_task, tuple(a.families.split(",")), tuple(int(x) for x in a.seeds.split(",")), a.epochs,
81
+ on_event=lambda ev: print(json.dumps(ev), file=sys.stderr), threads=a.threads)
82
+ md = render_markdown(s)
83
+ if a.report:
84
+ with open(a.report, "w", encoding="utf-8") as f:
85
+ f.write(md)
86
+ print(md)
87
+ return 0
88
+ except LiquidError as err:
89
+ print(json.dumps({"error": err.to_json()}), file=sys.stderr)
90
+ return 1
91
+ return 2
92
+
93
+
94
+ if __name__ == "__main__":
95
+ sys.exit(main())
@@ -0,0 +1,125 @@
1
+ """CfC vs GRU (vs LTC research, vs a script baseline) on identical synthetic splits."""
2
+
3
+ import json
4
+ import statistics
5
+ from pathlib import Path
6
+
7
+ from .data import encode_episode, split_episodes, verify_bundle
8
+ from .evaluate import evaluate_encoded, evaluate_markov
9
+ from .export import runtime_ref
10
+ from .synth import make_bundle
11
+ from .train import DEFAULTS, train
12
+
13
+ KEYS = ["action_accuracy", "target_accuracy", "step_accuracy", "episode_success", "external_write_recall", "latency_ms_p50", "latency_ms_p95"]
14
+
15
+
16
+ def run(out_dir, per_task=160, families=("cfc", "gru", "ltc"), seeds=(0, 1, 2), epochs=None, shift_per_task=40, on_event=None, threads=4):
17
+ out = Path(out_dir)
18
+ out.mkdir(parents=True, exist_ok=True)
19
+ main = make_bundle(per_task=per_task, seed=7)
20
+ shift = make_bundle(per_task=shift_per_task, seed=11, shifted=True)
21
+ verify_bundle(main)
22
+ verify_bundle(shift)
23
+ (out / "bench-bundle.json").write_text(json.dumps(main, separators=(",", ":")), "utf-8")
24
+ (out / "bench-shift-bundle.json").write_text(json.dumps(shift, separators=(",", ":")), "utf-8")
25
+ splits = split_episodes(main)
26
+ held = {}
27
+ for e in splits["test"]:
28
+ held[e["task"]] = held.get(e["task"], 0) + 1
29
+ results = {}
30
+ hp_base = dict(DEFAULTS)
31
+ if epochs:
32
+ hp_base["epochs"] = epochs
33
+ for fam in families:
34
+ runs = []
35
+ for seed in seeds:
36
+ hp = {**hp_base, "seed": seed}
37
+ model, info = train(main, fam, hp, "cpu", threads, on_event)
38
+ test = evaluate_encoded(model, info["encoded"]["test"])
39
+ enc_shift = [encode_episode(e, hp["feature_dim"]) for e in shift["episodes"]]
40
+ sh = evaluate_encoded(model, enc_shift, timing=False)
41
+ runs.append({"seed": seed, "train_seconds": info["train_seconds"], "epochs_run": info["epochs_run"], "params": model.param_count(), "test": test, "shift": sh})
42
+ if on_event:
43
+ on_event({"event": "bench_run", "family": fam, "seed": seed, "step_accuracy": test["step_accuracy"], "episode_success": test["episode_success"], "shift_episode_success": sh["episode_success"]})
44
+ results[fam] = runs
45
+ markov_test = evaluate_markov(splits["train"], splits["test"])
46
+ markov_shift = evaluate_markov(splits["train"], shift["episodes"])
47
+ summary = {
48
+ "runtime": runtime_ref(),
49
+ "dataset_hash": main["manifest"]["dataset_hash"],
50
+ "shift_dataset_hash": shift["manifest"]["dataset_hash"],
51
+ "split_counts": main["manifest"]["splits"]["counts"],
52
+ "held_out_per_task": held,
53
+ "shift_episodes": len(shift["episodes"]),
54
+ "hyperparameters": hp_base,
55
+ "seeds": list(seeds),
56
+ "families": {},
57
+ "script_baseline": {"test": markov_test, "shift": markov_shift},
58
+ }
59
+ for fam, runs in results.items():
60
+ agg = {}
61
+ for k in KEYS:
62
+ vals = [r["test"].get(k, 0.0) for r in runs]
63
+ agg[k] = {"mean": round(statistics.mean(vals), 4), "min": round(min(vals), 4), "max": round(max(vals), 4)}
64
+ agg["shift_episode_success"] = {"mean": round(statistics.mean(r["shift"]["episode_success"] for r in runs), 4)}
65
+ agg["shift_step_accuracy"] = {"mean": round(statistics.mean(r["shift"]["step_accuracy"] for r in runs), 4)}
66
+ agg["train_seconds"] = {"mean": round(statistics.mean(r["train_seconds"] for r in runs), 2)}
67
+ agg["params"] = runs[0]["params"]
68
+ agg["by_task_episode_success"] = {
69
+ t: round(statistics.mean(r["test"]["by_task"][t]["episode_success"] for r in runs), 4) for t in runs[0]["test"]["by_task"]
70
+ }
71
+ summary["families"][fam] = {"aggregate": agg, "runs": runs}
72
+ (out / "benchmark.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", "utf-8")
73
+ return summary
74
+
75
+
76
+ def render_markdown(s):
77
+ fams = list(s["families"])
78
+ lines = [
79
+ "# Temporal policy benchmark: CfC vs GRU",
80
+ "",
81
+ "Generated by `python -m infernet_liquid bench` on synthetic mock desktop traces",
82
+ f"(runtime {s['runtime']['name']} {s['runtime']['version']}, torch {s['runtime']['torch']}, ncps {s['runtime']['ncps']}, CPU).",
83
+ "",
84
+ f"- Dataset `{s['dataset_hash']}`: splits {s['split_counts']}, split by recording session.",
85
+ f"- Held-out test episodes per workflow: {s['held_out_per_task']}.",
86
+ f"- Layout-shift holdout: {s['shift_episodes']} episodes on an unseen layout (renamed controls, reordered) `{s['shift_dataset_hash']}`.",
87
+ f"- Same splits, features, hidden size ({s['hyperparameters']['hidden_units']}), optimiser and epoch budget for every family; seeds {s['seeds']}; mean (min to max) over seeds.",
88
+ "",
89
+ "Predeclared metric: **episode_success** on held-out test episodes (every step's action type and target right under teacher forcing).",
90
+ "Secondary: step_accuracy, layout-shift episode success, external_write_recall (an external write predicted as one, so approval gating fires), batch-1 step latency.",
91
+ "",
92
+ "| model | params | episode success | step acc | target acc | ext-write recall | shift episode success | latency p50 / p95 ms | train s |",
93
+ "|---|---|---|---|---|---|---|---|---|",
94
+ ]
95
+ for f in fams:
96
+ a = s["families"][f]["aggregate"]
97
+ r = lambda k: f"{a[k]['mean']:.3f} ({a[k]['min']:.3f} to {a[k]['max']:.3f})"
98
+ label = f + (" (research)" if f == "ltc" else " (baseline)" if f == "gru" else "")
99
+ lines.append(f"| {label} | {a['params']} | {r('episode_success')} | {r('step_accuracy')} | {a['target_accuracy']['mean']:.3f} | {a['external_write_recall']['mean']:.3f} | {a['shift_episode_success']['mean']:.3f} | {a['latency_ms_p50']['mean']:.3f} / {a['latency_ms_p95']['mean']:.3f} | {a['train_seconds']['mean']:.1f} |")
100
+ m = s["script_baseline"]
101
+ lines.append(f"| script (majority next action) | 0 | {m['test']['episode_success']:.3f} | {m['test']['step_accuracy']:.3f} | {m['test']['target_accuracy']:.3f} | {m['test']['external_write_recall']:.3f} | {m['shift']['episode_success']:.3f} | n/a | 0 |")
102
+ lines += ["", "Per-workflow episode success (mean over seeds):", "", "| model | " + " | ".join(s["families"][fams[0]]["aggregate"]["by_task_episode_success"]) + " |", "|---|" + "---|" * len(s["families"][fams[0]]["aggregate"]["by_task_episode_success"])]
103
+ for f in fams:
104
+ bt = s["families"][f]["aggregate"]["by_task_episode_success"]
105
+ lines.append(f"| {f} | " + " | ".join(f"{v:.3f}" for v in bt.values()) + " |")
106
+ if "cfc" in s["families"] and "gru" in s["families"]:
107
+ c = s["families"]["cfc"]["aggregate"]
108
+ g = s["families"]["gru"]["aggregate"]
109
+ d = c["episode_success"]["mean"] - g["episode_success"]["mean"]
110
+ if c["episode_success"]["min"] > g["episode_success"]["max"]:
111
+ verdict = "CfC beats GRU on every seed: CfC may be selected for these tasks."
112
+ elif g["episode_success"]["min"] > c["episode_success"]["max"]:
113
+ verdict = "GRU beats CfC on every seed: keep GRU."
114
+ else:
115
+ verdict = "Neither beats the other on every seed: keep GRU (cheaper, simpler)."
116
+ lines += [
117
+ "",
118
+ "## Verdict",
119
+ "",
120
+ f"{verdict} CfC minus GRU mean episode success: {d:+.3f}.",
121
+ "Rule (pipeline PRD): a liquid model is chosen only when it beats the baseline on the predeclared metric on every seed.",
122
+ "",
123
+ "Limits: synthetic traces from a generator with three layouts. They exercise the pipeline and the comparison method; they say nothing about real desktops. Rerun on reviewed W3BS episodes before choosing a family for a real task.",
124
+ ]
125
+ return "\n".join(lines) + "\n"
@@ -0,0 +1,63 @@
1
+ """Build sealed episodes and DatasetBundleV1 files; mirrors src/dataset.js byte for byte."""
2
+
3
+ from .canonical import dataset_hash, episode_hash, sha256_hex
4
+ from .secretscan import SCANNER_VERSION
5
+ from . import SCHEMA_VERSION
6
+
7
+
8
+ def seal_episode(ep):
9
+ e = {k: v for k, v in ep.items() if k != "content_hash"}
10
+ e["schema_version"] = SCHEMA_VERSION
11
+ e["kind"] = "episode"
12
+ e["content_hash"] = episode_hash(e)
13
+ return e
14
+
15
+
16
+ def _group(ep, strategy):
17
+ if strategy == "by_task":
18
+ return ep["task"]
19
+ if strategy == "by_variant":
20
+ return ep.get("variant") or ep.get("split_group") or ep["episode_id"]
21
+ return ep.get("split_group") or ep["episode_id"]
22
+
23
+
24
+ def _bucket(seed, group):
25
+ return int(sha256_hex(f"{seed}:{group}")[:8], 16) / 0x100000000
26
+
27
+
28
+ def assign_splits(episodes, strategy="by_session", seed=0, validation=0.15, test=0.2):
29
+ out = []
30
+ for ep in episodes:
31
+ g = _group(ep, strategy)
32
+ b = _bucket(seed, g)
33
+ split = "test" if b < test else "validation" if b < test + validation else "train"
34
+ out.append((ep, split, g))
35
+ return out
36
+
37
+
38
+ def build_bundle(episodes, dataset_id, created_at, consent, name=None, strategy="by_session", seed=0, validation=0.15, test=0.2, provenance=None, license="CC0-1.0"):
39
+ assigned = assign_splits(episodes, strategy, seed, validation, test)
40
+ counts = {"train": 0, "validation": 0, "test": 0}
41
+ for _, split, _ in assigned:
42
+ counts[split] += 1
43
+ manifest = {
44
+ "schema_version": SCHEMA_VERSION,
45
+ "kind": "dataset_manifest",
46
+ "dataset_id": dataset_id,
47
+ "version": 1,
48
+ "created_at": created_at,
49
+ "episodes": [
50
+ {"episode_id": ep["episode_id"], "content_hash": ep["content_hash"], "task": ep["task"], "split": split, "split_group": str(g)[:128]}
51
+ for ep, split, g in assigned
52
+ ],
53
+ "splits": {"strategy": strategy, "seed": seed, "counts": counts},
54
+ "license": license,
55
+ "provenance": provenance or {"source": "synthetic", "generator": "infernet-liquid synth"},
56
+ "consent": consent,
57
+ "retention": {"days": None, "delete_on_revoke": True},
58
+ "redaction": {"profile": "strict", "secret_scan": "passed", "scanner_version": SCANNER_VERSION},
59
+ }
60
+ if name:
61
+ manifest["name"] = name
62
+ manifest["dataset_hash"] = dataset_hash(manifest)
63
+ return {"schema_version": SCHEMA_VERSION, "kind": "dataset_bundle", "manifest": manifest, "episodes": episodes}
@@ -0,0 +1,75 @@
1
+ """Canonical JSON identical to src/canonical.js (sorted keys, no spaces, JS number format)."""
2
+
3
+ import hashlib
4
+ import json
5
+ import math
6
+
7
+
8
+ def _num(v):
9
+ if isinstance(v, bool):
10
+ raise TypeError("bool is not a number")
11
+ if isinstance(v, int):
12
+ return str(v)
13
+ if not math.isfinite(v):
14
+ raise TypeError("canonical JSON: non-finite number")
15
+ if v == int(v) and abs(v) < 1e21:
16
+ return str(int(v))
17
+ r = repr(v)
18
+ if "e" in r:
19
+ mant, exp = r.split("e")
20
+ sign = "-" if exp.startswith("-") else "+"
21
+ r = f"{mant}e{sign}{int(exp.lstrip('+-'))}"
22
+ return r
23
+
24
+
25
+ def canonicalize(value):
26
+ if value is None:
27
+ return "null"
28
+ if value is True:
29
+ return "true"
30
+ if value is False:
31
+ return "false"
32
+ if isinstance(value, (int, float)):
33
+ return _num(value)
34
+ if isinstance(value, str):
35
+ return json.dumps(value, ensure_ascii=False)
36
+ if isinstance(value, (list, tuple)):
37
+ return "[" + ",".join(canonicalize(v) for v in value) + "]"
38
+ if isinstance(value, dict):
39
+ keys = sorted(k for k, v in value.items())
40
+ return "{" + ",".join(json.dumps(k, ensure_ascii=False) + ":" + canonicalize(value[k]) for k in keys) + "}"
41
+ raise TypeError(f"canonical JSON: unsupported type {type(value).__name__}")
42
+
43
+
44
+ def sha256_hex(data):
45
+ if isinstance(data, str):
46
+ data = data.encode("utf-8")
47
+ return hashlib.sha256(data).hexdigest()
48
+
49
+
50
+ def hash_canonical(value):
51
+ return "sha256:" + sha256_hex(canonicalize(value))
52
+
53
+
54
+ def hash_without(value, fields):
55
+ return hash_canonical({k: v for k, v in value.items() if k not in fields})
56
+
57
+
58
+ def episode_hash(ep):
59
+ return hash_without(ep, ("content_hash",))
60
+
61
+
62
+ def dataset_hash(manifest):
63
+ return hash_canonical({"episodes": manifest["episodes"], "splits": manifest["splits"]})
64
+
65
+
66
+ def manifest_hash(manifest):
67
+ return hash_without(manifest, ("manifest_hash", "signature"))
68
+
69
+
70
+ def file_sha256(path):
71
+ h = hashlib.sha256()
72
+ with open(path, "rb") as f:
73
+ for chunk in iter(lambda: f.read(1 << 20), b""):
74
+ h.update(chunk)
75
+ return "sha256:" + h.hexdigest()
@@ -0,0 +1,121 @@
1
+ """Load, verify and tensorise DatasetBundleV1 files."""
2
+
3
+ import json
4
+
5
+ import torch
6
+
7
+ from .canonical import dataset_hash, episode_hash
8
+ from .errors import LiquidError
9
+ from .features import ACTION_TYPES, ELEMENT_DIM, MAX_ELEMENTS, encode_element, encode_step, target_index
10
+ from .schema import validate
11
+ from .secretscan import scan_secrets
12
+
13
+
14
+ def load_json(path):
15
+ try:
16
+ with open(path, encoding="utf-8") as f:
17
+ return json.load(f)
18
+ except FileNotFoundError:
19
+ raise LiquidError("E_BAD_REQUEST", f"file not found: {path}")
20
+ except json.JSONDecodeError as e:
21
+ raise LiquidError("E_SCHEMA", f"{path} is not JSON: {e}")
22
+
23
+
24
+ def verify_bundle(bundle):
25
+ """Schema, every hash, manifest/episode agreement and the secret scan. Returns the dataset hash."""
26
+ errors = validate(bundle, "dataset_bundle")
27
+ if errors:
28
+ raise LiquidError("E_SCHEMA", "dataset bundle failed liquid-agent/v1 validation: " + "; ".join(f"{p or '/'} {m}" for p, m in errors[:5]))
29
+ eps = {e["episode_id"]: e for e in bundle["episodes"]}
30
+ if len(eps) != len(bundle["episodes"]):
31
+ raise LiquidError("E_SCHEMA", "duplicate episode_id in bundle")
32
+ manifest = bundle["manifest"]
33
+ if len(manifest["episodes"]) != len(eps):
34
+ raise LiquidError("E_HASH_MISMATCH", "bundle carries episodes the manifest does not name")
35
+ for entry in manifest["episodes"]:
36
+ ep = eps.get(entry["episode_id"])
37
+ if ep is None:
38
+ raise LiquidError("E_HASH_MISMATCH", f"manifest names {entry['episode_id']} but the bundle lacks it")
39
+ if ep["content_hash"] != entry["content_hash"] or episode_hash(ep) != ep["content_hash"]:
40
+ raise LiquidError("E_HASH_MISMATCH", f"episode {entry['episode_id']} does not match its hash")
41
+ if dataset_hash(manifest) != manifest["dataset_hash"]:
42
+ raise LiquidError("E_HASH_MISMATCH", "dataset_hash does not match the manifest")
43
+ hits = scan_secrets(bundle["episodes"])
44
+ if hits:
45
+ raise LiquidError("E_SECRET_DETECTED", f"dataset contains what looks like a secret at {hits[0]['path']} ({','.join(hits[0]['kinds'])})")
46
+ return manifest["dataset_hash"]
47
+
48
+
49
+ def split_episodes(bundle):
50
+ eps = {e["episode_id"]: e for e in bundle["episodes"]}
51
+ out = {"train": [], "validation": [], "test": []}
52
+ for entry in bundle["manifest"]["episodes"]:
53
+ out[entry["split"]].append(eps[entry["episode_id"]])
54
+ return out
55
+
56
+
57
+ def encode_episode(ep, feature_dim):
58
+ xs, dts, elems, masks, y_type, y_ptr, needs = [], [], [], [], [], [], []
59
+ hidden_targets = 0
60
+ prev, prev_t = None, None
61
+ for step in ep["steps"]:
62
+ dt_ms = 0 if prev_t is None else max(0, step["t_ms"] - prev_t)
63
+ obs = step["observation"]
64
+ xs.append(encode_step(ep["task"], obs, prev, dt_ms, feature_dim))
65
+ dts.append([min(dt_ms / 1000.0, 60.0)])
66
+ els = (obs.get("elements") or [])[:MAX_ELEMENTS]
67
+ elems.append([encode_element(el, i) for i, el in enumerate(els)])
68
+ masks.append(len(els))
69
+ y_type.append(ACTION_TYPES.index(step["action"]["type"]))
70
+ needs.append(bool(step["action"].get("target")))
71
+ ti = target_index(obs, step["action"])
72
+ if ti is None:
73
+ hidden_targets += 1
74
+ ti = -100
75
+ elif ti == -1:
76
+ ti = -100
77
+ y_ptr.append(ti)
78
+ prev, prev_t = step["action"], step["t_ms"]
79
+ return {
80
+ "episode_id": ep["episode_id"],
81
+ "task": ep["task"],
82
+ "variant": ep.get("variant"),
83
+ "x": xs,
84
+ "dt": dts,
85
+ "elems": elems,
86
+ "n_elems": masks,
87
+ "y_type": y_type,
88
+ "y_ptr": y_ptr,
89
+ "needs_target": needs,
90
+ "hidden_targets": hidden_targets,
91
+ }
92
+
93
+
94
+ def collate(encoded):
95
+ """Pad a list of encoded episodes into batch tensors."""
96
+ B = len(encoded)
97
+ T = max(len(e["x"]) for e in encoded)
98
+ F = len(encoded[0]["x"][0])
99
+ E = max(1, max(max(e["n_elems"]) if e["n_elems"] else 0 for e in encoded))
100
+ x = torch.zeros(B, T, F)
101
+ dt = torch.zeros(B, T, 1)
102
+ elems = torch.zeros(B, T, E, ELEMENT_DIM)
103
+ emask = torch.zeros(B, T, E, dtype=torch.bool)
104
+ smask = torch.zeros(B, T, dtype=torch.bool)
105
+ y_type = torch.full((B, T), -100, dtype=torch.long)
106
+ y_ptr = torch.full((B, T), -100, dtype=torch.long)
107
+ for b, e in enumerate(encoded):
108
+ t = len(e["x"])
109
+ x[b, :t] = torch.tensor(e["x"])
110
+ dt[b, :t] = torch.tensor(e["dt"])
111
+ smask[b, :t] = True
112
+ y_type[b, :t] = torch.tensor(e["y_type"])
113
+ y_ptr[b, :t] = torch.tensor(e["y_ptr"])
114
+ for s, (vecs, n) in enumerate(zip(e["elems"], e["n_elems"])):
115
+ if n:
116
+ elems[b, s, :n] = torch.tensor(vecs)
117
+ emask[b, s, :n] = True
118
+ # A step with no visible elements still needs one finite pointer slot.
119
+ empty = ~emask.any(-1)
120
+ emask[..., 0] |= empty
121
+ return {"x": x, "dt": dt, "elems": elems, "emask": emask, "smask": smask, "y_type": y_type, "y_ptr": y_ptr}
@@ -0,0 +1,13 @@
1
+ """Failure taxonomy shared with src/contract.js ERROR_CODES."""
2
+
3
+ RETRIABLE = {"E_CAPACITY", "E_TIMEOUT", "E_RUNTIME"}
4
+
5
+
6
+ class LiquidError(Exception):
7
+ def __init__(self, code, message, **extra):
8
+ super().__init__(message)
9
+ self.code = code
10
+ self.extra = extra
11
+
12
+ def to_json(self):
13
+ return {"code": self.code, "message": str(self), "retriable": self.code in RETRIABLE, **self.extra}
@@ -0,0 +1,145 @@
1
+ """Metrics: per-step and per-episode accuracy, external-write recall, stateful latency."""
2
+
3
+ import time
4
+ from collections import Counter, defaultdict
5
+
6
+ import torch
7
+
8
+ from .data import collate
9
+ from .features import ACTION_TYPES, EXTERNAL_WRITE, MAX_ELEMENTS, words
10
+ from .train import forward_batch
11
+
12
+ EXT = {ACTION_TYPES.index(t) for t in EXTERNAL_WRITE}
13
+
14
+
15
+ def _pct(xs, q):
16
+ if not xs:
17
+ return 0.0
18
+ s = sorted(xs)
19
+ k = min(len(s) - 1, max(0, int(round(q / 100.0 * (len(s) - 1)))))
20
+ return s[k]
21
+
22
+
23
+ class Tally:
24
+ def __init__(self):
25
+ self.steps = self.type_ok = self.ptr_total = self.ptr_ok = self.step_ok = 0
26
+ self.episodes = self.ep_ok = 0
27
+ self.ext_total = self.ext_caught = 0
28
+ self.by_task = defaultdict(lambda: [0, 0, 0, 0]) # steps, step_ok, episodes, ep_ok
29
+
30
+ def episode(self, task, preds_type, preds_ptr, y_type, y_ptr, needs):
31
+ all_ok = True
32
+ bt = self.by_task[task]
33
+ for pt, pp, yt, yp, nd in zip(preds_type, preds_ptr, y_type, y_ptr, needs):
34
+ self.steps += 1
35
+ bt[0] += 1
36
+ tok = pt == yt
37
+ self.type_ok += tok
38
+ pok = True
39
+ if nd:
40
+ self.ptr_total += 1
41
+ pok = yp != -100 and pp == yp
42
+ self.ptr_ok += pok
43
+ ok = tok and pok
44
+ self.step_ok += ok
45
+ bt[1] += ok
46
+ all_ok &= ok
47
+ if yt in EXT:
48
+ self.ext_total += 1
49
+ self.ext_caught += pt in EXT
50
+ self.episodes += 1
51
+ bt[2] += 1
52
+ self.ep_ok += all_ok
53
+ bt[3] += all_ok
54
+
55
+ def metrics(self):
56
+ r = lambda a, b: round(a / b, 4) if b else 0.0
57
+ return {
58
+ "steps": self.steps,
59
+ "episodes": self.episodes,
60
+ "action_accuracy": r(self.type_ok, self.steps),
61
+ "target_accuracy": r(self.ptr_ok, self.ptr_total),
62
+ "step_accuracy": r(self.step_ok, self.steps),
63
+ "episode_success": r(self.ep_ok, self.episodes),
64
+ "external_write_recall": r(self.ext_caught, self.ext_total) if self.ext_total else 1.0,
65
+ "by_task": {t: {"step_accuracy": r(v[1], v[0]), "episode_success": r(v[3], v[2]), "episodes": v[2]} for t, v in sorted(self.by_task.items())},
66
+ }
67
+
68
+
69
+ @torch.no_grad()
70
+ def evaluate_encoded(model, encoded, device="cpu", timing=True, batch_size=64):
71
+ was_training = model.training
72
+ model.eval()
73
+ tally = Tally()
74
+ for i in range(0, len(encoded), batch_size):
75
+ chunk = encoded[i:i + batch_size]
76
+ batch = {k: v.to(device) for k, v in collate(chunk).items()}
77
+ tl, pl = forward_batch(model, batch)
78
+ pt = tl.argmax(-1).cpu().tolist()
79
+ pp = pl.argmax(-1).cpu().tolist()
80
+ for b, e in enumerate(chunk):
81
+ n = len(e["x"])
82
+ tally.episode(e["task"], pt[b][:n], pp[b][:n], e["y_type"], e["y_ptr"], e["needs_target"])
83
+ m = tally.metrics()
84
+ if timing and encoded:
85
+ m.update(step_latency(model, encoded[: min(20, len(encoded))]))
86
+ if was_training:
87
+ model.train()
88
+ return m
89
+
90
+
91
+ @torch.no_grad()
92
+ def step_latency(model, encoded):
93
+ """Batch-1 stateful step latency, the serving path: one forward per observed step."""
94
+ lat = []
95
+ for e in encoded:
96
+ batch = collate([e])
97
+ h = model.init_state(1)
98
+ for t in range(len(e["x"])):
99
+ t0 = time.perf_counter()
100
+ out, h = model.step(batch["x"][:, t], h, batch["dt"][:, t])
101
+ model.heads(out, batch["elems"][:, t], batch["emask"][:, t])
102
+ lat.append((time.perf_counter() - t0) * 1000.0)
103
+ return {"latency_ms_p50": round(_pct(lat, 50), 4), "latency_ms_p95": round(_pct(lat, 95), 4)}
104
+
105
+
106
+ def evaluate_markov(train_eps, test_eps):
107
+ """Script baseline: majority next action given (task, previous action type); target by word overlap."""
108
+ nxt = defaultdict(Counter)
109
+ tw = defaultdict(Counter)
110
+ for ep in train_eps:
111
+ prev = "<start>"
112
+ for st in ep["steps"]:
113
+ a = st["action"]
114
+ nxt[(ep["task"], prev)][a["type"]] += 1
115
+ if a.get("target"):
116
+ tw[(ep["task"], a["type"])].update(words(a["target"].get("name")))
117
+ prev = a["type"]
118
+ tally = Tally()
119
+ for ep in test_eps:
120
+ prev = "<start>"
121
+ pts, pps, yts, yps, nds = [], [], [], [], []
122
+ for st in ep["steps"]:
123
+ a = st["action"]
124
+ c = nxt.get((ep["task"], prev))
125
+ ptype = c.most_common(1)[0][0] if c else "wait"
126
+ els = (st["observation"].get("elements") or [])[:MAX_ELEMENTS]
127
+ best, best_s = 0, -1.0
128
+ for i, el in enumerate(els):
129
+ ws = words(el.get("name"))
130
+ s = sum(tw[(ep["task"], ptype)][w] for w in ws) / (len(ws) or 1)
131
+ if s > best_s:
132
+ best, best_s = i, s
133
+ from .features import target_index
134
+
135
+ ti = target_index(st["observation"], a)
136
+ pts.append(ACTION_TYPES.index(ptype))
137
+ pps.append(best)
138
+ yts.append(ACTION_TYPES.index(a["type"]))
139
+ yps.append(-100 if ti in (None, -1) else ti)
140
+ nds.append(bool(a.get("target")))
141
+ prev = a["type"]
142
+ tally.episode(ep["task"], pts, pps, yts, yps, nds)
143
+ m = tally.metrics()
144
+ m.update({"latency_ms_p50": 0.0, "latency_ms_p95": 0.0})
145
+ return m