dsh-eval-guard 0.7.4 → 0.7.6

package/assets/neural-pendulum/rl-feasibility/models.py

@@ -1,124 +0,0 @@
-"""Networks: PPO actor-critic MLP, SAC squashed-Gaussian actor, twin Q.
-
-PPO: 2x128 tanh, separate actor and critic, state-independent log_std capped
-to [-2.3, -0.25] (the recipe of the sibling rounds).
-SAC: 2x256 tanh actor with tanh-Gaussian head and a shared-trunk twin Q (two
-output heads).  Sharing the critic trunk keeps the CPU update affordable while
-preserving the double-Q target.
-"""
-from __future__ import annotations
-
-import math
-
-import numpy as np
-import torch
-import torch.nn as nn
-
-
-class MLP(nn.Module):
-    def __init__(self, in_dim, hidden=128, depth=2, out_dim=1, out_gain=0.01):
-        super().__init__()
-        layers = []
-        d = in_dim
-        for _ in range(depth):
-            lin = nn.Linear(d, hidden)
-            nn.init.orthogonal_(lin.weight, gain=np.sqrt(2))
-            nn.init.zeros_(lin.bias)
-            layers += [lin, nn.Tanh()]
-            d = hidden
-        self.body = nn.Sequential(*layers)
-        self.out = nn.Linear(d, out_dim)
-        nn.init.orthogonal_(self.out.weight, gain=out_gain)
-        nn.init.zeros_(self.out.bias)
-
-    def forward(self, x):
-        return self.out(self.body(x))
-
-
-# ------------------------------- PPO ---------------------------------------
-LOG_STD_MIN, LOG_STD_MAX = -2.3, -0.25
-
-
-class ActorCritic(nn.Module):
-    def __init__(self, obs_dim, act_dim=2, hidden=128, depth=2):
-        super().__init__()
-        self.act_dim = act_dim
-        self.actor = MLP(obs_dim, hidden, depth, act_dim)
-        self.critic = MLP(obs_dim, hidden, depth, 1, out_gain=1.0)
-        self.log_std = nn.Parameter(torch.full((act_dim,), -0.7))
-
-    def dist(self, obs):
-        mean = self.actor(obs)
-        std = self.effective_log_std().exp().expand_as(mean)
-        return torch.distributions.Normal(mean, std)
-
-    def effective_log_std(self):
-        return self.log_std.clamp(LOG_STD_MIN, LOG_STD_MAX)
-
-    def value(self, obs):
-        return self.critic(obs).squeeze(-1)
-
-    def deterministic_action(self, obs):
-        return self.actor(obs)
-
-
-# ------------------------------- SAC ---------------------------------------
-SAC_LOG_STD_MIN, SAC_LOG_STD_MAX = -5.0, 2.0
-
-
-class SquashedGaussianActor(nn.Module):
-    def __init__(self, in_dim, act_dim=2, hidden=256, depth=2):
-        super().__init__()
-        self.body = nn.Sequential()
-        d = in_dim
-        for _ in range(depth):
-            lin = nn.Linear(d, hidden)
-            nn.init.orthogonal_(lin.weight, gain=math.sqrt(2))
-            nn.init.zeros_(lin.bias)
-            self.body.extend([lin, nn.Tanh()])
-            d = hidden
-        self.mean = nn.Linear(hidden, act_dim)
-        self.log_std = nn.Linear(hidden, act_dim)
-        nn.init.orthogonal_(self.mean.weight, gain=0.01)
-        nn.init.zeros_(self.mean.bias)
-        nn.init.orthogonal_(self.log_std.weight, gain=0.01)
-        nn.init.zeros_(self.log_std.bias)
-        self.act_dim = act_dim
-
-    def forward(self, obs):
-        h = self.body(obs)
-        return self.mean(h), self.log_std(h).clamp(SAC_LOG_STD_MIN, SAC_LOG_STD_MAX)
-
-    def sample(self, obs, det=False):
-        if det:
-            return torch.tanh(self.mean(self.body(obs))), None
-        mean, log_std = self(obs)
-        std = log_std.exp()
-        eps = torch.randn_like(mean)
-        x = mean + std * eps
-        a = torch.tanh(x)
-        logp = (-0.5 * eps ** 2 - log_std - 0.5 * math.log(2 * math.pi)).sum(-1)
-        logp = logp - torch.log(1 - a ** 2 + 1e-6).sum(-1)
-        return a, logp
-
-
-class TwinQ(nn.Module):
-    """Shared trunk, two Q heads; forward returns (B, 2)."""
-
-    def __init__(self, in_dim, act_dim=2, hidden=256, depth=2):
-        super().__init__()
-        self.body = nn.Sequential()
-        d = in_dim + act_dim
-        for _ in range(depth):
-            lin = nn.Linear(d, hidden)
-            nn.init.orthogonal_(lin.weight, gain=math.sqrt(2))
-            nn.init.zeros_(lin.bias)
-            self.body.extend([lin, nn.Tanh()])
-            d = hidden
-        self.q = nn.Linear(hidden, 2)
-        nn.init.orthogonal_(self.q.weight, gain=1.0)
-        nn.init.zeros_(self.q.bias)
-
-    def forward(self, obs, act):
-        x = torch.cat([obs, act], dim=-1)
-        return self.q(self.body(x))

package/assets/neural-pendulum/rl-feasibility/obs.py

@@ -1,205 +0,0 @@
-"""Policy-side angle observation for the 3-link pendulum (switchable modes).
-
-The simulator only exposes the three joint angles theta.  The observation is
-built entirely from that angle stream (no omega, no tip, no energy):
-
-    a_i(t)  = wrap_to_pi(theta_i(t) - pi)                  (upright-relative)
-    da_i(t) = wrap_to_pi(a_i(t) - a_i(t-1)) / DT,          da_i(0) = 0 at reset
-
-    frame(t) = [cos a1, sin a1, cos a2, sin a2, cos a3, sin a3,
-                da1/5, da2/5, da3/5]                        (9 dims, diff=True)
-    frame(t) = [cos a1, sin a1, cos a2, sin a2, cos a3, sin a3]
-                                                            (6 dims, diff=False)
-
-    obs(t)   = H frames, oldest first, flattened
-
-Modes (--obs):
-    full      H=4, diff=True   -> 36 dims  (the original observation)
-    angle     H=4, diff=False  -> 24 dims  (cos/sin only, no manual velocity)
-    full_h1   H=1, diff=True   ->  9 dims  (single frame incl. da/5)
-    angle_h1  H=1, diff=False  ->  6 dims  (single frame cos/sin only)
-    full_h2   H=2, diff=True   -> 18 dims  (extra mode, optional)
-    angle_h2  H=2, diff=False  -> 12 dims  (extra mode, optional)
-
-Even when diff=False the builder still computes da internally, because the
-trainer-side reward uses it; the channel is only left out of the policy
-observation.  theta is used in radians exactly as produced by the simulator
-(never divided by pi).  The history is initialised by repeating the reset
-frame H times.
-"""
-from __future__ import annotations
-
-import numpy as np
-
-DT = 0.01
-H = 4
-VEL_SCALE = 5.0
-
-# mode -> (history length, whether the manual-difference velocity is observed)
-OBS_MODES = {
-    "full": dict(h=4, diff=True),
-    "angle": dict(h=4, diff=False),
-    "full_h1": dict(h=1, diff=True),
-    "angle_h1": dict(h=1, diff=False),
-    "full_h2": dict(h=2, diff=True),
-    "angle_h2": dict(h=2, diff=False),
-    "angle_h8": dict(h=8, diff=False),   # extra: can longer history replace da?
-}
-
-
-def frame_dim(diff: bool) -> int:
-    return 9 if diff else 6
-
-
-def obs_spec(mode: str = "full") -> dict:
-    if mode not in OBS_MODES:
-        raise ValueError(f"unknown obs mode {mode!r}; choose from {sorted(OBS_MODES)}")
-    c = OBS_MODES[mode]
-    fd = frame_dim(c["diff"])
-    notes = ("cos/sin(a) + da/5, H=%d, a=wrap(theta-pi), da is the manual "
-             "finite difference, da(0)=0" % c["h"])
-    if not c["diff"]:
-        notes = ("cos/sin(a) only (NO velocity channel), H=%d, "
-                 "a=wrap(theta-pi)" % c["h"])
-    return dict(mode=mode, enc="sincos", diff=bool(c["diff"]), h=int(c["h"]),
-                vel_scale=VEL_SCALE, dt=DT, frame_dim=fd,
-                dim=int(c["h"] * fd), notes=notes)
-
-
-# Backwards-compatible constants for the original full observation.
-FRAME_DIM = frame_dim(True)
-OBS_DIM = obs_spec("full")["dim"]
-
-
-def wrap_to_pi(a):
-    return (a + np.pi) % (2.0 * np.pi) - np.pi
-
-
-def frame_np(a, da, diff=True):
-    a = np.asarray(a, dtype=np.float64)
-    cs = np.empty(6, dtype=np.float64)
-    cs[0::2] = np.cos(a)
-    cs[1::2] = np.sin(a)
-    if not diff:
-        return cs
-    return np.concatenate([cs, np.asarray(da, dtype=np.float64) / VEL_SCALE])
-
-
-class NumpyAngleObs:
-    """Float64 single-env builder.  reset/update take theta only."""
-
-    def __init__(self, h: int = H, vel_scale: float = VEL_SCALE, dt: float = DT,
-                 diff: bool = True):
-        self.h = int(h)
-        self.diff = bool(diff)
-        self.fdim = frame_dim(self.diff)
-        self.vel_scale = float(vel_scale)
-        self.dt = float(dt)
-        self.a_prev = np.zeros(3)
-        self.da = np.zeros(3)
-        self.frames = np.zeros((self.h, self.fdim))
-        self._obs = np.zeros(self.h * self.fdim, dtype=np.float32)
-
-    def _frame(self, a, da):
-        cs = np.empty(6, dtype=np.float64)
-        cs[0::2] = np.cos(a)
-        cs[1::2] = np.sin(a)
-        if not self.diff:
-            return cs
-        return np.concatenate([cs, da / self.vel_scale])
-
-    def _flatten(self):
-        self._obs = self.frames.reshape(-1).astype(np.float32)
-        return self._obs
-
-    def reset(self, theta):
-        a = wrap_to_pi(np.asarray(theta, dtype=np.float64).reshape(3) - np.pi)
-        self.a_prev = a.copy()
-        self.da = np.zeros(3)
-        f0 = self._frame(a, self.da)
-        self.frames = np.tile(f0, (self.h, 1))
-        return self._flatten()
-
-    @property
-    def obs(self):
-        return self._obs
-
-    def update(self, theta):
-        a = wrap_to_pi(np.asarray(theta, dtype=np.float64).reshape(3) - np.pi)
-        self.da = wrap_to_pi(a - self.a_prev) / self.dt
-        self.a_prev = a.copy()
-        f = self._frame(a, self.da)
-        if self.h > 1:
-            self.frames = np.roll(self.frames, -1, axis=0)
-        self.frames[-1] = f
-        return self._flatten()
-
-
-class TorchAngleObs:
-    """Batched float32 builder.  reset/update take theta only."""
-
-    def __init__(self, n: int, h: int = H, vel_scale: float = VEL_SCALE,
-                 dt: float = DT, device="cpu", diff: bool = True):
-        import torch
-        self.torch = torch
-        self.n = int(n)
-        self.h = int(h)
-        self.diff = bool(diff)
-        self.fdim = frame_dim(self.diff)
-        self.vel_scale = float(vel_scale)
-        self.dt = float(dt)
-        self.device = device
-        self.a_prev = torch.zeros(n, 3, device=device)
-        self.da = torch.zeros(n, 3, device=device)
-        self.frames = torch.zeros(n, self.h, self.fdim, device=device)
-
-    def _frame(self, a, da):
-        t = self.torch
-        cs = t.stack([t.cos(a), t.sin(a)], dim=2).reshape(a.shape[0], 6)
-        if not self.diff:
-            return cs
-        return t.cat([cs, da / self.vel_scale], dim=1)
-
-    def obs(self):
-        return self.frames.reshape(self.n, -1)
-
-    def reset(self, theta, idx=None):
-        t = self.torch
-        theta = t.as_tensor(theta, dtype=t.float32, device=self.device).reshape(-1, 3)
-        if idx is None:
-            a = t.remainder(theta - np.pi + np.pi, 2 * np.pi) - np.pi
-            self.a_prev = a.clone()
-            self.da = t.zeros_like(a)
-            f0 = self._frame(a, self.da)
-            self.frames = f0.unsqueeze(1).expand(-1, self.h, -1).clone()
-            return self.obs()
-        idx = t.as_tensor(idx, dtype=t.long, device=self.device).flatten()
-        a = t.remainder(theta - np.pi + np.pi, 2 * np.pi) - np.pi
-        self.a_prev[idx] = a
-        self.da[idx] = 0.0
-        f0 = self._frame(a, t.zeros_like(a))
-        self.frames[idx] = f0.unsqueeze(1).expand(-1, self.h, -1)
-        return self.obs()
-
-    def update(self, theta):
-        t = self.torch
-        theta = t.as_tensor(theta, dtype=t.float32, device=self.device).reshape(-1, 3)
-        a = t.remainder(theta - np.pi + np.pi, 2 * np.pi) - np.pi
-        self.da = (t.remainder(a - self.a_prev + np.pi, 2 * np.pi) - np.pi) / self.dt
-        self.a_prev = a.clone()
-        f = self._frame(a, self.da)
-        if self.h > 1:
-            self.frames = t.cat([self.frames[:, 1:], f.unsqueeze(1)], dim=1)
-        else:
-            self.frames[:, -1] = f
-        return self.obs()
-
-
-def make_numpy_obs(mode: str = "full", **kw):
-    c = OBS_MODES[mode]
-    return NumpyAngleObs(h=c["h"], diff=c["diff"], **kw)
-
-
-def make_torch_obs(n: int, mode: str = "full", **kw):
-    c = OBS_MODES[mode]
-    return TorchAngleObs(n, h=c["h"], diff=c["diff"], **kw)

package/assets/neural-pendulum/rl-feasibility/plot_selected.py

@@ -1,55 +0,0 @@
-"""Plot selected eval t_can curves (PPO left, SAC right)."""
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-import matplotlib
-matplotlib.use("Agg")
-import matplotlib.pyplot as plt
-
-ROOT = Path(__file__).resolve().parent
-RES = ROOT / "results"
-
-PPO = ["ppo_full_base_rand", "ppo_full_base_rand_s1",
-       "ppo_full_base_rand_s2", "ppo_angle_base_rand", "ppo_angle_h8_rand",
-       "ppo_full_h1_rand", "ppo_full_h1_mix", "ppo_full_base_fixed"]
-SAC = ["sac_full_base_rand", "sac_full_base_fixed", "sac_angle_base_rand"]
-
-
-def load(exp):
-    p = RES / (exp + ".json")
-    if not p.exists():
-        return None
-    with open(p) as f:
-        return json.load(f)
-
-
-def main():
-    fig, axes = plt.subplots(1, 2, figsize=(14, 5.0), sharey=True)
-    for ax, names, title in [(axes[0], PPO, "PPO"), (axes[1], SAC, "SAC")]:
-        for exp in names:
-            r = load(exp)
-            if not r:
-                continue
-            c = r.get("eval_curve") or []
-            if not c:
-                continue
-            x = [v["global_steps"] / 1e6 for v in c]
-            y = [v["t_can"] for v in c]
-            ax.plot(x, y, lw=1.3, label=f"{exp} (best {r['best_t_can']:.2f}s)")
-        ax.axhline(10.0, color="k", ls=":", lw=1)
-        ax.set_title(f"{title}: fixed-initial eval t_can")
-        ax.set_xlabel("environment steps (M)")
-        ax.set_ylabel("t_can (s)")
-        ax.grid(alpha=0.3)
-        ax.legend(fontsize=7)
-    fig.tight_layout()
-    out = RES / "curves_selected.png"
-    fig.savefig(out, dpi=130)
-    plt.close(fig)
-    print(out)
-
-
-if __name__ == "__main__":
-    main()

package/assets/neural-pendulum/rl-feasibility/reference_controller.py

@@ -1,112 +0,0 @@
-"""Non-RL feasibility check: a pole-placement stabiliser around upright.
-
-This is NOT a learned policy and NOT part of the RL results.  It answers a
-single question: is the upright hold physically reachable with the fixed
-torque mapping (tau1, -tau3, tau3) and |u| <= 1?
-
-The controller uses privileged full state (theta and the simulator omega),
-so it is an upper bound / existence proof only.  It is deliberately simple:
-    u = clip(-K [wrap(theta-pi); omega], -1, 1)
-with K from pole placement on the linearisation at upright.
-
-Writes results/reference_controller.json.
-"""
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-import numpy as np
-from scipy.signal import place_poles
-
-from sim import EPISODE_STEPS, N_ACT, Params, PendulumSim, make_torch_consts
-from task import EVAL_OMEGA0, EVAL_THETA0, NumpyTracker, sample_train_init, wrap_to_pi
-
-ROOT = Path(__file__).resolve().parent
-
-
-def design(p: Params, poles=(-5.0, -6.0, -7.0, -8.0, -9.0, -10.0)):
-    M = p.A
-    K = np.diag(p.g * p.gvec)
-    Minv = np.linalg.inv(M)
-    B = np.zeros((3, 2))
-    B[0, 0] = 1.0
-    B[1, 1] = -1.0
-    B[2, 1] = 1.0
-    Be = Minv @ B
-    A = np.zeros((6, 6))
-    A[:3, 3:] = np.eye(3)
-    A[3:, :3] = Minv @ K
-    B6 = np.vstack([np.zeros((3, 2)), Be])
-    res = place_poles(A, B6, np.asarray(poles, dtype=float))
-    return res.gain_matrix
-
-
-def rollout(p, gain, theta0, omega0, episode_steps=EPISODE_STEPS,
-            alpha=0.5):
-    sim = PendulumSim(p, episode_steps=episode_steps)
-    tr = NumpyTracker()
-    theta = sim.reset(theta0, omega0)
-    tr.reset()
-    sat = 0
-    uf = np.zeros(2)
-    for _ in range(episode_steps):
-        x = np.r_[wrap_to_pi(theta - np.pi), sim._omega]
-        raw = np.clip(-gain @ x, -1.0, 1.0)
-        # first-order low-pass on the torque command: the unfiltered
-        # pole-placement controller chatters at 100 Hz and only holds ~1 s.
-        uf = alpha * raw + (1.0 - alpha) * uf
-        sat += int(np.abs(uf).max() > 1.0)
-        theta = sim.step(uf)
-        tr.update(theta, 0.0)
-    summ = tr.summary()
-    return dict(t_can=float(summ["t_can"]),
-                ever_up=float(summ["ever_up"]),
-                final_ang_err=float(np.abs(wrap_to_pi(theta - np.pi)).max()),
-                sat_frac=float(sat / episode_steps),
-                final_theta=theta.tolist())
-
-
-def main():
-    p = Params()
-    gain = design(p)
-    # This is a LOCAL stabiliser: start near upright (theta = pi + offset).
-    fixed_theta0 = np.pi + EVAL_THETA0
-    fixed = rollout(p, gain, fixed_theta0, EVAL_OMEGA0)
-    rng = np.random.default_rng(0)
-    th = np.pi + rng.uniform(-0.05, 0.05, size=(32, 3))
-    om = np.zeros((32, 3))
-    rows = [rollout(p, gain, th[i], om[i]) for i in range(th.shape[0])]
-    tc = np.array([r["t_can"] for r in rows])
-    out = dict(
-        note=("Non-RL existence check.  Pole-placement stabiliser using "
-              "privileged full state (theta and simulator omega); not a "
-              "learned policy.  Shows whether the upright hold is physically "
-              "reachable with the fixed torque mapping.  The torque command is "
-              "low-pass filtered (alpha=0.5); the unfiltered controller "
-              "chatters and only holds ~1 s."),
-        poles=[-5.0, -6.0, -7.0, -8.0, -9.0, -10.0],
-        action_filter_alpha=0.5,
-        gain=gain.tolist(),
-        fixed_start=dict(theta0=fixed_theta0.tolist(),
-                         offset_from_upright=EVAL_THETA0.tolist(),
-                         omega0=EVAL_OMEGA0.tolist(), **fixed),
-        random_starts=dict(n=len(rows), success_rate=float((tc >= 10.0).mean()),
-                           mean_t_can=float(tc.mean()),
-                           median_t_can=float(np.median(tc)),
-                           max_t_can=float(tc.max()),
-                           rows=rows),
-    )
-    out_path = ROOT / "results" / "reference_controller.json"
-    with open(out_path, "w") as f:
-        json.dump(out, f, indent=2)
-    print(json.dumps(dict(fixed_t_can=fixed["t_can"],
-                          fixed_final_err=fixed["final_ang_err"],
-                          fixed_sat=fixed["sat_frac"],
-                          random_success=out["random_starts"]["success_rate"],
-                          random_mean_t_can=out["random_starts"]["mean_t_can"]),
-                     indent=2))
-
-
-if __name__ == "__main__":
-    main()