@flowsilver/algoviz 0.0.0-stage → 0.4.0

package/examples/frozenlake/check.mjs

@@ -0,0 +1,265 @@
+#!/usr/bin/env node
+// Validate recorded FrozenLake Q-learning traces independently of the Python recorder.
+// The environment rules and Q-learning update below are a separate implementation of
+// Gymnasium FrozenLake-v1 with is_slippery=False.
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { isDeepStrictEqual, parseArgs } from "node:util";
+import { loadReplay } from "../../packages/core/index.js";
+
+const here = fileURLToPath(new URL(".", import.meta.url));
+const MAPS = {
+  "4x4": ["SFFF", "FHFH", "FFFH", "HFFG"],
+  "8x8": ["SFFFFFFF", "FFFFFFFF", "FFFHFFFF", "FFFFFHFF", "FFFHFFFF", "FHHFFFHF", "FHFFHFHF", "FFFHFFFG"],
+};
+const TIME_LIMITS = { "4x4": 100, "8x8": 200 };
+const ACTIONS = ["LEFT", "DOWN", "RIGHT", "UP"];
+const MOVES = { LEFT: [0, -1], DOWN: [1, 0], RIGHT: [0, 1], UP: [-1, 0] };
+const REWARDS = { G: 1, H: 0, F: 0, S: 0 };
+const TOTALS = { goal: "goals", hole: "holes", truncated: "truncations" };
+const STATE_KEYS = ["run", "map", "episode", "t", "cell", "status", "episodeReturn", "path", "last", "q", "totals"];
+const LAST_KEYS = ["from", "action", "explored", "to", "reward", "terminated", "truncated"];
+
+export class FixtureError extends Error {
+  /**
+   * @param {string} message
+   * @param {{ run: string, source: string, line?: number, episode?: number, step?: number }} location
+   */
+  constructor(message, { run, source, line, episode, step }) {
+    const where = [`${run} ${line === undefined ? source : `${source}:${line}`}`];
+    if (episode !== undefined) where.push(`episode ${episode}`);
+    if (step !== undefined) where.push(`step ${step}`);
+    super(`${where.join(" ")}: ${message}`);
+    this.name = "FixtureError";
+    Object.assign(this, { run, source, line, episode, step });
+  }
+}
+
+const show = (value) => JSON.stringify(value);
+
+function argmax(values) {
+  const best = Math.max(...values);
+  return ACTIONS.filter((_, index) => values[index] === best);
+}
+
+function outcomeOf(tile, terminated, truncated) {
+  if (terminated) return tile === "G" ? "goal" : "hole";
+  return truncated ? "truncated" : "running";
+}
+
+/** Compare the recorded state with the expected one, field by field, in a fixed order. */
+function compareState(actual, expected) {
+  const extra = Object.keys(actual).filter((key) => !STATE_KEYS.includes(key));
+  if (extra.length) return `unexpected state field ${show(extra[0])}`;
+  for (const key of STATE_KEYS) {
+    if (key === "last" && expected.last && actual.last && typeof actual.last === "object") {
+      for (const field of LAST_KEYS) {
+        if (!isDeepStrictEqual(actual.last[field], expected.last[field])) {
+          return `${field} is ${show(actual.last[field])}, expected ${show(expected.last[field])}`;
+        }
+      }
+      const extraLast = Object.keys(actual.last).filter((field) => !LAST_KEYS.includes(field));
+      if (extraLast.length) return `unexpected last-transition field ${show(extraLast[0])}`;
+      continue;
+    }
+    if (!isDeepStrictEqual(actual[key], expected[key])) {
+      if (key === "q") {
+        for (const [cell, row] of expected.q.entries()) {
+          for (const [index, value] of row.entries()) {
+            const found = actual.q?.[cell]?.[index];
+            if (!Object.is(found, value)) return `Q[${cell}][${ACTIONS[index]}] is ${show(found)}, expected ${show(value)}`;
+          }
+        }
+      }
+      return `${key} is ${show(actual[key])}, expected ${show(expected[key])}`;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Validate one recorded run and return its per-episode summaries and transitions.
+ * @param {{ traceText: string, specText?: string, source?: string, settings: any, run: any }} input
+ */
+export function checkRun({ traceText, specText, source = "trace.jsonl", settings, run }) {
+  const map = MAPS[settings.map];
+  const limit = TIME_LIMITS[settings.map];
+  if (!map) throw new Error(`Unsupported FrozenLake map ${show(settings.map)}`);
+  const ncol = map[0].length;
+  const tile = (cell) => map[Math.floor(cell / ncol)][cell % ncol];
+  const move = (cell, action) => {
+    const [dr, dc] = MOVES[action];
+    const row = Math.min(Math.max(Math.floor(cell / ncol) + dr, 0), map.length - 1);
+    const col = Math.min(Math.max((cell % ncol) + dc, 0), ncol - 1);
+    return row * ncol + col;
+  };
+
+  const replay = loadReplay(traceText, { specText, traceSource: source });
+  const fail = (message, frame) => {
+    throw new FixtureError(message, {
+      run: run.id, source, line: frame.step + 1, episode: frame.state.episode, step: frame.state.t,
+    });
+  };
+
+  const initial = {
+    run: { id: run.id, seed: run.seed, epsilon: run.epsilon, alpha: settings.alpha, gamma: settings.gamma },
+    map: [...map],
+    episode: 0, t: 0, cell: 0, status: "running", episodeReturn: 0, path: [0], last: null,
+    q: Array.from({ length: map.length * ncol }, () => ACTIONS.map(() => 0)),
+    totals: { transitions: 0, goals: 0, holes: 0, truncations: 0 },
+  };
+  let frame = replay.current;
+  const initialError = compareState(frame.state, initial);
+  if (initialError) fail(`initial state: ${initialError}`, frame);
+
+  const episodes = [];
+  /** @type {any[]} */
+  let transitions = [];
+  for (let index = 1; index <= replay.lastStep; index++) {
+    const previous = frame.state;
+    frame = replay.stepForward();
+    const state = frame.state;
+    let expected;
+    if (previous.status === "running") {
+      const action = state.last?.action;
+      if (!ACTIONS.includes(action)) fail(`action is ${show(action)}, expected one of ${ACTIONS.join(", ")}`, frame);
+      if (typeof state.last.explored !== "boolean") fail(`explored is ${show(state.last.explored)}, expected a boolean`, frame);
+      const t = previous.t + 1;
+      const to = move(previous.cell, action);
+      const reward = REWARDS[tile(to)];
+      const terminated = "GH".includes(tile(to));
+      // Gymnasium's TimeLimit wrapper truncates at the limit even when the step also terminates.
+      const truncated = t >= limit;
+      const status = outcomeOf(tile(to), terminated, truncated);
+      const a = ACTIONS.indexOf(action);
+      const q = structuredClone(previous.q);
+      const target = terminated ? reward : reward + settings.gamma * Math.max(...previous.q[to]);
+      q[previous.cell][a] = previous.q[previous.cell][a] + settings.alpha * (target - previous.q[previous.cell][a]);
+      const totals = { ...previous.totals, transitions: previous.totals.transitions + 1 };
+      if (status !== "running") totals[TOTALS[status]] += 1;
+      expected = {
+        ...previous, episode: previous.episode, t, cell: to, status,
+        episodeReturn: previous.episodeReturn + reward, path: [...previous.path, to],
+        last: { from: previous.cell, action, explored: state.last.explored, to, reward, terminated, truncated },
+        q, totals,
+      };
+      const error = compareState(state, expected);
+      if (error) fail(error, frame);
+      const greedy = argmax(previous.q[previous.cell]);
+      if (!state.last.explored && !greedy.includes(action)) {
+        fail(`greedy action ${action} is not among the highest-valued actions ${greedy.join(", ")}`, frame);
+      }
+      transitions.push({
+        traceStep: frame.step, t, from: previous.cell, values: previous.q[previous.cell], greedy, action, explored: state.last.explored,
+        cell: to, reward, status, episodeReturn: expected.episodeReturn,
+      });
+      if (status !== "running") {
+        episodes.push({
+          episode: state.episode, steps: t, return: expected.episodeReturn, outcome: status,
+          actions: transitions.map((step) => step.action[0]).join(""), endStep: frame.step, transitions,
+        });
+        transitions = [];
+      }
+    } else {
+      expected = { ...previous, episode: previous.episode + 1, t: 0, cell: 0, status: "running", episodeReturn: 0, path: [0], last: null };
+      if (state.cell !== 0) fail(`reset placed the agent at cell ${show(state.cell)}, expected the start cell 0`, frame);
+      const error = compareState(state, expected);
+      if (error) fail(`reset: ${error}`, frame);
+    }
+  }
+  if (frame.state.status === "running") fail("the trace ends before the episode terminates or truncates", frame);
+  const total = frame.state.totals.transitions;
+  if (total < settings.minTransitions) fail(`recorded ${total} transitions, expected at least ${settings.minTransitions}`, frame);
+  return { run: run.id, states: replay.lastStep + 1, transitions: total, episodes };
+}
+
+/** Compare trace-derived episode summaries with those of a separate uninstrumented run. */
+export function checkSummaries(result, uninstrumented, source = "trace.jsonl") {
+  const recorded = result.episodes;
+  for (let index = 0; index < Math.max(recorded.length, uninstrumented.length); index++) {
+    const episode = recorded[index];
+    const expected = uninstrumented[index];
+    const summary = episode && { episode: episode.episode, steps: episode.steps, return: episode.return, outcome: episode.outcome, actions: episode.actions };
+    if (!isDeepStrictEqual(summary, expected)) {
+      throw new FixtureError(
+        `trace summary ${show(summary ?? "missing")} differs from the uninstrumented run ${show(expected ?? "missing")}`,
+        { run: result.run, source, line: episode && episode.endStep + 1, episode: episode?.episode ?? expected?.episode },
+      );
+    }
+  }
+}
+
+/** Align one episode of two checked runs by step within the episode and find the first differing action. */
+export function compareEpisode(results, episode) {
+  const [a, b] = results.map((result) => {
+    const found = result.episodes[episode];
+    if (!found) throw new Error(`${result.run} has no episode ${episode}`);
+    return found;
+  });
+  let step = 1;
+  while (step <= Math.min(a.steps, b.steps) && a.transitions[step - 1].action === b.transitions[step - 1].action) step++;
+  const describe = (result, found) => {
+    const { traceStep, ...transition } = found.transitions[step - 1] ?? {};
+    return [result.run, found.transitions[step - 1] ? { traceStep, ...transition } : { ended: true, steps: found.steps, outcome: found.outcome }];
+  };
+  return {
+    episode,
+    firstDifference: step > Math.max(a.steps, b.steps) ? null : {
+      step,
+      runs: Object.fromEntries([describe(results[0], a), describe(results[1], b)]),
+    },
+    outcomes: Object.fromEntries(results.map((result, index) => {
+      const found = [a, b][index];
+      const resetStep = found.endStep - found.steps;
+      return [result.run, { resetStep, steps: found.steps, return: found.return, outcome: found.outcome, actions: found.actions }];
+    })),
+  };
+}
+
+function uninstrumentedSummaries(runId, runsPath) {
+  const python = process.env.ALGOVIZ_PYTHON ?? "python";
+  const output = execFileSync(python, ["-B", join(here, "frozenlake.py"), runId, "--runs", runsPath], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
+  return JSON.parse(output).episodes;
+}
+
+/**
+ * Check every configured run recorded under `directory` and compare them at `episode`.
+ * @param {string} directory
+ * @param {{ runsPath?: string, episode?: number }} [options]
+ */
+export function checkRecording(directory, { runsPath = join(here, "runs.json"), episode } = {}) {
+  const config = JSON.parse(readFileSync(runsPath, "utf8"));
+  const results = config.runs.map((run) => {
+    const source = `${run.id}/trace.jsonl`;
+    const traceBytes = readFileSync(join(directory, run.id, "trace.jsonl"));
+    const specText = readFileSync(join(directory, run.id, "visualization.json"), "utf8");
+    const result = checkRun({ traceText: traceBytes.toString("utf8"), specText, source, settings: config.settings, run });
+    checkSummaries(result, uninstrumentedSummaries(run.id, runsPath), source);
+    return { ...result, sha256: createHash("sha256").update(traceBytes).digest("hex") };
+  });
+  return {
+    runs: Object.fromEntries(results.map(({ run, states, transitions, episodes, sha256 }) => [run, { states, transitions, episodes: episodes.length, sha256 }])),
+    comparison: episode === undefined ? undefined : compareEpisode(results, episode),
+  };
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+  const { positionals, values } = parseArgs({ allowPositionals: true, options: { runs: { type: "string" }, episode: { type: "string" } } });
+  if (positionals.length !== 1) {
+    console.error("Usage: node examples/frozenlake/check.mjs <recorded-directory> [--episode <n>] [--runs <runs.json>]");
+    process.exit(2);
+  }
+  const expected = JSON.parse(readFileSync(join(here, "expected.json"), "utf8"));
+  try {
+    const report = checkRecording(positionals[0], {
+      runsPath: values.runs, episode: values.episode === undefined ? expected.comparison.episode : Number(values.episode),
+    });
+    console.log(JSON.stringify(report, null, 2));
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : error);
+    process.exitCode = 1;
+  }
+}

package/examples/frozenlake/comparison.json

@@ -0,0 +1,8 @@
+{
+  "version": 1,
+  "layout": "column",
+  "views": [
+    { "id": "frozenlake-grid", "type": "custom", "renderer": "frozenlake", "source": "" },
+    { "id": "episode-path", "type": "array", "source": "/path" }
+  ]
+}

package/examples/frozenlake/expected.json

@@ -0,0 +1,80 @@
+{
+  "runs": {
+    "epsilon-0.1": {
+      "states": 11557,
+      "transitions": 10000,
+      "episodes": 1557,
+      "sha256": "0ac9fe0b4507c12d2e72f773a2ec3d2750d0c60f7a095cf8a38c60a8bc9b6270"
+    },
+    "epsilon-0.3": {
+      "states": 11448,
+      "transitions": 10000,
+      "episodes": 1448,
+      "sha256": "a37137cc8e8f5bda15ffca4a68cf0d6d51cd2ce83102ab3a29bcddb56b5224ec"
+    }
+  },
+  "comparison": {
+    "episode": 1008,
+    "firstDifference": {
+      "step": 5,
+      "runs": {
+        "epsilon-0.1": {
+          "traceStep": 7517,
+          "t": 5,
+          "from": 13,
+          "values": [
+            0,
+            0.902499973103404,
+            0.95,
+            0.8573749743000758
+          ],
+          "greedy": [
+            "RIGHT"
+          ],
+          "action": "RIGHT",
+          "explored": false,
+          "cell": 14,
+          "reward": 0,
+          "status": "running",
+          "episodeReturn": 0
+        },
+        "epsilon-0.3": {
+          "traceStep": 8089,
+          "t": 5,
+          "from": 13,
+          "values": [
+            0,
+            0.9024999999999999,
+            0.95,
+            0.8573749999999998
+          ],
+          "greedy": [
+            "RIGHT"
+          ],
+          "action": "LEFT",
+          "explored": true,
+          "cell": 12,
+          "reward": 0,
+          "status": "hole",
+          "episodeReturn": 0
+        }
+      }
+    },
+    "outcomes": {
+      "epsilon-0.1": {
+        "resetStep": 7512,
+        "steps": 6,
+        "return": 1,
+        "outcome": "goal",
+        "actions": "DDRDRR"
+      },
+      "epsilon-0.3": {
+        "resetStep": 8084,
+        "steps": 5,
+        "return": 0,
+        "outcome": "hole",
+        "actions": "DDRDL"
+      }
+    }
+  }
+}

package/examples/frozenlake/frozenlake.py

@@ -0,0 +1,138 @@
+"""Deterministic FrozenLake and tabular Q-learning using only the Python standard library.
+
+The environment follows Gymnasium's FrozenLake-v1 with ``is_slippery=False``: the
+default maps, action encoding, observation ``row * ncol + col``, reward schedule
+``(1, 0, 0)``, termination on a hole or the goal, and truncation by the time limit.
+A move off the grid leaves the agent in place.
+
+Run this file directly for the uninstrumented summaries of one configured run.
+"""
+
+import argparse
+import json
+import random
+from pathlib import Path
+
+
+MAPS = {
+    "4x4": ("SFFF", "FHFH", "FFFH", "HFFG"),
+    "8x8": (
+        "SFFFFFFF", "FFFFFFFF", "FFFHFFFF", "FFFFFHFF",
+        "FFFHFFFF", "FHHFFFHF", "FHFFHFHF", "FFFHFFFG",
+    ),
+}
+MAX_EPISODE_STEPS = {"4x4": 100, "8x8": 200}
+ACTIONS = ("LEFT", "DOWN", "RIGHT", "UP")
+REWARD_SCHEDULE = {"G": 1, "H": 0, "F": 0, "S": 0}
+
+HERE = Path(__file__).resolve().parent
+
+
+class FrozenLake:
+    def __init__(self, map_name):
+        self.desc = MAPS[map_name]
+        self.nrow = len(self.desc)
+        self.ncol = len(self.desc[0])
+        self.max_episode_steps = MAX_EPISODE_STEPS[map_name]
+        self.state = 0
+        self.elapsed = 0
+
+    def reset(self):
+        self.state = 0
+        self.elapsed = 0
+        return self.state
+
+    def step(self, action):
+        row, col = divmod(self.state, self.ncol)
+        if action == 0:
+            col = max(col - 1, 0)
+        elif action == 1:
+            row = min(row + 1, self.nrow - 1)
+        elif action == 2:
+            col = min(col + 1, self.ncol - 1)
+        elif action == 3:
+            row = max(row - 1, 0)
+        else:
+            raise ValueError(f"invalid action {action!r}")
+        self.state = row * self.ncol + col
+        tile = self.desc[row][col]
+        self.elapsed += 1
+        terminated = tile in "GH"
+        # Gymnasium's TimeLimit wrapper truncates at the limit even on a terminal step.
+        truncated = self.elapsed >= self.max_episode_steps
+        return self.state, REWARD_SCHEDULE[tile], terminated, truncated
+
+
+def outcome(env, state, terminated, truncated):
+    if terminated:
+        return "goal" if env.desc[state // env.ncol][state % env.ncol] == "G" else "hole"
+    return "truncated" if truncated else "running"
+
+
+def train(settings, run, observer=None):
+    """Train until at least ``minTransitions`` transitions, finishing the last episode.
+
+    ``observer`` receives read-only copies of each reset and transition. It must not
+    draw from the random generator or change the Q-table.
+    """
+    rng = random.Random(run["seed"])
+    env = FrozenLake(settings["map"])
+    alpha, gamma, epsilon = settings["alpha"], settings["gamma"], run["epsilon"]
+    q = [[0.0] * len(ACTIONS) for _ in range(env.nrow * env.ncol)]
+    summaries = []
+    transitions = 0
+    while transitions < settings["minTransitions"]:
+        episode = len(summaries)
+        state = env.reset()
+        episode_return = 0
+        actions = []
+        if observer:
+            observer.reset(episode, state)
+        while True:
+            explored = rng.random() < epsilon
+            if explored:
+                action = rng.randrange(len(ACTIONS))
+            else:
+                best = max(q[state])
+                action = rng.choice([index for index, value in enumerate(q[state]) if value == best])
+            next_state, reward, terminated, truncated = env.step(action)
+            target = reward if terminated else reward + gamma * max(q[next_state])
+            q[state][action] += alpha * (target - q[state][action])
+            episode_return += reward
+            actions.append(ACTIONS[action][0])
+            transitions += 1
+            result = outcome(env, next_state, terminated, truncated)
+            if observer:
+                observer.transition({
+                    "episode": episode, "t": env.elapsed, "from": state, "action": action,
+                    "explored": explored, "to": next_state, "reward": reward,
+                    "terminated": terminated, "truncated": truncated, "outcome": result,
+                    "q": q[state][action],
+                })
+            state = next_state
+            if terminated or truncated:
+                break
+        summaries.append({"episode": episode, "steps": env.elapsed, "return": episode_return,
+                          "outcome": result, "actions": "".join(actions)})
+    return summaries
+
+
+def load_runs(path=HERE / "runs.json"):
+    return json.loads(Path(path).read_text(encoding="utf-8"))
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Print per-episode summaries of one uninstrumented run.")
+    parser.add_argument("run", help="run id from runs.json")
+    parser.add_argument("--runs", type=Path, default=HERE / "runs.json")
+    args = parser.parse_args()
+    config = load_runs(args.runs)
+    runs = {run["id"]: run for run in config["runs"]}
+    if args.run not in runs:
+        parser.error(f"unknown run {args.run!r}; choose from {', '.join(runs)}")
+    summaries = train(config["settings"], runs[args.run])
+    print(json.dumps({"run": args.run, "episodes": summaries}, separators=(",", ":")))
+
+
+if __name__ == "__main__":
+    main()

package/examples/frozenlake/grid.js

@@ -0,0 +1,57 @@
+const tiles = { S: "Start", F: "Ice", H: "Hole", G: "Goal" };
+const actions = ["Left", "Down", "Right", "Up"];
+const styles = `
+  .lake { padding: 1rem; }
+  .lake-grid { display: grid; gap: .3rem; max-width: 24rem; margin: 0 auto; }
+  .lake-cell { display: grid; place-content: center; gap: .1rem; min-height: 2.7rem; text-align: center;
+    border: 1px solid #cbd9df; border-radius: .35rem; background: #eff7fa; color: #214155; }
+  .lake-cell[data-tile="S"] { background: #f4f1e8; }
+  .lake-cell[data-tile="G"] { background: #dff3db; }
+  .lake-cell[data-tile="H"] { background: #e1e7ed; }
+  .lake-cell[data-visited="true"] { box-shadow: inset 0 -3px #51a6b6; }
+  .lake-cell[data-agent="true"] { outline: 2px solid #135974; outline-offset: -2px; }
+  .lake-cell small { font: .72rem var(--mono, monospace); }
+  .lake-path { overflow-wrap: anywhere; color: var(--muted); font: .78rem/1.5 var(--mono, monospace); }
+  .lake-values { display: flex; flex-wrap: wrap; gap: .35rem .8rem; margin: 0; font: .8rem var(--mono, monospace); }
+  .lake-values div { display: flex; gap: .35rem; }
+  .lake-values dd { margin: 0; font-weight: 700; }
+`;
+
+function element(tag, text, className) {
+  const node = document.createElement(tag);
+  if (className) node.className = className;
+  if (text !== undefined) node.textContent = text;
+  return node;
+}
+
+export default function renderFrozenLake({ map, cell, path, q }) {
+  const wrapper = element("div", undefined, "lake");
+  wrapper.append(element("style", styles));
+  const grid = element("div", undefined, "lake-grid");
+  grid.setAttribute("role", "group");
+  grid.setAttribute("aria-label", "FrozenLake grid");
+  grid.style.gridTemplateColumns = `repeat(${map[0].length}, minmax(0, 1fr))`;
+  const visited = new Set(path);
+  for (const [index, tile] of Array.from(map.join("")).entries()) {
+    const atAgent = index === cell;
+    const square = element("div", undefined, "lake-cell");
+    Object.assign(square.dataset, { cell: index, tile, visited: visited.has(index), agent: atAgent });
+    square.append(element("small", index), element("small", atAgent ? `Agent · ${tiles[tile]}` : tiles[tile]));
+    grid.append(square);
+  }
+  const pathLabel = element("p", `Path ${path.join(" → ")}`, "lake-path");
+  pathLabel.dataset.path = JSON.stringify(path);
+  const values = element("dl", undefined, "lake-values");
+  values.setAttribute("aria-label", `Action values at cell ${cell}`);
+  for (const [index, action] of actions.entries()) {
+    const value = q[cell][index];
+    const entry = element("div");
+    const label = element("dt", action);
+    const display = element("dd", Number(value.toFixed(4)));
+    Object.assign(display.dataset, { action: action.toUpperCase(), q: value });
+    entry.append(label, display);
+    values.append(entry);
+  }
+  wrapper.append(grid, pathLabel, values);
+  return wrapper;
+}

package/examples/frozenlake/record.py

@@ -0,0 +1,83 @@
+"""Record each configured FrozenLake Q-learning run as an AlgoViz v1 trace."""
+
+import argparse
+import json
+from pathlib import Path
+
+from algoviz_trace import TraceWriter
+from frozenlake import ACTIONS, MAPS, HERE, load_runs, train
+
+
+class RunObserver:
+    """Mirror resets and transitions into the observable state without touching the learner."""
+
+    def __init__(self, file, settings, run):
+        self.file = file
+        self.writer = None
+        self.state = {
+            "run": {"id": run["id"], "seed": run["seed"], "epsilon": run["epsilon"],
+                    "alpha": settings["alpha"], "gamma": settings["gamma"]},
+            "map": list(MAPS[settings["map"]]),
+            "episode": 0,
+            "t": 0,
+            "cell": 0,
+            "status": "running",
+            "episodeReturn": 0,
+            "path": [0],
+            "last": None,
+            "q": [[0.0] * len(ACTIONS) for _ in range(len(MAPS[settings["map"]]) * len(MAPS[settings["map"]][0]))],
+            "totals": {"transitions": 0, "goals": 0, "holes": 0, "truncations": 0},
+        }
+
+    def reset(self, episode, cell):
+        state = self.state
+        state.update(episode=episode, t=0, cell=cell, status="running", episodeReturn=0, path=[cell], last=None)
+        if self.writer is None:
+            self.writer = TraceWriter(self.file, state)
+        else:
+            self.writer.step(state, [{"type": "episode.reset", "data": {"episode": episode}}])
+
+    def transition(self, change):
+        state = self.state
+        state["t"] = change["t"]
+        state["cell"] = change["to"]
+        state["status"] = change["outcome"]
+        state["episodeReturn"] += change["reward"]
+        state["path"].append(change["to"])
+        state["last"] = {
+            "from": change["from"], "action": ACTIONS[change["action"]], "explored": change["explored"],
+            "to": change["to"], "reward": change["reward"],
+            "terminated": change["terminated"], "truncated": change["truncated"],
+        }
+        state["q"][change["from"]][change["action"]] = change["q"]
+        totals = state["totals"]
+        totals["transitions"] += 1
+        events = None
+        if change["outcome"] != "running":
+            totals[{"goal": "goals", "hole": "holes", "truncated": "truncations"}[change["outcome"]]] += 1
+            events = [{"type": "episode.end", "data": {"episode": change["episode"], "outcome": change["outcome"]}}]
+        self.writer.step(state, events)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("output", type=Path, help="directory that receives one folder per run")
+    parser.add_argument("--runs", type=Path, default=HERE / "runs.json")
+    args = parser.parse_args()
+    config = load_runs(args.runs)
+    spec = (HERE / "visualization.json").read_text(encoding="utf-8")
+    report = []
+    for run in config["runs"]:
+        directory = args.output / run["id"]
+        directory.mkdir(parents=True, exist_ok=True)
+        (directory / "visualization.json").write_text(spec, encoding="utf-8", newline="\n")
+        with (directory / "trace.jsonl").open("w", encoding="utf-8", newline="\n") as file:
+            observer = RunObserver(file, config["settings"], run)
+            summaries = train(config["settings"], run, observer)
+        report.append({"run": run["id"], "episodes": len(summaries),
+                       "transitions": observer.state["totals"]["transitions"], "states": observer.writer.steps + 1})
+    print(json.dumps(report))
+
+
+if __name__ == "__main__":
+    main()

package/examples/frozenlake/runs.json

@@ -0,0 +1,12 @@
+{
+  "settings": {
+    "map": "4x4",
+    "alpha": 0.5,
+    "gamma": 0.95,
+    "minTransitions": 10000
+  },
+  "runs": [
+    { "id": "epsilon-0.1", "seed": 7, "epsilon": 0.1 },
+    { "id": "epsilon-0.3", "seed": 7, "epsilon": 0.3 }
+  ]
+}

package/examples/frozenlake/visualization.json

@@ -0,0 +1,8 @@
+{
+  "version": 1,
+  "layout": "row",
+  "views": [
+    { "id": "episode-path", "type": "array", "source": "/path" },
+    { "id": "action-values", "type": "array", "source": "/q" }
+  ]
+}