@looop-games/cli 0.1.34 → 0.1.36

package/lib/replay-export.mjs

@@ -0,0 +1,515 @@
+// The recorded session, turned into a table.
+//
+// A recording stores what the player PRESSED — the world it started from, plus
+// every input stamped with the tick it arrived on. No position, no speed, no
+// score appears anywhere in it. So nothing can point a query at a recording:
+// everything worth asking about has to be re-derived by running the game
+// forward. That is what this does. It steps the session through the game's own
+// runtime and writes one row per (tick, entity), with every declared field as a
+// column — after which the questions are ordinary SQL, and asking a new one
+// costs nothing because the sim does not have to run again.
+//
+// Two tables come out:
+//
+//   <stream>.state.parquet   one row per tick per entity, the wide sparse one
+//   <stream>.inputs.parquet  one row per wire send — already rows, no replay
+//
+// The state table is wide and mostly repeated (a real session measured 8.7% of
+// rows and 1.3% of cells changing between consecutive ticks). Delta-encoding it
+// was considered and rejected: Parquet's columnar compression already absorbs
+// the redundancy — 225 MB of JSONL becomes ~3 MB — and deltas would push
+// last-value-carried-forward semantics into every query anybody writes.
+import { mkdirSync, renameSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { ParquetWriter, fileWriter } from 'hyparquet-writer';
+
+// The columns every state row carries, before any declared field. `def` is a
+// column rather than a prefix on every field name — which is what lets a query
+// say `WHERE def IN ('prop','saucer')` and read one `body_x`.
+export const STATE_KEYS = ['tick', 'id', 'def', 'owner'];
+const STATE_KEY_TYPES = { tick: 'DOUBLE', id: 'STRING', def: 'STRING', owner: 'STRING' };
+
+// The columns every input row carries.
+// `tick` is when a send ARRIVED. A predicting client stamps `targetTick` and the
+// room holds the send in its jitter buffer until exactly that tick, so for any
+// predicted input the tick it arrives at is not the tick its effect shows up at
+// in the state table. Without this column a join on `tick` mis-attributes every
+// one of them, silently.
+const INPUT_KEYS = ['tick', 'target_tick', 'conn', 'seq', 'entity', 'channel', 'payload'];
+const INPUT_KEY_TYPES = {
+  tick: 'DOUBLE', target_tick: 'DOUBLE', conn: 'STRING', seq: 'DOUBLE',
+  entity: 'STRING', channel: 'STRING', payload: 'JSON',
+};
+
+/**
+ * The logical column type a JS value implies.
+ *
+ * DOUBLE for every number, never INT: the engine has one numeric type (a JS
+ * double), and a column typed from whole-looking values is a column that throws
+ * a thousand rows later when a fraction arrives. JSON for anything that is not
+ * flat, and for anything that says nothing — a cell has to be a scalar, so a
+ * list or an object rides as text rather than being dropped.
+ */
+function typeOf(value) {
+  if (value === null || value === undefined) return 'JSON';
+  switch (typeof value) {
+    case 'number': return 'DOUBLE';
+    case 'boolean': return 'BOOLEAN';
+    case 'string': return 'STRING';
+    default: return 'JSON';
+  }
+}
+
+// Two declarations of one column name that disagree. One Parquet column cannot
+// be both a double and a string, and dropping it would lose both kinds' data
+// silently — so the column widens to the one type that holds anything.
+const merge = (a, b) => (a === b ? a : 'JSON');
+
+/**
+ * The state table's shape, read off the lowered bundle.
+ *
+ * Types come from what each field was DECLARED as, never from a sample of what
+ * rows happen to hold. That is the whole reason this reads the manifests rather
+ * than the world: a sample is a guess that is right until it is not, and the
+ * failure lands mid-write, after the replay has already run.
+ *
+ * @param {{ manifests: Array<{component: string, owns: object}> }} bundle
+ * @returns {{ columns: Array<{name: string, type: string}>, conflicts: string[] }}
+ */
+export function tableSchema(bundle) {
+  const declared = new Map();
+  const conflicts = new Set();
+  for (const manifest of bundle?.manifests ?? []) {
+    for (const [field, spec] of Object.entries(manifest.owns ?? {})) {
+      const type = typeOf(spec?.default);
+      if (!declared.has(field)) {
+        declared.set(field, type);
+        continue;
+      }
+      const merged = merge(declared.get(field), type);
+      if (merged !== declared.get(field) || merged !== type) conflicts.add(field);
+      declared.set(field, merged);
+    }
+  }
+  return {
+    columns: [
+      ...STATE_KEYS.map((name) => ({ name, type: STATE_KEY_TYPES[name] })),
+      // Sorted so the same game always exports the same column order, whatever
+      // order its definitions happened to be lowered in.
+      ...[...declared.keys()].sort().map((name) => ({ name, type: declared.get(name) })),
+    ],
+    conflicts: [...conflicts].sort(),
+  };
+}
+
+/**
+ * One cell, as its column's type — or null.
+ *
+ * A value that does not fit is null rather than coerced. Turning a string into
+ * 0, or a NaN into "some number", would put a value in the table that the
+ * session never held, and the whole point of this table is that a question
+ * asked of it gets the sim's real answer.
+ */
+export function coerce(value, type) {
+  if (value === null || value === undefined) return null;
+  switch (type) {
+    // Number.isFinite rejects NaN and ±Infinity: Parquet has no way to carry
+    // them as doubles that any reader agrees on.
+    case 'DOUBLE': return typeof value === 'number' && Number.isFinite(value) ? value : null;
+    case 'BOOLEAN': return typeof value === 'boolean' ? value : null;
+    case 'STRING': return typeof value === 'string' ? value : null;
+    case 'JSON':
+      try { return JSON.stringify(value) ?? null; } catch { return null; }
+    default: return null;
+  }
+}
+
+/**
+ * One state row: the fixed keys, then every declared field this entity holds.
+ *
+ * A record key is `<kind>.<field>`; the kind is dropped because `def` already
+ * carries it. A key the schema does not know is skipped rather than added —
+ * Parquet needs its columns fixed before the first row is written, so a column
+ * discovered mid-session could not be honoured anyway.
+ */
+export function stateRow(entity, tick, columnIndex, tally) {
+  // The key columns go through `coerce` like every other cell. They used to be
+  // assigned straight through, so an id that was not a string reached the
+  // writer as one and threw a raw type error — after the entire replay had
+  // already been paid for.
+  const row = {
+    tick: coerce(tick, 'DOUBLE'),
+    id: coerce(entity.id, 'STRING'),
+    def: coerce(entity.def, 'STRING'),
+    owner: coerce(entity.owner, 'STRING'),
+  };
+  for (const [key, value] of Object.entries(entity.record ?? {})) {
+    const field = key.slice(key.indexOf('.') + 1);
+    const type = columnIndex.get(field);
+    if (!type) continue;
+    row[field] = coerce(value, type);
+    // NaN and ±Infinity are real values a world can hold (M2 carries them out
+    // of band through a keyframe for exactly that reason) and Parquet has no
+    // double that means them. Null is the right answer and a SILENT null is
+    // not: null already means "this kind has no such field", so a dropped NaN
+    // would be indistinguishable from an absent column.
+    if (tally && row[field] === null && type === 'DOUBLE' && typeof value === 'number') tally.undoubleable += 1;
+  }
+  return row;
+}
+
+/**
+ * The input table: one row per accepted wire send.
+ *
+ * No replay is needed — the recording already holds these as rows. A payload
+ * has no declared shape, so unlike the state table its columns are typed by
+ * reading EVERY event first. That is not the sampling rule 1 forbids: the whole
+ * list is in hand before a row is written, so this is an exact read of the data
+ * rather than a guess from its first few values.
+ *
+ * Each scalar payload key becomes `p_<key>`, and the whole payload also rides
+ * as JSON — so a question about a field that did not flatten is still one
+ * `json_extract` away rather than a re-export.
+ */
+// How many payload keys may become their own column.
+//
+// A recording is UNTRUSTED input. The segment endpoint takes what a room sends
+// and cannot demand an Origin (a multiplayer room posts from a worker that has
+// none), so anything that can reach the dev server can append events — and one
+// column per distinct key, uncapped, turns a 151 KB post into a multi-gigabyte
+// out-of-memory abort that names neither the export nor the recording.
+//
+// Capping loses nothing: the whole payload is on every row as JSON anyway, so a
+// question about a rare key is one `json_extract` away instead of a column. The
+// keys that survive are the ones that appear MOST OFTEN, because a key on every
+// event is the one worth a column and a key on one event is the one that is
+// not.
+const MAX_PAYLOAD_COLUMNS = 256;
+
+export function inputSchema(session) {
+  const sends = (session?.events ?? []).filter((e) => e?.k === 'w');
+  const payloadTypes = new Map();
+  const seen = new Map();
+  for (const send of sends) {
+    for (const [key, value] of Object.entries(send.payload ?? {})) {
+      const type = typeOf(value);
+      payloadTypes.set(key, payloadTypes.has(key) ? merge(payloadTypes.get(key), type) : type);
+      seen.set(key, (seen.get(key) ?? 0) + 1);
+    }
+  }
+  const ranked = [...seen.keys()].sort((a, b) => (seen.get(b) - seen.get(a)) || a.localeCompare(b));
+  // Sorted by name after ranking, so the same recording always exports the same
+  // column order.
+  const keys = ranked.slice(0, MAX_PAYLOAD_COLUMNS).sort();
+  const columns = [
+    ...INPUT_KEYS.map((name) => ({ name, type: INPUT_KEY_TYPES[name] })),
+    ...keys.map((k) => ({ name: `p_${k}`, type: payloadTypes.get(k) })),
+  ];
+  return { columns, keys, types: payloadTypes, sends, truncatedKeys: ranked.length - keys.length };
+}
+
+/** One input row: the fixed columns, then each flattened payload key. */
+export function inputRow(send, keys, types) {
+  const row = {
+    tick: coerce(send.t, 'DOUBLE'),
+    // Null, not a copy of `tick`: an input with no target tick was never
+    // held, and repeating arrival here would invent a fact.
+    target_tick: coerce(send.targetTick, 'DOUBLE'),
+    conn: coerce(send.connId, 'STRING'),
+    seq: coerce(send.seq, 'DOUBLE'),
+    entity: coerce(send.entityId, 'STRING'),
+    channel: coerce(send.channel, 'STRING'),
+    payload: coerce(send.payload ?? null, 'JSON'),
+  };
+  // Every column on every row, so a field an event did not carry reads as
+  // null rather than being absent — a Parquet column has one length.
+  for (const k of keys) row[`p_${k}`] = coerce(send.payload?.[k], types.get(k));
+  return row;
+}
+
+export function inputTable(session) {
+  const { columns, keys, types, sends, truncatedKeys } = inputSchema(session);
+  return { columns, rows: sends.map((send) => inputRow(send, keys, types)), truncatedKeys };
+}
+
+// The logical types above, as the Parquet schema elements the writer wants.
+// Every column is OPTIONAL because the table is deliberately SPARSE: a kind
+// that does not declare a field leaves it null on that row, and null is a
+// different answer from zero.
+const PARQUET_TYPE = {
+  DOUBLE: { type: 'DOUBLE' },
+  BOOLEAN: { type: 'BOOLEAN' },
+  STRING: { type: 'BYTE_ARRAY', converted_type: 'UTF8' },
+  JSON: { type: 'BYTE_ARRAY', converted_type: 'JSON' },
+};
+
+function schemaElements(columns) {
+  return [
+    { name: 'root', num_children: columns.length },
+    ...columns.map((c) => ({ name: c.name, repetition_type: 'OPTIONAL', ...PARQUET_TYPE[c.type] })),
+  ];
+}
+
+// A row-at-a-time front end over a columnar writer: rows go in, and every time
+// `rowsPerGroup` of them have arrived the batch is handed over as one row group
+// and dropped. Nothing accumulates across groups, which is the whole point —
+// materialising every column of every tick before writing is what made the
+// prototype cost 19 s and a gigabyte for a 27-second session.
+// A row group is held whole in memory before it is encoded, and its cost is
+// rows x COLUMNS — so a fixed row count means a 40-column game and a
+// 5,000-column one buffer two orders of magnitude apart. Budgeting cells keeps
+// the ceiling flat whatever the table's width. An explicit rowsPerGroup (a
+// test, a caller who knows) is obeyed as given.
+export const CELL_BUDGET = 8_000_000;
+export const DEFAULT_ROWS_PER_GROUP = 20_000;
+
+function groupFor(columns, requested) {
+  if (requested != null) return requested;
+  return Math.max(500, Math.min(DEFAULT_ROWS_PER_GROUP, Math.floor(CELL_BUDGET / Math.max(1, columns.length))));
+}
+
+// Written under a temporary name and moved into place only once the file is
+// complete. The writer streams straight to disk, so a throw partway through
+// would otherwise leave a truncated file sitting at the canonical name — every
+// later read of which fails with "not a parquet file", and the previous good
+// export of that session is already gone. A partial answer under the name of a
+// whole one is the worst of the three outcomes.
+function rowSink({ file, columns, rowsPerGroup }) {
+  const partial = `${file}.partial`;
+  const writer = fileWriter(partial);
+  const pq = new ParquetWriter({ writer, schema: schemaElements(columns) });
+  // Name → column position, so a row is placed by index rather than by looking
+  // each of the table's columns up on it. The table is WIDE and each row is
+  // narrow — 443 columns, ~49 of them set on any given entity — so reading
+  // every column name off every row costs a property lookup for the ~90% that
+  // are absent. Filling nulls and writing only what is there is the same table
+  // for about a third of the work.
+  const at = new Map(columns.map((c, i) => [c.name, i]));
+  let batch = columns.map(() => []);
+  let rows = 0;
+  const flush = () => {
+    if (!batch[0].length) return;
+    pq.write({
+      columnData: columns.map((c, i) => ({ name: c.name, data: batch[i] })),
+      // One group per batch: the batch IS the group, so the writer is never
+      // asked to hold more than what has already been handed to it.
+      rowGroupSize: batch[0].length,
+    });
+    batch = columns.map(() => []);
+  };
+  return {
+    push(row) {
+      const index = batch[0].length;
+      for (let i = 0; i < columns.length; i++) batch[i].push(null);
+      for (const name in row) {
+        const i = at.get(name);
+        if (i !== undefined && row[name] !== undefined) batch[i][index] = row[name];
+      }
+      rows += 1;
+      if (batch[0].length >= rowsPerGroup) flush();
+    },
+    finish() {
+      flush();
+      pq.finish();
+      renameSync(partial, file);
+      return rows;
+    },
+    // Called when the run did not get that far. Best-effort: a leftover
+    // `.partial` is untidy, an unreadable `.parquet` is a wrong answer.
+    abandon() {
+      try { rmSync(partial, { force: true }); } catch { /* nothing to clean up */ }
+    },
+  };
+}
+
+/**
+ * Where the engine's headless replay door lives, in the engine THIS GAME is
+ * pinned to.
+ *
+ * Imported from the resolved bundle rather than by a relative path, for the
+ * same reason the recording's reassembler is: a game can sit on a new CLI and
+ * an older engine, and the honest answer to that is a sentence saying so — not
+ * a module-resolution stack trace that reads, to the person holding it, as "my
+ * recording is broken".
+ */
+async function loadOpenReplay(sharedDir) {
+  const at = join(sharedDir, 'framework', 'replay-open.mjs');
+  try {
+    return (await import(pathToFileURL(at).href)).openReplay;
+  } catch (e) {
+    throw new Error(
+      "this game's pinned engine cannot open a recording headlessly, so there is nothing to export — "
+      + `run \`looop update\` and try again (${e?.message ?? e})`,
+    );
+  }
+}
+
+/**
+ * Export one recorded session as two Parquet tables.
+ *
+ * The state table is the expensive one and the reason this exists: a recording
+ * holds inputs, so the only way to know where anything WAS is to run the game
+ * forward again. Doing that once and writing the result down is what turns
+ * every later question into SQL instead of another replay.
+ *
+ * @param {object} opts
+ * @param {string} opts.dir - the game folder
+ * @param {string} [opts.stream] - which recording; defaults to the most recent
+ * @param {string} opts.outDir - where the .parquet files go
+ * @param {string} [opts.sharedDir] - the resolved engine bundle's shared/ tree
+ * @param {number} [opts.rowsPerGroup]
+ * @param {string} [opts.engineVersion] - the engine this export is running on,
+ *   compared against the one that recorded the session
+ * @param {(o: object) => Promise<object>} [opts.openReplayImpl] - seam for tests
+ * @param {(msg: string) => void} [opts.log]
+ */
+export async function exportSession({
+  dir, stream, outDir, sharedDir, rowsPerGroup, engineVersion, openReplayImpl, log = () => {},
+}) {
+  const openReplay = openReplayImpl ?? await loadOpenReplay(sharedDir);
+  // The engine's own warnings — a recording missing segments, a session tuned
+  // live mid-play — are written to this sink. Without it they go to the floor
+  // and the export looks clean while describing a world that is not the one
+  // that was played.
+  const opened = await openReplay({ dir, stream, log });
+  const { replayer, bundle, session } = opened;
+  const chosen = opened.stream ?? stream ?? null;
+
+  // Which engine wrote this recording, against the one about to replay it. The
+  // header has carried engineVersion since the first recording and nothing read
+  // it, so the only way a reader learned the two disagreed was a divergence —
+  // which does not fire if the change altered nothing the checksums cover.
+  //
+  // Reported before the replay runs, not after: the replay is the slow part and
+  // this answer is already known. An absent version is UNKNOWN, not different —
+  // every recording made before this shipped has none, and warning on those
+  // would fire on every old session still on disk.
+  // A recording's header is NOT first-party text. Segments reach the dev server
+  // over a lane that deliberately does not require an Origin (a multiplayer
+  // room's worker sends none), and the store validates only the four fields it
+  // routes on — format, seq, stream, events. Everything else is spread through
+  // the stitcher verbatim, so anything printed from a header is attacker-
+  // reachable by whoever can post a segment.
+  //
+  // Printing is what makes that matter. `ESC[2J ESC[H` clears the screen and
+  // repaints, and the line being repainted here is the one a creator — or the
+  // coding agent reading their terminal — uses to decide whether to trust the
+  // exported numbers. An export with no session named takes the newest
+  // recording, so a segment posted a moment ago is the default target.
+  //
+  // So: REJECT, don't sanitize. A value that is not a plain version-or-digest
+  // token is treated as unknown, which is a state this code already handles
+  // honestly — and an unreadable field is genuinely unknown, not a mismatch.
+  // The charset covers semver including pre-release labels (`0.2.0-riven`) and
+  // hex digests, and admits no control character, escape or whitespace.
+  const TOKEN = /^[A-Za-z0-9._+-]{1,64}$/;
+  // `__PUBLISH_VERSION__` is the placeholder the release build substitutes. A
+  // room served straight off a checkout never went through that build, so every
+  // session recorded under `looop dev` carries the token verbatim — reading it
+  // as a version would report a mismatch on every single dev export, and a
+  // warning that fires every time is one nobody reads by the third time.
+  const known = (v) => (typeof v === 'string' && TOKEN.test(v) && v !== '__PUBLISH_VERSION__' ? v : null);
+  const recordedEngine = known(session?.header?.engineVersion);
+  const runningEngine = known(engineVersion);
+  if (recordedEngine && runningEngine && recordedEngine !== runningEngine) {
+    log(`  ⚠ this session was recorded by engine ${recordedEngine} and is being replayed on `
+      + `${runningEngine}. A replay runs the code as it is now, so where the two engines `
+      + 'simulate differently the table holds a world nobody played.');
+  }
+  // And the same question for the game's own code, which is the half that
+  // actually moves. engineVersion changes once per release; a fingerprint
+  // changes the moment a tick does.
+  // Same provenance, same rule: this one comes off the header too.
+  const recordedCode = known(session?.header?.codeFingerprint);
+  const runningCode = known(bundle?.fingerprint);
+  const codeChanged = Boolean(recordedCode && runningCode && recordedCode !== runningCode);
+  if (codeChanged) {
+    log('  ⚠ this game has changed since the session was recorded (the simulation it lowers to '
+      + `is ${runningCode}; the recording was made by ${recordedCode}). The table below is what `
+      + 'your game does with those inputs TODAY, which is not what happened when you played.');
+  }
+
+  const { columns, conflicts } = tableSchema(bundle);
+  if (conflicts.length) {
+    log(`note: ${conflicts.length} field name(s) are declared with different types by different entity kinds `
+      + `and ride as JSON text — ${conflicts.join(', ')}`);
+  }
+  const columnIndex = new Map(columns.map((c) => [c.name, c.type]));
+
+  mkdirSync(outDir, { recursive: true });
+  const stateFile = join(outDir, `${chosen ?? 'session'}.state.parquet`);
+  const inputFile = join(outDir, `${chosen ?? 'session'}.inputs.parquet`);
+
+  const sink = rowSink({ file: stateFile, columns, rowsPerGroup: groupFor(columns, rowsPerGroup) });
+  const tally = { undoubleable: 0 };
+  let ticks = 0;
+  let rows;
+  try {
+  // The world BEFORE any step is a real tick of the session — it is the
+  // keyframe the replay entered at — so it is written before the loop, not
+  // skipped into it.
+    for (;;) {
+      const { tick, entities } = replayer.runtime.serializeEntities();
+      for (const entity of entities) sink.push(stateRow(entity, tick, columnIndex, tally));
+      ticks += 1;
+      if (replayer.atEnd) break;
+      replayer.step();
+    }
+    rows = sink.finish();
+  } catch (e) {
+    sink.abandon();
+    throw e;
+  }
+
+  // Streamed like the state table rather than materialised: the row count comes
+  // from a recording, and a recording is not this machine's to trust.
+  const inputs = inputSchema(session);
+  const inputSink = rowSink({ file: inputFile, columns: inputs.columns, rowsPerGroup: groupFor(inputs.columns, rowsPerGroup) });
+  let inputRows;
+  try {
+    for (const send of inputs.sends) inputSink.push(inputRow(send, inputs.keys, inputs.types));
+    inputRows = inputSink.finish();
+  } catch (e) {
+    inputSink.abandon();
+    throw e;
+  }
+  if (inputs.truncatedKeys) {
+    log(`note: ${inputs.truncatedKeys} rarely-used payload field(s) did not get their own column `
+      + `(the cap is ${MAX_PAYLOAD_COLUMNS}) — they are still in the \`payload\` JSON column on every row`);
+  }
+
+  return {
+    stream: chosen,
+    stateFile,
+    inputFile,
+    ticks,
+    rows,
+    columns: columns.length,
+    inputs: inputRows,
+    truncatedKeys: inputs.truncatedKeys,
+    // Three separate ways the table can describe a world that is not the one
+    // that was played, kept apart because they have different causes and
+    // different answers.
+    divergences: replayer.divergences?.length ?? 0,
+    gaps: replayer.gaps?.length ?? 0,
+    unsupported: replayer.unsupported?.length ?? 0,
+    // How many times playback was put back onto the recorded state by a
+    // keyframe. This is the counterweight to `divergences`: it is what bounds a
+    // drift to one keyframe interval instead of the rest of the session, so a
+    // reader deciding whether to trust the table needs both numbers.
+    reanchors: replayer.reanchors?.length ?? 0,
+    // Counts alone cannot say whether the damage was undone: a re-anchor at
+    // tick 4 tells a reader nothing about a divergence at tick 900. These two
+    // let the caller compare the last drift against the last correction after
+    // it, which is the question actually being asked.
+    lastDivergenceTick: replayer.divergences?.at(-1)?.tick ?? null,
+    lastReanchorTick: replayer.reanchors?.at(-1)?.tick ?? null,
+    recordedEngine,
+    runningEngine,
+    recordedCode,
+    runningCode,
+    codeChanged,
+    undoubleable: tally.undoubleable,
+  };
+}