@trolleroof/tui 0.2.4 → 0.2.6

package/src/recording/trace-recorder.ts

@@ -0,0 +1,560 @@
+import { createHash } from "node:crypto";
+import { buildTraceChunk, estimatedBytes } from "./chunks.js";
+import {
+  resolveRecordingConfig,
+  type RecordingConfig,
+} from "./config.js";
+import {
+  createChunkId,
+  createEventId,
+  createGameId,
+  createObservationId,
+} from "./ids.js";
+import {
+  RecordingStore,
+  type ReadonlyRecoveryChunkIndex,
+  type RecoveryChunkIndex,
+} from "./store.js";
+import {
+  encodeTerminalObservation,
+  type TerminalGrid,
+} from "./terminal-grid.js";
+import type { ChunkTransport } from "./transport.js";
+import { isTraceEvent } from "./types.js";
+import type {
+  CheckpointReason,
+  EngineCheckpointTraceEvent,
+  EpisodeEndedTraceEvent,
+  EpisodeEndReason,
+  EpisodeStartedTraceEvent,
+  ObservationTraceEvent,
+  OutcomeChangedTraceEvent,
+  RecordingHealth,
+  TraceActor,
+  TraceEvent,
+  TraceVisibility,
+} from "./types.js";
+
+export interface TraceRecorderOptions {
+  store: RecordingStore;
+  userId: string;
+  gameType: string;
+  gameVersion: string;
+  rulesetId: string;
+  seed: number;
+  initialState: unknown;
+  controlSchema: readonly string[];
+  viewport: { width: number; height: number };
+  applicationVersion?: string;
+  config?: Partial<RecordingConfig>;
+  /** @deprecated Uploads are coordinated by RecordingSyncCoordinator. */
+  transport?: ChunkTransport;
+  onChunkPersisted?: () => void;
+  wallTimeMs?: () => number;
+  monotonicTimeMs?: () => number;
+  createGameId?: () => string;
+  createEventId?: () => string;
+  createChunkId?: () => string;
+  createObservationId?: () => string;
+}
+
+export interface RecordObservationInput {
+  grid: TerminalGrid;
+  simulationTick?: number;
+  causedByEventId?: string;
+  captureTiming?: {
+    generated_time_us: number;
+    delivered_time_us: number;
+    dropped_observations: number;
+  };
+}
+
+export interface RecordControlInput {
+  control: string;
+  action: string;
+  actor: TraceActor;
+  simulationTick?: number;
+  receivedTimeUs?: number;
+}
+
+export interface RecordOutcomeInput {
+  scoreBefore: number;
+  scoreAfter: number;
+  engineReward: number;
+  terminal: boolean;
+  causedByEventId: string;
+  simulationTick?: number;
+  progress?: Record<string, number | string | boolean | null>;
+  performance?: {
+    action_received_us: number;
+    action_applied_us: number;
+    action_completed_us: number;
+    queueing_delay_us: number;
+    simulation_tick_duration_us: number;
+  };
+}
+
+export interface RecordCheckpointInput {
+  state: unknown;
+  reason: CheckpointReason;
+  simulationTick?: number;
+  seed?: number;
+  randomEffects?: unknown;
+  causedByEventId?: string;
+}
+
+export interface EndEpisodeInput {
+  reason: EpisodeEndReason;
+  finalScore: number;
+  completed: boolean;
+  simulationTick?: number;
+}
+
+export interface TraceRecoveryOptions {
+  applicationVersion?: string;
+  wallTimeMs?: () => number;
+}
+
+export interface TraceRecoveryResult {
+  games: number;
+  chunks: number;
+  events: number;
+}
+
+function hashState(state: unknown): string {
+  return `sha256:${createHash("sha256").update(JSON.stringify(state)).digest("hex")}`;
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+export class InteractionTraceRecorder {
+  readonly gameId: string;
+  readonly userId: string;
+  readonly gameType: string;
+
+  private readonly store: RecordingStore;
+  private readonly gameVersion: string;
+  private readonly applicationVersion: string;
+  private readonly config: RecordingConfig;
+  private readonly wallTimeMs: () => number;
+  private readonly monotonicTimeMs: () => number;
+  private readonly nextEventId: () => string;
+  private readonly nextChunkId: () => string;
+  private readonly nextObservationId: () => string;
+  private readonly onChunkPersisted: () => void;
+  private readonly buffered: TraceEvent[] = [];
+  private monotonicOrigin: number | null = null;
+  private currentChunkId: string;
+  private nextEventSequence = 1;
+  private nextChunkSequence = 1;
+  private flushTimer: ReturnType<typeof setTimeout> | null = null;
+  private previousObservation: { observationId: string; grid: TerminalGrid } | undefined;
+  private ended = false;
+  private recordingHealth: RecordingHealth = { status: "healthy" };
+
+  constructor(options: TraceRecorderOptions) {
+    this.store = options.store;
+    this.userId = options.userId;
+    this.gameType = options.gameType;
+    this.gameVersion = options.gameVersion;
+    this.applicationVersion = options.applicationVersion ?? "dev";
+    this.config = resolveRecordingConfig(options.config);
+    this.wallTimeMs = options.wallTimeMs ?? Date.now;
+    this.monotonicTimeMs = options.monotonicTimeMs ?? (() => performance.now());
+    this.gameId = (options.createGameId ?? createGameId)();
+    this.nextEventId = options.createEventId ?? createEventId;
+    this.nextChunkId = options.createChunkId ?? createChunkId;
+    this.nextObservationId = options.createObservationId ?? createObservationId;
+    this.onChunkPersisted = options.onChunkPersisted ?? (() => undefined);
+    this.currentChunkId = this.nextChunkId();
+
+    const started: EpisodeStartedTraceEvent = {
+      ...this.envelope("policy"),
+      event_type: "episode_started",
+      visibility: "policy",
+      payload: {
+        game_type: this.gameType,
+        game_version: this.gameVersion,
+        ruleset_id: options.rulesetId,
+        telemetry_version: 2,
+        platform: "tui",
+        viewport: { ...options.viewport },
+        control_schema: [...options.controlSchema],
+        input_semantics: "terminal_impulse",
+      },
+    };
+    this.append(started);
+    this.recordCheckpoint({
+      state: options.initialState,
+      reason: "episode_start",
+      seed: options.seed,
+      simulationTick: 0,
+    });
+  }
+
+  get lastObservationId(): string | null {
+    return this.previousObservation?.observationId ?? null;
+  }
+
+  monotonicTimeUs(): number {
+    return this.elapsedUs();
+  }
+
+  health(): RecordingHealth {
+    return { ...this.recordingHealth };
+  }
+
+  recordObservation(input: RecordObservationInput): string {
+    const observationId = this.nextObservationId();
+    const payload = encodeTerminalObservation({
+      observationId,
+      grid: input.grid,
+      previous: this.previousObservation,
+      causedByEventId: input.causedByEventId,
+      captureTiming: input.captureTiming,
+    });
+    const event: ObservationTraceEvent = {
+      ...this.envelope("policy", input.simulationTick),
+      event_type: "observation",
+      visibility: "policy",
+      payload,
+    };
+    this.append(event);
+    this.previousObservation = { observationId, grid: input.grid };
+    return observationId;
+  }
+
+  recordControl(input: RecordControlInput): string {
+    const event = {
+      ...this.envelope("policy", input.simulationTick),
+      event_type: "control_changed" as const,
+      visibility: "policy" as const,
+      payload: {
+        based_on_observation_id: this.lastObservationId,
+        control: input.control,
+        action: input.action,
+        kind: "impulse" as const,
+        actor: input.actor,
+        ...(input.receivedTimeUs === undefined
+          ? {}
+          : { action_received_us: input.receivedTimeUs }),
+      },
+    };
+    this.append(event);
+    return event.event_id;
+  }
+
+  recordControlCheckpoint(heldControls: readonly string[], simulationTick?: number): string {
+    const event = {
+      ...this.envelope("policy", simulationTick),
+      event_type: "control_checkpoint" as const,
+      visibility: "policy" as const,
+      payload: {
+        held_controls: [...heldControls],
+        input_semantics: "terminal_impulse" as const,
+      },
+    };
+    this.append(event);
+    return event.event_id;
+  }
+
+  recordOutcome(input: RecordOutcomeInput): string {
+    const event: OutcomeChangedTraceEvent = {
+      ...this.envelope("policy", input.simulationTick),
+      event_type: "outcome_changed",
+      visibility: "policy",
+      payload: {
+        score_before: input.scoreBefore,
+        score_after: input.scoreAfter,
+        score_delta: input.scoreAfter - input.scoreBefore,
+        engine_reward: input.engineReward,
+        reward_spec_id: "engine_v1",
+        terminal: input.terminal,
+        caused_by_event_id: input.causedByEventId,
+        ...(input.progress ? { progress: { ...input.progress } } : {}),
+        ...(input.performance ? { performance: { ...input.performance } } : {}),
+      },
+    };
+    this.append(event);
+    return event.event_id;
+  }
+
+  recordCheckpoint(input: RecordCheckpointInput): string | null {
+    try {
+      const clonedState = structuredClone(input.state);
+      const event: EngineCheckpointTraceEvent = {
+        ...this.envelope("privileged", input.simulationTick),
+        event_type: "engine_checkpoint",
+        visibility: "privileged",
+        payload: {
+          state_version: 1,
+          reason: input.reason,
+          state: clonedState,
+          state_hash: hashState(clonedState),
+          ...(input.seed === undefined ? {} : { seed: input.seed }),
+          ...(input.randomEffects === undefined
+            ? {}
+            : { random_effects: structuredClone(input.randomEffects) }),
+          ...(input.causedByEventId
+            ? { caused_by_event_id: input.causedByEventId }
+            : {}),
+        },
+      };
+      this.append(event);
+      return event.event_id;
+    } catch (error) {
+      this.degrade(error);
+      return null;
+    }
+  }
+
+  endEpisode(input: EndEpisodeInput): EpisodeEndedTraceEvent | null {
+    if (this.ended) return null;
+    const event: EpisodeEndedTraceEvent = {
+      ...this.envelope("policy", input.simulationTick),
+      event_type: "episode_ended",
+      visibility: "policy",
+      payload: {
+        reason: input.reason,
+        final_event_sequence: this.nextEventSequence - 1,
+        final_score: input.finalScore,
+        duration_us: this.elapsedUs(),
+        completed: input.completed,
+      },
+    };
+    event.payload.final_event_sequence = event.event_sequence;
+    this.append(event);
+    this.ended = true;
+    this.flush();
+    if (this.buffered.length === 0) {
+      this.attempt(() => this.store.finalizeGame(this.gameId));
+    }
+    return event;
+  }
+
+  flush(): void {
+    this.clearFlushTimer();
+    try {
+      this.flushBuffered();
+    } catch (error) {
+      this.degrade(error);
+    }
+    if (this.buffered.length > 0) this.scheduleFlush();
+  }
+
+  shutdown(): void {
+    this.flush();
+  }
+
+  /** @deprecated Uploads are coordinated by RecordingSyncCoordinator. */
+  async drainUploads(): Promise<void> {
+    this.flush();
+  }
+
+  private envelope(visibility: TraceVisibility, simulationTick?: number) {
+    return {
+      schema_version: 2 as const,
+      user_id: this.userId,
+      game_id: this.gameId,
+      chunk_id: this.currentChunkId,
+      event_id: this.nextEventId(),
+      event_sequence: this.nextEventSequence++,
+      event_version: 1 as const,
+      visibility,
+      client_time_ms: this.wallTimeMs(),
+      monotonic_time_us: this.elapsedUs(),
+      ...(simulationTick === undefined ? {} : { simulation_tick: simulationTick }),
+    };
+  }
+
+  private elapsedUs(): number {
+    const now = this.monotonicTimeMs();
+    this.monotonicOrigin ??= now;
+    return Math.max(0, Math.round((now - this.monotonicOrigin) * 1_000));
+  }
+
+  private append(event: TraceEvent): void {
+    if (this.ended) return;
+    this.attempt(() => this.store.appendEvent(this.gameId, event));
+    this.buffered.push(event);
+    if (
+      this.buffered.length >= this.config.maxEventsPerChunk ||
+      estimatedBytes(this.buffered) >= this.config.maxEstimatedChunkBytes
+    ) {
+      this.flush();
+    } else {
+      this.scheduleFlush();
+    }
+  }
+
+  private flushBuffered(): void {
+    if (this.buffered.length === 0) return;
+    const events = [...this.buffered];
+    const chunk = buildTraceChunk({
+      userId: this.userId,
+      gameId: this.gameId,
+      gameType: this.gameType,
+      gameVersion: this.gameVersion,
+      chunkId: this.currentChunkId,
+      chunkSequence: this.nextChunkSequence,
+      createdAtMs: this.wallTimeMs(),
+      applicationVersion: this.applicationVersion,
+      events,
+    });
+    if (!this.attempt(() => this.store.persistChunk(chunk))) return;
+    this.attempt(this.onChunkPersisted);
+    this.buffered.splice(0, events.length);
+    this.nextChunkSequence++;
+    this.currentChunkId = this.nextChunkId();
+  }
+
+  private scheduleFlush(): void {
+    if (this.flushTimer || this.ended) return;
+    this.flushTimer = setTimeout(() => {
+      this.flushTimer = null;
+      this.flush();
+    }, this.config.flushIntervalMs);
+    this.flushTimer.unref?.();
+  }
+
+  private clearFlushTimer(): void {
+    if (!this.flushTimer) return;
+    clearTimeout(this.flushTimer);
+    this.flushTimer = null;
+  }
+
+  private attempt<T>(operation: () => T): T | undefined {
+    try {
+      return operation();
+    } catch (error) {
+      this.degrade(error);
+      return undefined;
+    }
+  }
+
+  private degrade(error: unknown): void {
+    this.recordingHealth = {
+      status: "degraded",
+      last_error: errorMessage(error),
+    };
+  }
+}
+
+export function recoverActiveTraces(
+  store: RecordingStore,
+  options: TraceRecoveryOptions = {},
+): TraceRecoveryResult {
+  const persisted = [...store.listPending(), ...store.listFailed(), ...store.listCompletedChunks()];
+  return recoverActiveTraceGames(
+    store,
+    options,
+    store.listActiveGameIds(),
+    indexTraceChunksByGame(persisted),
+  );
+}
+
+export async function recoverActiveTracesBatched(
+  store: RecordingStore,
+  options: TraceRecoveryOptions & {
+    batchSize?: number;
+    yieldControl?: () => Promise<void>;
+    signal?: AbortSignal;
+    persistedByGame?: ReadonlyRecoveryChunkIndex;
+  } = {},
+): Promise<TraceRecoveryResult> {
+  const batchSize = Math.max(1, options.batchSize ?? 16);
+  const yieldControl = options.yieldControl ?? (() =>
+    new Promise<void>((resolve) => setTimeout(resolve, 0))
+  );
+  const persistedByGame = options.persistedByGame ?? await store.buildRecoveryChunkIndex({
+    batchSize,
+    yieldControl,
+    signal: options.signal,
+  });
+  if (options.signal?.aborted) return { games: 0, chunks: 0, events: 0 };
+  const total: TraceRecoveryResult = { games: 0, chunks: 0, events: 0 };
+  for await (const gameId of store.iterateActiveGameIdsBatched({
+    batchSize,
+    yieldControl,
+    signal: options.signal,
+  })) {
+    if (options.signal?.aborted) break;
+    const batch = recoverActiveTraceGames(
+      store,
+      options,
+      [gameId],
+      persistedByGame,
+    );
+    total.games += batch.games;
+    total.chunks += batch.chunks;
+    total.events += batch.events;
+  }
+  return total;
+}
+
+function recoverActiveTraceGames(
+  store: RecordingStore,
+  options: TraceRecoveryOptions,
+  gameIds: string[],
+  persistedByGame: ReadonlyRecoveryChunkIndex,
+): TraceRecoveryResult {
+  const result: TraceRecoveryResult = { games: 0, chunks: 0, events: 0 };
+  const wallTimeMs = options.wallTimeMs ?? Date.now;
+
+  for (const gameId of gameIds) {
+    const events = store.readActiveRecordedEvents(gameId).filter(isTraceEvent);
+    if (events.length === 0) continue;
+    const started = events.find(
+      (event): event is EpisodeStartedTraceEvent => event.event_type === "episode_started",
+    );
+    if (!started) continue;
+
+    const existing = (persistedByGame.get(gameId) ?? []).filter(
+      (item) => item.chunk.schema_version === 2,
+    );
+    const existingIds = new Set(existing.map((item) => item.chunk.chunk_id));
+    const groups: Array<{ chunkId: string; events: TraceEvent[] }> = [];
+    for (const event of events) {
+      const current = groups.at(-1);
+      if (current?.chunkId === event.chunk_id) current.events.push(event);
+      else groups.push({ chunkId: event.chunk_id, events: [event] });
+    }
+
+    let recoveredGame = false;
+    groups.forEach((group, index) => {
+      if (existingIds.has(group.chunkId)) return;
+      store.persistChunk(buildTraceChunk({
+        userId: started.user_id,
+        gameId,
+        gameType: started.payload.game_type,
+        gameVersion: started.payload.game_version,
+        chunkId: group.chunkId,
+        chunkSequence: index + 1,
+        createdAtMs: wallTimeMs(),
+        applicationVersion: options.applicationVersion ?? "dev",
+        events: group.events,
+      }));
+      recoveredGame = true;
+      result.chunks++;
+      result.events += group.events.length;
+    });
+    if (recoveredGame) result.games++;
+    if (events.at(-1)?.event_type === "episode_ended") store.finalizeGame(gameId);
+  }
+  return result;
+}
+
+function indexTraceChunksByGame(
+  persisted: ReturnType<RecordingStore["listPending"]>,
+): RecoveryChunkIndex {
+  const persistedByGame: RecoveryChunkIndex = new Map();
+  for (const item of persisted) {
+    if (item.chunk.schema_version !== 2) continue;
+    const game = persistedByGame.get(item.chunk.game_id) ?? [];
+    game.push(item);
+    persistedByGame.set(item.chunk.game_id, game);
+  }
+  return persistedByGame;
+}