@trolleroof/tui 0.2.4 → 0.2.6

package/src/recording/store.ts

@@ -0,0 +1,305 @@
+import { randomUUID } from "node:crypto";
+import {
+  appendFileSync,
+  existsSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  writeFileSync,
+} from "node:fs";
+import { opendir } from "node:fs/promises";
+import { basename, dirname, join } from "node:path";
+import type {
+  GameEvent,
+  PersistedChunk,
+  RecordedEvent,
+  RecordedEventChunk,
+} from "./types.js";
+import { ensurePrivateDirectory, ensurePrivateFile } from "./permissions.js";
+
+const CHUNK_SEQUENCE_WIDTH = 12;
+
+export interface CompletedGame {
+  gameId: string;
+  events: RecordedEvent[];
+}
+
+export type RecoveryChunkIndex = Map<string, PersistedChunk[]>;
+export type ReadonlyRecoveryChunkIndex = ReadonlyMap<string, readonly PersistedChunk[]>;
+
+export interface BatchedStoreOptions {
+  batchSize?: number;
+  yieldControl?: () => Promise<void>;
+  signal?: AbortSignal;
+}
+
+export class RecordingStore {
+  constructor(readonly root: string) {
+    ensurePrivateDirectory(root);
+    for (const directory of [
+      "active",
+      "pending",
+      "failed",
+      "completed",
+      "completed/games",
+      "completed/chunks",
+    ]) {
+      ensurePrivateDirectory(join(root, directory));
+    }
+  }
+
+  activePath(gameId: string): string {
+    return join(this.root, "active", `${gameId}.jsonl`);
+  }
+
+  appendEvent(gameId: string, event: RecordedEvent): void {
+    const path = this.activePath(gameId);
+    appendFileSync(path, `${JSON.stringify(event)}\n`, { mode: 0o600 });
+    ensurePrivateFile(path);
+  }
+
+  readActiveEvents(gameId: string): GameEvent[] {
+    return this.readActiveRecordedEvents(gameId) as GameEvent[];
+  }
+
+  readActiveRecordedEvents(gameId: string): RecordedEvent[] {
+    const path = this.activePath(gameId);
+    if (!existsSync(path)) return [];
+    ensurePrivateFile(path);
+    const lines = readFileSync(path, "utf8")
+      .split("\n")
+      .filter(Boolean);
+    return lines.flatMap((line, index) => {
+      try {
+        return [JSON.parse(line) as RecordedEvent];
+      } catch (error) {
+        if (index === lines.length - 1) return [];
+        throw error;
+      }
+    });
+  }
+
+  listActiveGameIds(): string[] {
+    return readdirSync(join(this.root, "active"))
+      .filter((name) => name.endsWith(".jsonl"))
+      .map((name) => name.slice(0, -".jsonl".length))
+      .sort();
+  }
+
+  persistChunk(chunk: RecordedEventChunk): PersistedChunk {
+    const bytes = JSON.stringify(chunk);
+    const directory = join(this.root, "pending", chunk.game_id);
+    ensurePrivateDirectory(directory);
+    const name = `${String(chunk.chunk_sequence).padStart(CHUNK_SEQUENCE_WIDTH, "0")}-${chunk.chunk_id}.json`;
+    const path = join(directory, name);
+    if (existsSync(path)) {
+      ensurePrivateFile(path);
+      const existing = readFileSync(path, "utf8");
+      if (existing !== bytes) throw new Error(`chunk ${chunk.chunk_id} has conflicting bytes`);
+      return { chunk, path, bytes: existing };
+    }
+    const temporary = `${path}.tmp-${process.pid}-${randomUUID()}`;
+    writeFileSync(temporary, bytes, { flag: "wx", mode: 0o600 });
+    renameSync(temporary, path);
+    ensurePrivateFile(path);
+    return { chunk, path, bytes };
+  }
+
+  listPending(): PersistedChunk[] {
+    return this.listChunks(join(this.root, "pending"));
+  }
+
+  listFailed(): PersistedChunk[] {
+    return this.listChunks(join(this.root, "failed"));
+  }
+
+  listCompletedChunks(): PersistedChunk[] {
+    return this.listChunks(join(this.root, "completed", "chunks"));
+  }
+
+  async listRecoveryChunksBatched(
+    options: BatchedStoreOptions = {},
+  ): Promise<PersistedChunk[]> {
+    const chunks: PersistedChunk[] = [];
+    for await (const persisted of this.iterateRecoveryChunksBatched(options)) {
+      chunks.push(persisted);
+    }
+    return this.sortChunks(chunks);
+  }
+
+  async buildRecoveryChunkIndex(
+    options: BatchedStoreOptions = {},
+  ): Promise<RecoveryChunkIndex> {
+    const chunksByGame: RecoveryChunkIndex = new Map();
+    for await (const persisted of this.iterateRecoveryChunksBatched(options)) {
+      const chunks = chunksByGame.get(persisted.chunk.game_id) ?? [];
+      chunks.push(persisted);
+      chunksByGame.set(persisted.chunk.game_id, chunks);
+    }
+    return chunksByGame;
+  }
+
+  async *iterateRecoveryChunksBatched(
+    options: BatchedStoreOptions = {},
+  ): AsyncGenerator<PersistedChunk> {
+    const batchSize = Math.max(1, options.batchSize ?? 16);
+    const yieldControl = options.yieldControl ?? (() =>
+      new Promise<void>((resolve) => setTimeout(resolve, 0))
+    );
+    let processed = 0;
+    for (const base of [
+      join(this.root, "pending"),
+      join(this.root, "failed"),
+      join(this.root, "completed", "chunks"),
+    ]) {
+      if (options.signal?.aborted || !existsSync(base)) return;
+      const games = await opendir(base);
+      for await (const game of games) {
+        if (options.signal?.aborted) return;
+        if (!game.isDirectory()) continue;
+        const directory = join(base, game.name);
+        ensurePrivateDirectory(directory);
+        const files = await opendir(directory);
+        for await (const file of files) {
+          if (options.signal?.aborted) return;
+          if (!file.isFile() || !file.name.endsWith(".json")) continue;
+          if (processed > 0 && processed % batchSize === 0) {
+            await yieldControl();
+            if (options.signal?.aborted) return;
+          }
+          yield this.readChunk(join(directory, file.name));
+          processed++;
+        }
+      }
+    }
+  }
+
+  async *iterateActiveGameIdsBatched(
+    options: BatchedStoreOptions = {},
+  ): AsyncGenerator<string> {
+    const directory = join(this.root, "active");
+    const batchSize = Math.max(1, options.batchSize ?? 16);
+    const yieldControl = options.yieldControl ?? (() =>
+      new Promise<void>((resolve) => setTimeout(resolve, 0))
+    );
+    let processed = 0;
+    const files = await opendir(directory);
+    for await (const file of files) {
+      if (options.signal?.aborted) return;
+      if (!file.isFile() || !file.name.endsWith(".jsonl")) continue;
+      if (processed > 0 && processed % batchSize === 0) {
+        await yieldControl();
+        if (options.signal?.aborted) return;
+      }
+      processed++;
+      yield file.name.slice(0, -".jsonl".length);
+    }
+  }
+
+  listCompletedGames(): CompletedGame[] {
+    return this.listCompletedGameIds().map((gameId) => this.readCompletedGame(gameId));
+  }
+
+  listCompletedGameIds(): string[] {
+    ensurePrivateDirectory(join(this.root, "completed"));
+    const directory = join(this.root, "completed", "games");
+    ensurePrivateDirectory(directory);
+    return readdirSync(directory)
+      .filter((name) => name.endsWith(".jsonl"))
+      .sort()
+      .map((name) => name.slice(0, -".jsonl".length));
+  }
+
+  readCompletedGame(gameId: string): CompletedGame {
+    const path = join(this.root, "completed", "games", `${gameId}.jsonl`);
+    ensurePrivateFile(path);
+    const events = readFileSync(path, "utf8")
+      .split("\n")
+      .filter(Boolean)
+      .map((line) => JSON.parse(line) as RecordedEvent);
+    return { gameId, events };
+  }
+
+  async *iterateCompletedGamesBatched(
+    options: BatchedStoreOptions = {},
+  ): AsyncGenerator<CompletedGame> {
+    const directory = join(this.root, "completed", "games");
+    ensurePrivateDirectory(directory);
+    const batchSize = Math.max(1, options.batchSize ?? 16);
+    const yieldControl = options.yieldControl ?? (() =>
+      new Promise<void>((resolve) => setTimeout(resolve, 0))
+    );
+    let processed = 0;
+    const files = await opendir(directory);
+    for await (const file of files) {
+      if (options.signal?.aborted) return;
+      if (!file.isFile() || !file.name.endsWith(".jsonl")) continue;
+      if (processed > 0 && processed % batchSize === 0) {
+        await yieldControl();
+        if (options.signal?.aborted) return;
+      }
+      const gameId = file.name.slice(0, -".jsonl".length);
+      yield this.readCompletedGame(gameId);
+      processed++;
+    }
+  }
+
+  acknowledge(persisted: PersistedChunk): string {
+    return this.moveChunk(persisted, join(this.root, "completed", "chunks"));
+  }
+
+  markFailed(persisted: PersistedChunk, reason: string): string {
+    const destination = this.moveChunk(persisted, join(this.root, "failed"));
+    writeFileSync(`${destination}.error.txt`, `${reason}\n`, { mode: 0o600 });
+    ensurePrivateFile(`${destination}.error.txt`);
+    return destination;
+  }
+
+  finalizeGame(gameId: string): string | null {
+    const source = this.activePath(gameId);
+    if (!existsSync(source)) return null;
+    const destination = join(this.root, "completed", "games", `${gameId}.jsonl`);
+    renameSync(source, destination);
+    ensurePrivateFile(destination);
+    return destination;
+  }
+
+  private moveChunk(persisted: PersistedChunk, base: string): string {
+    const destination = join(base, persisted.chunk.game_id, basename(persisted.path));
+    ensurePrivateDirectory(dirname(destination));
+    renameSync(persisted.path, destination);
+    ensurePrivateFile(destination);
+    return destination;
+  }
+
+  private listChunks(base: string): PersistedChunk[] {
+    return this.sortChunks(this.chunkPaths(base).map((path) => this.readChunk(path)));
+  }
+
+  private chunkPaths(base: string): string[] {
+    const paths: string[] = [];
+    if (!existsSync(base)) return paths;
+    for (const game of readdirSync(base, { withFileTypes: true })) {
+      if (!game.isDirectory()) continue;
+      const directory = join(base, game.name);
+      ensurePrivateDirectory(directory);
+      for (const name of readdirSync(directory).filter((entry) => entry.endsWith(".json")).sort()) {
+        paths.push(join(directory, name));
+      }
+    }
+    return paths;
+  }
+
+  protected readChunk(path: string): PersistedChunk {
+    ensurePrivateFile(path);
+    const bytes = readFileSync(path, "utf8");
+    return { chunk: JSON.parse(bytes) as RecordedEventChunk, path, bytes };
+  }
+
+  private sortChunks(chunks: PersistedChunk[]): PersistedChunk[] {
+    return chunks.sort((a, b) =>
+      a.chunk.game_id.localeCompare(b.chunk.game_id) ||
+      a.chunk.chunk_sequence - b.chunk.chunk_sequence,
+    );
+  }
+}

package/src/recording/terminal-grid.test.ts

@@ -0,0 +1,88 @@
+import { describe, expect, it } from "vitest";
+import {
+  encodeTerminalObservation,
+  parseTerminalGrid,
+} from "./terminal-grid.js";
+
+describe("terminal grid capture", () => {
+  it("preserves glyphs, 256-color styling, attributes, and viewport dimensions", () => {
+    const grid = parseTerminalGrid([
+      "\x1b[1;48;5;45;38;5;22m T \x1b[0mX",
+    ], { width: 5, height: 2 });
+
+    expect(grid.width).toBe(5);
+    expect(grid.height).toBe(2);
+    expect(grid.cells.slice(0, 5)).toEqual([
+      { glyph: " ", foreground: 22, background: 45, attributes: 1, width: 1 },
+      { glyph: "T", foreground: 22, background: 45, attributes: 1, width: 1 },
+      { glyph: " ", foreground: 22, background: 45, attributes: 1, width: 1 },
+      { glyph: "X", foreground: null, background: null, attributes: 0, width: 1 },
+      { glyph: " ", foreground: null, background: null, attributes: 0, width: 1 },
+    ]);
+  });
+
+  it("emits a sparse full frame followed by a causally linked delta", () => {
+    const first = parseTerminalGrid([
+      "\x1b[48;5;45m T \x1b[0m",
+    ], { width: 4, height: 2 });
+    const full = encodeTerminalObservation({
+      observationId: "obs_1",
+      grid: first,
+      captureTiming: {
+        generated_time_us: 100,
+        delivered_time_us: 125,
+        dropped_observations: 0,
+      },
+    });
+    expect(full.representation).toBe("terminal_grid_v1");
+    if (full.representation !== "terminal_grid_v1") throw new Error("expected full frame");
+    expect(full.cells).toHaveLength(3);
+    expect(full.width).toBe(4);
+    expect(full.height).toBe(2);
+    expect(full.state_hash).toMatch(/^sha256:/);
+    expect(full.capture_timing).toEqual({
+      generated_time_us: 100,
+      delivered_time_us: 125,
+      dropped_observations: 0,
+    });
+
+    const second = parseTerminalGrid([
+      "\x1b[48;5;45m I \x1b[0m",
+    ], { width: 4, height: 2 });
+    const delta = encodeTerminalObservation({
+      observationId: "obs_2",
+      grid: second,
+      previous: { observationId: "obs_1", grid: first },
+      causedByEventId: "evt_control",
+    });
+    expect(delta).toMatchObject({
+      observation_id: "obs_2",
+      representation: "terminal_grid_delta_v1",
+      base_observation_id: "obs_1",
+      caused_by_event_id: "evt_control",
+      changed_cells: [{
+        x: 1,
+        y: 0,
+        cell: {
+          glyph: "I",
+          foreground: null,
+          background: 45,
+          attributes: 0,
+          width: 1,
+        },
+      }],
+    });
+  });
+
+  it("records an empty delta when an accepted control does not change the screen", () => {
+    const grid = parseTerminalGrid(["same"], { width: 4, height: 1 });
+    const delta = encodeTerminalObservation({
+      observationId: "obs_2",
+      grid,
+      previous: { observationId: "obs_1", grid },
+      causedByEventId: "evt_noop",
+    });
+    if (delta.representation !== "terminal_grid_delta_v1") throw new Error("expected delta");
+    expect(delta.changed_cells).toEqual([]);
+  });
+});

package/src/recording/terminal-grid.ts

@@ -0,0 +1,248 @@
+import { createHash } from "node:crypto";
+import type {
+  TerminalCell,
+  TerminalColor,
+  TerminalObservation,
+  ObservationCaptureTiming,
+} from "./types.js";
+
+const BOLD = 1;
+const DIM = 2;
+const ITALIC = 4;
+const UNDERLINE = 8;
+const INVERSE = 16;
+const GRAPHEME_SEGMENTER = typeof Intl.Segmenter === "function"
+  ? new Intl.Segmenter(undefined, { granularity: "grapheme" })
+  : null;
+
+const DEFAULT_CELL: TerminalCell = {
+  glyph: " ",
+  foreground: null,
+  background: null,
+  attributes: 0,
+  width: 1,
+};
+
+interface Style {
+  foreground: TerminalColor;
+  background: TerminalColor;
+  attributes: number;
+}
+
+export interface TerminalViewport {
+  width: number;
+  height: number;
+}
+
+export interface TerminalGrid {
+  width: number;
+  height: number;
+  cells: TerminalCell[];
+}
+
+interface EncodeObservationInput {
+  observationId: string;
+  grid: TerminalGrid;
+  previous?: { observationId: string; grid: TerminalGrid };
+  causedByEventId?: string;
+  captureTiming?: ObservationCaptureTiming;
+}
+
+function cloneDefaultCell(): TerminalCell {
+  return { ...DEFAULT_CELL };
+}
+
+function basicColor(code: number, background: boolean): number | null {
+  const base = background ? 40 : 30;
+  const bright = background ? 100 : 90;
+  if (code >= base && code <= base + 7) return code - base;
+  if (code >= bright && code <= bright + 7) return code - bright + 8;
+  return null;
+}
+
+function rgbColor(red: number, green: number, blue: number): string {
+  return `#${[red, green, blue].map((value) =>
+    Math.max(0, Math.min(255, value)).toString(16).padStart(2, "0")
+  ).join("")}`;
+}
+
+function applySgr(style: Style, parameters: string): void {
+  const codes = parameters === "" ? [0] : parameters.split(";").map(Number);
+  for (let index = 0; index < codes.length; index++) {
+    const code = Number.isFinite(codes[index]) ? codes[index]! : 0;
+    if (code === 0) {
+      style.foreground = null;
+      style.background = null;
+      style.attributes = 0;
+    } else if (code === 1) style.attributes |= BOLD;
+    else if (code === 2) style.attributes |= DIM;
+    else if (code === 3) style.attributes |= ITALIC;
+    else if (code === 4) style.attributes |= UNDERLINE;
+    else if (code === 7) style.attributes |= INVERSE;
+    else if (code === 22) style.attributes &= ~(BOLD | DIM);
+    else if (code === 23) style.attributes &= ~ITALIC;
+    else if (code === 24) style.attributes &= ~UNDERLINE;
+    else if (code === 27) style.attributes &= ~INVERSE;
+    else if (code === 39) style.foreground = null;
+    else if (code === 49) style.background = null;
+    else if ((code >= 30 && code <= 37) || (code >= 90 && code <= 97)) {
+      style.foreground = basicColor(code, false);
+    } else if ((code >= 40 && code <= 47) || (code >= 100 && code <= 107)) {
+      style.background = basicColor(code, true);
+    } else if ((code === 38 || code === 48) && codes[index + 1] === 5) {
+      const color = codes[index + 2];
+      if (Number.isInteger(color)) {
+        if (code === 38) style.foreground = color!;
+        else style.background = color!;
+      }
+      index += 2;
+    } else if ((code === 38 || code === 48) && codes[index + 1] === 2) {
+      const [red, green, blue] = codes.slice(index + 2, index + 5);
+      if ([red, green, blue].every(Number.isInteger)) {
+        const color = rgbColor(red!, green!, blue!);
+        if (code === 38) style.foreground = color;
+        else style.background = color;
+      }
+      index += 4;
+    }
+  }
+}
+
+function cellWidth(glyph: string): number {
+  const codePoint = glyph.codePointAt(0) ?? 0;
+  if (codePoint === 0) return 0;
+  if (
+    codePoint >= 0x1100 && (
+      codePoint <= 0x115f ||
+      codePoint === 0x2329 || codePoint === 0x232a ||
+      (codePoint >= 0x2e80 && codePoint <= 0xa4cf) ||
+      (codePoint >= 0xac00 && codePoint <= 0xd7a3) ||
+      (codePoint >= 0xf900 && codePoint <= 0xfaff) ||
+      (codePoint >= 0xfe10 && codePoint <= 0xfe19) ||
+      (codePoint >= 0xfe30 && codePoint <= 0xfe6f) ||
+      (codePoint >= 0xff00 && codePoint <= 0xff60) ||
+      (codePoint >= 0x1f300 && codePoint <= 0x1faff)
+    )
+  ) return 2;
+  return 1;
+}
+
+function graphemes(value: string): string[] {
+  if (GRAPHEME_SEGMENTER) {
+    return Array.from(GRAPHEME_SEGMENTER.segment(value), ({ segment }) => segment);
+  }
+  return Array.from(value);
+}
+
+function sameCell(left: TerminalCell, right: TerminalCell): boolean {
+  return left.glyph === right.glyph &&
+    left.foreground === right.foreground &&
+    left.background === right.background &&
+    left.attributes === right.attributes &&
+    left.width === right.width;
+}
+
+function isDefaultCell(cell: TerminalCell): boolean {
+  return sameCell(cell, DEFAULT_CELL);
+}
+
+function gridHash(grid: TerminalGrid): string {
+  return `sha256:${createHash("sha256").update(JSON.stringify(grid)).digest("hex")}`;
+}
+
+export function parseTerminalGrid(
+  lines: readonly string[],
+  viewport: TerminalViewport,
+): TerminalGrid {
+  if (!Number.isInteger(viewport.width) || viewport.width <= 0) {
+    throw new Error("terminal viewport width must be a positive integer");
+  }
+  if (!Number.isInteger(viewport.height) || viewport.height <= 0) {
+    throw new Error("terminal viewport height must be a positive integer");
+  }
+  const cells = Array.from(
+    { length: viewport.width * viewport.height },
+    cloneDefaultCell,
+  );
+  const style: Style = { foreground: null, background: null, attributes: 0 };
+
+  for (let y = 0; y < Math.min(lines.length, viewport.height); y++) {
+    const line = lines[y]!;
+    let x = 0;
+    let cursor = 0;
+    while (cursor < line.length && x < viewport.width) {
+      if (line[cursor] === "\x1b" && line[cursor + 1] === "[") {
+        const match = /^\x1b\[([0-9;?]*)([@-~])/.exec(line.slice(cursor));
+        if (match) {
+          if (match[2] === "m") applySgr(style, match[1]!);
+          cursor += match[0].length;
+          continue;
+        }
+      }
+      const nextEscape = line.indexOf("\x1b", cursor);
+      const plain = line.slice(cursor, nextEscape === -1 ? line.length : nextEscape);
+      for (const glyph of graphemes(plain)) {
+        if (x >= viewport.width) break;
+        const width = cellWidth(glyph);
+        cells[y * viewport.width + x] = { glyph, ...style, width };
+        if (width === 2 && x + 1 < viewport.width) {
+          cells[y * viewport.width + x + 1] = {
+            glyph: "",
+            ...style,
+            width: 0,
+          };
+        }
+        x += Math.max(1, width);
+      }
+      cursor += plain.length || 1;
+    }
+  }
+  return { width: viewport.width, height: viewport.height, cells };
+}
+
+export function encodeTerminalObservation(
+  input: EncodeObservationInput,
+): TerminalObservation {
+  const common = {
+    observation_id: input.observationId,
+    width: input.grid.width,
+    height: input.grid.height,
+    state_hash: gridHash(input.grid),
+    ...(input.causedByEventId
+      ? { caused_by_event_id: input.causedByEventId }
+      : {}),
+    ...(input.captureTiming ? { capture_timing: { ...input.captureTiming } } : {}),
+  };
+  if (
+    input.previous &&
+    input.previous.grid.width === input.grid.width &&
+    input.previous.grid.height === input.grid.height
+  ) {
+    const changed = input.grid.cells.flatMap((cell, index) => {
+      if (sameCell(cell, input.previous!.grid.cells[index]!)) return [];
+      return [{
+        x: index % input.grid.width,
+        y: Math.floor(index / input.grid.width),
+        cell: isDefaultCell(cell) ? null : { ...cell },
+      }];
+    });
+    return {
+      ...common,
+      representation: "terminal_grid_delta_v1",
+      base_observation_id: input.previous.observationId,
+      changed_cells: changed,
+    };
+  }
+  return {
+    ...common,
+    representation: "terminal_grid_v1",
+    cells: input.grid.cells.flatMap((cell, index) => {
+      if (isDefaultCell(cell)) return [];
+      return [{
+        x: index % input.grid.width,
+        y: Math.floor(index / input.grid.width),
+        ...cell,
+      }];
+    }),
+  };
+}