flingit 0.0.63 → 0.0.66

package/node_modules/flingflow/src/store-sqlite.ts

@@ -0,0 +1,498 @@
+import Database from "better-sqlite3";
+import { ok, err, type Result } from "neverthrow";
+import type { SimulationTask } from "determined";
+import type { WorkflowEvent } from "./types.js";
+import {
+  UniqueConstraintError,
+  WorkflowAlreadyActiveError,
+  WorkflowIdMismatchError,
+  type EventStore,
+  type EventStoreCheck,
+  type AppendError,
+  type AppendIfNoActiveError,
+  type PruneResult,
+} from "./store.js";
+
+function toError(e: unknown): Error {
+  return e instanceof Error ? e : new Error(String(e));
+}
+
+interface EventRow {
+  run_id: string;
+  workflow_id: string;
+  seq: number;
+  event_type: string;
+  payload: string;
+  created_at: number;
+}
+
+function isSqliteConstraintError(e: unknown): boolean {
+  if (!(e instanceof Error)) return false;
+  const sqliteErr = e as Error & { code?: string };
+  return sqliteErr.code === "SQLITE_CONSTRAINT_PRIMARYKEY" || e.message.includes("UNIQUE constraint failed");
+}
+
+export class SqliteEventStore implements EventStore {
+  private db: Database.Database;
+  private checkFn: EventStoreCheck | undefined;
+  private table: string;
+
+  constructor(pathOrDb: string | Database.Database, options?: { check?: EventStoreCheck; tableName?: string }) {
+    if (typeof pathOrDb === "string") {
+      this.db = new Database(pathOrDb);
+    } else {
+      this.db = pathOrDb;
+    }
+    this.checkFn = options?.check;
+    this.table = options?.tableName ?? "workflow_events";
+  }
+
+  // eslint-disable-next-line @typescript-eslint/require-await -- interface requires async
+  async initialize(): Promise<Result<void, Error>> {
+    try {
+      this.db.pragma("journal_mode = WAL");
+      this.db.exec(`
+        CREATE TABLE IF NOT EXISTS ${this.table} (
+          run_id TEXT NOT NULL,
+          workflow_id TEXT NOT NULL,
+          seq INTEGER NOT NULL,
+          event_type TEXT NOT NULL,
+          payload TEXT NOT NULL,
+          created_at INTEGER NOT NULL,
+          PRIMARY KEY (run_id, seq)
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_${this.table}_last
+        ON ${this.table}(run_id, seq DESC);
+
+        CREATE INDEX IF NOT EXISTS idx_${this.table}_workflow
+        ON ${this.table}(workflow_id);
+      `);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+    return ok(undefined);
+  }
+
+  private toRow(event: WorkflowEvent): EventRow {
+    return {
+      run_id: event.run_id,
+      workflow_id: event.workflow_id,
+      seq: event.seq,
+      event_type: event.event_type,
+      payload: JSON.stringify(event.payload),
+      created_at: event.created_at,
+    };
+  }
+
+  private fromRow(row: EventRow): WorkflowEvent {
+    return {
+      run_id: row.run_id,
+      workflow_id: row.workflow_id,
+      seq: row.seq,
+      event_type: row.event_type,
+      payload: JSON.parse(row.payload) as WorkflowEvent["payload"],
+      created_at: row.created_at,
+    } as WorkflowEvent;
+  }
+
+  private collectEvents(runId: string): WorkflowEvent[] {
+    const rows = this.db.prepare(
+      `SELECT * FROM ${this.table} WHERE run_id = ? ORDER BY seq ASC`
+    ).all(runId) as EventRow[];
+    return rows.map((row) => this.fromRow(row));
+  }
+
+  private runCheck(runId: string): void {
+    if (this.checkFn === undefined) return;
+    this.checkFn(runId, this.collectEvents(runId));
+  }
+
+  private validateWorkflowId(event: WorkflowEvent): WorkflowIdMismatchError | undefined {
+    const row = this.db.prepare(
+      `SELECT workflow_id FROM ${this.table} WHERE run_id = ? LIMIT 1`
+    ).get(event.run_id) as { workflow_id: string } | undefined;
+    if (row !== undefined && row.workflow_id !== event.workflow_id) {
+      return new WorkflowIdMismatchError(event.run_id, row.workflow_id, event.workflow_id);
+    }
+    return undefined;
+  }
+
+  async append(event: WorkflowEvent, sim: SimulationTask): Promise<Result<void, AppendError>> {
+    try {
+      await sim.failpoint("store.append:before", event.run_id, event.seq);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      const mismatch = this.validateWorkflowId(event);
+      if (mismatch !== undefined) return err(mismatch);
+
+      const row = this.toRow(event);
+      try {
+        this.db.prepare(`
+          INSERT INTO ${this.table} (run_id, workflow_id, seq, event_type, payload, created_at)
+          VALUES (@run_id, @workflow_id, @seq, @event_type, @payload, @created_at)
+        `).run(row);
+      } catch (e: unknown) {
+        if (isSqliteConstraintError(e)) {
+          return err(new UniqueConstraintError(event.run_id, event.seq));
+        }
+        return err(toError(e));
+      }
+
+      this.runCheck(event.run_id);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.append:after", event.run_id, event.seq);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(undefined);
+  }
+
+  async appendBatch(events: WorkflowEvent[], sim: SimulationTask): Promise<Result<void, AppendError>> {
+    const runId = events[0]?.run_id ?? "batch";
+
+    try {
+      await sim.failpoint("store.appendBatch:before", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      // Validate workflow_id consistency
+      for (const event of events) {
+        const mismatch = this.validateWorkflowId(event);
+        if (mismatch !== undefined) return err(mismatch);
+      }
+
+      const insert = this.db.prepare(`
+        INSERT INTO ${this.table} (run_id, workflow_id, seq, event_type, payload, created_at)
+        VALUES (@run_id, @workflow_id, @seq, @event_type, @payload, @created_at)
+      `);
+
+      const transaction = this.db.transaction((evts: WorkflowEvent[]) => {
+        for (const event of evts) {
+          try {
+            insert.run(this.toRow(event));
+          } catch (e: unknown) {
+            if (isSqliteConstraintError(e)) {
+              throw new UniqueConstraintError(event.run_id, event.seq);
+            }
+            throw e;
+          }
+        }
+      });
+
+      try {
+        transaction(events);
+      } catch (e: unknown) {
+        if (e instanceof UniqueConstraintError) {
+          return err(e);
+        }
+        return err(toError(e));
+      }
+
+      const runIds = new Set(events.map(e => e.run_id));
+      for (const rid of runIds) {
+        this.runCheck(rid);
+      }
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.appendBatch:after", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(undefined);
+  }
+
+  async appendIfNoActiveWorkflow(
+    workflowId: string,
+    event: WorkflowEvent,
+    sim: SimulationTask,
+  ): Promise<Result<void, AppendIfNoActiveError>> {
+    try {
+      await sim.failpoint("store.appendIfNoActiveWorkflow:before", workflowId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      // Atomic check + insert in a single transaction
+      const insertStmt = this.db.prepare(`
+        INSERT INTO ${this.table} (run_id, workflow_id, seq, event_type, payload, created_at)
+        VALUES (@run_id, @workflow_id, @seq, @event_type, @payload, @created_at)
+      `);
+
+      const checkStmt = this.db.prepare(`
+        SELECT e.run_id FROM ${this.table} e
+        INNER JOIN (
+          SELECT run_id, MAX(seq) as max_seq FROM ${this.table} GROUP BY run_id
+        ) latest ON e.run_id = latest.run_id AND e.seq = latest.max_seq
+        WHERE e.workflow_id = ?
+          AND e.event_type NOT IN ('workflow_completed', 'workflow_failed')
+        LIMIT 1
+      `);
+
+      type CheckResult = Result<void, WorkflowAlreadyActiveError | UniqueConstraintError | Error>;
+
+      const transaction = this.db.transaction((): CheckResult => {
+        const active = checkStmt.get(workflowId) as { run_id: string } | undefined;
+        if (active !== undefined) {
+          return err(new WorkflowAlreadyActiveError(workflowId, active.run_id));
+        }
+        try {
+          insertStmt.run(this.toRow(event));
+        } catch (e: unknown) {
+          if (isSqliteConstraintError(e)) {
+            return err(new UniqueConstraintError(event.run_id, event.seq));
+          }
+          return err(toError(e));
+        }
+        return ok(undefined);
+      });
+
+      const result = transaction();
+      if (result.isErr()) {
+        return result;
+      }
+
+      this.runCheck(event.run_id);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.appendIfNoActiveWorkflow:after", workflowId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(undefined);
+  }
+
+  async getLastEvent(runId: string, sim: SimulationTask): Promise<Result<WorkflowEvent | null, Error>> {
+    try {
+      await sim.failpoint("store.getLastEvent:before", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let result: WorkflowEvent | null;
+    try {
+      const row = this.db.prepare(`
+        SELECT * FROM ${this.table}
+        WHERE run_id = ?
+        ORDER BY seq DESC
+        LIMIT 1
+      `).get(runId) as EventRow | undefined;
+      result = row !== undefined ? this.fromRow(row) : null;
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.getLastEvent:after", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(result);
+  }
+
+  async getEvent(runId: string, seq: number, sim: SimulationTask): Promise<Result<WorkflowEvent | null, Error>> {
+    try {
+      await sim.failpoint("store.getEvent:before", runId, seq);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let result: WorkflowEvent | null;
+    try {
+      const row = this.db.prepare(`
+        SELECT * FROM ${this.table}
+        WHERE run_id = ? AND seq = ?
+      `).get(runId, seq) as EventRow | undefined;
+      result = row !== undefined ? this.fromRow(row) : null;
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.getEvent:after", runId, seq);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(result);
+  }
+
+  async getCompletedEvents(runId: string, sim: SimulationTask): Promise<Result<WorkflowEvent[], Error>> {
+    try {
+      await sim.failpoint("store.getCompletedEvents:before", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let results: WorkflowEvent[];
+    try {
+      const rows = this.db.prepare(`
+        SELECT * FROM ${this.table}
+        WHERE run_id = ? AND event_type = 'step_completed'
+        ORDER BY seq DESC
+      `).all(runId) as EventRow[];
+      results = rows.map((row) => this.fromRow(row));
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.getCompletedEvents:after", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(results);
+  }
+
+  async getAllEvents(runId: string, sim: SimulationTask): Promise<Result<WorkflowEvent[], Error>> {
+    try {
+      await sim.failpoint("store.getAllEvents:before", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let results: WorkflowEvent[];
+    try {
+      results = this.collectEvents(runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.getAllEvents:after", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(results);
+  }
+
+  async getRecentRuns(limit: number, sim: SimulationTask): Promise<Result<WorkflowEvent[], Error>> {
+    try {
+      await sim.failpoint("store.getRecentRuns:before");
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let results: WorkflowEvent[];
+    try {
+      const rows = this.db.prepare(
+        `SELECT * FROM ${this.table} WHERE seq = 0 ORDER BY created_at DESC LIMIT ?`
+      ).all(limit) as EventRow[];
+      results = rows.map((row) => this.fromRow(row));
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.getRecentRuns:after");
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(results);
+  }
+
+  async getActiveWorkflows(sim: SimulationTask): Promise<Result<WorkflowEvent[], Error>> {
+    try {
+      await sim.failpoint("store.getActiveWorkflows:before");
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let results: WorkflowEvent[];
+    try {
+      const rows = this.db.prepare(`
+        SELECT e.* FROM ${this.table} e
+        INNER JOIN (
+          SELECT run_id, MAX(seq) as max_seq
+          FROM ${this.table}
+          GROUP BY run_id
+        ) latest ON e.run_id = latest.run_id AND e.seq = latest.max_seq
+        WHERE e.event_type NOT IN ('workflow_completed', 'workflow_failed')
+      `).all() as EventRow[];
+      results = rows.map((row) => this.fromRow(row));
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.getActiveWorkflows:after");
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(results);
+  }
+
+  async pruneTerminalRunsOlderThan(cutoff: number, sim: SimulationTask): Promise<Result<PruneResult, Error>> {
+    try {
+      await sim.failpoint("store.pruneTerminalRunsOlderThan:before", cutoff);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let result: PruneResult;
+    try {
+      const runIds = this.db.prepare(`
+        SELECT e.run_id FROM ${this.table} e
+        INNER JOIN (
+          SELECT run_id, MAX(seq) as max_seq
+          FROM ${this.table}
+          GROUP BY run_id
+        ) latest ON e.run_id = latest.run_id AND e.seq = latest.max_seq
+        WHERE e.event_type IN ('workflow_completed', 'workflow_failed')
+          AND e.created_at < ?
+      `).all(cutoff) as { run_id: string }[];
+
+      if (runIds.length === 0) {
+        result = { prunedRuns: 0, prunedEvents: 0 };
+      } else {
+        const deleteStmt = this.db.prepare(`DELETE FROM ${this.table} WHERE run_id = ?`);
+        const transaction = this.db.transaction((ids: string[]): PruneResult => {
+          let prunedEvents = 0;
+          for (const runId of ids) {
+            prunedEvents += deleteStmt.run(runId).changes;
+          }
+          return { prunedRuns: ids.length, prunedEvents };
+        });
+        result = transaction(runIds.map((row) => row.run_id));
+      }
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.pruneTerminalRunsOlderThan:after", cutoff);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(result);
+  }
+
+  close(): void {
+    this.db.close();
+  }
+}

package/node_modules/flingflow/src/store.ts

@@ -0,0 +1,61 @@
+import type { Result } from "neverthrow";
+import type { SimulationTask } from "determined";
+import type { WorkflowEvent } from "./types.js";
+
+export class UniqueConstraintError extends Error {
+  constructor(runId: string, seq: number) {
+    super(`Duplicate event: run_id=${runId}, seq=${seq}`);
+    this.name = "UniqueConstraintError";
+  }
+}
+
+export class WorkflowAlreadyActiveError extends Error {
+  public readonly activeRunId: string;
+  constructor(workflowId: string, activeRunId: string) {
+    super(`Workflow ${workflowId} already has an active run: ${activeRunId}`);
+    this.name = "WorkflowAlreadyActiveError";
+    this.activeRunId = activeRunId;
+  }
+}
+
+export class WorkflowIdMismatchError extends Error {
+  constructor(runId: string, expected: string, got: string) {
+    super(`Workflow ID mismatch for run ${runId}: expected ${expected}, got ${got}`);
+    this.name = "WorkflowIdMismatchError";
+  }
+}
+
+export type EventStoreCheck = (runId: string, allEvents: WorkflowEvent[]) => void;
+
+export type AppendError = UniqueConstraintError | WorkflowIdMismatchError | Error;
+export type AppendIfNoActiveError = WorkflowAlreadyActiveError | UniqueConstraintError | Error;
+
+export interface PruneResult {
+  prunedRuns: number;
+  prunedEvents: number;
+}
+
+/**
+ * Event payloads are stored as JSON. This means:
+ * - `undefined` values in objects are silently dropped
+ * - `Date` objects are serialized as strings and NOT restored as Dates
+ * - `NaN`, `Infinity`, `-Infinity` become `null`
+ * - `bigint` values will throw — they are not JSON-serializable
+ * - Prototype chains and class instances are reduced to plain objects
+ *
+ * Keep payloads to JSON-safe primitive types (string, number, boolean, null)
+ * and plain objects/arrays thereof.
+ */
+export interface EventStore {
+  initialize(): Promise<Result<void, Error>>;
+  append(event: WorkflowEvent, sim: SimulationTask): Promise<Result<void, AppendError>>;
+  appendBatch(events: WorkflowEvent[], sim: SimulationTask): Promise<Result<void, AppendError>>;
+  appendIfNoActiveWorkflow(workflowId: string, event: WorkflowEvent, sim: SimulationTask): Promise<Result<void, AppendIfNoActiveError>>;
+  getLastEvent(runId: string, sim: SimulationTask): Promise<Result<WorkflowEvent | null, Error>>;
+  getEvent(runId: string, seq: number, sim: SimulationTask): Promise<Result<WorkflowEvent | null, Error>>;
+  getCompletedEvents(runId: string, sim: SimulationTask): Promise<Result<WorkflowEvent[], Error>>;
+  getAllEvents(runId: string, sim: SimulationTask): Promise<Result<WorkflowEvent[], Error>>;
+  getActiveWorkflows(sim: SimulationTask): Promise<Result<WorkflowEvent[], Error>>;
+  getRecentRuns(limit: number, sim: SimulationTask): Promise<Result<WorkflowEvent[], Error>>;
+  pruneTerminalRunsOlderThan(cutoff: number, sim: SimulationTask): Promise<Result<PruneResult, Error>>;
+}