flingit 0.0.63 → 0.0.66

package/node_modules/flingflow/src/recovery.ts

@@ -0,0 +1,144 @@
+import { ok, err, type Result } from "neverthrow";
+import type { SimulationTask } from "determined";
+import { UniqueConstraintError, type EventStore } from "./store.js";
+import type { Clock } from "./clock.js";
+import type { WorkflowRegistry } from "./registry.js";
+import { resolveTransition } from "./transitions.js";
+import { calculateBackoffMS } from "./backoff.js";
+import type { WorkflowConfig, WorkflowCreatedPayload, WorkflowEvent } from "./types.js";
+
+function toError(e: unknown): Error {
+  if (e instanceof Error) return e;
+  return new Error(String(e));
+}
+
+function extractConfig(payload: WorkflowCreatedPayload): WorkflowConfig {
+  return payload.config;
+}
+
+export interface RecoveryConfig {
+  store: EventStore;
+  registry: WorkflowRegistry;
+  clock: Clock;
+}
+
+export type RecoveryReason =
+  | "timeout"          // step_started but exceeded stepTimeoutMs
+  | "stuck_created"    // workflow_created but no step was ever started
+  | "stuck_completed"  // step_completed but next step was never claimed
+  | "stuck_failed"     // step_failed (retryable) but retry was never started
+  | "stuck_terminal";  // step_completed/step_failed but workflow was never finalized
+
+export interface RecoveryResult {
+  runId: string;
+  reason: RecoveryReason;
+}
+
+export async function runRecovery(
+  config: RecoveryConfig,
+  sim: SimulationTask,
+): Promise<Result<RecoveryResult[], Error>> {
+  const { store, registry, clock } = config;
+  const recovered: RecoveryResult[] = [];
+
+  try {
+    await sim.checkpoint("recovery:before-getActiveWorkflows");
+  } catch (e) {
+    return err(toError(e));
+  }
+
+  const activeResult = await store.getActiveWorkflows(sim);
+  if (activeResult.isErr()) return err(activeResult.error);
+  const activeWorkflows = activeResult.value;
+
+  for (const lastEvent of activeWorkflows) {
+    const runId = lastEvent.run_id;
+    const workflowId = lastEvent.workflow_id;
+
+    try {
+      await sim.checkpoint("recovery:per-workflow", runId);
+    } catch (e) {
+      return err(toError(e));
+    }
+
+    // Get workflow definition for timeout and maxAttempts
+    const event0Result = await store.getEvent(runId, 0, sim);
+    if (event0Result.isErr()) return err(event0Result.error);
+    const event0 = event0Result.value;
+
+    if (event0?.event_type !== "workflow_created") continue;
+    const workflowName = event0.payload.workflowName;
+    const def = registry.get(workflowName);
+    if (!def) continue;
+
+    const config = extractConfig(event0.payload);
+
+    if (lastEvent.event_type === "step_started") {
+      // Check for timeout
+      const elapsed = clock.now() - lastEvent.created_at;
+      if (elapsed >= config.stepTimeoutMS) {
+        const payload = lastEvent.payload;
+        const terminal = payload.attempt >= config.maxAttempts;
+        const failedEvent: WorkflowEvent = {
+          run_id: runId,
+          workflow_id: workflowId,
+          seq: lastEvent.seq + 1,
+          event_type: "step_failed",
+          payload: {
+            step: payload.step,
+            attempt: payload.attempt,
+            reason: "timeout",
+            retryable: true,
+          },
+          created_at: clock.now(),
+        };
+
+        if (terminal) {
+          const workflowFailedEvent: WorkflowEvent = {
+            run_id: runId,
+            workflow_id: workflowId,
+            seq: lastEvent.seq + 2,
+            event_type: "workflow_failed",
+            payload: { reason: `max attempts (${config.maxAttempts}) exceeded: timeout`, cause: "max_attempts_exceeded" as const },
+            created_at: clock.now(),
+          };
+          const result = await store.appendBatch([failedEvent, workflowFailedEvent], sim);
+          if (result.isErr()) {
+            if (result.error instanceof UniqueConstraintError) continue;
+            return err(result.error);
+          }
+          recovered.push({ runId, reason: "timeout" });
+        } else {
+          const result = await store.append(failedEvent, sim);
+          if (result.isErr()) {
+            if (result.error instanceof UniqueConstraintError) continue;
+            return err(result.error);
+          }
+          recovered.push({ runId, reason: "timeout" });
+        }
+      }
+    } else {
+      // workflow_created, step_completed, or step_failed (retryable)
+      // These should have a step running but don't — they're stuck
+      const transition = resolveTransition(lastEvent, config.maxAttempts);
+
+      if (transition.action === "start_step") {
+        // Check if within backoff window (step_failed only — not stuck, just waiting)
+        if (lastEvent.event_type === "step_failed") {
+          const backoffMS = calculateBackoffMS(config, transition.attempt);
+          const elapsed = clock.now() - lastEvent.created_at;
+          if (elapsed < backoffMS) continue;
+        }
+        const reason: RecoveryReason =
+          lastEvent.event_type === "workflow_created" ? "stuck_created" :
+          lastEvent.event_type === "step_completed" ? "stuck_completed" :
+          "stuck_failed";
+        recovered.push({ runId, reason });
+      } else if (transition.action === "complete_workflow" || transition.action === "fail_workflow") {
+        recovered.push({ runId, reason: "stuck_terminal" });
+      }
+    }
+  }
+
+  return ok(recovered);
+}

package/node_modules/flingflow/src/registry.ts

@@ -0,0 +1,52 @@
+import type { WorkflowFn } from "./types.js";
+
+export interface WorkflowDefinition {
+  name: string;
+  fn: WorkflowFn;
+  maxAttempts: number;
+  initialBackoffMS: number;
+  backoffMultiplier: number;
+  maxBackoffMS: number;
+  stepTimeoutMS: number;
+}
+
+const DEFAULT_MAX_ATTEMPTS = 5;
+const DEFAULT_INITIAL_BACKOFF_MS = 1000;
+const DEFAULT_BACKOFF_MULTIPLIER = 2;
+const DEFAULT_MAX_BACKOFF_MS = 60_000;
+const DEFAULT_STEP_TIMEOUT_MS = 300_000;
+
+export class WorkflowRegistry {
+  private workflows = new Map<string, WorkflowDefinition>();
+
+  register(def: {
+    name: string;
+    fn: WorkflowFn;
+    maxAttempts?: number;
+    initialBackoffMS?: number;
+    backoffMultiplier?: number;
+    maxBackoffMS?: number;
+    stepTimeoutMS?: number;
+  }): void {
+    if (this.workflows.has(def.name)) {
+      throw new Error(`Workflow "${def.name}" is already registered`);
+    }
+    this.workflows.set(def.name, {
+      name: def.name,
+      fn: def.fn,
+      maxAttempts: def.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
+      initialBackoffMS: def.initialBackoffMS ?? DEFAULT_INITIAL_BACKOFF_MS,
+      backoffMultiplier: def.backoffMultiplier ?? DEFAULT_BACKOFF_MULTIPLIER,
+      maxBackoffMS: def.maxBackoffMS ?? DEFAULT_MAX_BACKOFF_MS,
+      stepTimeoutMS: def.stepTimeoutMS ?? DEFAULT_STEP_TIMEOUT_MS,
+    });
+  }
+
+  get(name: string): WorkflowDefinition | undefined {
+    return this.workflows.get(name);
+  }
+
+  has(name: string): boolean {
+    return this.workflows.has(name);
+  }
+}

package/node_modules/flingflow/src/store-memory.ts

@@ -0,0 +1,418 @@
+import { ok, err, type Result } from "neverthrow";
+import type { SimulationTask } from "determined";
+import { defined } from "@glideapps/ts-necessities";
+import { TERMINAL_EVENT_TYPES, type WorkflowEvent } from "./types.js";
+import {
+  UniqueConstraintError,
+  WorkflowAlreadyActiveError,
+  WorkflowIdMismatchError,
+  type EventStore,
+  type EventStoreCheck,
+  type AppendError,
+  type AppendIfNoActiveError,
+  type PruneResult,
+} from "./store.js";
+
+function toError(e: unknown): Error {
+  return e instanceof Error ? e : new Error(String(e));
+}
+
+export class MemoryEventStore implements EventStore {
+  private events = new Map<string, Map<number, string>>();
+  private checkFn: EventStoreCheck | undefined;
+
+  constructor(options?: { check?: EventStoreCheck; tableName?: string }) {
+    this.checkFn = options?.check;
+    // tableName is accepted for API parity with SqliteEventStore but not used in-memory
+  }
+
+  // eslint-disable-next-line @typescript-eslint/require-await -- interface requires async
+  async initialize(): Promise<Result<void, Error>> {
+    return ok(undefined);
+  }
+
+  private getRunMap(runId: string): Map<number, string> {
+    let map = this.events.get(runId);
+    if (!map) {
+      map = new Map();
+      this.events.set(runId, map);
+    }
+    return map;
+  }
+
+  private serialize(event: WorkflowEvent): string {
+    return JSON.stringify(event);
+  }
+
+  private deserialize(json: string): WorkflowEvent {
+    return JSON.parse(json) as WorkflowEvent;
+  }
+
+  private collectEvents(runId: string): WorkflowEvent[] {
+    const map = this.events.get(runId);
+    if (!map) return [];
+    const results: WorkflowEvent[] = [];
+    for (const [, json] of map) {
+      results.push(this.deserialize(json));
+    }
+    results.sort((a, b) => a.seq - b.seq);
+    return results;
+  }
+
+  private runCheck(runId: string): void {
+    if (this.checkFn === undefined) return;
+    this.checkFn(runId, this.collectEvents(runId));
+  }
+
+  private validateWorkflowId(event: WorkflowEvent): WorkflowIdMismatchError | undefined {
+    const map = this.events.get(event.run_id);
+    if (!map || map.size === 0) return undefined;
+    // Check workflow_id of any existing event
+    const firstJson = defined(map.get(Math.min(...map.keys())));
+    const existing = this.deserialize(firstJson);
+    if (existing.workflow_id !== event.workflow_id) {
+      return new WorkflowIdMismatchError(event.run_id, existing.workflow_id, event.workflow_id);
+    }
+    return undefined;
+  }
+
+  private getLastEventSync(runId: string): WorkflowEvent | null {
+    const map = this.events.get(runId);
+    if (!map || map.size === 0) return null;
+    const maxSeq = Math.max(...map.keys());
+    return this.deserialize(defined(map.get(maxSeq)));
+  }
+
+  async append(event: WorkflowEvent, sim: SimulationTask): Promise<Result<void, AppendError>> {
+    try {
+      await sim.failpoint("store.append:before", event.run_id, event.seq);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      const mismatch = this.validateWorkflowId(event);
+      if (mismatch !== undefined) return err(mismatch);
+
+      const map = this.getRunMap(event.run_id);
+      if (map.has(event.seq)) {
+        return err(new UniqueConstraintError(event.run_id, event.seq));
+      }
+      map.set(event.seq, this.serialize(event));
+
+      this.runCheck(event.run_id);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.append:after", event.run_id, event.seq);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(undefined);
+  }
+
+  async appendBatch(events: WorkflowEvent[], sim: SimulationTask): Promise<Result<void, AppendError>> {
+    const runId = events[0]?.run_id ?? "batch";
+
+    try {
+      await sim.failpoint("store.appendBatch:before", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      // Validate workflow_id consistency
+      for (const event of events) {
+        const mismatch = this.validateWorkflowId(event);
+        if (mismatch !== undefined) return err(mismatch);
+      }
+
+      // Check all first (atomic: no partial inserts)
+      for (const event of events) {
+        const map = this.getRunMap(event.run_id);
+        if (map.has(event.seq)) {
+          return err(new UniqueConstraintError(event.run_id, event.seq));
+        }
+      }
+      // Then insert all
+      for (const event of events) {
+        const map = this.getRunMap(event.run_id);
+        map.set(event.seq, this.serialize(event));
+      }
+
+      const runIds = new Set(events.map(e => e.run_id));
+      for (const rid of runIds) {
+        this.runCheck(rid);
+      }
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.appendBatch:after", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(undefined);
+  }
+
+  async appendIfNoActiveWorkflow(
+    workflowId: string,
+    event: WorkflowEvent,
+    sim: SimulationTask,
+  ): Promise<Result<void, AppendIfNoActiveError>> {
+    try {
+      await sim.failpoint("store.appendIfNoActiveWorkflow:before", workflowId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      // Atomic check + insert: no awaits between check and insert
+      for (const [runId, map] of this.events) {
+        if (map.size === 0) continue;
+        const maxSeq = Math.max(...map.keys());
+        const last = this.deserialize(defined(map.get(maxSeq)));
+        if (last.workflow_id === workflowId && !TERMINAL_EVENT_TYPES.includes(last.event_type)) {
+          return err(new WorkflowAlreadyActiveError(workflowId, runId));
+        }
+      }
+
+      const map = this.getRunMap(event.run_id);
+      if (map.has(event.seq)) {
+        return err(new UniqueConstraintError(event.run_id, event.seq));
+      }
+      map.set(event.seq, this.serialize(event));
+
+      this.runCheck(event.run_id);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.appendIfNoActiveWorkflow:after", workflowId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(undefined);
+  }
+
+  async getLastEvent(runId: string, sim: SimulationTask): Promise<Result<WorkflowEvent | null, Error>> {
+    try {
+      await sim.failpoint("store.getLastEvent:before", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let result: WorkflowEvent | null;
+    try {
+      result = this.getLastEventSync(runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.getLastEvent:after", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(result);
+  }
+
+  async getEvent(runId: string, seq: number, sim: SimulationTask): Promise<Result<WorkflowEvent | null, Error>> {
+    try {
+      await sim.failpoint("store.getEvent:before", runId, seq);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let result: WorkflowEvent | null;
+    try {
+      const map = this.events.get(runId);
+      result = null;
+      if (map) {
+        const json = map.get(seq);
+        if (json !== undefined) result = this.deserialize(json);
+      }
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.getEvent:after", runId, seq);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(result);
+  }
+
+  async getCompletedEvents(runId: string, sim: SimulationTask): Promise<Result<WorkflowEvent[], Error>> {
+    try {
+      await sim.failpoint("store.getCompletedEvents:before", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let results: WorkflowEvent[];
+    try {
+      const map = this.events.get(runId);
+      if (!map) {
+        try {
+          await sim.failpoint("store.getCompletedEvents:after", runId);
+        } catch (e: unknown) {
+          return err(toError(e));
+        }
+        return ok([]);
+      }
+      results = [];
+      for (const [, json] of map) {
+        const event = this.deserialize(json);
+        if (event.event_type === "step_completed") {
+          results.push(event);
+        }
+      }
+      results.sort((a, b) => b.seq - a.seq);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.getCompletedEvents:after", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(results);
+  }
+
+  async getAllEvents(runId: string, sim: SimulationTask): Promise<Result<WorkflowEvent[], Error>> {
+    try {
+      await sim.failpoint("store.getAllEvents:before", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let results: WorkflowEvent[];
+    try {
+      results = this.collectEvents(runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.getAllEvents:after", runId);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(results);
+  }
+
+  async getRecentRuns(limit: number, sim: SimulationTask): Promise<Result<WorkflowEvent[], Error>> {
+    try {
+      await sim.failpoint("store.getRecentRuns:before");
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let results: WorkflowEvent[];
+    try {
+      results = [];
+      for (const [, map] of this.events) {
+        const json = map.get(0);
+        if (json !== undefined) {
+          results.push(this.deserialize(json));
+        }
+      }
+      results.sort((a, b) => b.created_at - a.created_at);
+      results = results.slice(0, limit);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.getRecentRuns:after");
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(results);
+  }
+
+  async getActiveWorkflows(sim: SimulationTask): Promise<Result<WorkflowEvent[], Error>> {
+    try {
+      await sim.failpoint("store.getActiveWorkflows:before");
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let results: WorkflowEvent[];
+    try {
+      results = [];
+      for (const [, map] of this.events) {
+        if (map.size === 0) continue;
+        const maxSeq = Math.max(...map.keys());
+        const last = this.deserialize(defined(map.get(maxSeq)));
+        if (!TERMINAL_EVENT_TYPES.includes(last.event_type)) {
+          results.push(last);
+        }
+      }
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.getActiveWorkflows:after");
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(results);
+  }
+
+  async pruneTerminalRunsOlderThan(cutoff: number, sim: SimulationTask): Promise<Result<PruneResult, Error>> {
+    try {
+      await sim.failpoint("store.pruneTerminalRunsOlderThan:before", cutoff);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    let result: PruneResult;
+    try {
+      let prunedRuns = 0;
+      let prunedEvents = 0;
+
+      for (const [runId, map] of this.events) {
+        if (map.size === 0) continue;
+        const maxSeq = Math.max(...map.keys());
+        const last = this.deserialize(defined(map.get(maxSeq)));
+        if (!TERMINAL_EVENT_TYPES.includes(last.event_type) || last.created_at >= cutoff) {
+          continue;
+        }
+
+        prunedRuns++;
+        prunedEvents += map.size;
+        this.events.delete(runId);
+      }
+
+      result = { prunedRuns, prunedEvents };
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    try {
+      await sim.failpoint("store.pruneTerminalRunsOlderThan:after", cutoff);
+    } catch (e: unknown) {
+      return err(toError(e));
+    }
+
+    return ok(result);
+  }
+}