flingit 0.0.63 → 0.0.65

package/node_modules/flingflow/test/recovery.test.ts

@@ -0,0 +1,176 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { runRecovery } from "../src/recovery.js";
+import { Engine } from "../src/engine.js";
+import { MemoryEventStore } from "../src/store-memory.js";
+import { WorkflowRegistry } from "../src/registry.js";
+import { ManualClock } from "../src/clock.js";
+import type { EventStore } from "../src/store.js";
+import type { WorkflowConfig } from "../src/types.js";
+import { noSim } from "./fixtures.js";
+
+const TEST_WF_CONFIG: WorkflowConfig = { maxAttempts: 3, initialBackoffMS: 0, backoffMultiplier: 2, maxBackoffMS: 60_000, stepTimeoutMS: 5000 };
+
+describe("recovery", () => {
+  let store: EventStore;
+  let registry: WorkflowRegistry;
+  let clock: ManualClock;
+  let idCounter: number;
+
+  beforeEach(async () => {
+    store = new MemoryEventStore();
+    (await store.initialize())._unsafeUnwrap();
+    registry = new WorkflowRegistry();
+    clock = new ManualClock(1000);
+    idCounter = 0;
+
+    registry.register({
+      name: "test-wf",
+      fn: async (_ctx) => ({ done: true, result: "ok" }),
+      maxAttempts: 3,
+      stepTimeoutMS: 5000,
+      initialBackoffMS: 0,
+    });
+  });
+
+  it("detects timed-out step_started", async () => {
+    // Manually create a workflow with a step_started that's timed out
+    (await store.appendBatch([
+      { run_id: "run-1", workflow_id: "wf-1", seq: 0, event_type: "workflow_created", payload: { workflowName: "test-wf", writes: {}, config: TEST_WF_CONFIG }, created_at: 1000 },
+      { run_id: "run-1", workflow_id: "wf-1", seq: 1, event_type: "step_started", payload: { step: "start", attempt: 1 }, created_at: 1000 },
+    ], noSim))._unsafeUnwrap();
+
+    // Not yet timed out
+    clock.set(5999);
+    let recovered = (await runRecovery({ store, registry, clock }, noSim))._unsafeUnwrap();
+    expect(recovered).toHaveLength(0);
+
+    // Now timed out
+    clock.set(6000);
+    recovered = (await runRecovery({ store, registry, clock }, noSim))._unsafeUnwrap();
+    expect(recovered).toEqual([{ runId: "run-1", reason: "timeout" }]);
+
+    // step_failed should have been appended
+    const events = (await store.getAllEvents("run-1", noSim))._unsafeUnwrap();
+    expect(events).toHaveLength(3);
+    expect(events[2].event_type).toBe("step_failed");
+    expect((events[2].payload as any).reason).toBe("timeout");
+    expect((events[2].payload as any).retryable).toBe(true);
+  });
+
+  it("detects stuck workflow at workflow_created", async () => {
+    (await store.append({
+      run_id: "run-stuck", workflow_id: "wf-stuck", seq: 0, event_type: "workflow_created",
+      payload: { workflowName: "test-wf", writes: {}, config: TEST_WF_CONFIG }, created_at: 1000,
+    }, noSim))._unsafeUnwrap();
+
+    const recovered = (await runRecovery({ store, registry, clock }, noSim))._unsafeUnwrap();
+    expect(recovered).toEqual([{ runId: "run-stuck", reason: "stuck_created" }]);
+  });
+
+  it("detects stuck workflow at step_completed with next_step", async () => {
+    (await store.appendBatch([
+      { run_id: "run-2", workflow_id: "wf-2", seq: 0, event_type: "workflow_created", payload: { workflowName: "test-wf", writes: {}, config: TEST_WF_CONFIG }, created_at: 1000 },
+      { run_id: "run-2", workflow_id: "wf-2", seq: 1, event_type: "step_started", payload: { step: "start", attempt: 1 }, created_at: 1000 },
+      { run_id: "run-2", workflow_id: "wf-2", seq: 2, event_type: "step_completed", payload: { step: "start", writes: {}, next_step: "two" }, created_at: 1000 },
+    ], noSim))._unsafeUnwrap();
+
+    const recovered = (await runRecovery({ store, registry, clock }, noSim))._unsafeUnwrap();
+    expect(recovered).toEqual([{ runId: "run-2", reason: "stuck_completed" }]);
+  });
+
+  it("detects stuck workflow at retryable step_failed", async () => {
+    (await store.appendBatch([
+      { run_id: "run-3", workflow_id: "wf-3", seq: 0, event_type: "workflow_created", payload: { workflowName: "test-wf", writes: {}, config: TEST_WF_CONFIG }, created_at: 1000 },
+      { run_id: "run-3", workflow_id: "wf-3", seq: 1, event_type: "step_started", payload: { step: "start", attempt: 1 }, created_at: 1000 },
+      { run_id: "run-3", workflow_id: "wf-3", seq: 2, event_type: "step_failed", payload: { step: "start", attempt: 1, reason: "oops", retryable: true }, created_at: 1000 },
+    ], noSim))._unsafeUnwrap();
+
+    const recovered = (await runRecovery({ store, registry, clock }, noSim))._unsafeUnwrap();
+    expect(recovered).toEqual([{ runId: "run-3", reason: "stuck_failed" }]);
+  });
+
+  it("ignores terminal workflows", async () => {
+    (await store.appendBatch([
+      { run_id: "run-done", workflow_id: "wf-done", seq: 0, event_type: "workflow_created", payload: { workflowName: "test-wf", writes: {}, config: TEST_WF_CONFIG }, created_at: 1000 },
+      { run_id: "run-done", workflow_id: "wf-done", seq: 1, event_type: "step_started", payload: { step: "start", attempt: 1 }, created_at: 1001 },
+      { run_id: "run-done", workflow_id: "wf-done", seq: 2, event_type: "step_completed", payload: { step: "start", writes: {} }, created_at: 1002 },
+      { run_id: "run-done", workflow_id: "wf-done", seq: 3, event_type: "workflow_completed", payload: { result: "ok" }, created_at: 1003 },
+    ], noSim))._unsafeUnwrap();
+
+    const recovered = (await runRecovery({ store, registry, clock }, noSim))._unsafeUnwrap();
+    expect(recovered).toHaveLength(0);
+  });
+
+  it("ignores non-timed-out in-progress steps", async () => {
+    (await store.appendBatch([
+      { run_id: "run-running", workflow_id: "wf-running", seq: 0, event_type: "workflow_created", payload: { workflowName: "test-wf", writes: {}, config: TEST_WF_CONFIG }, created_at: 1000 },
+      { run_id: "run-running", workflow_id: "wf-running", seq: 1, event_type: "step_started", payload: { step: "start", attempt: 1 }, created_at: 1000 },
+    ], noSim))._unsafeUnwrap();
+
+    clock.set(3000); // Only 2s elapsed, timeout is 5s
+    const recovered = (await runRecovery({ store, registry, clock }, noSim))._unsafeUnwrap();
+    expect(recovered).toHaveLength(0);
+  });
+
+  it("recovery + engine integration: timed-out workflow recovers and completes", async () => {
+    const engine = new Engine({
+      store,
+      registry,
+      clock,
+      generateId: () => `run-${++idCounter}`,
+    });
+
+    // Simulate a step that was claimed but the worker died
+    (await store.appendBatch([
+      { run_id: "run-10", workflow_id: "wf-10", seq: 0, event_type: "workflow_created", payload: { workflowName: "test-wf", writes: {}, config: TEST_WF_CONFIG }, created_at: 1000 },
+      { run_id: "run-10", workflow_id: "wf-10", seq: 1, event_type: "step_started", payload: { step: "start", attempt: 1 }, created_at: 1000 },
+    ], noSim))._unsafeUnwrap();
+
+    // Advance past timeout
+    clock.set(7000);
+
+    // Recovery marks it as failed
+    const recovered = (await runRecovery({ store, registry, clock }, noSim))._unsafeUnwrap();
+    expect(recovered).toEqual([{ runId: "run-10", reason: "timeout" }]);
+
+    // Engine retries and completes
+    (await engine.runToCompletion("run-10", noSim))._unsafeUnwrap();
+
+    const events = (await store.getAllEvents("run-10", noSim))._unsafeUnwrap();
+    const last = events[events.length - 1];
+    expect(last.event_type).toBe("workflow_completed");
+  });
+
+  it("detects step_completed without next_step as needing workflow_completed", async () => {
+    (await store.appendBatch([
+      { run_id: "run-4", workflow_id: "wf-4", seq: 0, event_type: "workflow_created", payload: { workflowName: "test-wf", writes: {}, config: TEST_WF_CONFIG }, created_at: 1000 },
+      { run_id: "run-4", workflow_id: "wf-4", seq: 1, event_type: "step_started", payload: { step: "start", attempt: 1 }, created_at: 1000 },
+      { run_id: "run-4", workflow_id: "wf-4", seq: 2, event_type: "step_completed", payload: { step: "start", writes: {} }, created_at: 1000 },
+    ], noSim))._unsafeUnwrap();
+
+    const recovered = (await runRecovery({ store, registry, clock }, noSim))._unsafeUnwrap();
+    expect(recovered).toEqual([{ runId: "run-4", reason: "stuck_terminal" }]);
+  });
+
+  it("detects non-retryable step_failed as needing workflow_failed", async () => {
+    (await store.appendBatch([
+      { run_id: "run-5", workflow_id: "wf-5", seq: 0, event_type: "workflow_created", payload: { workflowName: "test-wf", writes: {}, config: TEST_WF_CONFIG }, created_at: 1000 },
+      { run_id: "run-5", workflow_id: "wf-5", seq: 1, event_type: "step_started", payload: { step: "start", attempt: 1 }, created_at: 1000 },
+      { run_id: "run-5", workflow_id: "wf-5", seq: 2, event_type: "step_failed", payload: { step: "start", attempt: 1, reason: "fatal", retryable: false }, created_at: 1000 },
+    ], noSim))._unsafeUnwrap();
+
+    const recovered = (await runRecovery({ store, registry, clock }, noSim))._unsafeUnwrap();
+    expect(recovered).toEqual([{ runId: "run-5", reason: "stuck_terminal" }]);
+  });
+
+  it("detects max-attempts-exhausted step_failed as needing workflow_failed", async () => {
+    (await store.appendBatch([
+      { run_id: "run-6", workflow_id: "wf-6", seq: 0, event_type: "workflow_created", payload: { workflowName: "test-wf", writes: {}, config: TEST_WF_CONFIG }, created_at: 1000 },
+      { run_id: "run-6", workflow_id: "wf-6", seq: 1, event_type: "step_started", payload: { step: "start", attempt: 3 }, created_at: 1000 },
+      { run_id: "run-6", workflow_id: "wf-6", seq: 2, event_type: "step_failed", payload: { step: "start", attempt: 3, reason: "oops", retryable: true }, created_at: 1000 },
+    ], noSim))._unsafeUnwrap();
+
+    const recovered = (await runRecovery({ store, registry, clock }, noSim))._unsafeUnwrap();
+    expect(recovered).toEqual([{ runId: "run-6", reason: "stuck_terminal" }]);
+  });
+});

package/node_modules/flingflow/test/simulation.test.ts

@@ -0,0 +1,395 @@
+import { describe, it, expect } from "vitest";
+import {
+  SimulationImpl,
+  SimpleEntropySource,
+  type SimulationTask,
+  type TaskSpec,
+} from "determined";
+import { Engine } from "../src/engine.js";
+import { MemoryEventStore } from "../src/store-memory.js";
+import { SqliteEventStore } from "../src/store-sqlite.js";
+import { WorkflowRegistry } from "../src/registry.js";
+import { ManualClock } from "../src/clock.js";
+import { runRecovery } from "../src/recovery.js";
+import type { EventStore, EventStoreCheck } from "../src/store.js";
+import type { Continuation, Ctx, WorkflowEvent } from "../src/types.js";
+import { noSim } from "./fixtures.js";
+
+function makeRegistry(): WorkflowRegistry {
+  const registry = new WorkflowRegistry();
+
+  registry.register({
+    name: "counter",
+    fn: async (ctx: Ctx): Promise<Continuation> => {
+      switch (ctx.step) {
+        case "start": {
+          ctx.set("count", 0);
+          return { step: "increment" };
+        }
+        case "increment": {
+          const count = (await ctx.get("count")) as number;
+          if (count >= 3) {
+            return { done: true, result: { finalCount: count } };
+          }
+          ctx.set("count", count + 1);
+          return { step: "increment" };
+        }
+        default:
+          throw new Error(`Unknown step: ${ctx.step}`);
+      }
+    },
+    maxAttempts: 3,
+    stepTimeoutMS: 30_000,
+    initialBackoffMS: 0,
+  });
+
+  registry.register({
+    name: "simple",
+    fn: async (ctx: Ctx): Promise<Continuation> => {
+      const data = await ctx.get("seed") ?? await ctx.get("test");
+      ctx.set("done", true);
+      return { done: true, result: { data } };
+    },
+    maxAttempts: 3,
+    stepTimeoutMS: 30_000,
+    initialBackoffMS: 0,
+  });
+
+  return registry;
+}
+
+const validTransitions: Record<string, string[]> = {
+  workflow_created: ["step_started"],
+  step_started: ["step_completed", "step_failed"],
+  step_completed: ["step_started", "workflow_completed"],
+  step_failed: ["step_started", "workflow_failed"],
+};
+
+function makeCheck(): EventStoreCheck {
+  return (runId: string, events: WorkflowEvent[]) => {
+    if (events.length === 0) return;
+
+    // Ascending seq with no gaps
+    for (let i = 0; i < events.length; i++) {
+      if (events[i].seq !== i) {
+        throw new Error(
+          `Check failed [${runId}]: expected seq ${i}, got ${events[i].seq}`,
+        );
+      }
+    }
+
+    // First event must be workflow_created
+    if (events[0].event_type !== "workflow_created") {
+      throw new Error(
+        `Check failed [${runId}]: first event is ${events[0].event_type}, expected workflow_created`,
+      );
+    }
+
+    // All transitions are valid
+    for (let i = 0; i < events.length - 1; i++) {
+      const from = events[i].event_type;
+      const to = events[i + 1].event_type;
+      const allowed = validTransitions[from];
+      if (!allowed?.includes(to)) {
+        throw new Error(
+          `Check failed [${runId}]: invalid transition ${from} → ${to} at seq ${i}→${i + 1}`,
+        );
+      }
+    }
+
+    // At most one terminal event
+    const terminals = events.filter(
+      (e) =>
+        e.event_type === "workflow_completed" ||
+        e.event_type === "workflow_failed",
+    );
+    if (terminals.length > 1) {
+      throw new Error(
+        `Check failed [${runId}]: ${terminals.length} terminal events`,
+      );
+    }
+  };
+}
+
+function validateEventTrace(events: WorkflowEvent[]): void {
+  // Check ascending seq
+  for (let i = 0; i < events.length; i++) {
+    if (events[i].seq !== i) {
+      throw new Error(`Expected seq ${i}, got ${events[i].seq}`);
+    }
+  }
+
+  // Check valid transitions
+  for (let i = 0; i < events.length - 1; i++) {
+    const from = events[i].event_type;
+    const to = events[i + 1].event_type;
+    const allowed = validTransitions[from];
+    if (!allowed?.includes(to)) {
+      throw new Error(
+        `Invalid transition: ${from} → ${to} at seq ${i}→${i + 1}`,
+      );
+    }
+  }
+
+  // Check terminal
+  const last = events[events.length - 1];
+  if (
+    last.event_type !== "workflow_completed" &&
+    last.event_type !== "workflow_failed"
+  ) {
+    throw new Error(`Workflow not terminal: last event is ${last.event_type}`);
+  }
+}
+
+function simulationTests(
+  name: string,
+  createStore: () => Promise<EventStore>,
+) {
+  describe(`simulation tests (${name})`, () => {
+    it("concurrent workers: exactly one claims each step", async () => {
+      const store = await createStore();
+      const registry = makeRegistry();
+      const clock = new ManualClock(1000);
+      let idCounter = 0;
+
+      const engine = new Engine({
+        store,
+        registry,
+        clock,
+        generateId: () => `run-${++idCounter}`,
+      });
+
+      const runId = (await engine.createWorkflow("counter-wf", "counter", {}, noSim))._unsafeUnwrap();
+
+      const logger = { log: () => {}, error: () => {} };
+      const entropy = new SimpleEntropySource();
+
+      const sim = new SimulationImpl(logger, entropy, 0);
+
+      const workerSpec = (workerName: string): TaskSpec<void> => ({
+        name: workerName,
+        f: async (task: SimulationTask) => {
+          for (let i = 0; i < 20; i++) {
+            const result = await engine.executeStep(runId, task);
+            if (result.isErr()) continue; // treat err as crash
+          }
+        },
+      });
+
+      const result = await sim.runTasks([
+        workerSpec("worker-1"),
+        workerSpec("worker-2"),
+        workerSpec("worker-3"),
+      ] as const);
+
+      expect(result.isOk()).toBe(true);
+
+      const events = (await store.getAllEvents(runId, noSim))._unsafeUnwrap();
+      validateEventTrace(events);
+
+      const last = events[events.length - 1];
+      expect(last.event_type).toBe("workflow_completed");
+      expect((last.payload as any).result).toEqual({ finalCount: 3 });
+    });
+
+    it("concurrent workers across multiple seeds", async () => {
+      for (let seed = 0; seed < 100; seed++) {
+        const store = await createStore();
+        const registry = makeRegistry();
+        const clock = new ManualClock(1000);
+        let idCounter = 0;
+
+        const engine = new Engine({
+          store,
+          registry,
+          clock,
+          generateId: () => `run-${++idCounter}`,
+        });
+
+        const runId = (await engine.createWorkflow("simple-wf", "simple", {
+          seed,
+        }, noSim))._unsafeUnwrap();
+
+        const logger = { log: () => {}, error: () => {} };
+
+        const entropy = {
+          random: () => {
+            const x = Math.sin(seed * 9301 + 49297) * 49297;
+            return x - Math.floor(x);
+          },
+        };
+
+        const sim = new SimulationImpl(logger, entropy, 0);
+
+        const workerSpec = (workerName: string): TaskSpec<void> => ({
+          name: workerName,
+          f: async (task: SimulationTask) => {
+            for (let i = 0; i < 10; i++) {
+              const result = await engine.executeStep(runId, task);
+              if (result.isErr()) continue; // treat err as crash
+            }
+          },
+        });
+
+        const result = await sim.runTasks([
+          workerSpec("w1"),
+          workerSpec("w2"),
+        ] as const);
+
+        expect(result.isOk(), `Seed ${seed} failed`).toBe(true);
+
+        const events = (await store.getAllEvents(runId, noSim))._unsafeUnwrap();
+        validateEventTrace(events);
+
+        const last = events[events.length - 1];
+        expect(last.event_type).toBe("workflow_completed");
+      }
+    });
+
+    it("failure injection with recovery", async () => {
+      const store = await createStore();
+      const registry = makeRegistry();
+      const clock = new ManualClock(1000);
+      let idCounter = 0;
+
+      const engine = new Engine({
+        store,
+        registry,
+        clock,
+        generateId: () => `run-${++idCounter}`,
+      });
+
+      const runId = (await engine.createWorkflow("counter-wf", "counter", {}, noSim))._unsafeUnwrap();
+
+      const logger = { log: () => {}, error: () => {} };
+      const entropy = new SimpleEntropySource();
+
+      const sim = new SimulationImpl(logger, entropy, 0.3);
+
+      const workerSpec = (workerName: string): TaskSpec<void> => ({
+        name: workerName,
+        f: async (task: SimulationTask) => {
+          for (let i = 0; i < 50; i++) {
+            const result = await engine.executeStep(runId, task);
+            if (result.isErr()) {
+              // Err — simulate crash, just keep trying
+            }
+          }
+        },
+      });
+
+      const recoverySpec: TaskSpec<void> = {
+        name: "recovery",
+        f: async (task: SimulationTask) => {
+          for (let i = 0; i < 30; i++) {
+            clock.advance(35_000);
+
+            try {
+              const recoveryResult = await runRecovery(
+                { store, registry, clock },
+                task,
+              );
+              if (recoveryResult.isErr()) continue; // treat err as crash
+              const recovered = recoveryResult.value;
+
+              for (const r of recovered) {
+                const stepResult = await engine.executeStep(r.runId, task);
+                if (stepResult.isErr()) {
+                  // failpoint
+                }
+              }
+            } catch {
+              // failpoint in store during recovery
+            }
+          }
+        },
+      };
+
+      await sim.runTasks([
+        workerSpec("worker-1"),
+        workerSpec("worker-2"),
+        recoverySpec,
+      ] as const);
+
+      const events = (await store.getAllEvents(runId, noSim))._unsafeUnwrap();
+
+      if (events.length > 0) {
+        for (let i = 0; i < events.length; i++) {
+          expect(events[i].seq).toBe(i);
+        }
+
+        for (let i = 0; i < events.length - 1; i++) {
+          const from = events[i].event_type;
+          const to = events[i + 1].event_type;
+          const allowed = validTransitions[from];
+          expect(allowed, `No transitions defined from ${from}`).toBeDefined();
+          expect(
+            allowed,
+            `Invalid transition: ${from} → ${to} at seq ${i}→${i + 1}`,
+          ).toContain(to);
+        }
+      }
+    });
+
+    it("appendBatch atomicity under concurrency", async () => {
+      const store = await createStore();
+      const registry = makeRegistry();
+      const clock = new ManualClock(1000);
+      let idCounter = 0;
+
+      const engine = new Engine({
+        store,
+        registry,
+        clock,
+        generateId: () => `run-${++idCounter}`,
+      });
+
+      const runId = (await engine.createWorkflow("simple-wf", "simple", { test: true }, noSim))._unsafeUnwrap();
+
+      const logger = { log: () => {}, error: () => {} };
+      const entropy = new SimpleEntropySource();
+      const sim = new SimulationImpl(logger, entropy, 0);
+
+      const workerSpec = (workerName: string): TaskSpec<void> => ({
+        name: workerName,
+        f: async (task: SimulationTask) => {
+          for (let i = 0; i < 10; i++) {
+            const result = await engine.executeStep(runId, task);
+            if (result.isErr()) continue; // treat err as crash
+          }
+        },
+      });
+
+      const result = await sim.runTasks([
+        workerSpec("w1"),
+        workerSpec("w2"),
+        workerSpec("w3"),
+      ] as const);
+
+      expect(result.isOk()).toBe(true);
+
+      const events = (await store.getAllEvents(runId, noSim))._unsafeUnwrap();
+      validateEventTrace(events);
+
+      const terminals = events.filter(
+        (e) =>
+          e.event_type === "workflow_completed" ||
+          e.event_type === "workflow_failed",
+      );
+      expect(terminals).toHaveLength(1);
+    });
+  });
+}
+
+// Run against both store implementations, with check callback
+simulationTests("MemoryEventStore", async () => {
+  const store = new MemoryEventStore({ check: makeCheck() });
+  (await store.initialize())._unsafeUnwrap();
+  return store;
+});
+
+simulationTests("SqliteEventStore", async () => {
+  const store = new SqliteEventStore(":memory:", { check: makeCheck() });
+  (await store.initialize())._unsafeUnwrap();
+  return store;
+});