flingit 0.0.62 → 0.0.65

package/node_modules/flingflow/src/stress.ts

@@ -0,0 +1,423 @@
+import {
+  SimulationImpl,
+  NoSimulationTask,
+  type EntropySource,
+  type SimulationTask,
+  type TaskSpec,
+} from "determined";
+import { defined } from "@glideapps/ts-necessities";
+import { Engine } from "./engine.js";
+import { MemoryEventStore } from "./store-memory.js";
+import { SqliteEventStore } from "./store-sqlite.js";
+import { WorkflowRegistry } from "./registry.js";
+import { ManualClock } from "./clock.js";
+import { runRecovery } from "./recovery.js";
+import type { EventStore, EventStoreCheck } from "./store.js";
+import type { Continuation, Ctx, StepFailedPayload, WorkflowEvent } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// CLI argument parsing
+// ---------------------------------------------------------------------------
+
+const args = process.argv.slice(2);
+
+function getArg(name: string, defaultValue: string): string {
+  const idx = args.indexOf(name);
+  if (idx === -1 || idx + 1 >= args.length) return defaultValue;
+  return defined(args[idx + 1]);
+}
+
+const count = parseInt(getArg("--count", "10"));
+const storeType = getArg("--store", "memory") as "memory" | "sqlite";
+const numWorkers = parseInt(getArg("--workers", "4"));
+const numSeeds = parseInt(getArg("--seeds", "50"));
+const failureProbability = parseFloat(getArg("--failure", "0.1"));
+const maxAttempts = parseInt(getArg("--max-attempts", "5"));
+const stepTimeoutMs = 5_000;
+
+// ---------------------------------------------------------------------------
+// Seeded entropy source (LCG)
+// ---------------------------------------------------------------------------
+
+function seededEntropy(seed: number): EntropySource {
+  let state = seed | 0;
+  return {
+    random(_reason: string): number {
+      state = (Math.imul(state, 1664525) + 1013904223) | 0;
+      return (state >>> 0) / 0x100000000;
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// In-memory key-value store with failpoints
+// ---------------------------------------------------------------------------
+
+const noSim = new NoSimulationTask("kv", false);
+
+class SimKV {
+  private data = new Map<string, unknown>();
+
+  async get(key: string, sim: SimulationTask): Promise<unknown> {
+    await sim.failpoint("kv.get:before", key);
+    const value = this.data.get(key);
+    await sim.failpoint("kv.get:after", key);
+    return value;
+  }
+
+  async set(key: string, value: unknown, sim: SimulationTask): Promise<void> {
+    await sim.failpoint("kv.set:before", key);
+    this.data.set(key, value);
+    await sim.failpoint("kv.set:after", key);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Event-log consistency check (called on every store mutation)
+// ---------------------------------------------------------------------------
+
+const validTransitions: Partial<Record<string, string[]>> = {
+  workflow_created: ["step_started"],
+  step_started: ["step_completed", "step_failed"],
+  step_completed: ["step_started", "workflow_completed"],
+  step_failed: ["step_started", "workflow_failed"],
+};
+
+function makeCheck(): EventStoreCheck {
+  return (runId: string, events: WorkflowEvent[]) => {
+    if (events.length === 0) return;
+    for (let i = 0; i < events.length; i++) {
+      const event = defined(events[i]);
+      if (event.seq !== i) {
+        throw new Error(`Check [${runId}]: expected seq ${i}, got ${event.seq}`);
+      }
+    }
+    if (defined(events[0]).event_type !== "workflow_created") {
+      throw new Error(`Check [${runId}]: first event is ${defined(events[0]).event_type}`);
+    }
+    for (let i = 0; i < events.length - 1; i++) {
+      const from = defined(events[i]).event_type;
+      const to = defined(events[i + 1]).event_type;
+      const allowed = validTransitions[from];
+      if (!allowed?.includes(to)) {
+        throw new Error(`Check [${runId}]: invalid transition ${from} → ${to} at seq ${i}→${i + 1}`);
+      }
+    }
+    const terminals = events.filter(
+      (e) => e.event_type === "workflow_completed" || e.event_type === "workflow_failed",
+    );
+    if (terminals.length > 1) {
+      throw new Error(`Check [${runId}]: ${terminals.length} terminal events`);
+    }
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Sum-of-squares workflow
+// ---------------------------------------------------------------------------
+
+function parseStep(step: string): { phase: "set" | "sum-init" | "add"; index: number } {
+  if (step === "start") return { phase: "set", index: 1 };
+  if (step === "sum-init") return { phase: "sum-init", index: 0 };
+  if (step.startsWith("set-")) return { phase: "set", index: parseInt(step.slice(4)) };
+  if (step.startsWith("add-")) return { phase: "add", index: parseInt(step.slice(4)) };
+  throw new Error(`Unknown step: ${step}`);
+}
+
+function makeSumOfSquaresWorkflow(kv: SimKV): (ctx: Ctx) => Promise<Continuation> {
+  return async (ctx: Ctx): Promise<Continuation> => {
+    const sim: SimulationTask = ctx.sim;
+    const n = await ctx.get("count") as number;
+    const parsed = parseStep(ctx.step);
+
+    switch (parsed.phase) {
+      case "set": {
+        const i = parsed.index;
+        await kv.set(String(i), i * i, sim);
+        if (i < n) return { step: `set-${i + 1}` };
+        return { step: "sum-init" };
+      }
+      case "sum-init": {
+        ctx.set("sum", 0);
+        return { step: "add-1" };
+      }
+      case "add": {
+        const i = parsed.index;
+        const value = (await kv.get(String(i), sim)) as number;
+        const sum = ((await ctx.get("sum")) as number) + value;
+        ctx.set("sum", sum);
+        if (i < n) return { step: `add-${i + 1}` };
+        return { done: true, result: { sum } };
+      }
+    }
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Store factory
+// ---------------------------------------------------------------------------
+
+function createStore(check: EventStoreCheck): EventStore {
+  if (storeType === "sqlite") {
+    return new SqliteEventStore(":memory:", { check });
+  }
+  return new MemoryEventStore({ check });
+}
+
+// ---------------------------------------------------------------------------
+// Run a single seed
+// ---------------------------------------------------------------------------
+
+async function runSeed(seed: number): Promise<{ ok: boolean; detail?: string }> {
+  const kv = new SimKV();
+  const store = createStore(makeCheck());
+  const initResult = await store.initialize();
+  if (initResult.isErr()) {
+    return { ok: false, detail: `initialize failed: ${initResult.error.message}` };
+  }
+
+  const registry = new WorkflowRegistry();
+  registry.register({
+    name: "sum-of-squares",
+    fn: makeSumOfSquaresWorkflow(kv),
+    maxAttempts,
+    initialBackoffMS: 0,
+    stepTimeoutMS: stepTimeoutMs,
+  });
+
+  const clock = new ManualClock(1_000_000);
+  let idCounter = 0;
+
+  const engine = new Engine({
+    store,
+    registry,
+    clock,
+    generateId: () => `run-${++idCounter}`,
+  });
+
+  const createResult = await engine.createWorkflow("sum-of-squares-wf", "sum-of-squares", { count }, noSim);
+  if (createResult.isErr()) {
+    return { ok: false, detail: `createWorkflow failed: ${createResult.error.message}` };
+  }
+  const runId = createResult.value;
+
+  // Total steps: count (set) + 1 (sum-init) + count (add) = 2*count + 1
+  // ~16 failpoints per step attempt (store reads/writes + engine + kv).
+  // Scale iterations so we have enough budget at higher failure rates.
+  const totalSteps = 2 * count + 1;
+  const failpointsPerStep = 16;
+  const attemptsPerStep = Math.ceil(1 / Math.pow(1 - failureProbability, failpointsPerStep));
+  const iterationsPerWorker = Math.max(1000, totalSteps * attemptsPerStep * 5);
+
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  const logger = { log: () => {}, error: () => {} };
+  const entropy = seededEntropy(seed);
+  const sim = new SimulationImpl(logger, entropy, failureProbability);
+
+  const specs: TaskSpec<void>[] = [];
+
+  for (let i = 0; i < numWorkers; i++) {
+    specs.push({
+      name: `worker-${i}`,
+      f: async (task: SimulationTask) => {
+        for (let j = 0; j < iterationsPerWorker; j++) {
+          try {
+            const stepResult = await engine.executeStep(runId, task);
+            if (stepResult.isErr()) {
+              // Result error — treat as crash, keep going
+              continue;
+            }
+            const ran = stepResult.value;
+            if (ran === false) {
+              // Nothing claimable — workflow may be done. Check.
+              const lastResult = await store.getLastEvent(runId, task);
+              if (lastResult.isErr()) continue; // treat as crash
+              const last = lastResult.value;
+              if (
+                last?.event_type === "workflow_completed" ||
+                last?.event_type === "workflow_failed"
+              ) {
+                break;
+              }
+            }
+          } catch {
+            // failpoint crash — keep going
+          }
+        }
+      },
+    });
+  }
+
+  specs.push({
+    name: "recovery",
+    f: async (task: SimulationTask) => {
+      for (;;) {
+        try {
+          const lastResult = await store.getLastEvent(runId, task);
+          if (lastResult.isErr()) continue; // treat as crash
+          const last = lastResult.value;
+          if (
+            last?.event_type === "workflow_completed" ||
+            last?.event_type === "workflow_failed"
+          ) {
+            break;
+          }
+          // Advance the clock so that timed-out steps get detected.
+          // This is always safe: recovery only acts on genuinely
+          // stuck workflows (timed-out step_started, or states
+          // where the next step should have been claimed but wasn't).
+          clock.advance(stepTimeoutMs + 1000);
+          const recoveryResult = await runRecovery({ store, registry, clock }, task);
+          if (recoveryResult.isErr()) continue; // treat as crash
+          const recovered = recoveryResult.value;
+          for (const r of recovered) {
+            const reasonChar: Record<string, string> = {
+              timeout: "T",
+              stuck_created: "C",
+              stuck_completed: "N",
+              stuck_failed: "F",
+              stuck_terminal: "X",
+            };
+            process.stderr.write(reasonChar[r.reason] ?? "?");
+            try {
+              const execResult = await engine.executeStep(r.runId, task);
+              if (execResult.isErr()) {
+                // failpoint — treat as crash
+              }
+            } catch {
+              // failpoint
+            }
+          }
+        } catch {
+          // failpoint in recovery store ops
+        }
+      }
+    },
+  });
+
+  await sim.runTasks(specs as readonly TaskSpec<void>[]);
+
+  // Validate result
+  const eventsResult = await store.getAllEvents(runId, noSim);
+  if (eventsResult.isErr()) {
+    return { ok: false, detail: `getAllEvents failed: ${eventsResult.error.message}` };
+  }
+  const events = eventsResult.value;
+  if (events.length === 0) {
+    return { ok: false, detail: "no events" };
+  }
+
+  // Validate the trace structure: sequential numbering
+  for (let i = 0; i < events.length; i++) {
+    if (defined(events[i]).seq !== i) {
+      return { ok: false, detail: `seq gap at ${i}` };
+    }
+  }
+
+  // Validate transitions
+  for (let i = 0; i < events.length - 1; i++) {
+    const from = defined(events[i]).event_type;
+    const to = defined(events[i + 1]).event_type;
+    const allowed = validTransitions[from];
+    if (!allowed?.includes(to)) {
+      return { ok: false, detail: `invalid transition ${from} → ${to} at ${i}` };
+    }
+  }
+
+  // Validate per-step retry chains: attempt numbers must be sequential
+  // and never exceed maxAttempts
+  for (let i = 0; i < events.length; i++) {
+    const event = defined(events[i]);
+    if (event.event_type !== "step_started") continue;
+    const payload = event.payload;
+
+    if (payload.attempt > maxAttempts) {
+      return { ok: false, detail: `step "${payload.step}" attempt ${payload.attempt} exceeds maxAttempts ${maxAttempts} at seq ${i}` };
+    }
+
+    // Check that the attempt number is consistent with the preceding event
+    const prev = defined(events[i - 1]);
+    if (prev.event_type === "workflow_created" || prev.event_type === "step_completed") {
+      if (payload.attempt !== 1) {
+        return { ok: false, detail: `step "${payload.step}" should start at attempt 1 after ${prev.event_type}, got ${payload.attempt} at seq ${i}` };
+      }
+    } else if (prev.event_type === "step_failed") {
+      const prevPayload = prev.payload;
+      if (payload.attempt !== prevPayload.attempt + 1) {
+        return { ok: false, detail: `step "${payload.step}" attempt ${payload.attempt} should be ${prevPayload.attempt + 1} after step_failed at seq ${i}` };
+      }
+    }
+  }
+
+  // Check terminal event
+  const last = defined(events[events.length - 1]);
+
+  if (last.event_type === "workflow_failed") {
+    const payload = last.payload;
+    if (payload.cause === "max_attempts_exceeded") {
+      // Find the most recent step_failed event
+      let lastStepFailed: WorkflowEvent | undefined;
+      for (let i = events.length - 1; i >= 0; i--) {
+        if (defined(events[i]).event_type === "step_failed") {
+          lastStepFailed = defined(events[i]);
+          break;
+        }
+      }
+      if (lastStepFailed === undefined) {
+        return { ok: false, detail: "workflow_failed with max_attempts_exceeded but no step_failed found" };
+      }
+      const failedPayload = lastStepFailed.payload as StepFailedPayload;
+      if (failedPayload.attempt !== maxAttempts) {
+        return { ok: false, detail: `workflow_failed with max_attempts_exceeded but last step_failed attempt is ${failedPayload.attempt}, expected ${maxAttempts}` };
+      }
+      return { ok: true };
+    }
+    return { ok: false, detail: `workflow failed: ${payload.reason} (cause: ${payload.cause})` };
+  }
+
+  if (last.event_type !== "workflow_completed") {
+    return { ok: false, detail: `not terminal (last=${last.event_type}, ${events.length} events)` };
+  }
+
+  const result = last.payload.result as { sum: number };
+  const expected = (count * (count + 1) * (2 * count + 1)) / 6;
+  if (result.sum !== expected) {
+    return { ok: false, detail: `wrong result: got ${result.sum}, expected ${expected}` };
+  }
+
+  return { ok: true };
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+async function main(): Promise<void> {
+  console.log(
+    `stress test: count=${count} store=${storeType} workers=${numWorkers} ` +
+      `seeds=${numSeeds} failure=${failureProbability}`,
+  );
+
+  let passed = 0;
+  let failed = 0;
+
+  for (let seed = 0; seed < numSeeds; seed++) {
+    const { ok, detail } = await runSeed(seed);
+    if (ok) {
+      passed++;
+      process.stdout.write(".");
+    } else {
+      failed++;
+      process.stdout.write("\n");
+      console.error(`  FAIL seed ${seed}: ${detail}`);
+    }
+  }
+
+  console.log(`\n\n${passed} passed, ${failed} failed (${numSeeds} seeds)`);
+  if (failed > 0) process.exit(1);
+}
+
+main().catch((err: unknown) => {
+  console.error(err);
+  process.exit(1);
+});

package/node_modules/flingflow/src/transitions.ts

@@ -0,0 +1,38 @@
+import type { WorkflowEvent } from "./types.js";
+
+export type TransitionResult =
+  | { action: "start_step"; step: string; attempt: number }
+  | { action: "complete_workflow" }
+  | { action: "fail_workflow"; reason: string }
+  | { action: "none" };
+
+export function resolveTransition(lastEvent: WorkflowEvent, maxAttempts: number): TransitionResult {
+  switch (lastEvent.event_type) {
+    case "workflow_created":
+      return { action: "start_step", step: "start", attempt: 1 };
+
+    case "step_completed": {
+      const { next_step } = lastEvent.payload;
+      if (next_step !== undefined) {
+        return { action: "start_step", step: next_step, attempt: 1 };
+      }
+      return { action: "complete_workflow" };
+    }
+
+    case "step_failed": {
+      const { retryable, attempt, step } = lastEvent.payload;
+      if (!retryable) {
+        return { action: "fail_workflow", reason: lastEvent.payload.reason };
+      }
+      if (attempt >= maxAttempts) {
+        return { action: "fail_workflow", reason: `max attempts (${maxAttempts}) exceeded: ${lastEvent.payload.reason}` };
+      }
+      return { action: "start_step", step, attempt: attempt + 1 };
+    }
+
+    case "step_started":
+    case "workflow_completed":
+    case "workflow_failed":
+      return { action: "none" };
+  }
+}

package/node_modules/flingflow/src/types.ts

@@ -0,0 +1,84 @@
+import { ApplicationFailure, isApplicationFailure, type SimulationTask } from "determined";
+
+// Re-export determined types used by consumers
+export type { SimulationTask };
+export { ApplicationFailure, isApplicationFailure };
+
+export class NonRetryableError extends ApplicationFailure {
+  constructor(message: string) {
+    super(message, undefined, true);
+  }
+}
+
+// --- Workflow config (sealed into workflow_created event) ---
+
+export interface WorkflowConfig {
+  maxAttempts: number;
+  initialBackoffMS: number;
+  backoffMultiplier: number;
+  maxBackoffMS: number;
+  stepTimeoutMS: number;
+}
+
+// --- Step outcome (returned by executeStep) ---
+
+export type StepOutcome =
+  | false                        // nothing to claim (terminal/done)
+  | true                         // step executed, continue immediately
+  | { backoffMS: number };       // step failed (retryable), wait before retry (backoffMS > 0)
+
+// --- Core workflow types ---
+
+export type Continuation =
+  | { step: string }
+  | { done: true; result: unknown };
+
+export interface Ctx {
+  workflowId: string;
+  runId: string;
+  step: string;
+  sim: SimulationTask;
+  get(key: string): Promise<unknown>;
+  set(key: string, value: unknown): void;
+}
+
+export type WorkflowFn = (ctx: Ctx) => Promise<Continuation>;
+
+// --- Typed event payloads ---
+
+export interface WorkflowCreatedPayload { workflowName: string; writes: Record<string, unknown>; config: WorkflowConfig }
+export interface StepStartedPayload { step: string; attempt: number }
+export interface StepCompletedPayload { step: string; writes: Record<string, unknown>; next_step?: string }
+export interface StepFailedPayload { step: string; attempt: number; reason: string; retryable: boolean }
+export interface WorkflowCompletedPayload { result: unknown }
+export type WorkflowFailureCause = "max_attempts_exceeded" | "non_retryable";
+export interface WorkflowFailedPayload { reason: string; cause: WorkflowFailureCause }
+
+// --- Discriminated union for WorkflowEvent ---
+
+export type WorkflowEvent =
+  | { run_id: string; workflow_id: string; seq: number; event_type: "workflow_created"; payload: WorkflowCreatedPayload; created_at: number }
+  | { run_id: string; workflow_id: string; seq: number; event_type: "step_started"; payload: StepStartedPayload; created_at: number }
+  | { run_id: string; workflow_id: string; seq: number; event_type: "step_completed"; payload: StepCompletedPayload; created_at: number }
+  | { run_id: string; workflow_id: string; seq: number; event_type: "step_failed"; payload: StepFailedPayload; created_at: number }
+  | { run_id: string; workflow_id: string; seq: number; event_type: "workflow_completed"; payload: WorkflowCompletedPayload; created_at: number }
+  | { run_id: string; workflow_id: string; seq: number; event_type: "workflow_failed"; payload: WorkflowFailedPayload; created_at: number };
+
+export type EventType = WorkflowEvent["event_type"];
+
+// --- Terminal event types ---
+
+export const TERMINAL_EVENT_TYPES: readonly string[] = ["workflow_completed", "workflow_failed"];
+
+// --- Claim result ---
+
+export interface ClaimResult {
+  workflowId: string;
+  runId: string;
+  workflowName: string;
+  step: string;
+  attempt: number;
+  seq: number;
+  initialWrites: Record<string, unknown>;
+  config: WorkflowConfig;
+}

package/node_modules/flingflow/test/backoff.test.ts

@@ -0,0 +1,54 @@
+import { describe, it, expect } from "vitest";
+import { calculateBackoffMS } from "../src/backoff.js";
+import type { WorkflowConfig } from "../src/types.js";
+
+const config: WorkflowConfig = {
+  maxAttempts: 5,
+  initialBackoffMS: 1000,
+  backoffMultiplier: 2,
+  maxBackoffMS: 60_000,
+  stepTimeoutMS: 300_000,
+};
+
+describe("calculateBackoffMS", () => {
+  it("returns 0 for attempt 1 (first try)", () => {
+    expect(calculateBackoffMS(config, 1)).toBe(0);
+  });
+
+  it("returns initialBackoffMS for attempt 2", () => {
+    expect(calculateBackoffMS(config, 2)).toBe(1000);
+  });
+
+  it("returns initialBackoffMS * multiplier for attempt 3", () => {
+    expect(calculateBackoffMS(config, 3)).toBe(2000);
+  });
+
+  it("grows exponentially", () => {
+    expect(calculateBackoffMS(config, 4)).toBe(4000);
+    expect(calculateBackoffMS(config, 5)).toBe(8000);
+  });
+
+  it("caps at maxBackoffMS", () => {
+    // With multiplier 2 and initial 1000: attempt 8 → 1000 * 2^6 = 64000, capped at 60000
+    expect(calculateBackoffMS(config, 8)).toBe(60_000);
+    expect(calculateBackoffMS(config, 20)).toBe(60_000);
+  });
+
+  it("returns 0 for attempt <= 0", () => {
+    expect(calculateBackoffMS(config, 0)).toBe(0);
+    expect(calculateBackoffMS(config, -1)).toBe(0);
+  });
+
+  it("works with initialBackoffMS: 0 (no backoff)", () => {
+    const noBackoff = { ...config, initialBackoffMS: 0 };
+    expect(calculateBackoffMS(noBackoff, 2)).toBe(0);
+    expect(calculateBackoffMS(noBackoff, 5)).toBe(0);
+  });
+
+  it("works with backoffMultiplier: 1 (constant delay)", () => {
+    const constant = { ...config, backoffMultiplier: 1 };
+    expect(calculateBackoffMS(constant, 2)).toBe(1000);
+    expect(calculateBackoffMS(constant, 3)).toBe(1000);
+    expect(calculateBackoffMS(constant, 10)).toBe(1000);
+  });
+});