flingit 0.0.63 → 0.0.65

package/node_modules/flingflow/eslint.config.js

@@ -0,0 +1,94 @@
+import tseslint from "typescript-eslint";
+
+export default tseslint.config(
+  {
+    ignores: ["dist/**", "node_modules/**", "eslint.config.js"],
+  },
+  ...tseslint.configs.strictTypeChecked,
+  ...tseslint.configs.stylisticTypeChecked,
+  {
+    languageOptions: {
+      parserOptions: {
+        project: "tsconfig.lint.json",
+        tsconfigRootDir: import.meta.dirname,
+      },
+    },
+    rules: {
+      // Non-null assertions are forbidden — use `defined()` from @glideapps/ts-necessities
+      "@typescript-eslint/no-non-null-assertion": "error",
+
+      // Strictness
+      "@typescript-eslint/no-explicit-any": "error",
+      "@typescript-eslint/no-unsafe-argument": "error",
+      "@typescript-eslint/no-unsafe-assignment": "error",
+      "@typescript-eslint/no-unsafe-call": "error",
+      "@typescript-eslint/no-unsafe-member-access": "error",
+      "@typescript-eslint/no-unsafe-return": "error",
+      "@typescript-eslint/explicit-function-return-type": ["error", {
+        allowExpressions: true,
+        allowTypedFunctionExpressions: true,
+      }],
+      "@typescript-eslint/explicit-module-boundary-types": "error",
+      "@typescript-eslint/no-floating-promises": "error",
+      "@typescript-eslint/no-misused-promises": "error",
+      "@typescript-eslint/await-thenable": "error",
+      "@typescript-eslint/require-await": "error",
+      "@typescript-eslint/no-unnecessary-condition": "error",
+      "@typescript-eslint/strict-boolean-expressions": ["error", {
+        allowNullableBoolean: true,
+      }],
+      "@typescript-eslint/switch-exhaustiveness-check": "error",
+      "@typescript-eslint/consistent-type-imports": ["error", {
+        prefer: "type-imports",
+      }],
+      "@typescript-eslint/no-import-type-side-effects": "error",
+      "@typescript-eslint/no-unused-vars": ["error", {
+        argsIgnorePattern: "^_",
+      }],
+      "@typescript-eslint/prefer-nullish-coalescing": "error",
+      "@typescript-eslint/prefer-optional-chain": "error",
+      "@typescript-eslint/no-unnecessary-type-assertion": "error",
+      "@typescript-eslint/no-redundant-type-constituents": "error",
+      "@typescript-eslint/restrict-template-expressions": ["error", {
+        allowNumber: true,
+        allowBoolean: true,
+      }],
+
+      // Core ESLint
+      "eqeqeq": ["error", "always"],
+      "no-eval": "error",
+      "no-implied-eval": "off",
+      "@typescript-eslint/no-implied-eval": "error",
+      "no-var": "error",
+      "prefer-const": "error",
+      "no-throw-literal": "off",
+      "@typescript-eslint/only-throw-error": "error",
+      "no-constant-condition": "error",
+      "no-debugger": "error",
+      "no-duplicate-imports": "error",
+    },
+  },
+  // Relaxed rules for tests
+  {
+    files: ["test/**/*.ts"],
+    rules: {
+      "@typescript-eslint/no-non-null-assertion": "off",
+      "@typescript-eslint/no-explicit-any": "off",
+      "@typescript-eslint/no-unsafe-argument": "off",
+      "@typescript-eslint/no-unsafe-assignment": "off",
+      "@typescript-eslint/no-unsafe-call": "off",
+      "@typescript-eslint/no-unsafe-member-access": "off",
+      "@typescript-eslint/no-unsafe-return": "off",
+      "@typescript-eslint/no-floating-promises": "off",
+      "@typescript-eslint/strict-boolean-expressions": "off",
+      "@typescript-eslint/explicit-function-return-type": "off",
+      "@typescript-eslint/explicit-module-boundary-types": "off",
+      "@typescript-eslint/no-empty-function": "off",
+      "@typescript-eslint/require-await": "off",
+      "@typescript-eslint/no-unused-vars": ["error", {
+        argsIgnorePattern: "^_",
+        varsIgnorePattern: "^_",
+      }],
+    },
+  },
+);

package/node_modules/flingflow/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "flingflow",
+  "version": "0.1.0",
+  "description": "Durable execution engine with explicit continuations and event sourcing",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./engine": {
+      "types": "./dist/engine.d.ts",
+      "import": "./dist/engine.js"
+    },
+    "./registry": {
+      "types": "./dist/registry.d.ts",
+      "import": "./dist/registry.js"
+    },
+    "./store": {
+      "types": "./dist/store.d.ts",
+      "import": "./dist/store.js"
+    },
+    "./types": {
+      "types": "./dist/types.d.ts",
+      "import": "./dist/types.js"
+    },
+    "./context": {
+      "types": "./dist/context.d.ts",
+      "import": "./dist/context.js"
+    },
+    "./clock": {
+      "types": "./dist/clock.d.ts",
+      "import": "./dist/clock.js"
+    },
+    "./recovery": {
+      "types": "./dist/recovery.d.ts",
+      "import": "./dist/recovery.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "stress": "tsx src/stress.ts",
+    "typecheck": "tsc -p tsconfig.typecheck.json",
+    "lint": "eslint ."
+  },
+  "dependencies": {
+    "@glideapps/ts-necessities": "^2.4.0",
+    "better-sqlite3": "^12.8.0",
+    "determined": "^0.1.0",
+    "neverthrow": "^8.2.0"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "@typescript-eslint/eslint-plugin": "^8.57.1",
+    "@typescript-eslint/parser": "^8.57.1",
+    "eslint": "^10.1.0",
+    "tsx": "^4.19.4",
+    "typescript": "^5.8.3",
+    "typescript-eslint": "^8.57.1",
+    "vitest": "^3.1.1"
+  }
+}

package/node_modules/flingflow/src/backoff.ts

@@ -0,0 +1,14 @@
+import type { WorkflowConfig } from "./types.js";
+
+/**
+ * Calculate the backoff delay for a given attempt number.
+ *
+ * Attempt 1 has no delay (first try).
+ * Attempt 2 has `initialBackoffMS` delay.
+ * Attempt N has `min(initialBackoffMS * backoffMultiplier^(N-2), maxBackoffMS)`.
+ */
+export function calculateBackoffMS(config: WorkflowConfig, attempt: number): number {
+  if (attempt <= 1) return 0;
+  const raw = config.initialBackoffMS * Math.pow(config.backoffMultiplier, attempt - 2);
+  return Math.min(raw, config.maxBackoffMS);
+}

package/node_modules/flingflow/src/clock.ts

@@ -0,0 +1,29 @@
+export interface Clock {
+  now(): number;
+}
+
+export class SystemClock implements Clock {
+  now(): number {
+    return Date.now();
+  }
+}
+
+export class ManualClock implements Clock {
+  private _now: number;
+
+  constructor(initial: number) {
+    this._now = initial;
+  }
+
+  now(): number {
+    return this._now;
+  }
+
+  advance(ms: number): void {
+    this._now += ms;
+  }
+
+  set(ms: number): void {
+    this._now = ms;
+  }
+}

package/node_modules/flingflow/src/context.ts

@@ -0,0 +1,60 @@
+import { ok, err, type Result } from "neverthrow";
+import type { SimulationTask } from "determined";
+import type { EventStore } from "./store.js";
+import type { Ctx, StepCompletedPayload } from "./types.js";
+
+export async function buildContext(
+  store: EventStore,
+  runId: string,
+  workflowId: string,
+  step: string,
+  initialWrites: Record<string, unknown>,
+  sim: SimulationTask,
+): Promise<Result<{ ctx: Ctx; getPendingWrites: () => Record<string, unknown> }, Error>> {
+  // Load scratchpad from step_completed events (newest first, first match wins)
+  const completedResult = await store.getCompletedEvents(runId, sim);
+  if (completedResult.isErr()) return err(completedResult.error);
+  const completedEvents = completedResult.value;
+
+  const scratchpad = new Map<string, unknown>();
+
+  for (const event of completedEvents) {
+    const payload = event.payload as StepCompletedPayload;
+    for (const [key, value] of Object.entries(payload.writes)) {
+      if (!scratchpad.has(key)) {
+        scratchpad.set(key, value);
+      }
+    }
+  }
+
+  // Apply initial writes (lowest priority — step writes take precedence)
+  for (const [key, value] of Object.entries(initialWrites)) {
+    if (!scratchpad.has(key)) {
+      scratchpad.set(key, value);
+    }
+  }
+
+  const pendingWrites: Record<string, unknown> = {};
+
+  const ctx: Ctx = {
+    workflowId,
+    runId,
+    step,
+    sim,
+    get(key: string): Promise<unknown> {
+      // Pending writes take priority
+      if (key in pendingWrites) {
+        return Promise.resolve(pendingWrites[key]);
+      }
+      return Promise.resolve(scratchpad.get(key));
+    },
+    set(key: string, value: unknown): void {
+      pendingWrites[key] = value;
+    },
+  };
+
+  return ok({
+    ctx,
+    getPendingWrites: () => ({ ...pendingWrites }),
+  });
+}

package/node_modules/flingflow/src/engine.ts

@@ -0,0 +1,367 @@
+import { ok, err, type Result } from "neverthrow";
+import { isApplicationFailure, type SimulationTask } from "determined";
+import { UniqueConstraintError, type WorkflowAlreadyActiveError, type EventStore } from "./store.js";
+import { SystemClock, type Clock } from "./clock.js";
+import type { WorkflowRegistry } from "./registry.js";
+import { resolveTransition } from "./transitions.js";
+import { buildContext } from "./context.js";
+import { calculateBackoffMS } from "./backoff.js";
+import type {
+  ClaimResult,
+  Continuation,
+  StepOutcome,
+  WorkflowConfig,
+  WorkflowCreatedPayload,
+  WorkflowEvent,
+} from "./types.js";
+
+function toError(e: unknown): Error {
+  if (e instanceof Error) return e;
+  return new Error(String(e));
+}
+
+export class UnknownWorkflowError extends Error {
+  constructor(name: string) {
+    super(`Unknown workflow: ${name}`);
+    this.name = "UnknownWorkflowError";
+  }
+}
+
+export type CreateWorkflowError = WorkflowAlreadyActiveError | UnknownWorkflowError | Error;
+
+export interface EngineConfig {
+  store: EventStore;
+  registry: WorkflowRegistry;
+  clock?: Clock;
+  generateId?: () => string;
+}
+
+function extractConfig(payload: WorkflowCreatedPayload): WorkflowConfig {
+  return payload.config;
+}
+
+export class Engine {
+  private store: EventStore;
+  private registry: WorkflowRegistry;
+  private clock: Clock;
+  private generateId: () => string;
+
+  constructor(config: EngineConfig) {
+    this.store = config.store;
+    this.registry = config.registry;
+    this.clock = config.clock ?? new SystemClock();
+    this.generateId = config.generateId ?? (() => crypto.randomUUID());
+  }
+
+  async initialize(): Promise<Result<void, Error>> {
+    return this.store.initialize();
+  }
+
+  async createWorkflow(
+    workflowId: string,
+    name: string,
+    writes: Record<string, unknown>,
+    sim: SimulationTask,
+  ): Promise<Result<string, CreateWorkflowError>> {
+    const def = this.registry.get(name);
+    if (!def) {
+      return err(new UnknownWorkflowError(name));
+    }
+
+    const config: WorkflowConfig = {
+      maxAttempts: def.maxAttempts,
+      initialBackoffMS: def.initialBackoffMS,
+      backoffMultiplier: def.backoffMultiplier,
+      maxBackoffMS: def.maxBackoffMS,
+      stepTimeoutMS: def.stepTimeoutMS,
+    };
+
+    const runId = this.generateId();
+
+    try {
+      await sim.checkpoint("createWorkflow:before-append", runId);
+    } catch (e) {
+      return err(toError(e));
+    }
+
+    const event: WorkflowEvent = {
+      run_id: runId,
+      workflow_id: workflowId,
+      seq: 0,
+      event_type: "workflow_created",
+      payload: { workflowName: name, writes, config },
+      created_at: this.clock.now(),
+    };
+
+    const result = await this.store.appendIfNoActiveWorkflow(workflowId, event, sim);
+    if (result.isErr()) {
+      return err(result.error);
+    }
+
+    try {
+      await sim.checkpoint("createWorkflow:after-append", runId);
+    } catch (e) {
+      return err(toError(e));
+    }
+
+    return ok(runId);
+  }
+
+  async claimStep(
+    runId: string,
+    sim: SimulationTask,
+  ): Promise<Result<ClaimResult | null, Error>> {
+    const lastEventResult = await this.store.getLastEvent(runId, sim);
+    if (lastEventResult.isErr()) return err(lastEventResult.error);
+    const lastEvent = lastEventResult.value;
+
+    try {
+      await sim.checkpoint("claimStep:after-getLastEvent", runId);
+    } catch (e) {
+      return err(toError(e));
+    }
+
+    if (!lastEvent) return ok(null);
+
+    const workflowId = lastEvent.workflow_id;
+
+    // Get workflow definition and config from event 0
+    const event0Result = await this.store.getEvent(runId, 0, sim);
+    if (event0Result.isErr()) return err(event0Result.error);
+    const event0 = event0Result.value;
+
+    if (event0?.event_type !== "workflow_created") return ok(null);
+    const workflowName = event0.payload.workflowName;
+    const def = this.registry.get(workflowName);
+    if (!def) return ok(null);
+
+    const config = extractConfig(event0.payload);
+    const transition = resolveTransition(lastEvent, config.maxAttempts);
+
+    // Unreachable: commitStep/executeStep write terminal events atomically
+    // with step_completed/step_failed via appendBatch. Seeing these transitions
+    // means the event log has been corrupted externally.
+    if (transition.action === "complete_workflow" || transition.action === "fail_workflow") {
+      return err(new Error(
+        `Invariant violation: unexpected transition "${transition.action}" for run ${runId} ` +
+        `(last event: ${lastEvent.event_type} seq ${lastEvent.seq})`
+      ));
+    }
+
+    if (transition.action !== "start_step") return ok(null);
+
+    // Enforce backoff: if this is a retry after step_failed, check if enough time has elapsed
+    if (lastEvent.event_type === "step_failed") {
+      const backoffMS = calculateBackoffMS(config, transition.attempt);
+      const elapsed = this.clock.now() - lastEvent.created_at;
+      if (elapsed < backoffMS) return ok(null);
+    }
+
+    const stepStartedEvent: WorkflowEvent = {
+      run_id: runId,
+      workflow_id: workflowId,
+      seq: lastEvent.seq + 1,
+      event_type: "step_started",
+      payload: { step: transition.step, attempt: transition.attempt },
+      created_at: this.clock.now(),
+    };
+
+    const appendResult = await this.store.append(stepStartedEvent, sim);
+    if (appendResult.isErr()) return err(appendResult.error);
+
+    try {
+      await sim.checkpoint("claimStep:after-append", runId);
+    } catch (e) {
+      return err(toError(e));
+    }
+
+    const initialWrites = event0.payload.writes;
+
+    return ok({
+      workflowId,
+      runId,
+      workflowName,
+      step: transition.step,
+      attempt: transition.attempt,
+      seq: stepStartedEvent.seq,
+      initialWrites,
+      config,
+    });
+  }
+
+  async executeStep(
+    runId: string,
+    sim: SimulationTask,
+  ): Promise<Result<StepOutcome, Error>> {
+    const claimResult = await this.claimStep(runId, sim);
+    if (claimResult.isErr()) {
+      // UniqueConstraintError = normal contention (another worker claimed the step).
+      // Return true: the workflow is still active, caller should keep trying.
+      if (claimResult.error instanceof UniqueConstraintError) return ok(true);
+      return err(claimResult.error);
+    }
+    const claim = claimResult.value;
+    if (!claim) return ok(false);
+
+    const def = this.registry.get(claim.workflowName);
+    if (!def) return ok(false);
+
+    try {
+      await sim.failpoint("executeStep:before-fn", runId);
+    } catch (e) {
+      return err(toError(e));
+    }
+
+    const buildResult = await buildContext(
+      this.store,
+      runId,
+      claim.workflowId,
+      claim.step,
+      claim.initialWrites,
+      sim,
+    );
+    if (buildResult.isErr()) return err(buildResult.error);
+    const { ctx, getPendingWrites } = buildResult.value;
+
+    // Only catch errors from the workflow function itself.
+    // Failpoint/store errors propagate out (they represent crashes).
+    let continuation: Continuation;
+    try {
+      continuation = await def.fn(ctx);
+    } catch (fnErr: unknown) {
+      const retryable = !isApplicationFailure(fnErr) || !fnErr.nonRetryable;
+      const reason = fnErr instanceof Error ? fnErr.message : String(fnErr);
+      const terminal = !retryable || claim.attempt >= claim.config.maxAttempts;
+
+      const failedEvent: WorkflowEvent = {
+        run_id: runId,
+        workflow_id: claim.workflowId,
+        seq: claim.seq + 1,
+        event_type: "step_failed",
+        payload: {
+          step: claim.step,
+          attempt: claim.attempt,
+          reason,
+          retryable,
+        },
+        created_at: this.clock.now(),
+      };
+
+      if (terminal) {
+        const terminalReason = !retryable
+          ? reason
+          : `max attempts (${claim.config.maxAttempts}) exceeded: ${reason}`;
+        const workflowFailedEvent: WorkflowEvent = {
+          run_id: runId,
+          workflow_id: claim.workflowId,
+          seq: claim.seq + 2,
+          event_type: "workflow_failed",
+          payload: { reason: terminalReason, cause: retryable ? "max_attempts_exceeded" as const : "non_retryable" as const },
+          created_at: this.clock.now(),
+        };
+        const result = await this.store.appendBatch([failedEvent, workflowFailedEvent], sim);
+        if (result.isErr()) {
+          // UniqueConstraintError = recovery already wrote step_failed at this seq
+          if (result.error instanceof UniqueConstraintError) return ok(true);
+          return err(result.error);
+        }
+      } else {
+        const result = await this.store.append(failedEvent, sim);
+        if (result.isErr()) {
+          if (result.error instanceof UniqueConstraintError) return ok(true);
+          return err(result.error);
+        }
+        // Return backoff info for non-terminal retryable failures
+        const nextBackoffMS = calculateBackoffMS(claim.config, claim.attempt + 1);
+        if (nextBackoffMS > 0) return ok({ backoffMS: nextBackoffMS });
+      }
+
+      return ok(true);
+    }
+
+    try {
+      await sim.failpoint("executeStep:after-fn", runId);
+    } catch (e) {
+      return err(toError(e));
+    }
+
+    const commitResult = await this.commitStep(runId, claim.workflowId, claim.seq, claim.step, getPendingWrites(), continuation, sim);
+    if (commitResult.isErr()) {
+      // UniqueConstraintError = recovery wrote step_failed at our seq while we were executing
+      if (commitResult.error instanceof UniqueConstraintError) return ok(true);
+      return err(commitResult.error);
+    }
+
+    return ok(true);
+  }
+
+  private async commitStep(
+    runId: string,
+    workflowId: string,
+    claimSeq: number,
+    step: string,
+    writes: Record<string, unknown>,
+    continuation: Continuation,
+    sim: SimulationTask,
+  ): Promise<Result<void, Error>> {
+    try {
+      await sim.failpoint("commitStep:before-append", runId);
+    } catch (e) {
+      return err(toError(e));
+    }
+
+    if ("done" in continuation) {
+      // eslint-disable-next-line @typescript-eslint/no-unnecessary-boolean-literal-compare, @typescript-eslint/no-unnecessary-condition -- runtime guard against invalid continuations
+      if (continuation.done !== true) {
+        return err(new Error("Invalid continuation: done must be true"));
+      }
+      const events: WorkflowEvent[] = [
+        {
+          run_id: runId,
+          workflow_id: workflowId,
+          seq: claimSeq + 1,
+          event_type: "step_completed",
+          payload: { step, writes },
+          created_at: this.clock.now(),
+        },
+        {
+          run_id: runId,
+          workflow_id: workflowId,
+          seq: claimSeq + 2,
+          event_type: "workflow_completed",
+          payload: { result: continuation.result },
+          created_at: this.clock.now(),
+        },
+      ];
+      const result = await this.store.appendBatch(events, sim);
+      if (result.isErr()) return err(result.error);
+    } else {
+      const event: WorkflowEvent = {
+        run_id: runId,
+        workflow_id: workflowId,
+        seq: claimSeq + 1,
+        event_type: "step_completed",
+        payload: { step, writes, next_step: continuation.step },
+        created_at: this.clock.now(),
+      };
+      const result = await this.store.append(event, sim);
+      if (result.isErr()) return err(result.error);
+    }
+
+    return ok(undefined);
+  }
+
+  async runToCompletion(
+    runId: string,
+    sim: SimulationTask,
+  ): Promise<Result<void, Error>> {
+    for (;;) {
+      const stepResult = await this.executeStep(runId, sim);
+      if (stepResult.isErr()) return err(stepResult.error);
+      const outcome = stepResult.value;
+      // false = terminal/done, { backoffMS } = backoff pending (caller must wait)
+      if (outcome === false || typeof outcome === "object") break;
+    }
+    return ok(undefined);
+  }
+}

package/node_modules/flingflow/src/index.ts

@@ -0,0 +1,52 @@
+// Core types
+export type {
+  Continuation,
+  Ctx,
+  WorkflowFn,
+  WorkflowEvent,
+  EventType,
+  ClaimResult,
+  StepOutcome,
+  WorkflowConfig,
+  WorkflowCreatedPayload,
+  StepStartedPayload,
+  StepCompletedPayload,
+  StepFailedPayload,
+  WorkflowCompletedPayload,
+  WorkflowFailedPayload,
+  WorkflowFailureCause,
+  SimulationTask,
+} from "./types.js";
+export { ApplicationFailure, isApplicationFailure, NonRetryableError, TERMINAL_EVENT_TYPES } from "./types.js";
+
+// Backoff
+export { calculateBackoffMS } from "./backoff.js";
+
+// Clock
+export type { Clock } from "./clock.js";
+export { SystemClock, ManualClock } from "./clock.js";
+
+// Transitions
+export type { TransitionResult } from "./transitions.js";
+export { resolveTransition } from "./transitions.js";
+
+// Store
+export type { EventStore, EventStoreCheck, AppendError, AppendIfNoActiveError } from "./store.js";
+export { UniqueConstraintError, WorkflowAlreadyActiveError, WorkflowIdMismatchError } from "./store.js";
+export { MemoryEventStore } from "./store-memory.js";
+export { SqliteEventStore } from "./store-sqlite.js";
+
+// Context
+export { buildContext } from "./context.js";
+
+// Registry
+export type { WorkflowDefinition } from "./registry.js";
+export { WorkflowRegistry } from "./registry.js";
+
+// Engine
+export type { EngineConfig, CreateWorkflowError } from "./engine.js";
+export { Engine, UnknownWorkflowError } from "./engine.js";
+
+// Recovery
+export type { RecoveryConfig, RecoveryReason, RecoveryResult } from "./recovery.js";
+export { runRecovery } from "./recovery.js";