@sonamu-kit/tasks 0.0.1

package/src/execution.ts

@@ -0,0 +1,291 @@
+import type { Backend } from "./backend";
+import type { DurationString } from "./core/duration";
+import { serializeError } from "./core/error";
+import type { JsonValue } from "./core/json";
+import type { StepAttempt, StepAttemptCache } from "./core/step";
+import {
+  addToStepAttemptCache,
+  calculateSleepResumeAt,
+  createSleepContext,
+  createStepAttemptCacheFromAttempts,
+  getCachedStepAttempt,
+  normalizeStepOutput,
+} from "./core/step";
+import type { WorkflowRun } from "./core/workflow";
+
+/**
+ * Config for an individual step defined with `step.run()`.
+ */
+export interface StepFunctionConfig {
+  /**
+   * The name of the step.
+   */
+  name: string;
+}
+
+/**
+ * Represents the API for defining steps within a workflow. Used within a
+ * workflow handler to define steps by calling `step.run()`.
+ */
+export interface StepApi {
+  run<Output>(config: Readonly<StepFunctionConfig>, fn: StepFunction<Output>): Promise<Output>;
+  sleep(name: string, duration: DurationString): Promise<void>;
+}
+
+/**
+ * The step definition (defined by the user) that executes user code. Can return
+ * undefined (e.g., when using `return;`) which will be converted to null.
+ */
+export type StepFunction<Output> = () => Promise<Output | undefined> | Output | undefined;
+
+/**
+ * Params passed to a workflow function for the user to use when defining steps.
+ */
+export interface WorkflowFunctionParams<Input> {
+  input: Input;
+  step: StepApi;
+  version: string | null;
+}
+
+/**
+ * The workflow definition's function (defined by the user) that the user uses
+ * to define the workflow's steps.
+ */
+export type WorkflowFunction<Input, Output> = (
+  params: Readonly<WorkflowFunctionParams<Input>>,
+) => Promise<Output> | Output;
+
+/**
+ * Signal thrown when a workflow needs to sleep. Contains the time when the
+ * workflow should resume.
+ */
+class SleepSignal extends Error {
+  readonly resumeAt: Date;
+
+  constructor(resumeAt: Readonly<Date>) {
+    super("SleepSignal");
+    this.name = "SleepSignal";
+    this.resumeAt = resumeAt;
+  }
+}
+
+/**
+ * Configures the options for a StepExecutor.
+ */
+export interface StepExecutorOptions {
+  backend: Backend;
+  workflowRunId: string;
+  workerId: string;
+  attempts: StepAttempt[];
+}
+
+/**
+ * Replays prior step attempts and persists new ones while memoizing
+ * deterministic step outputs.
+ */
+export class StepExecutor implements StepApi {
+  private readonly backend: Backend;
+  private readonly workflowRunId: string;
+  private readonly workerId: string;
+  private cache: StepAttemptCache;
+
+  constructor(options: Readonly<StepExecutorOptions>) {
+    this.backend = options.backend;
+    this.workflowRunId = options.workflowRunId;
+    this.workerId = options.workerId;
+
+    this.cache = createStepAttemptCacheFromAttempts(options.attempts);
+  }
+
+  async run<Output>(
+    config: Readonly<StepFunctionConfig>,
+    fn: StepFunction<Output>,
+  ): Promise<Output> {
+    const { name } = config;
+
+    // return cached result if available
+    const existingAttempt = getCachedStepAttempt(this.cache, name);
+    if (existingAttempt) {
+      return existingAttempt.output as Output;
+    }
+
+    // not in cache, create new step attempt
+    const attempt = await this.backend.createStepAttempt({
+      workflowRunId: this.workflowRunId,
+      workerId: this.workerId,
+      stepName: name,
+      kind: "function",
+      config: {},
+      context: null,
+    });
+
+    try {
+      // execute step function
+      const result = await fn();
+      const output = normalizeStepOutput(result);
+
+      // mark success
+      const savedAttempt = await this.backend.completeStepAttempt({
+        workflowRunId: this.workflowRunId,
+        stepAttemptId: attempt.id,
+        workerId: this.workerId,
+        output,
+      });
+
+      // cache result
+      this.cache = addToStepAttemptCache(this.cache, savedAttempt);
+
+      return savedAttempt.output as Output;
+    } catch (error) {
+      // mark failure
+      await this.backend.failStepAttempt({
+        workflowRunId: this.workflowRunId,
+        stepAttemptId: attempt.id,
+        workerId: this.workerId,
+        error: serializeError(error),
+      });
+      throw error;
+    }
+  }
+
+  async sleep(name: string, duration: DurationString): Promise<void> {
+    // return cached result if this sleep already completed
+    const existingAttempt = getCachedStepAttempt(this.cache, name);
+    if (existingAttempt) return;
+
+    // create new step attempt for the sleep
+    const result = calculateSleepResumeAt(duration);
+    if (!result.ok) {
+      throw result.error;
+    }
+    const resumeAt = result.value;
+    const context = createSleepContext(resumeAt);
+
+    await this.backend.createStepAttempt({
+      workflowRunId: this.workflowRunId,
+      workerId: this.workerId,
+      stepName: name,
+      kind: "sleep",
+      config: {},
+      context,
+    });
+
+    // throw sleep signal to trigger postponement
+    // we do not mark the step as completed here; it will be updated
+    // when the workflow resumes
+    throw new SleepSignal(resumeAt);
+  }
+}
+
+/**
+ * Parameters for the workflow execution use case.
+ */
+export interface ExecuteWorkflowParams {
+  backend: Backend;
+  workflowRun: WorkflowRun;
+  workflowFn: WorkflowFunction<unknown, unknown>;
+  workflowVersion: string | null;
+  workerId: string;
+}
+
+/**
+ * Execute a workflow run. This is the core application use case that handles:
+ * - Loading step history
+ * - Handling sleeping steps
+ * - Creating the step executor
+ * - Executing the workflow function
+ * - Completing, failing, or sleeping the workflow run based on the outcome
+ * @param params - The execution parameters
+ */
+export async function executeWorkflow(params: Readonly<ExecuteWorkflowParams>): Promise<void> {
+  const { backend, workflowRun, workflowFn, workflowVersion, workerId } = params;
+
+  try {
+    // load all pages of step history
+    const attempts: StepAttempt[] = [];
+    let cursor: string | undefined;
+    do {
+      const response = await backend.listStepAttempts({
+        workflowRunId: workflowRun.id,
+        ...(cursor ? { after: cursor } : {}),
+        limit: 1000,
+      });
+      attempts.push(...response.data);
+      cursor = response.pagination.next ?? undefined;
+    } while (cursor);
+
+    // mark any sleep steps as completed if their sleep duration has elapsed,
+    // or rethrow SleepSignal if still sleeping
+    for (let i = 0; i < attempts.length; i++) {
+      const attempt = attempts[i];
+      if (!attempt) continue;
+
+      if (
+        attempt.status === "running" &&
+        attempt.kind === "sleep" &&
+        attempt.context?.kind === "sleep"
+      ) {
+        const now = Date.now();
+        const resumeAt = new Date(attempt.context.resumeAt);
+        const resumeAtMs = resumeAt.getTime();
+
+        if (now < resumeAtMs) {
+          // sleep duration HAS NOT elapsed yet, throw signal to put workflow
+          // back to sleep
+          throw new SleepSignal(resumeAt);
+        }
+
+        // sleep duration HAS elapsed, mark the step as completed and continue
+        const completed = await backend.completeStepAttempt({
+          workflowRunId: workflowRun.id,
+          stepAttemptId: attempt.id,
+          workerId,
+          output: null,
+        });
+
+        // update cache w/ completed attempt
+        attempts[i] = completed;
+      }
+    }
+
+    // create step executor
+    const executor = new StepExecutor({
+      backend,
+      workflowRunId: workflowRun.id,
+      workerId,
+      attempts,
+    });
+
+    // execute workflow
+    const output = await workflowFn({
+      input: workflowRun.input as unknown,
+      step: executor,
+      version: workflowVersion,
+    });
+
+    // mark success
+    await backend.completeWorkflowRun({
+      workflowRunId: workflowRun.id,
+      workerId,
+      output: (output ?? null) as JsonValue,
+    });
+  } catch (error) {
+    // handle sleep signal by setting workflow to sleeping status
+    if (error instanceof SleepSignal) {
+      await backend.sleepWorkflowRun({
+        workflowRunId: workflowRun.id,
+        workerId,
+        availableAt: error.resumeAt,
+      });
+
+      return;
+    }
+
+    // mark failure
+    await backend.failWorkflowRun({
+      workflowRunId: workflowRun.id,
+      workerId,
+      error: serializeError(error),
+    });
+  }
+}

package/src/index.ts

@@ -0,0 +1,7 @@
+export type { OpenWorkflowOptions } from "./client";
+export { declareWorkflow, OpenWorkflow } from "./client";
+export type { OpenWorkflowConfig, WorkerConfig } from "./config";
+export { defineConfig } from "./config";
+export { BackendPostgres } from "./database/backend";
+export type { WorkerOptions } from "./worker";
+export { Worker } from "./worker";

package/src/internal.ts

@@ -0,0 +1,11 @@
+export * from "./backend";
+export type * from "./client";
+export { loadConfig } from "./config";
+export type { DurationString } from "./core/duration";
+export type { JsonValue } from "./core/json";
+export { DEFAULT_RETRY_POLICY } from "./core/retry";
+export type { StandardSchemaV1 } from "./core/schema";
+export type { StepAttempt } from "./core/step";
+export type { SchemaInput, SchemaOutput, WorkflowRun } from "./core/workflow";
+export type { StepApi, WorkflowFunction } from "./execution";
+export type { WorkflowSpec } from "./workflow";

package/src/practices/01-remote-workflow.ts

@@ -0,0 +1,61 @@
+import assert from "node:assert";
+import { randomUUID } from "node:crypto";
+import { BackendPostgres, OpenWorkflow } from "../";
+import { KNEX_GLOBAL_CONFIG } from "../testing/connection";
+
+let _backend: BackendPostgres | null = null;
+
+async function getBackend(): Promise<BackendPostgres> {
+  if (_backend !== null) {
+    return _backend;
+  }
+
+  _backend = await BackendPostgres.connect(KNEX_GLOBAL_CONFIG, {
+    runMigrations: true,
+    namespaceId: randomUUID(),
+  });
+
+  return _backend;
+}
+
+async function practice() {
+  const backend = await getBackend();
+  const ow = new OpenWorkflow({ backend });
+
+  const sampleWorkflow = ow.defineWorkflow({ name: "sample-workflow" }, async ({ step }) => {
+    const { result: result1 } = await step.run({ name: "test-1" }, async () => {
+      return {
+        result: ["Result from test-1"],
+      };
+    });
+
+    await step.run({ name: "test-2" }, async () => {
+      return await new Promise<void>((resolve) => setTimeout(resolve, 1000));
+    });
+
+    return await step.run({ name: "test-3" }, async () => {
+      return {
+        result: [...result1, "Result from test-3"],
+      };
+    });
+  });
+
+  // create a worker that will listen to the channel and process the workflow runs
+  const worker = ow.newWorker({ concurrency: 3 });
+  await worker.start();
+
+  const handle = await sampleWorkflow.run();
+
+  await new Promise((resolve) => setTimeout(resolve, 3000));
+  await worker.tick();
+
+  const result = await handle.result();
+  assert.deepEqual(result, {
+    result: ["Result from test-1", "Result from test-3"],
+  });
+
+  await worker.stop();
+  await backend.stop();
+}
+
+await practice();

package/src/registry.test.ts

@@ -0,0 +1,122 @@
+import { describe, expect, test } from "vitest";
+import { WorkflowRegistry } from "./registry";
+import { defineWorkflow } from "./workflow";
+
+describe("WorkflowRegistry", () => {
+  describe("register", () => {
+    test("registers a workflow without version", () => {
+      const registry = new WorkflowRegistry();
+      const workflow = createMockWorkflow("my-workflow");
+
+      registry.register(workflow);
+
+      expect(registry.get("my-workflow", null)).toBe(workflow);
+    });
+
+    test("registers a workflow with version", () => {
+      const registry = new WorkflowRegistry();
+      const workflow = createMockWorkflow("my-workflow", "v1");
+
+      registry.register(workflow);
+
+      expect(registry.get("my-workflow", "v1")).toBe(workflow);
+    });
+
+    test("registers multiple versions of the same workflow", () => {
+      const registry = new WorkflowRegistry();
+      const v1 = createMockWorkflow("my-workflow", "v1");
+      const v2 = createMockWorkflow("my-workflow", "v2");
+
+      registry.register(v1);
+      registry.register(v2);
+
+      expect(registry.get("my-workflow", "v1")).toBe(v1);
+      expect(registry.get("my-workflow", "v2")).toBe(v2);
+    });
+
+    test("registers different workflows with same version", () => {
+      const registry = new WorkflowRegistry();
+      const workflow1 = createMockWorkflow("workflow-a", "v1");
+      const workflow2 = createMockWorkflow("workflow-b", "v1");
+
+      registry.register(workflow1);
+      registry.register(workflow2);
+
+      expect(registry.get("workflow-a", "v1")).toBe(workflow1);
+      expect(registry.get("workflow-b", "v1")).toBe(workflow2);
+    });
+
+    test("throws when registering duplicate unversioned workflow", () => {
+      const registry = new WorkflowRegistry();
+      registry.register(createMockWorkflow("my-workflow"));
+
+      expect(() => {
+        registry.register(createMockWorkflow("my-workflow"));
+      }).toThrow('Workflow "my-workflow" is already registered');
+    });
+
+    test("throws when registering duplicate versioned workflow", () => {
+      const registry = new WorkflowRegistry();
+      registry.register(createMockWorkflow("my-workflow", "v1"));
+
+      expect(() => {
+        registry.register(createMockWorkflow("my-workflow", "v1"));
+      }).toThrow('Workflow "my-workflow" (version: v1) is already registered');
+    });
+
+    test("allows same name with different versions", () => {
+      const registry = new WorkflowRegistry();
+      const versioned = createMockWorkflow("my-workflow", "v1");
+      const unversioned = createMockWorkflow("my-workflow");
+
+      registry.register(versioned);
+      registry.register(unversioned);
+
+      expect(registry.get("my-workflow", "v1")).toBe(versioned);
+      expect(registry.get("my-workflow", null)).toBe(unversioned);
+    });
+  });
+
+  describe("get", () => {
+    test("returns undefined for non-existent workflow", () => {
+      const registry = new WorkflowRegistry();
+
+      expect(registry.get("non-existent", null)).toBeUndefined();
+    });
+
+    test("returns undefined for wrong version", () => {
+      const registry = new WorkflowRegistry();
+      registry.register(createMockWorkflow("my-workflow", "v1"));
+
+      expect(registry.get("my-workflow", "v2")).toBeUndefined();
+      expect(registry.get("my-workflow", null)).toBeUndefined();
+    });
+
+    test("returns undefined for versioned lookup on unversioned workflow", () => {
+      const registry = new WorkflowRegistry();
+      registry.register(createMockWorkflow("my-workflow"));
+
+      expect(registry.get("my-workflow", "v1")).toBeUndefined();
+    });
+
+    test("returns the registered workflow", () => {
+      const registry = new WorkflowRegistry();
+      const workflow = createMockWorkflow("my-workflow");
+      registry.register(workflow);
+
+      expect(registry.get("my-workflow", null)).toBe(workflow);
+    });
+  });
+});
+
+function createMockWorkflow(name: string, version?: string) {
+  return defineWorkflow(
+    {
+      name,
+      ...(version && { version }),
+    },
+    async () => {
+      // no-op
+    },
+  );
+}

package/src/registry.ts

@@ -0,0 +1,65 @@
+import type { Workflow } from "./workflow";
+
+/**
+ * A registry for storing and retrieving workflows by name and version.
+ * Provides a centralized way to manage workflow registrations.
+ */
+export class WorkflowRegistry {
+  private readonly workflows = new Map<string, Workflow<unknown, unknown, unknown>>();
+
+  /**
+   * Register a workflow in the registry.
+   * @param workflow - The workflow to register
+   * @throws {Error} If a workflow with the same name and version is already registered
+   */
+  register(workflow: Workflow<unknown, unknown, unknown>): void {
+    const name = workflow.spec.name;
+    const version = workflow.spec.version ?? null;
+    const key = registryKey(name, version);
+    if (this.workflows.has(key)) {
+      const versionStr = version ? ` (version: ${version})` : "";
+      throw new Error(`Workflow "${name}"${versionStr} is already registered`);
+    }
+    this.workflows.set(key, workflow);
+  }
+
+  /**
+   * Get a workflow from the registry by name and version.
+   * @param name - The workflow name
+   * @param version - The workflow version (null for unversioned)
+   * @returns The workflow if found, undefined otherwise
+   */
+  get(name: string, version: string | null): Workflow<unknown, unknown, unknown> | undefined {
+    const key = registryKey(name, version);
+    return this.workflows.get(key);
+  }
+
+  /**
+   * Get a workflow from the registry by name and version.
+   * @param name - The workflow name
+   * @param version - The workflow version (null for unversioned)
+   * @returns The workflow if found, undefined otherwise
+   */
+  has(name: string, version: string | null): boolean {
+    const key = registryKey(name, version);
+    return this.workflows.has(key);
+  }
+
+  /**
+   * Remove a workflow from the registry by name and version.
+   */
+  remove(name: string, version: string | null): void {
+    const key = registryKey(name, version);
+    this.workflows.delete(key);
+  }
+}
+
+/**
+ * Build a registry key from name and version.
+ * @param name - Workflow name
+ * @param version - Workflow version (or null)
+ * @returns Registry key
+ */
+function registryKey(name: string, version: string | null): string {
+  return version ? `${name}@${version}` : name;
+}

package/src/testing/connection.ts

@@ -0,0 +1,44 @@
+import { randomUUID } from "node:crypto";
+import knex, { type Knex } from "knex";
+import { BackendPostgres } from "../database/backend";
+import { migrate as baseMigrate, DEFAULT_SCHEMA } from "../database/base";
+
+let backend: BackendPostgres | null = null;
+
+export const KNEX_GLOBAL_CONFIG: Knex.Config = {
+  client: "pg",
+  connection: {
+    host: "127.0.0.1",
+    port: 5432,
+    user: "postgres",
+    password: "miomock123",
+    database: "postgres",
+  },
+  pool: {
+    max: 50,
+  },
+} as const;
+
+export async function migrate(): Promise<void> {
+  await baseMigrate(knex(KNEX_GLOBAL_CONFIG), DEFAULT_SCHEMA);
+}
+
+export async function createBackend(): Promise<BackendPostgres> {
+  if (backend) {
+    return backend;
+  }
+
+  backend = await BackendPostgres.connect(KNEX_GLOBAL_CONFIG, {
+    namespaceId: randomUUID(),
+  });
+
+  return backend;
+}
+
+export async function stopBackend(): Promise<void> {
+  if (backend) {
+    await backend.stop();
+  }
+
+  backend = null;
+}