@aikirun/worker 0.5.3

package/README.md

@@ -0,0 +1,149 @@
+# @aikirun/worker
+
+Worker SDK for Aiki durable execution platform - execute workflows and tasks with durable state management and automatic
+recovery.
+
+## Installation
+
+```bash
+npm install @aikirun/worker
+```
+
+## Quick Start
+
+### Create and Spawn a Worker
+
+```typescript
+import { worker } from "@aikirun/worker";
+import { client } from "@aikirun/client";
+import { onboardingWorkflowV1 } from "./workflows.ts";
+
+// Define worker
+const aikiWorker = worker({
+	id: "worker-1",
+	workflows: [onboardingWorkflowV1],
+	subscriber: { type: "redis_streams" },
+	opts: {
+		maxConcurrentWorkflowRuns: 10,
+	},
+});
+
+// Initialize client
+const aikiClient = await client({
+	url: "http://localhost:9090",
+	redis: { host: "localhost", port: 6379 },
+});
+
+// Spawn worker
+const handle = await aikiWorker.spawn(aiki);
+```
+
+### Graceful Shutdown
+
+```typescript
+import process from "node:process";
+
+const shutdown = async () => {
+	await handle.stop();
+	await aiki.close();
+	process.exit(0);
+};
+
+process.on("SIGINT", shutdown);
+process.on("SIGTERM", shutdown);
+```
+
+## Features
+
+- **Durable Execution** - Automatically resume from failures without losing progress
+- **Horizontal Scaling** - Multiple workers process workflows in parallel
+- **State Management** - Persist execution state at each step
+- **Automatic Recovery** - Detect stuck workflows and retry automatically
+- **Polling Strategies** - Adaptive polling with configurable backoff
+- **Graceful Shutdown** - Clean worker termination with in-flight workflow handling
+
+## Horizontal Scaling
+
+Scale workers by creating separate definitions to isolate workflows or shard by key:
+
+```typescript
+// Separate workers by workflow type
+const orderWorker = worker({ id: "orders", workflows: [orderWorkflowV1] });
+const emailWorker = worker({ id: "emails", workflows: [emailWorkflowV1] });
+
+await orderWorker.spawn(client);
+await emailWorker.spawn(client);
+```
+
+```typescript
+// Shard workers by key (reuse base definition with different shards)
+const orderWorker = worker({ id: "order-processor", workflows: [orderWorkflowV1] });
+
+await orderWorker.with().opt("shardKeys", ["us-east", "us-west"]).spawn(client);
+await orderWorker.with().opt("shardKeys", ["eu-west"]).spawn(client);
+```
+
+## Worker Configuration
+
+### Params (required for worker identity)
+
+```typescript
+interface WorkerParams {
+	id: string; // Unique worker ID
+	workflows: WorkflowVersion[]; // Workflow versions to execute
+	subscriber?: SubscriberStrategy; // Message subscriber (default: redis_streams)
+}
+```
+
+### Options (via `opts` param or `with()` builder)
+
+```typescript
+interface WorkerOptions {
+	maxConcurrentWorkflowRuns?: number; // Concurrency limit (default: 1)
+	workflowRun?: {
+		heartbeatIntervalMs?: number; // Heartbeat interval (default: 30s)
+	};
+	gracefulShutdownTimeoutMs?: number; // Shutdown timeout (default: 5s)
+	shardKeys?: string[]; // Optional shard keys for distributed work
+}
+```
+
+## Workflow Registration
+
+Workers receive workflow versions through the `workflows` param:
+
+```typescript
+const aikiWorker = worker({
+	id: "worker-1",
+	workflows: [workflowV1, workflowV2, anotherWorkflowV1],
+	subscriber: { type: "redis_streams" },
+});
+```
+
+The worker automatically discovers and executes the registered workflow versions.
+
+## State Persistence
+
+Workers store execution state at each step:
+
+- Task completion status
+- Sleep/wait checkpoints
+- Event acknowledgments
+- Child workflow results
+
+This allows workflows to resume from the exact point of failure.
+
+## Related Packages
+
+- [@aikirun/client](https://www.npmjs.com/package/@aikirun/client) - Start workflows
+- [@aikirun/workflow](https://www.npmjs.com/package/@aikirun/workflow) - Define workflows
+- [@aikirun/task](https://www.npmjs.com/package/@aikirun/task) - Define tasks
+- [@aikirun/types](https://www.npmjs.com/package/@aikirun/types) - Type definitions
+
+## Changelog
+
+See the [CHANGELOG](https://github.com/aikirun/aiki/blob/main/CHANGELOG.md) for version history.
+
+## License
+
+Apache-2.0

package/dist/index.d.ts

@@ -0,0 +1,96 @@
+import { Client, SubscriberStrategy } from '@aikirun/client';
+import { WorkflowVersion } from '@aikirun/workflow';
+
+type NonEmptyArray<T> = [T, ...T[]];
+
+type NonArrayObject<T> = T extends object ? (T extends ReadonlyArray<unknown> ? never : T) : never;
+type IsSubtype<SubT, SuperT> = SubT extends SuperT ? true : false;
+type And<T extends NonEmptyArray<boolean>> = T extends [infer First, ...infer Rest] ? false extends First ? false : Rest extends NonEmptyArray<boolean> ? And<Rest> : true : never;
+type Or<T extends NonEmptyArray<boolean>> = T extends [infer First, ...infer Rest] ? true extends First ? true : Rest extends NonEmptyArray<boolean> ? Or<Rest> : false : never;
+type PathFromObject<T, IncludeArrayKeys extends boolean = false> = T extends T ? PathFromObjectInternal<T, IncludeArrayKeys> : never;
+type PathFromObjectInternal<T, IncludeArrayKeys extends boolean> = And<[
+    IsSubtype<T, object>,
+    Or<[IncludeArrayKeys, NonArrayObject<T> extends never ? false : true]>
+]> extends true ? {
+    [K in Exclude<keyof T, symbol>]-?: And<[
+        IsSubtype<NonNullable<T[K]>, object>,
+        Or<[IncludeArrayKeys, NonArrayObject<NonNullable<T[K]>> extends never ? false : true]>
+    ]> extends true ? K | `${K}.${PathFromObjectInternal<NonNullable<T[K]>, IncludeArrayKeys>}` : K;
+}[Exclude<keyof T, symbol>] : "";
+type ExtractObjectType<T> = T extends object ? T : never;
+type TypeOfValueAtPath<T extends object, Path extends PathFromObject<T>> = Path extends keyof T ? T[Path] : Path extends `${infer First}.${infer Rest}` ? First extends keyof T ? undefined extends T[First] ? Rest extends PathFromObject<ExtractObjectType<T[First]>> ? TypeOfValueAtPath<ExtractObjectType<T[First]>, Rest> | undefined : never : Rest extends PathFromObject<ExtractObjectType<T[First]>> ? TypeOfValueAtPath<ExtractObjectType<T[First]>, Rest> : never : never : never;
+
+/**
+ * Creates an Aiki worker definition for executing workflows.
+ *
+ * Worker definitions are static and reusable. Call `spawn(client)` to begin
+ * execution, which returns a handle for controlling the running worker.
+ *
+ * @param params - Worker configuration parameters
+ * @param params.id - Unique worker ID for identification and monitoring
+ * @param params.workflows - Array of workflow versions this worker can execute
+ * @param params.subscriber - Message subscriber strategy (default: redis_streams)
+ * @returns Worker definition, call spawn(client) to begin execution
+ *
+ * @example
+ * ```typescript
+ * export const myWorker = worker({
+ *   id: "order-worker",
+ *   workflows: [orderWorkflowV1, paymentWorkflowV1],
+ *   opts: {
+ *     maxConcurrentWorkflowRuns: 10,
+ *   },
+ * });
+ *
+ * const handle = await myWorker.spawn(client);
+ *
+ * process.on("SIGINT", async () => {
+ *   await handle.stop();
+ *   await client.close();
+ * });
+ * ```
+ */
+declare function worker(params: WorkerParams): Worker;
+interface WorkerParams {
+    id: string;
+    workflows: WorkflowVersion<any, any, any>[];
+    subscriber?: SubscriberStrategy;
+    opts?: WorkerOptions;
+}
+interface WorkerOptions {
+    maxConcurrentWorkflowRuns?: number;
+    workflowRun?: {
+        heartbeatIntervalMs?: number;
+        /**
+         * Threshold for spinning vs persisting delays (default: 10ms).
+         *
+         * Delays <= threshold: In-memory wait (fast, no history, not durable)
+         * Delays > threshold: Server state transition (history recorded, durable)
+         *
+         * Set to 0 to record all delays in transition history.
+         */
+        spinThresholdMs?: number;
+    };
+    gracefulShutdownTimeoutMs?: number;
+    /**
+     * Optional array of shardKeys this worker should process.
+     * When provided, the worker will only subscribe to sharded streams.
+     * When omitted, the worker subscribes to default streams.
+     */
+    shardKeys?: string[];
+}
+interface WorkerBuilder {
+    opt<Path extends PathFromObject<WorkerOptions>>(path: Path, value: TypeOfValueAtPath<WorkerOptions, Path>): WorkerBuilder;
+    spawn: Worker["spawn"];
+}
+interface Worker {
+    id: string;
+    with(): WorkerBuilder;
+    spawn: <AppContext>(client: Client<AppContext>) => Promise<WorkerHandle>;
+}
+interface WorkerHandle {
+    id: string;
+    stop: () => Promise<void>;
+}
+
+export { type Worker, type WorkerParams, worker };

package/dist/index.js

@@ -0,0 +1,351 @@
+// ../../lib/array/utils.ts
+function isNonEmptyArray(value) {
+  return value.length > 0;
+}
+
+// ../../lib/async/delay.ts
+function delay(ms, options) {
+  const abortSignal = options?.abortSignal;
+  if (abortSignal?.aborted) {
+    return Promise.reject(abortSignal.reason);
+  }
+  return new Promise((resolve, reject) => {
+    const abort = () => {
+      clearTimeout(timeout);
+      reject(abortSignal?.reason);
+    };
+    const timeout = setTimeout(() => {
+      abortSignal?.removeEventListener("abort", abort);
+      resolve();
+    }, ms);
+    abortSignal?.addEventListener("abort", abort, { once: true });
+  });
+}
+
+// ../../lib/async/fire-and-forget.ts
+function fireAndForget(promise, onError) {
+  promise.catch((error) => {
+    onError(error instanceof Error ? error : new Error(String(error)));
+  });
+}
+
+// ../../lib/error/conflict.ts
+function isServerConflictError(error) {
+  if (error === null || typeof error !== "object") {
+    return false;
+  }
+  if ("code" in error && error.code === "CONFLICT") {
+    return true;
+  }
+  if ("status" in error && error.status === 409) {
+    return true;
+  }
+  if (error instanceof Error && error.name === "ConflictError") {
+    return true;
+  }
+  if ("data" in error && error.data !== null && typeof error.data === "object" && "status" in error.data && error.data.status === 409) {
+    return true;
+  }
+  return false;
+}
+
+// ../../lib/object/overrider.ts
+function set(obj, path, value) {
+  const keys = path.split(".");
+  let currentValue = obj;
+  for (let i = 0; i < keys.length - 1; i++) {
+    const key = keys[i];
+    currentValue = currentValue[key];
+    if (currentValue === void 0 || currentValue === null) {
+      currentValue = {};
+      currentValue[key] = currentValue;
+    }
+  }
+  const lastKey = keys[keys.length - 1];
+  currentValue[lastKey] = value;
+}
+var objectOverrider = (defaultObj) => (obj) => {
+  const createBuilder = (overrides) => ({
+    with: (path, value) => createBuilder([...overrides, { path: `${path}`, value }]),
+    build: () => {
+      const clonedObject = structuredClone(obj ?? defaultObj);
+      for (const { path, value } of overrides) {
+        set(clonedObject, path, value);
+      }
+      return clonedObject;
+    }
+  });
+  return createBuilder([]);
+};
+
+// worker.ts
+import { INTERNAL } from "@aikirun/types/symbols";
+import { TaskFailedError } from "@aikirun/types/task";
+import {
+  WorkflowRunCancelledError,
+  WorkflowRunFailedError,
+  WorkflowRunNotExecutableError,
+  WorkflowRunSuspendedError
+} from "@aikirun/types/workflow-run";
+import {
+  createWorkflowRunSleeper,
+  workflowRegistry,
+  workflowRunHandle
+} from "@aikirun/workflow";
+function worker(params) {
+  return new WorkerImpl(params);
+}
+var WorkerImpl = class _WorkerImpl {
+  constructor(params) {
+    this.params = params;
+    this.id = params.id;
+  }
+  id;
+  with() {
+    const optsOverrider = objectOverrider(this.params.opts ?? {});
+    const createBuilder = (optsBuilder) => ({
+      opt: (path, value) => createBuilder(optsBuilder.with(path, value)),
+      spawn: (client) => new _WorkerImpl({ ...this.params, opts: optsBuilder.build() }).spawn(client)
+    });
+    return createBuilder(optsOverrider());
+  }
+  async spawn(client) {
+    const handle = new WorkerHandleImpl(client, this.params);
+    await handle._start();
+    return handle;
+  }
+};
+var WorkerHandleImpl = class {
+  constructor(client, params) {
+    this.client = client;
+    this.params = params;
+    this.id = params.id;
+    this.registry = workflowRegistry().addMany(this.params.workflows);
+    this.logger = client.logger.child({
+      "aiki.component": "worker",
+      "aiki.workerId": this.id
+    });
+  }
+  id;
+  registry;
+  logger;
+  abortController;
+  subscriberStrategy;
+  activeWorkflowRunsById = /* @__PURE__ */ new Map();
+  async _start() {
+    const subscriberStrategyBuilder = this.client[INTERNAL].subscriber.create(
+      this.params.subscriber ?? { type: "redis_streams" },
+      this.registry.getAll(),
+      this.params.opts?.shardKeys
+    );
+    this.subscriberStrategy = await subscriberStrategyBuilder.init(this.id, {
+      onError: (error) => this.handleNotificationError(error),
+      onStop: () => this.stop()
+    });
+    this.abortController = new AbortController();
+    const abortSignal = this.abortController.signal;
+    fireAndForget(this.poll(abortSignal), (error) => {
+      if (!abortSignal.aborted) {
+        this.logger.error("Unexpected error", {
+          "aiki.error": error.message
+        });
+      }
+    });
+  }
+  async stop() {
+    this.logger.info("Worker stopping");
+    this.abortController?.abort();
+    const activeWorkflowRuns = Array.from(this.activeWorkflowRunsById.values());
+    if (activeWorkflowRuns.length === 0) return;
+    const timeoutMs = this.params.opts?.gracefulShutdownTimeoutMs ?? 5e3;
+    if (timeoutMs > 0) {
+      await Promise.race([Promise.allSettled(activeWorkflowRuns.map((w) => w.executionPromise)), delay(timeoutMs)]);
+    }
+    const stillActive = Array.from(this.activeWorkflowRunsById.values());
+    if (stillActive.length > 0) {
+      const ids = stillActive.map((w) => w.run.id).join(", ");
+      this.logger.warn("Worker shutdown with active workflows", {
+        "aiki.activeWorkflowRunIds": ids
+      });
+    }
+    this.activeWorkflowRunsById.clear();
+  }
+  async poll(abortSignal) {
+    this.logger.info("Worker started");
+    if (!this.subscriberStrategy) {
+      throw new Error("Subscriber strategy not initialized");
+    }
+    let nextDelayMs = this.subscriberStrategy.getNextDelay({ type: "polled", foundWork: false });
+    let subscriberFailedAttempts = 0;
+    while (!abortSignal.aborted) {
+      await delay(nextDelayMs, { abortSignal });
+      const availableCapacity = (this.params.opts?.maxConcurrentWorkflowRuns ?? 1) - this.activeWorkflowRunsById.size;
+      if (availableCapacity <= 0) {
+        nextDelayMs = this.subscriberStrategy.getNextDelay({ type: "at_capacity" });
+        continue;
+      }
+      const nextBatchResponse = await this.fetchNextWorkflowRunBatch(availableCapacity);
+      if (!nextBatchResponse.success) {
+        subscriberFailedAttempts++;
+        nextDelayMs = this.subscriberStrategy.getNextDelay({
+          type: "retry",
+          attemptNumber: subscriberFailedAttempts
+        });
+        continue;
+      }
+      subscriberFailedAttempts = 0;
+      if (!isNonEmptyArray(nextBatchResponse.batch)) {
+        nextDelayMs = this.subscriberStrategy.getNextDelay({ type: "polled", foundWork: false });
+        continue;
+      }
+      await this.enqueueWorkflowRunBatch(nextBatchResponse.batch, abortSignal);
+      nextDelayMs = this.subscriberStrategy.getNextDelay({ type: "polled", foundWork: true });
+    }
+  }
+  async fetchNextWorkflowRunBatch(size) {
+    if (!this.subscriberStrategy) {
+      return {
+        success: false,
+        error: new Error("Subscriber strategy not initialized")
+      };
+    }
+    try {
+      const batch = await this.subscriberStrategy.getNextBatch(size);
+      return {
+        success: true,
+        batch
+      };
+    } catch (error) {
+      this.logger.error("Error getting next workflow runs batch", {
+        "aiki.error": error instanceof Error ? error.message : String(error)
+      });
+      return {
+        success: false,
+        error
+      };
+    }
+  }
+  async enqueueWorkflowRunBatch(batch, abortSignal) {
+    for (const { data, meta } of batch) {
+      const { workflowRunId } = data;
+      if (this.activeWorkflowRunsById.has(workflowRunId)) {
+        this.logger.info("Workflow already running", {
+          "aiki.workflowRunId": workflowRunId
+        });
+        continue;
+      }
+      const { run: workflowRun } = await this.client.api.workflowRun.getByIdV1({ id: workflowRunId });
+      if (!workflowRun) {
+        if (meta && this.subscriberStrategy?.acknowledge) {
+          await this.subscriberStrategy.acknowledge(workflowRunId, meta).catch(() => {
+          });
+        }
+        continue;
+      }
+      const workflowVersion = this.registry.get(
+        workflowRun.workflowId,
+        workflowRun.workflowVersionId
+      );
+      if (!workflowVersion) {
+        this.logger.warn("Workflow version not found", {
+          "aiki.workflowId": workflowRun.workflowId,
+          "aiki.workflowVersionId": workflowRun.workflowVersionId,
+          "aiki.workflowRunId": workflowRun.id
+        });
+        if (meta && this.subscriberStrategy?.acknowledge) {
+          await this.subscriberStrategy.acknowledge(workflowRunId, meta).catch(() => {
+          });
+        }
+        continue;
+      }
+      if (abortSignal.aborted) break;
+      const workflowExecutionPromise = this.executeWorkflow(workflowRun, workflowVersion, meta);
+      this.activeWorkflowRunsById.set(workflowRun.id, {
+        run: workflowRun,
+        executionPromise: workflowExecutionPromise,
+        meta
+      });
+    }
+  }
+  async executeWorkflow(workflowRun, workflowVersion, meta) {
+    const logger = this.logger.child({
+      "aiki.component": "workflow-execution",
+      "aiki.workflowId": workflowRun.workflowId,
+      "aiki.workflowVersionId": workflowRun.workflowVersionId,
+      "aiki.workflowRunId": workflowRun.id,
+      ...meta && {
+        "aiki.messageId": meta.messageId
+      }
+    });
+    let heartbeatInterval;
+    let shouldAcknowledge = false;
+    try {
+      const handle = await workflowRunHandle(this.client, workflowRun, logger);
+      const appContext = this.client[INTERNAL].contextFactory ? await this.client[INTERNAL].contextFactory(workflowRun) : null;
+      const heartbeat = this.subscriberStrategy?.heartbeat;
+      if (meta && heartbeat) {
+        heartbeatInterval = setInterval(() => {
+          try {
+            heartbeat(workflowRun.id, meta);
+          } catch (error) {
+            logger.warn("Failed to send heartbeat", {
+              "aiki.error": error instanceof Error ? error.message : String(error)
+            });
+          }
+        }, this.params.opts?.workflowRun?.heartbeatIntervalMs ?? 3e4);
+      }
+      await workflowVersion[INTERNAL].handler(
+        workflowRun.input,
+        {
+          id: workflowRun.id,
+          workflowId: workflowRun.workflowId,
+          workflowVersionId: workflowRun.workflowVersionId,
+          options: workflowRun.options,
+          logger,
+          sleep: createWorkflowRunSleeper(handle, logger, {
+            spinThresholdMs: this.params.opts?.workflowRun?.spinThresholdMs ?? 10
+          }),
+          [INTERNAL]: { handle }
+        },
+        appContext
+      );
+      shouldAcknowledge = true;
+    } catch (error) {
+      if (error instanceof WorkflowRunNotExecutableError || error instanceof WorkflowRunCancelledError || error instanceof WorkflowRunFailedError || error instanceof WorkflowRunSuspendedError || error instanceof TaskFailedError || isServerConflictError(error)) {
+        shouldAcknowledge = true;
+      } else {
+        logger.error("Unexpected error during workflow execution", {
+          "aiki.error": error instanceof Error ? error.message : String(error),
+          "aiki.stack": error instanceof Error ? error.stack : void 0
+        });
+        shouldAcknowledge = false;
+      }
+    } finally {
+      if (heartbeatInterval) clearInterval(heartbeatInterval);
+      if (meta && this.subscriberStrategy?.acknowledge) {
+        if (shouldAcknowledge) {
+          try {
+            await this.subscriberStrategy.acknowledge(workflowRun.id, meta);
+          } catch (error) {
+            logger.error("Failed to acknowledge message, it may be reprocessed", {
+              "aiki.errorType": "MESSAGE_ACK_FAILED",
+              "aiki.error": error instanceof Error ? error.message : String(error)
+            });
+          }
+        } else {
+          logger.debug("Message left in PEL for retry");
+        }
+      }
+      this.activeWorkflowRunsById.delete(workflowRun.id);
+    }
+  }
+  handleNotificationError(error) {
+    this.logger.warn("Notification error, falling back to polling", {
+      "aiki.error": error.message,
+      "aiki.stack": error.stack
+    });
+  }
+};
+export {
+  worker
+};

package/package.json

@@ -0,0 +1,29 @@
+{
+	"name": "@aikirun/worker",
+	"version": "0.5.3",
+	"description": "Worker SDK for Aiki - execute workflows and tasks with durable state management and automatic recovery",
+	"type": "module",
+	"main": "./dist/index.js",
+	"types": "./dist/index.d.ts",
+	"exports": {
+		".": {
+			"types": "./dist/index.d.ts",
+			"import": "./dist/index.js"
+		}
+	},
+	"files": [
+		"dist"
+	],
+	"scripts": {
+		"build": "tsup"
+	},
+	"dependencies": {
+		"@aikirun/types": "0.5.3",
+		"@aikirun/client": "0.5.3",
+		"@aikirun/workflow": "0.5.3"
+	},
+	"publishConfig": {
+		"access": "public"
+	},
+	"license": "Apache-2.0"
+}