@usehelical/workflows 0.0.1-alpha.1

package/README.md

@@ -0,0 +1,123 @@
+# Helical Workflows
+
+Simple, typesafe durable workflows without bundler magic
+
+> [!WARNING]  
+> This is a work in progress
+
+# Goals
+
+- simple & typesafe api
+- minimal deployment complexity (just postgres)
+- low latency (making it suitable for user-facing ai workflows)
+
+# Getting started
+
+## Writing workflows
+
+```ts
+import * as wf from 'helical-workflows/workflow';
+
+async function indexDocument(url: string) {
+  // idempotent retryable durable steps
+  const document = await wf.runStep(async () => fetchDocument(url), { maxRetries: 5 });
+  const parsedDocument = await wf.runStep(async () => parseDocument(document));
+  await wf.runStep(async () => upsertIntoDb(parsedDocument));
+}
+
+const indexDocumentWorkflow = defineWorkflow(indexDocument);
+
+const indexingQueue = defineQueue();
+
+async function processDocumentBatch(urls: string[]) {
+  const handles = [];
+
+  // enqueue all documents for indexing
+  for (const url of urls) {
+    const runHandle = await wf.enqueueWorkflow(indexingQueue, indexDocument, [url]);
+    handles.push(runHandle);
+  }
+
+  let successfulCount = 0;
+  let failedCount = 0;
+
+  // wait for all documents to be indexed
+  for (const handle of handles) {
+    try {
+      await handle.result();
+      successfulCount++;
+    } catch (error) {
+      failedCount++;
+    }
+  }
+
+  return { successfulCount, failedCount };
+}
+
+const processDocumentBatchWorkflow = defineWorkflow(processDocumentBatch);
+```
+
+## Running a workflow
+
+```ts
+// create instance and register workflows and queues
+const { runWorkflow } = createInstance({
+  workflows: { indexDocumentWorkflow, processDocumentBatchWorkflow },
+  queues: { indexingQueue },
+  options: { connectionString: 'dummy' },
+});
+
+const urls = ['a', 'b', 'c'];
+
+const run = await runWorkflow(processDocumentBatchWorkflow, urls);
+
+console.log(await run.result());
+```
+
+# Development roadmap
+
+## Messages
+
+Messages are a way of communicating with a workflow from the outside. This can be useful for human-in-the-loop workflows or for other signals. Messages are persisted in the database and atomically consumed adhering to once-and-only-once semantics.
+
+```ts
+const approvalMessage = defineMessage<ApprovalMessage>('approval');
+
+async function myWorkflowFunction() {
+  await wf.runStep(() => stageForApproval());
+  await wf.receiveMessage(approvalMessage);
+  await wf.runStep(() => publish());
+}
+```
+
+## State
+
+Workflow state is a way for workflows to expose state to the outside. The benefit of this state approach in comparison to other request handler approaches like in Temporal is that the workflow doesn’t have to be executing in order for the state to be retrievable. This makes it possible to retrieve state even if a workflow is already cancelled or finished.
+
+```ts
+async function myWorkflowFunction() {
+  await wf.setState({ progress: 50 });
+}
+```
+
+## Schedules
+
+Allow a workflow to be scheduled…
+
+## Streams
+
+Streams allow for publishing streaming data from workflows to the outside the primary use case for this is realtime streaming of LLM responses. I want to take a different approach to other workflow systems here and treat streams as side effects inside of workflows and not as durable outputs. Since LLM streaming responses can be quite quick persisting the token stream into the database is costly. My hypothesis is that stream persistence is not needed since resuming LLM streaming responses require [special handling](<[https://platform.claude.com/docs/en/build-with-claude/streaming#error-recovery](https://platform.claude.com/docs/en/build-with-claude/streaming#error-recovery)>) and in most cases it makes more sense to just regenerate the response. Usually non-idempotent side effects in workflows are prohibited to achieve durability guarantees. But in this case the correctness of the workflow doesn’t depend on the stream the stream is just a user experience optimisation.
+
+## Workflow suspension
+
+When implementing very long running workflows (days, months…) the workflow would still be the whole time in memory idling waiting for instance for a message or for a sleep() event. Workflow suspension would allow for workflows to be suspended when waiting and then replayed and continued when the event arrives. This happens at the cost of latency (since the workflow has to be replayed to reach its original state again after has been suspended) which is why I want to make this an optional feature.
+
+## Workflow version management
+
+A tricky topic. In production systems it is possible that workflows with old versions are still Pending or are Queued. I don’t know yet what is the best way to handle this but I’m leaning towards workflow versions that can be manually specified using a versionId or could be auto generated by creating a hash from the workflow code.
+
+## Chaos tests
+
+## OTEL Integration
+
+## CLI & Observability UI

package/dist/index.d.mts

@@ -0,0 +1,42 @@
+import { W as WorkflowStatus, a as WorkflowEntry, Q as QueueEntry, M as MessageDefinition, S as StateDefinition } from './state-CmpqDrnz.mjs';
+
+interface Run<TReturn = unknown> {
+    id: string;
+    status: () => Promise<WorkflowStatus>;
+    result: () => Promise<TReturn>;
+}
+
+type RunWorkflowOptions = {
+    timeout?: number;
+    deadline?: number;
+};
+
+type QueueWorkflowOptions = {
+    timeout?: number;
+    deadline?: number;
+    priority?: number;
+    partitionKey?: string;
+    deduplicationId?: string;
+};
+
+type CreateInstanceOptions = {
+    instanceId?: string;
+    connectionString: string;
+};
+type CreateInstanceParams = {
+    workflows: Record<string, WorkflowEntry>;
+    queues?: Record<string, QueueEntry>;
+    options: CreateInstanceOptions;
+};
+declare function createInstance(props: CreateInstanceParams): {
+    runWorkflow: <TArgs extends unknown[], TReturn>(wf: WorkflowEntry<TArgs, TReturn> | string, args?: TArgs, options?: RunWorkflowOptions) => Promise<Run<TReturn>>;
+    cancelRun: (runId: string) => Promise<void>;
+    resumeRun: (runId: string) => Promise<Run<void>>;
+    getRun: (runId: string) => Promise<Run<unknown>>;
+    queueWorkflow: <TArgs extends unknown[], TReturn>(queue: QueueEntry | string, wf: WorkflowEntry<TArgs, TReturn> | string, args?: TArgs, options?: QueueWorkflowOptions) => Promise<Run<TReturn>>;
+    sendMessage: <T>(target: Run | string, name: MessageDefinition<T>, data: T) => Promise<void>;
+    getState: <T>(target: Run | string, key: StateDefinition<T> | string) => Promise<T>;
+};
+type Instance = ReturnType<typeof createInstance>;
+
+export { type CreateInstanceParams, type Instance, createInstance };

package/dist/index.mjs

@@ -0,0 +1,3 @@
+export * from './runtime';
+//# sourceMappingURL=index.mjs.map
+//# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"index.mjs","sourcesContent":[]}

package/dist/state-CmpqDrnz.d.mts

@@ -0,0 +1,49 @@
+type QueueRateLimit = {
+    limitPerPeriod: number;
+    period: number;
+};
+type QueueOptions = {
+    workerConcurrency?: number;
+    concurrency?: number;
+    rateLimit?: QueueRateLimit;
+    priorityEnabled?: boolean;
+    partitioningEnabled?: boolean;
+    name?: string;
+};
+declare function defineQueue(options?: QueueOptions): QueueEntry;
+type QueueEntry = () => QueueOptions;
+type QueueDefinition = QueueOptions & {
+    name: string;
+};
+
+declare enum WorkflowStatus {
+    PENDING = "PENDING",
+    QUEUED = "QUEUED",
+    SUCCESS = "SUCCESS",
+    ERROR = "ERROR",
+    CANCELLED = "CANCELLED",
+    MAX_RECOVERY_ATTEMPTS_EXCEEDED = "MAX_RECOVERY_ATTEMPTS_EXCEEDED"
+}
+type WorkflowFunction<Args extends unknown[], R> = (...args: Args) => Promise<R> | R;
+type WorkflowDefinition<TArgs extends unknown[] = unknown[], TReturn = unknown> = {
+    fn: WorkflowFunction<TArgs, TReturn>;
+    maxRecoveryAttempts?: number;
+};
+declare function defineWorkflow<TArgs extends unknown[], TReturn>(fn: WorkflowFunction<TArgs, TReturn>, options?: {
+    maxRecoveryAttempts?: number;
+}): () => WorkflowDefinition<TArgs, TReturn>;
+type WorkflowEntry<TArgs extends unknown[] = unknown[], TReturn = unknown> = () => WorkflowDefinition<TArgs, TReturn>;
+
+type MessageDefinition<T> = {
+    name: string;
+    data?: T;
+};
+declare function defineMessage<T>(name: string): MessageDefinition<T>;
+
+interface StateDefinition<T> {
+    name: string;
+    data?: T;
+}
+declare function defineState<T>(name: string): StateDefinition<T>;
+
+export { type MessageDefinition as M, type QueueEntry as Q, type StateDefinition as S, WorkflowStatus as W, type WorkflowEntry as a, type QueueDefinition as b, type QueueOptions as c, type QueueRateLimit as d, type WorkflowDefinition as e, type WorkflowFunction as f, defineMessage as g, defineQueue as h, defineState as i, defineWorkflow as j };

package/dist/workflows.d.mts

@@ -0,0 +1,19 @@
+export { M as MessageDefinition, b as QueueDefinition, Q as QueueEntry, c as QueueOptions, d as QueueRateLimit, S as StateDefinition, e as WorkflowDefinition, a as WorkflowEntry, f as WorkflowFunction, W as WorkflowStatus, g as defineMessage, h as defineQueue, i as defineState, j as defineWorkflow } from './state-CmpqDrnz.mjs';
+
+type RetryConfig = {
+    maxRetries?: number;
+    retryDelay?: number;
+    backOffRate?: number;
+};
+type StepOptions = RetryConfig & {
+    name?: string;
+};
+type StepFunction<Args extends unknown[], R> = (...args: Args) => Promise<R> | R;
+type StepDefinition<TArgs extends unknown[], TReturn> = {
+    fn: StepFunction<TArgs, TReturn>;
+    args: TArgs;
+    options: StepOptions;
+};
+declare function defineStep<TArgs extends unknown[], TReturn>(fn: StepFunction<TArgs, TReturn>, options?: StepOptions): (...args: TArgs) => StepDefinition<TArgs, TReturn>;
+
+export { type RetryConfig, type StepDefinition, type StepFunction, defineStep };

package/dist/workflows.mjs

@@ -0,0 +1,7 @@
+export * from './message';
+export * from './queue';
+export * from './state';
+export * from './step';
+export * from './workflow';
+//# sourceMappingURL=workflows.mjs.map
+//# sourceMappingURL=workflows.mjs.map

package/dist/workflows.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"workflows.mjs","sourcesContent":[]}

package/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "@usehelical/workflows",
+  "version": "0.0.1-alpha.1",
+  "description": "simple typesage durable workflows without magic",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./workflows": {
+      "types": "./dist/workflows.d.ts",
+      "import": "./dist/workflows.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint .",
+    "fmt": "prettier --write .",
+    "build": "tsup",
+    "db:generate:types": "kysely-codegen --out-file ./core/internal/db/types.ts",
+    "db:migrate": "docker-compose exec -T postgres psql -U postgres -d postgres < core/internal/db/migrations/init.up.sql",
+    "db:destroy": "docker-compose exec -T postgres psql -U postgres -d postgres -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;'"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "AGPL 3.0",
+  "devDependencies": {
+    "@electric-sql/pglite": "^0.3.15",
+    "@eslint/js": "^9.39.2",
+    "@types/pg": "^8.16.0",
+    "@vitest/coverage-v8": "^4.0.18",
+    "eslint": "^9.39.2",
+    "globals": "^17.2.0",
+    "jiti": "^2.6.1",
+    "kysely-codegen": "^0.19.0",
+    "kysely-pglite": "^0.6.1",
+    "prettier": "^3.7.4",
+    "tsup": "^8.5.1",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "^8.54.0",
+    "uuid_ossp": "link:@electric-sql/pglite/contrib/uuid_ossp",
+    "vitest": "^4.0.17"
+  },
+  "dependencies": {
+    "kysely": "^0.28.9",
+    "pg": "^8.16.3",
+    "pino": "^10.2.0",
+    "serialize-error": "^13.0.1"
+  }
+}