flowcraft 1.0.0-beta.1

package/src/builder/collection.ts

@@ -0,0 +1,269 @@
+import type { NodeFunction } from '../functions'
+import type { AbstractNode, NodeArgs } from '../workflow'
+import { AbortError, Flow } from '../workflow'
+
+/**
+ * A `Flow` that creates a linear workflow from a sequence of nodes,
+ * automatically chaining them in order.
+ */
+export class SequenceFlow extends Flow {
+	/**
+	 * @param nodes A sequence of `Node` or `Flow` instances to be executed in order.
+	 */
+	constructor(...nodes: AbstractNode[]) {
+		if (nodes.length === 0) {
+			super()
+			return
+		}
+		super(nodes[0])
+		let current = nodes[0]
+		for (let i = 1; i < nodes.length; i++)
+			current = current.next(nodes[i])
+	}
+}
+
+/**
+ * A `Flow` that executes a collection of different nodes concurrently.
+ * This is the core of the "fan-out, fan-in" pattern for structural parallelism.
+ * After all parallel branches complete, the flow can proceed to a single successor.
+ */
+export class ParallelFlow extends Flow {
+	/**
+	 * @param nodesToRun The array of nodes to execute concurrently.
+	 */
+	constructor(protected nodesToRun: AbstractNode[]) {
+		super()
+	}
+
+	/**
+	 * Orchestrates the parallel execution of all nodes.
+	 * @internal
+	 */
+	async exec({ ctx, params, signal, logger, executor }: NodeArgs): Promise<void> {
+		if (this.nodesToRun.length === 0) {
+			logger.info('[ParallelFlow] No branches to execute in parallel.')
+			return
+		}
+
+		logger.info(`[ParallelFlow] Executing ${this.nodesToRun.length} branches in parallel...`)
+		const promises = this.nodesToRun.map(node =>
+			node._run({
+				ctx,
+				params: { ...params, ...node.params },
+				signal,
+				logger,
+				executor,
+			}),
+		)
+		const results = await Promise.allSettled(promises)
+		logger.info(`[ParallelFlow] ✓ All parallel branches finished.`)
+		// Check for and log any failures. A more robust implementation might
+		// collect these errors and decide on a specific failure action.
+		results.forEach((result) => {
+			if (result.status === 'rejected')
+				logger.error('[ParallelFlow] A parallel branch failed.', { error: result.reason })
+		})
+	}
+}
+
+/**
+ * An abstract `Flow` that processes a collection of items sequentially, one by one.
+ * Subclasses must implement the `prep` method to provide the items and the
+ * `nodeToRun` property to define the processing logic for each item.
+ */
+export abstract class BatchFlow extends Flow {
+	/**
+	 * The `Node` instance that will be executed for each item in the batch.
+	 * This must be implemented by any subclass.
+	 */
+	protected abstract nodeToRun: AbstractNode
+
+	constructor() {
+		super()
+	}
+
+	/**
+	 * (Abstract) Prepares the list of items to be processed.
+	 * This method is called once before the batch processing begins.
+	 * @param _args The arguments for the node, including `ctx` and `params`.
+	 * @returns An array or iterable of parameter objects, one for each item.
+	 * The `nodeToRun` will be executed once for each of these objects.
+	 */
+	async prep(_args: NodeArgs): Promise<Iterable<any>> {
+		return []
+	}
+
+	/**
+	 * Orchestrates the sequential execution of `nodeToRun` for each item.
+	 * @internal
+	 */
+	async exec(args: NodeArgs): Promise<null> {
+		if (!this.nodeToRun)
+			return null
+
+		const combinedParams = { ...this.params, ...args.params }
+		const batchParamsIterable = (await this.prep(args)) || []
+		const batchParamsList = Array.from(batchParamsIterable)
+		args.logger.info(`[BatchFlow] Starting sequential processing of ${batchParamsList.length} items.`)
+
+		for (const batchParams of batchParamsList) {
+			if (args.signal?.aborted)
+				throw new AbortError()
+
+			await this.nodeToRun._run({
+				ctx: args.ctx,
+				params: { ...combinedParams, ...batchParams },
+				signal: args.signal,
+				logger: args.logger,
+				executor: args.executor,
+			})
+		}
+		return null
+	}
+}
+
+/**
+ * An abstract `Flow` that processes a collection of items concurrently.
+ * Subclasses must implement the `prep` method to provide the items and the
+ * `nodeToRun` property to define the processing logic for each item.
+ * This provides a significant performance boost for I/O-bound tasks.
+ */
+export abstract class ParallelBatchFlow extends Flow {
+	/**
+	 * The `Node` instance that will be executed concurrently for each item in the batch.
+	 * This must be implemented by any subclass.
+	 */
+	protected abstract nodeToRun: AbstractNode
+
+	constructor() {
+		super()
+	}
+
+	/**
+	 * (Abstract) Prepares the list of items to be processed.
+	 * This method is called once before the batch processing begins.
+	 * @param _args The arguments for the node, including `ctx` and `params`.
+	 * @returns An array or iterable of parameter objects, one for each item.
+	 * The `nodeToRun` will be executed concurrently for each of these objects.
+	 */
+	async prep(_args: NodeArgs): Promise<Iterable<any>> {
+		return []
+	}
+
+	/**
+	 * Orchestrates the parallel execution of `nodeToRun` for each item.
+	 * @internal
+	 */
+	async exec(args: NodeArgs<any, void>): Promise<any> {
+		if (!this.nodeToRun)
+			return null
+
+		const combinedParams = { ...this.params, ...args.params }
+		const batchParamsIterable = (await this.prep(args)) || []
+		const batchParamsList = Array.from(batchParamsIterable)
+		args.logger.info(`[ParallelBatchFlow] Starting parallel processing of ${batchParamsList.length} items.`)
+
+		const promises = batchParamsList.map(batchParams =>
+			this.nodeToRun._run({
+				ctx: args.ctx,
+				params: { ...combinedParams, ...batchParams },
+				signal: args.signal,
+				logger: args.logger,
+				executor: args.executor,
+			}),
+		)
+
+		const results = await Promise.allSettled(promises)
+
+		// Optionally handle rejected promises, but don't rethrow to avoid halting the whole batch.
+		for (const result of results) {
+			if (result.status === 'rejected') {
+				args.logger.error('A parallel batch item failed.', { error: result.reason })
+			}
+		}
+
+		return results
+	}
+}
+
+/**
+ * Creates a flow that applies a mapping function to each item in a collection in parallel
+ * and returns a new array containing the results.
+ *
+ * @example
+ * const numbers = [1, 2, 3];
+ * const double = (n: number) => n * 2;
+ * const processingFlow = mapCollection(numbers, double);
+ * // When run, processingFlow's result will be [2, 4, 6]
+ *
+ * @param items The initial array of items of type `T`.
+ * @param fn An async or sync function that transforms an item from type `T` to type `U`.
+ * @returns A `Flow` instance that, when run, will output an array of type `U[]`.
+ */
+export function mapCollection<T, U>(items: T[], fn: NodeFunction<T, U>): Flow {
+	return new class extends Flow {
+		async exec(): Promise<U[]> {
+			// Using Promise.all to run the mapping function on all items concurrently.
+			const promises = items.map(item => fn(item))
+			return Promise.all(promises)
+		}
+	}()
+}
+
+/**
+ * Creates a flow that filters a collection based on a predicate function,
+ * returning a new array containing only the items that pass the predicate.
+ * The predicate is applied to all items concurrently.
+ *
+ * @example
+ * const users = [{ id: 1, admin: true }, { id: 2, admin: false }];
+ * const isAdmin = async (user: { admin: boolean }) => user.admin;
+ * const adminFilterFlow = filterCollection(users, isAdmin);
+ * // When run, the result will be [{ id: 1, admin: true }]
+ *
+ * @param items The initial array of items of type `T`.
+ * @param predicate An async or sync function that returns `true` or `false` for an item.
+ * @returns A `Flow` instance that, when run, will output a filtered array of type `T[]`.
+ */
+export function filterCollection<T>(items: T[], predicate: (item: T) => boolean | Promise<boolean>): Flow {
+	return new class extends Flow {
+		async exec(): Promise<T[]> {
+			const results = await Promise.all(items.map(item => predicate(item)))
+			return items.filter((_, index) => results[index])
+		}
+	}()
+}
+
+/**
+ * Creates a flow that reduces a collection to a single value by executing a
+ * reducer function sequentially for each item, similar to `Array.prototype.reduce()`.
+ *
+ * @example
+ * const numbers = [1, 2, 3, 4];
+ * const sumReducer = (acc: number, val: number) => acc + val;
+ * const sumFlow = reduceCollection(numbers, sumReducer, 0);
+ * // When run, the result will be 10.
+ *
+ * @param items The array of items to be reduced.
+ * @param reducer An async or sync function that processes the accumulator and the current item.
+ * @param initialValue The initial value for the accumulator.
+ * @returns A `Flow` instance that, when run, will output the final accumulated value of type `U`.
+ */
+export function reduceCollection<T, U>(
+	items: T[],
+	reducer: (accumulator: U, item: T) => U | Promise<U>,
+	initialValue: U,
+): Flow {
+	return new class extends Flow {
+		async exec(_args: NodeArgs): Promise<U> {
+			let accumulator = initialValue
+			for (const item of items) {
+				if (_args.signal?.aborted) {
+					throw new AbortError()
+				}
+				accumulator = await reducer(accumulator, item)
+			}
+			return accumulator
+		}
+	}()
+}

package/src/builder/graph.test.ts

@@ -0,0 +1,406 @@
+import type { AbstractNode, Logger, NodeArgs, RunOptions } from '../workflow'
+import type { NodeConstructorOptions, NodeRegistry, TypedWorkflowGraph, WorkflowGraph } from './graph.types'
+import { describe, expect, it, vi } from 'vitest'
+import { ConsoleLogger, contextKey, Node, TypedContext } from '../workflow'
+import { createNodeRegistry, GraphBuilder } from './graph'
+
+function createMockLogger(): Logger {
+	return {
+		debug: vi.fn(),
+		info: vi.fn(),
+		warn: vi.fn(),
+		error: vi.fn(),
+	}
+}
+
+const mockLogger = createMockLogger()
+const runOptions: RunOptions = { logger: mockLogger }
+
+describe('graphBuilder', () => {
+	const VALUE = contextKey<number>('value')
+	const PATH = contextKey<string[]>('path')
+
+	interface TestNodeTypeMap {
+		set: { value: number }
+		add: { value: number }
+		branch: { threshold: number }
+		logPath: { id: string }
+	}
+
+	class SetValueNode extends Node {
+		private value: number
+		constructor(options: NodeConstructorOptions<TestNodeTypeMap['set']>) {
+			super()
+			this.value = options.data.value
+		}
+
+		async prep({ ctx }: NodeArgs) {
+			ctx.set(VALUE, this.value)
+		}
+	}
+
+	class AddValueNode extends Node {
+		private valueToAdd: number
+		constructor(options: NodeConstructorOptions<TestNodeTypeMap['add']>) {
+			super()
+			this.valueToAdd = options.data.value
+		}
+
+		async prep({ ctx }: NodeArgs) {
+			const current = ctx.get(VALUE) ?? 0
+			ctx.set(VALUE, current + this.valueToAdd)
+		}
+	}
+
+	class ConditionalBranchNode extends Node<void, void, string> {
+		private threshold: number
+		constructor(options: NodeConstructorOptions<TestNodeTypeMap['branch']>) {
+			super()
+			this.threshold = options.data.threshold
+		}
+
+		async post({ ctx }: NodeArgs) {
+			const current = ctx.get(VALUE) ?? 0
+			return current > this.threshold ? 'over' : 'under'
+		}
+	}
+
+	class LogPathNode extends Node {
+		private pathId: string
+		constructor(options: NodeConstructorOptions<TestNodeTypeMap['logPath']>) {
+			super()
+			this.pathId = options.data.id
+		}
+
+		async prep({ ctx }: NodeArgs) {
+			const currentPath = ctx.get(PATH) ?? []
+			ctx.set(PATH, [...currentPath, this.pathId])
+		}
+	}
+
+	const testRegistry = createNodeRegistry({
+		set: SetValueNode,
+		add: AddValueNode,
+		branch: ConditionalBranchNode,
+		logPath: LogPathNode,
+	})
+
+	it('should build and run a complex graph with parallel fan-out', async () => {
+		const graph: TypedWorkflowGraph<TestNodeTypeMap> = {
+			nodes: [
+				{ id: 'start', type: 'set', data: { value: 10 } },
+				{ id: 'brancher', type: 'branch', data: { threshold: 15 } },
+				{ id: 'add-10', type: 'add', data: { value: 10 } },
+				{ id: 'log-A', type: 'logPath', data: { id: 'path_A' } }, // Parallel branch 1
+				{ id: 'log-B', type: 'logPath', data: { id: 'path_B' } }, // Parallel branch 2
+				{ id: 'final', type: 'add', data: { value: 1000 } },
+				// TRY THIS: Uncomment the lines below. TypeScript will throw an error!
+				// { id: 'error-test', type: 'add', data: { threshold: 99 } },
+				// { id: 'error-test', type: 'add', data: { WRONG_PROP: 99 } },
+			],
+			edges: [
+				{ source: 'start', target: 'brancher' },
+				{ source: 'brancher', target: 'add-10', action: 'under' },
+				// Fan-out from 'add-10'
+				{ source: 'add-10', target: 'log-A' },
+				{ source: 'add-10', target: 'log-B' },
+				// Fan-in to 'final'
+				{ source: 'log-A', target: 'final' },
+				{ source: 'log-B', target: 'final' },
+			],
+		}
+
+		const builder = new GraphBuilder(testRegistry)
+		const { flow } = builder.build(graph)
+		const ctx = new TypedContext()
+		await flow.run(ctx, runOptions)
+		expect(ctx.get(VALUE)).toBe(1020)
+		const path = ctx.get(PATH)
+		expect(path).toBeDefined()
+		expect(path).toHaveLength(2)
+		expect(path).toEqual(expect.arrayContaining(['path_A', 'path_B']))
+	})
+})
+
+describe('graphBuilder (no type safety)', () => {
+	const VALUE = contextKey<number>('value')
+	const PATH = contextKey<string[]>('path')
+
+	class SetValueNode extends Node {
+		private value: number
+		constructor(options: { data: { value: number } }) {
+			super()
+			this.value = options.data.value
+		}
+
+		async prep({ ctx }: NodeArgs) { ctx.set(VALUE, this.value) }
+	}
+
+	class AddValueNode extends Node {
+		private valueToAdd: number
+		constructor(options: { data: { value: number } }) {
+			super()
+			this.valueToAdd = options.data.value
+		}
+
+		async prep({ ctx }: NodeArgs) {
+			const current = ctx.get(VALUE) ?? 0
+			ctx.set(VALUE, current + this.valueToAdd)
+		}
+	}
+
+	class ConditionalBranchNode extends Node<void, void, string> {
+		private threshold: number
+		constructor(options: { data: { threshold: number } }) {
+			super()
+			this.threshold = options.data.threshold
+		}
+
+		async post({ ctx }: NodeArgs) {
+			const current = ctx.get(VALUE) ?? 0
+			return current > this.threshold ? 'over' : 'under'
+		}
+	}
+
+	class LogPathNode extends Node {
+		private pathId: string
+		constructor(options: { data: { id: string } }) {
+			super()
+			this.pathId = options.data.id
+		}
+
+		async prep({ ctx }: NodeArgs) {
+			const currentPath = ctx.get(PATH) ?? []
+			ctx.set(PATH, [...currentPath, this.pathId])
+		}
+	}
+
+	const testRegistry: NodeRegistry = new Map<string, new (...args: any[]) => AbstractNode>([
+		['set', SetValueNode],
+		['add', AddValueNode],
+		['branch', ConditionalBranchNode],
+		['logPath', LogPathNode],
+	])
+
+	it('should build and run a simple graph using the untyped API', async () => {
+		const graph: WorkflowGraph = {
+			nodes: [
+				{ id: 'start', type: 'set', data: { value: 10 } },
+				{ id: 'brancher', type: 'branch', data: { threshold: 15 } },
+				{ id: 'add-10', type: 'add', data: { value: 10 } },
+				{ id: 'log-A', type: 'logPath', data: { id: 'path_A' } },
+				{ id: 'log-B', type: 'logPath', data: { id: 'path_B' } },
+				{ id: 'final', type: 'add', data: { value: 1000 } },
+			],
+			edges: [
+				{ source: 'start', target: 'brancher' },
+				{ source: 'brancher', target: 'add-10', action: 'under' },
+				{ source: 'add-10', target: 'log-A' },
+				{ source: 'add-10', target: 'log-B' },
+				{ source: 'log-A', target: 'final' },
+				{ source: 'log-B', target: 'final' },
+			],
+		}
+		const builder = new GraphBuilder(testRegistry)
+		const { flow } = builder.build(graph)
+		const ctx = new TypedContext()
+		await flow.run(ctx)
+		expect(ctx.get(VALUE)).toBe(1020)
+		const path = ctx.get(PATH)
+		expect(path).toBeDefined()
+		expect(path).toHaveLength(2)
+		expect(path).toEqual(expect.arrayContaining(['path_A', 'path_B']))
+	})
+})
+
+describe('graphBuilder with sub-workflows', () => {
+	interface TestSubWorkflowNodeTypeMap {
+		append: { value: string }
+		final: Record<string, never>
+		custom_sub_workflow: {
+			workflowId: number
+			inputs: Record<string, string>
+			outputs: Record<string, string>
+		}
+	}
+
+	const PARENT_VALUE = contextKey<string>('parent_value')
+	const SUB_VALUE = contextKey<string>('sub_value')
+	const FINAL_VALUE = contextKey<string>('final_value')
+
+	class AppendStringNode extends Node {
+		private str: string
+		constructor(options: NodeConstructorOptions<TestSubWorkflowNodeTypeMap['append']>) {
+			super()
+			this.str = options.data.value
+		}
+
+		async exec({ prepRes }: NodeArgs<string>): Promise<string> {
+			return `${prepRes} -> ${this.str}`
+		}
+
+		async prep({ ctx }: NodeArgs): Promise<string> {
+			return ctx.get(SUB_VALUE) ?? ctx.get(PARENT_VALUE) ?? 'start'
+		}
+
+		async post({ ctx, execRes }: NodeArgs<string, string>) {
+			ctx.set(SUB_VALUE, execRes)
+		}
+	}
+
+	class FinalOutputNode extends Node {
+		constructor(_options: NodeConstructorOptions<TestSubWorkflowNodeTypeMap['final']>) {
+			super()
+		}
+
+		async prep({ ctx }: NodeArgs) {
+			const subResult = ctx.get(SUB_VALUE) ?? ''
+			ctx.set(FINAL_VALUE, `final: ${subResult}`)
+		}
+	}
+
+	const mockRegistry = {
+		registry: createNodeRegistry({
+			append: AppendStringNode,
+			final: FinalOutputNode,
+		}),
+		graphs: new Map<number, WorkflowGraph>([
+			[200, {
+				nodes: [
+					{ id: 'step_d', type: 'append', data: { value: 'D' } },
+					{ id: 'step_e', type: 'append', data: { value: 'E' } },
+				],
+				edges: [{ source: 'step_d', target: 'step_e' }],
+			}],
+		]),
+		getGraph(id: number) {
+			return this.graphs.get(id)
+		},
+	}
+
+	it('should correctly inline a sub-workflow and run the flattened graph', async () => {
+		const parentGraph: TypedWorkflowGraph<TestSubWorkflowNodeTypeMap> = {
+			nodes: [
+				{ id: 'step_a', type: 'append', data: { value: 'A' } },
+				{
+					id: 'the_sub',
+					type: 'custom_sub_workflow',
+					data: {
+						workflowId: 200,
+						inputs: { sub_value: 'parent_value' }, // PARENT_VALUE
+						outputs: { parent_value: 'sub_value' }, // SUB_VALUE
+					},
+				},
+				{ id: 'step_c', type: 'final', data: {} },
+			],
+			edges: [
+				{ source: 'step_a', target: 'the_sub' },
+				{ source: 'the_sub', target: 'step_c' },
+			],
+		}
+
+		const builder = new GraphBuilder(mockRegistry.registry, { registry: mockRegistry }, {
+			subWorkflowNodeTypes: ['custom_sub_workflow'],
+		})
+
+		const { flow } = builder.build(parentGraph)
+		const ctx = new TypedContext([
+			[PARENT_VALUE, 'start'],
+		])
+
+		await flow.run(ctx, runOptions)
+		expect(ctx.get(FINAL_VALUE)).toBe('final: start -> A -> D -> E')
+	})
+
+	it('should throw an error if a node with workflowId is not a registered sub-workflow type', () => {
+		const graphWithUndeclaredSub: TypedWorkflowGraph<TestSubWorkflowNodeTypeMap> = {
+			nodes: [
+				{ id: 'step_a', type: 'append', data: { value: 'A' } },
+				// This node has a `workflowId` but its type ('some_other_type') isn't in our list.
+				// Note: We cast to `any` because TS would correctly catch this error at compile time!
+				{ id: 'the_sub', type: 'some_other_type' as any, data: { workflowId: 200 } as any },
+			],
+			edges: [
+				{ source: 'step_a', target: 'the_sub' },
+			],
+		}
+
+		const builder = new GraphBuilder(mockRegistry.registry, { registry: mockRegistry }, {
+			subWorkflowNodeTypes: ['custom_sub_workflow'],
+		})
+
+		expect(() => builder.build(graphWithUndeclaredSub)).toThrow(
+			/Node with ID 'the_sub' and type 'some_other_type' contains a 'workflowId' property, but its type is not registered/,
+		)
+	})
+})
+
+describe('graphBuilder with parallel start nodes', () => {
+	const A = contextKey<string>('a')
+	const B = contextKey<string>('b')
+	const RESULT = contextKey<string>('result')
+
+	interface ParallelTestMap {
+		'set-value': { key: 'a' | 'b', value: string }
+		'combine': Record<string, never>
+	}
+
+	class SetValueNode extends Node {
+		private key: 'a' | 'b'
+		private value: string
+		constructor(options: NodeConstructorOptions<ParallelTestMap['set-value']>) {
+			super(options) // Pass options up for potential retries etc.
+			this.key = options.data.key
+			this.value = options.data.value
+		}
+
+		async exec({ ctx }: NodeArgs) {
+			const key = this.key === 'a' ? A : B
+			ctx.set(key, this.value)
+		}
+	}
+
+	class CombineNode extends Node {
+		constructor(options: NodeConstructorOptions<ParallelTestMap['combine']>) {
+			super(options)
+		}
+
+		async exec({ ctx }: NodeArgs) {
+			const valA = ctx.get(A)
+			const valB = ctx.get(B)
+			ctx.set(RESULT, `${valA}-${valB}`)
+		}
+	}
+
+	const parallelRegistry = createNodeRegistry<ParallelTestMap>({
+		'set-value': SetValueNode,
+		'combine': CombineNode,
+	})
+
+	it('should build and run a graph with multiple start nodes in parallel', async () => {
+		const graph: TypedWorkflowGraph<ParallelTestMap> = {
+			nodes: [
+				{ id: 'set-a', type: 'set-value', data: { key: 'a', value: 'Hello' } },
+				{ id: 'set-b', type: 'set-value', data: { key: 'b', value: 'World' } },
+				{ id: 'combiner', type: 'combine', data: {} },
+			],
+			edges: [
+				{ source: 'set-a', target: 'combiner' },
+				{ source: 'set-b', target: 'combiner' },
+			],
+		}
+
+		const builder = new GraphBuilder(
+			parallelRegistry,
+			{ registry: parallelRegistry },
+			{ subWorkflowNodeTypes: [] },
+			new ConsoleLogger(),
+		)
+		const { flow } = builder.build(graph)
+		const ctx = new TypedContext()
+
+		await flow.run(ctx, runOptions)
+
+		expect(ctx.get(RESULT)).toBe('Hello-World')
+	})
+})