flowcraft 1.0.0-beta.1

package/sandbox/6.rag/src/nodes/LoadAndChunkNode.ts

@@ -0,0 +1,40 @@
+import type { NodeArgs } from 'flowcraft'
+import type { RagNodeOptions } from '../types'
+import { promises as fs } from 'node:fs'
+import { Node } from 'flowcraft'
+import { DocumentChunk } from '../types'
+import { CHUNKS, DOCUMENT_PATH } from './index'
+
+export class LoadAndChunkNode extends Node<string, Map<string, DocumentChunk>> {
+	private filePath: string
+
+	constructor(options: RagNodeOptions<'load-and-chunk'>) {
+		super(options)
+		this.filePath = options.data.filePath
+	}
+
+	async exec({ ctx, logger }: NodeArgs): Promise<Map<string, DocumentChunk>> {
+		const path = ctx.get(DOCUMENT_PATH) || this.filePath
+		logger.info(`[LoadAndChunkNode] Reading and chunking file: ${path}`)
+
+		const content = await fs.readFile(path, 'utf-8')
+		const chunks = new Map<string, DocumentChunk>()
+
+		// Simple chunking strategy: split by paragraph.
+		// A real implementation might use a more sophisticated text splitter.
+		const paragraphs = content.split(/\n\s*\n/).filter(p => p.trim().length > 10)
+
+		for (const [i, paragraph] of paragraphs.entries()) {
+			const chunkId = `chunk_${i}`
+			const chunk = new DocumentChunk(chunkId, paragraph.trim(), this.filePath)
+			chunks.set(chunkId, chunk)
+		}
+
+		logger.info(`[LoadAndChunkNode] Created ${chunks.size} chunks.`)
+		return chunks
+	}
+
+	async post({ ctx, execRes }: NodeArgs<string, Map<string, DocumentChunk>>) {
+		ctx.set(CHUNKS, execRes)
+	}
+}

package/sandbox/6.rag/src/nodes/StoreInVectorDBNode.ts

@@ -0,0 +1,36 @@
+import type { NodeArgs, NodeOptions } from 'flowcraft'
+import type { DocumentChunk } from '../types'
+import { Node } from 'flowcraft'
+import { CHUNKS, EMBEDDINGS, VECTOR_DB } from './index'
+
+export class StoreInVectorDBNode extends Node {
+	constructor(options?: NodeOptions) {
+		super(options)
+	}
+
+	async exec({ ctx, logger }: NodeArgs) {
+		logger.info('[StoreInVectorDBNode] Simulating storage of chunks and vectors.')
+
+		const chunks = ctx.get(CHUNKS)
+		const embeddings = ctx.get(EMBEDDINGS)
+
+		if (!chunks || !embeddings) {
+			throw new Error('Missing chunks or embeddings in context.')
+		}
+
+		const db = new Map<string, { chunk: DocumentChunk, vector: number[] }>()
+
+		for (const [chunkId, chunk] of chunks.entries()) {
+			const vector = embeddings.get(chunkId)
+			if (vector) {
+				db.set(chunkId, { chunk, vector })
+			}
+			else {
+				logger.warn(`[StoreInVectorDBNode] No embedding found for chunk ID: ${chunkId}`)
+			}
+		}
+
+		logger.info(`[StoreInVectorDBNode] DB is ready with ${db.size} entries.`)
+		ctx.set(VECTOR_DB, db)
+	}
+}

package/sandbox/6.rag/src/nodes/VectorSearchNode.ts

@@ -0,0 +1,53 @@
+import type { NodeArgs } from 'flowcraft'
+import type { RagNodeOptions } from '../types'
+import { Node } from 'flowcraft'
+import { SearchResult } from '../types'
+import { cosineSimilarity, getEmbedding } from '../utils'
+import { SEARCH_RESULTS, VECTOR_DB } from './index'
+
+export class VectorSearchNode extends Node<void, SearchResult[]> {
+	private question: string
+	private topK: number
+
+	constructor(options: RagNodeOptions<'vector-search'>) {
+		super(options)
+		this.question = options.data.question
+		this.topK = options.data.topK
+	}
+
+	async exec({ ctx, logger }: NodeArgs): Promise<SearchResult[]> {
+		logger.info(`[VectorSearchNode] Performing vector search for question: "${this.question}"`)
+		const db = ctx.get(VECTOR_DB)
+		if (!db || db.size === 0) {
+			logger.warn('[VectorSearchNode] Vector DB is empty. Cannot perform search.')
+			return []
+		}
+
+		// 1. Get the embedding for the user's question.
+		const questionVector = await getEmbedding(this.question)
+
+		// 2. Calculate similarity between the question and all document chunks.
+		const similarities: { id: string, score: number }[] = []
+		for (const [chunkId, { vector }] of db.entries()) {
+			const score = cosineSimilarity(questionVector, vector)
+			similarities.push({ id: chunkId, score })
+		}
+
+		// 3. Sort by score and take the top K results.
+		similarities.sort((a, b) => b.score - a.score)
+		const topResults = similarities.slice(0, this.topK)
+
+		// 4. Create SearchResult instances.
+		const searchResults = topResults.map(({ id, score }) => {
+			const chunk = db.get(id)!.chunk
+			return new SearchResult(chunk, score)
+		})
+
+		logger.info(`[VectorSearchNode] Found ${searchResults.length} relevant results.`)
+		return searchResults
+	}
+
+	async post({ ctx, execRes }: NodeArgs<void, SearchResult[]>) {
+		ctx.set(SEARCH_RESULTS, execRes)
+	}
+}

package/sandbox/6.rag/src/nodes/index.ts

@@ -0,0 +1,28 @@
+import type { DocumentChunk, SearchResult } from '../types'
+import { contextKey } from 'flowcraft'
+
+// These are the type-safe keys we'll use to pass state through the workflow.
+export const DOCUMENT_PATH = contextKey<string>('document_path')
+export const CHUNKS = contextKey<Map<string, DocumentChunk>>('chunks')
+export const EMBEDDINGS = contextKey<Map<string, number[]>>('embeddings')
+export const VECTOR_DB = contextKey<Map<string, { chunk: DocumentChunk, vector: number[] }>>('vector_db')
+export const QUESTION = contextKey<string>('question')
+export const SEARCH_RESULTS = contextKey<SearchResult[]>('search_results')
+export const FINAL_ANSWER = contextKey<string>('final_answer')
+
+// A map to resolve string names from the JSON graph to our actual ContextKey symbols.
+export const keyRegistry = new Map<string, symbol>([
+	['document_path', DOCUMENT_PATH],
+	['chunks', CHUNKS],
+	['embeddings', EMBEDDINGS],
+	['vector_db', VECTOR_DB],
+	['question', QUESTION],
+	['search_results', SEARCH_RESULTS],
+	['final_answer', FINAL_ANSWER],
+])
+
+export * from './GenerateEmbeddingsNode'
+export * from './LLMProcessNode'
+export * from './LoadAndChunkNode'
+export * from './StoreInVectorDBNode'
+export * from './VectorSearchNode'

package/sandbox/6.rag/src/registry.ts

@@ -0,0 +1,23 @@
+import type { TypedNodeRegistry } from 'flowcraft'
+import type { RagNodeTypeMap } from './types'
+import { createNodeRegistry, GraphBuilder } from 'flowcraft'
+import {
+	GenerateEmbeddingsNode,
+	LLMProcessNode,
+	LoadAndChunkNode,
+	StoreInVectorDBNode,
+	VectorSearchNode,
+} from './nodes'
+
+// Create a type-safe registry that maps our node type strings to their classes.
+export const nodeRegistry = createNodeRegistry({
+	'load-and-chunk': LoadAndChunkNode,
+	'generate-embeddings': GenerateEmbeddingsNode,
+	'store-in-db': StoreInVectorDBNode,
+	'vector-search': VectorSearchNode,
+	'llm-process': LLMProcessNode,
+} as TypedNodeRegistry<RagNodeTypeMap>)
+
+// Instantiate the GraphBuilder with our registry.
+// This builder will be used in main.ts to construct the flow.
+export const ragGraphBuilder = new GraphBuilder<RagNodeTypeMap>(nodeRegistry)

package/sandbox/6.rag/src/types.ts

@@ -0,0 +1,44 @@
+import type { NodeConstructorOptions, NodeOptions } from 'flowcraft'
+
+// A class representing a single chunk of a document.
+// Using a class demonstrates a common data modeling pattern.
+export class DocumentChunk {
+	constructor(
+		public readonly id: string,
+		public readonly text: string,
+		public readonly source: string,
+		public readonly ingestedAt: Date = new Date(),
+	) { }
+}
+
+// A class representing a search result from the vector database.
+export class SearchResult {
+	constructor(
+		public readonly chunk: DocumentChunk,
+		public readonly score: number,
+	) { }
+}
+
+// A type-safe mapping of our graph node types to their expected `data` payloads.
+// This will be used by the GraphBuilder for compile-time validation.
+export interface RagNodeTypeMap {
+	'load-and-chunk': {
+		filePath: string
+	}
+	// This node will read from context, so its data payload is minimal.
+	'generate-embeddings': object
+	// This node will read from context, so its data payload is minimal.
+	'store-in-db': object
+	'vector-search': {
+		question: string
+		topK: number
+	}
+	'llm-process': {
+		promptTemplate: string
+		inputs: Record<string, string>
+	}
+}
+
+// The options object passed to each node's constructor by the GraphBuilder.
+export type RagNodeOptions<T extends keyof RagNodeTypeMap>
+	= & NodeConstructorOptions<RagNodeTypeMap[T]> & NodeOptions

package/sandbox/6.rag/src/utils.ts

@@ -0,0 +1,77 @@
+import OpenAI from 'openai'
+import SuperJSON from 'superjson'
+import 'dotenv/config'
+
+const openaiClient = new OpenAI()
+
+/**
+ * Calls the OpenAI Chat Completions API for generation tasks.
+ */
+export async function callLLM(prompt: string): Promise<string> {
+	console.log(`\n--- Sending to LLM for Generation ---\n${prompt}\n`)
+	try {
+		const response = await openaiClient.chat.completions.create({
+			model: 'gpt-4o-mini',
+			messages: [{ role: 'user', content: prompt }],
+			temperature: 0.1,
+		})
+		const result = response.choices[0].message.content || ''
+		console.log(
+			'--- Received from LLM ---',
+			'\n====================================================\n',
+			result,
+			'\n====================================================\n',
+		)
+		return result
+	}
+	catch (error: any) {
+		console.error('Error calling OpenAI API for generation:', error)
+		throw new Error(`OpenAI API call failed: ${error.message}`)
+	}
+}
+
+/**
+ * Calls the OpenAI Embeddings API.
+ */
+export async function getEmbedding(text: string): Promise<number[]> {
+	console.log(`[Embeddings API] Generating embedding for text: "${text.substring(0, 50)}..."`)
+	try {
+		const response = await openaiClient.embeddings.create({
+			model: 'text-embedding-3-small',
+			input: text.replace(/\n/g, ' '),
+		})
+		return response.data[0].embedding
+	}
+	catch (error: any) {
+		console.error('Error calling OpenAI Embeddings API:', error)
+		throw new Error(`OpenAI Embeddings API call failed: ${error.message}`)
+	}
+}
+
+/**
+ * Simulates cosine similarity between two vectors.
+ */
+export function cosineSimilarity(vecA: number[], vecB: number[]): number {
+	const dotProduct = vecA.reduce((sum, a, i) => sum + a * vecB[i], 0)
+	const magnitudeA = Math.sqrt(vecA.reduce((sum, a) => sum + a * a, 0))
+	const magnitudeB = Math.sqrt(vecB.reduce((sum, b) => sum + b * b, 0))
+	return dotProduct / (magnitudeA * magnitudeB)
+}
+
+/**
+ * Resolves a template string by replacing {{key}} with values from a data object.
+ */
+export function resolveTemplate(template: string, data: Record<string, any>): string {
+	return template.replace(/\{\{(.*?)\}\}/g, (_, key) => {
+		const value = data[key.trim()]
+		if (value === undefined || value === null) {
+			console.warn(`Template variable '{{${key.trim()}}}' not found in data.`)
+			return ''
+		}
+		// Use superjson to handle complex objects like our SearchResult class
+		if (typeof value === 'object')
+			return SuperJSON.stringify(value)
+
+		return String(value)
+	})
+}

package/sandbox/6.rag/tsconfig.json

@@ -0,0 +1,3 @@
+{
+	"extends": "../tsconfig.json"
+}

package/sandbox/tsconfig.json

@@ -0,0 +1,13 @@
+{
+	"compilerOptions": {
+		"target": "esnext",
+		"module": "esnext",
+		"moduleResolution": "bundler",
+		"strict": true,
+		"esModuleInterop": true,
+		"skipLibCheck": true
+	},
+	"include": [
+		"**/*.ts"
+	]
+}

package/src/builder/collection.test.ts

@@ -0,0 +1,287 @@
+import type { AbstractNode, Logger, NodeArgs, RunOptions } from '../workflow'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { contextKey, Flow, Node, TypedContext } from '../workflow'
+import {
+	BatchFlow,
+	filterCollection,
+	mapCollection,
+	ParallelBatchFlow,
+	ParallelFlow,
+	reduceCollection,
+	SequenceFlow,
+} from './collection'
+
+function createMockLogger(): Logger {
+	return {
+		debug: vi.fn(),
+		info: vi.fn(),
+		warn: vi.fn(),
+		error: vi.fn(),
+	}
+}
+
+let mockLogger = createMockLogger()
+let runOptions: RunOptions = { logger: mockLogger }
+
+afterEach(() => {
+	mockLogger = createMockLogger()
+	runOptions = { logger: mockLogger }
+})
+
+const PROCESSED_IDS = contextKey<number[]>('processed_ids')
+const BATCH_RESULTS = contextKey<string[]>('batch_results')
+const VALUE = contextKey<number>('value')
+const PATH = contextKey<string[]>('path')
+
+class ProcessItemNode extends Node {
+	async exec({ ctx, params }: NodeArgs) {
+		const id: number = params.id
+		const value: string = params.value
+
+		const processed = ctx.get(PROCESSED_IDS) ?? []
+		ctx.set(PROCESSED_IDS, [...processed, id])
+
+		const results = ctx.get(BATCH_RESULTS) ?? []
+		const newResult = `Item ${id}: Processed ${value}`
+		ctx.set(BATCH_RESULTS, [...results, newResult])
+	}
+}
+
+class AddNode extends Node {
+	constructor(private number: number) { super() }
+	async exec({ ctx }: NodeArgs) {
+		const current = ctx.get(VALUE) ?? 0
+		ctx.set(VALUE, current + this.number)
+		const path = ctx.get(PATH) ?? []
+		ctx.set(PATH, [...path, `add${this.number}`])
+	}
+}
+
+describe('sequenceFlow', () => {
+	it('should create and run a linear sequence of nodes', async () => {
+		const ctx = new TypedContext([[VALUE, 1]])
+		const flow = new SequenceFlow(
+			new AddNode(5), // 1 + 5 = 6
+			new AddNode(10), // 6 + 10 = 16
+		)
+		await flow.run(ctx, runOptions)
+		expect(ctx.get(VALUE)).toBe(16)
+	})
+})
+
+describe('batchFlow (Sequential)', () => {
+	class TestSequentialBatchFlow extends BatchFlow {
+		protected nodeToRun: AbstractNode = new ProcessItemNode()
+
+		constructor(private items: any[]) {
+			super()
+		}
+
+		async prep() {
+			return this.items
+		}
+	}
+	it('should process all items in the specified order', async () => {
+		const ctx = new TypedContext()
+		const flow = new TestSequentialBatchFlow([ // Use the updated test class
+			{ id: 1, value: 'A' },
+			{ id: 2, value: 'B' },
+			{ id: 3, value: 'C' },
+		])
+		await flow.run(ctx, runOptions)
+		expect(ctx.get(PROCESSED_IDS)).toEqual([1, 2, 3])
+		expect(ctx.get(BATCH_RESULTS)).toEqual([
+			'Item 1: Processed A',
+			'Item 2: Processed B',
+			'Item 3: Processed C',
+		])
+	})
+	it('should complete successfully with an empty batch', async () => {
+		const ctx = new TypedContext()
+		const flow = new TestSequentialBatchFlow([])
+		await flow.run(ctx, runOptions)
+		expect(ctx.get(PROCESSED_IDS)).toBeUndefined()
+		expect(ctx.get(BATCH_RESULTS)).toBeUndefined()
+		expect(mockLogger.info).toHaveBeenCalledWith(
+			'[BatchFlow] Starting sequential processing of 0 items.',
+		)
+	})
+	it('should pass parent flow parameters to each batch item', async () => {
+		const ctx = new TypedContext()
+		const flow = new TestSequentialBatchFlow([
+			{ id: 1 },
+			{ id: 2 },
+		])
+		flow.withParams({ value: 'shared' })
+		await flow.run(ctx, runOptions)
+		expect(ctx.get(BATCH_RESULTS)).toEqual([
+			'Item 1: Processed shared',
+			'Item 2: Processed shared',
+		])
+	})
+})
+
+describe('parallelBatchFlow', () => {
+	class TestParallelBatchFlow extends ParallelBatchFlow {
+		protected nodeToRun: AbstractNode = new ProcessItemNode()
+
+		constructor(private items: any[]) {
+			super()
+		}
+
+		async prep() {
+			return this.items
+		}
+	}
+	it('should process all items', async () => {
+		const ctx = new TypedContext()
+		const flow = new TestParallelBatchFlow([
+			{ id: 1, value: 'A' },
+			{ id: 2, value: 'B' },
+			{ id: 3, value: 'C' },
+		])
+		await flow.run(ctx, runOptions)
+		// In parallel, order is not guaranteed, so we check for presence and size.
+		const processedIds = ctx.get(PROCESSED_IDS)
+		expect(processedIds).toHaveLength(3)
+		expect(processedIds).toContain(1)
+		expect(processedIds).toContain(2)
+		expect(processedIds).toContain(3)
+		const results = ctx.get(BATCH_RESULTS)
+		expect(results).toHaveLength(3)
+		expect(results).toContain('Item 1: Processed A')
+		expect(results).toContain('Item 2: Processed B')
+		expect(results).toContain('Item 3: Processed C')
+	})
+	it('should complete successfully with an empty batch', async () => {
+		const ctx = new TypedContext()
+		const flow = new TestParallelBatchFlow([])
+		await flow.run(ctx, runOptions)
+		expect(ctx.get(PROCESSED_IDS)).toBeUndefined()
+		expect(ctx.get(BATCH_RESULTS)).toBeUndefined()
+		expect(mockLogger.info).toHaveBeenCalledWith(
+			'[ParallelBatchFlow] Starting parallel processing of 0 items.',
+		)
+	})
+})
+
+describe('parallelFlow', () => {
+	it('should run all nodes in parallel and then proceed', async () => {
+		const ctx = new TypedContext([[VALUE, 0]])
+		const pFlow = new ParallelFlow([
+			new AddNode(1),
+			new AddNode(10),
+			new AddNode(100),
+		])
+		const finalNode = new AddNode(1000)
+		pFlow.next(finalNode)
+
+		await new Flow(pFlow).run(ctx, runOptions)
+
+		// The parallel AddNodes will race. The final value depends on execution order,
+		// but the path should be correct. Let's verify the final step.
+		// Expected path: add1, add10, add100 (in any order), then add1000
+		const path = ctx.get(PATH)
+		expect(path).toHaveLength(4)
+		expect(path).toContain('add1')
+		expect(path).toContain('add10')
+		expect(path).toContain('add100')
+		expect(path![3]).toBe('add1000') // Final node must be last
+
+		// Verify the final sum
+		expect(ctx.get(VALUE)).toBe(1111)
+	})
+
+	it('should handle an empty parallel flow', async () => {
+		const ctx = new TypedContext()
+		const pFlow = new ParallelFlow([])
+		const finalNode = new AddNode(5)
+		pFlow.next(finalNode)
+
+		await new Flow(pFlow).run(ctx, runOptions)
+		expect(ctx.get(VALUE)).toBe(5)
+	})
+})
+
+describe('functionalHelpers', () => {
+	describe('mapCollection', () => {
+		it('should map items using a synchronous function', async () => {
+			const items = [1, 2, 3]
+			const double = (n: number) => n * 2
+			const flow = mapCollection(items, double)
+			const result = await flow.run(new TypedContext(), runOptions)
+			expect(result).toEqual([2, 4, 6])
+		})
+		it('should map items using an asynchronous function', async () => {
+			const items = ['a', 'b', 'c']
+			const toUpper = async (s: string) => {
+				await new Promise(resolve => setTimeout(resolve, 1))
+				return s.toUpperCase()
+			}
+			const flow = mapCollection(items, toUpper)
+			const result = await flow.run(new TypedContext(), runOptions)
+			expect(result).toEqual(['A', 'B', 'C'])
+		})
+		it('should handle an empty collection', async () => {
+			const items: number[] = []
+			const double = (n: number) => n * 2
+			const flow = mapCollection(items, double)
+			const result = await flow.run(new TypedContext(), runOptions)
+			expect(result).toEqual([])
+		})
+	})
+
+	describe('filterCollection', () => {
+		it('should filter items using a synchronous predicate', async () => {
+			const items = [1, 2, 3, 4, 5]
+			const isEven = (n: number) => n % 2 === 0
+			const flow = filterCollection(items, isEven)
+			const result = await flow.run(new TypedContext(), runOptions)
+			expect(result).toEqual([2, 4])
+		})
+		it('should filter items using an asynchronous predicate', async () => {
+			const items = ['short', 'long-word', 'tiny', 'another-long-word']
+			const isLong = async (s: string) => {
+				await new Promise(resolve => setTimeout(resolve, 1))
+				return s.length > 5
+			}
+			const flow = filterCollection(items, isLong)
+			const result = await flow.run(new TypedContext(), runOptions)
+			expect(result).toEqual(['long-word', 'another-long-word'])
+		})
+		it('should handle an empty collection', async () => {
+			const items: string[] = []
+			const isLong = (s: string) => s.length > 5
+			const flow = filterCollection(items, isLong)
+			const result = await flow.run(new TypedContext(), runOptions)
+			expect(result).toEqual([])
+		})
+	})
+
+	describe('reduceCollection', () => {
+		it('should reduce a collection using a synchronous reducer', async () => {
+			const items = [1, 2, 3, 4]
+			const sum = (acc: number, val: number) => acc + val
+			const flow = reduceCollection(items, sum, 0)
+			const result = await flow.run(new TypedContext(), runOptions)
+			expect(result).toBe(10)
+		})
+		it('should reduce a collection using an asynchronous reducer', async () => {
+			const items = ['a', 'b', 'c']
+			const concatUpper = async (acc: string, val: string) => {
+				await new Promise(resolve => setTimeout(resolve, 1))
+				return acc + val.toUpperCase()
+			}
+			const flow = reduceCollection(items, concatUpper, 'start:')
+			const result = await flow.run(new TypedContext(), runOptions)
+			expect(result).toBe('start:ABC')
+		})
+		it('should return the initial value for an empty collection', async () => {
+			const items: number[] = []
+			const sum = (acc: number, val: number) => acc + val
+			const flow = reduceCollection(items, sum, 100)
+			const result = await flow.run(new TypedContext(), runOptions)
+			expect(result).toBe(100)
+		})
+	})
+})