@pikku/core 0.10.1 → 0.11.0

package/dist/wirings/workflow/pikku-workflow-service.d.ts

@@ -0,0 +1,152 @@
+import { CoreConfig, CoreSingletonServices, CreateSessionServices, SerializedError } from '../../types/core.types.js';
+import type { StepState, WorkflowRun, WorkflowStatus, WorkflowService, WorkflowStepOptions } from './workflow.types.js';
+export declare class WorkflowServiceNotInitialized extends Error {
+}
+export declare class WorkflowStepNameNotString extends Error {
+    constructor(stepName: any);
+}
+/**
+ * Abstract workflow state service
+ * Implementations provide pluggable storage backends (SQLite, PostgreSQL, etc.)
+ * Combines orchestration and step execution
+ */
+export declare abstract class PikkuWorkflowService implements WorkflowService {
+    private config;
+    private singletonServices;
+    private createSessionServices;
+    constructor();
+    setServices(singletonServices: CoreSingletonServices, createSessionServices: CreateSessionServices, { workflow }: CoreConfig): void;
+    /**
+     * Create a new workflow run
+     * @param name - Workflow name
+     * @param input - Input data for the workflow
+     * @returns Run ID
+     */
+    abstract createRun(workflowName: string, input: any): Promise<string>;
+    /**
+     * Get a workflow run by ID
+     * @param id - Run ID
+     * @returns Workflow run or null if not found
+     */
+    abstract getRun(id: string): Promise<WorkflowRun | null>;
+    /**
+     * Get workflow run history (all step attempts in chronological order)
+     * @param runId - Run ID
+     * @returns Array of step states with step names, ordered oldest to newest
+     */
+    abstract getRunHistory(runId: string): Promise<Array<StepState & {
+        stepName: string;
+    }>>;
+    /**
+     * Update workflow run status
+     * @param id - Run ID
+     * @param status - New status
+     */
+    abstract updateRunStatus(id: string, status: WorkflowStatus, output?: any, error?: SerializedError): Promise<void>;
+    /**
+     * Insert initial step state (called by orchestrator)
+     * Creates pending step in both workflow_step and workflow_step_history
+     * @param runId - Run ID
+     * @param stepName - Step cache key
+     * @param rpcName - RPC function name
+     * @param data - Step input data
+     * @param stepOptions - Step options (retries, retryDelay)
+     * @returns Step state with generated stepId
+     */
+    abstract insertStepState(runId: string, stepName: string, rpcName: string | null, data: any, stepOptions?: WorkflowStepOptions): Promise<StepState>;
+    /**
+     * Get step state by cache key (read-only)
+     * @param runId - Run ID
+     * @param stepName - Step cache key (from workflow.do)
+     * @returns Step state with attemptCount calculated from history
+     */
+    abstract getStepState(runId: string, stepName: string): Promise<StepState>;
+    /**
+     * Mark step as running
+     * Updates both workflow_step and workflow_step_history
+     * @param stepId - Step ID
+     */
+    abstract setStepRunning(stepId: string): Promise<void>;
+    /**
+     * Mark step as scheduled (queued for execution)
+     * Updates both workflow_step and workflow_step_history
+     * @param stepId - Step ID
+     */
+    abstract setStepScheduled(stepId: string): Promise<void>;
+    /**
+     * Store step result and mark as succeeded
+     * Updates both workflow_step and workflow_step_history
+     * @param stepId - Step ID
+     * @param result - Step result
+     */
+    abstract setStepResult(stepId: string, result: any): Promise<void>;
+    /**
+     * Store step error and mark as failed
+     * Updates both workflow_step and workflow_step_history
+     * @param stepId - Step ID
+     * @param error - Error object
+     */
+    abstract setStepError(stepId: string, error: Error): Promise<void>;
+    /**
+     * Create a new retry attempt for a failed step
+     * Inserts new pending step in both workflow_step and workflow_step_history
+     * Resets status to 'pending' with new stepId
+     * Copies metadata (rpcName, data, retries, retryDelay) from failed attempt
+     * @param failedStepId - Failed step ID to copy from
+     * @returns New step state for the retry attempt
+     */
+    abstract createRetryAttempt(failedStepId: string, status: 'pending' | 'running'): Promise<StepState>;
+    /**
+     * Execute function within a run lock to prevent concurrent modifications
+     * @param id - Run ID
+     * @param fn - Function to execute
+     * @returns Function result
+     */
+    abstract withRunLock<T>(id: string, fn: () => Promise<T>): Promise<T>;
+    /**
+     * Close any open connections
+     */
+    abstract close(): Promise<void>;
+    /**
+     * Resume a paused workflow by triggering the orchestrator
+     * @param runId - Run ID
+     */
+    resumeWorkflow(runId: string): Promise<void>;
+    /**
+     * Execute a workflow sleep step completion
+     * Sets the step result to null and resumes the workflow
+     * @param data - Sleeper input data
+     */
+    executeWorkflowSleep(runId: string, stepId: string): Promise<void>;
+    /**
+     * Schedule orchestrator retry with delay
+     * @param runId - Run ID
+     * @param retryDelay - Delay in milliseconds or duration string (optional)
+     */
+    protected scheduleOrchestratorRetry(runId: string, retryDelay?: number | string): Promise<void>;
+    /**
+     * Start a new workflow run
+     */
+    startWorkflow<I>(name: string, input: I, rpcService: any): Promise<{
+        runId: string;
+    }>;
+    runWorkflowJob(runId: string, rpcService: any): Promise<void>;
+    /**
+     * Execute a single workflow step (called by worker)
+     * Handles idempotency, RPC execution, result storage, retry logic, and orchestrator triggering
+     */
+    executeWorkflowStep(runId: string, stepName: string, rpcName: string, data: any, rpcService: any): Promise<void>;
+    /**
+     * Orchestrate workflow execution (called by orchestrator)
+     * Runs workflow job and handles async exceptions
+     */
+    orchestrateWorkflow(runId: string, rpcService: any): Promise<void>;
+    private verifyQueueService;
+    private rpcStep;
+    private inlineStep;
+    private sleepStep;
+    private cancelWorkflow;
+    private createWorkflowInteraction;
+    private verifyStepName;
+    private getConfig;
+}

package/dist/wirings/workflow/pikku-workflow-service.js

@@ -0,0 +1,448 @@
+import { runPikkuFunc } from '../../function/function-runner.js';
+import { pikkuState } from '../../pikku-state.js';
+import { getDurationInMilliseconds } from '../../time-utils.js';
+import { PikkuWiringTypes, } from '../../types/core.types.js';
+import { WorkflowAsyncException, WorkflowCancelledException, WorkflowNotFoundError, WorkflowRunNotFound, } from './workflow-runner.js';
+export class WorkflowServiceNotInitialized extends Error {
+}
+export class WorkflowStepNameNotString extends Error {
+    constructor(stepName) {
+        super(`Workflow step name must be a string. Received: ${typeof stepName}`);
+    }
+}
+/**
+ * Abstract workflow state service
+ * Implementations provide pluggable storage backends (SQLite, PostgreSQL, etc.)
+ * Combines orchestration and step execution
+ */
+export class PikkuWorkflowService {
+    config;
+    singletonServices;
+    createSessionServices;
+    constructor() { }
+    setServices(singletonServices, createSessionServices, { workflow }) {
+        this.singletonServices = singletonServices;
+        this.createSessionServices = createSessionServices;
+        this.config = {
+            retries: workflow?.retries ?? 0,
+            retryDelay: workflow?.retryDelay ?? 0,
+            orchestratorQueueName: workflow?.orchestratorQueueName ?? 'pikku-workflow-orchestrator',
+            stepWorkerQueueName: workflow?.stepWorkerQueueName ?? 'pikku-workflow-step-worker',
+            sleeperRPCName: workflow?.sleeperRPCName ?? 'pikkuWorkflowStepSleeper',
+        };
+    }
+    // ============================================================================
+    // Workflow Lifecycle Methods
+    // ============================================================================
+    /**
+     * Resume a paused workflow by triggering the orchestrator
+     * @param runId - Run ID
+     */
+    async resumeWorkflow(runId) {
+        const queueService = this.verifyQueueService();
+        await queueService.add(this.getConfig().orchestratorQueueName, { runId });
+    }
+    /**
+     * Execute a workflow sleep step completion
+     * Sets the step result to null and resumes the workflow
+     * @param data - Sleeper input data
+     */
+    async executeWorkflowSleep(runId, stepId) {
+        await this.setStepResult(stepId, null);
+        await this.resumeWorkflow(runId);
+    }
+    /**
+     * Schedule orchestrator retry with delay
+     * @param runId - Run ID
+     * @param retryDelay - Delay in milliseconds or duration string (optional)
+     */
+    async scheduleOrchestratorRetry(runId, retryDelay) {
+        const queueService = this.verifyQueueService();
+        await queueService.add(this.getConfig().orchestratorQueueName, { runId }, retryDelay ? { delay: getDurationInMilliseconds(retryDelay) } : undefined);
+    }
+    /**
+     * Start a new workflow run
+     */
+    async startWorkflow(name, input, rpcService) {
+        const registrations = pikkuState('workflows', 'registrations');
+        const workflow = registrations.get(name);
+        if (!workflow) {
+            throw new WorkflowNotFoundError(name);
+        }
+        // Create workflow run in state
+        const runId = await this.createRun(name, input);
+        // If queue service is available, use remote execution (queue-based)
+        // Otherwise, execute directly (inline/synchronous)
+        if (this.singletonServices?.queueService) {
+            // Queue orchestrator to start the workflow
+            await this.resumeWorkflow(runId);
+        }
+        else {
+            // No queue service - execute directly via runWorkflowJob
+            await this.runWorkflowJob(runId, rpcService);
+        }
+        return { runId };
+    }
+    async runWorkflowJob(runId, rpcService) {
+        // Get the run
+        const run = await this.getRun(runId);
+        if (!run) {
+            throw new WorkflowRunNotFound(runId);
+        }
+        // Get workflow registration
+        const registrations = pikkuState('workflows', 'registrations');
+        const workflow = registrations.get(run.workflow);
+        if (!workflow) {
+            throw new WorkflowNotFoundError(run.workflow);
+        }
+        // Use lock to prevent concurrent execution
+        await this.withRunLock(runId, async () => {
+            // Create workflow interaction object
+            const workflowInteraction = this.createWorkflowInteraction(run.workflow, runId, rpcService);
+            // Get function metadata
+            const meta = pikkuState('workflows', 'meta');
+            const workflowMeta = meta[run.workflow];
+            // Execute workflow function with workflow interaction
+            try {
+                const getAllServices = () => {
+                    let sessionServices = {};
+                    const interaction = { workflow: workflowInteraction };
+                    if (this.createSessionServices) {
+                        sessionServices = this.createSessionServices(this.singletonServices, interaction, undefined);
+                    }
+                    return {
+                        ...this.singletonServices,
+                        ...sessionServices,
+                        ...interaction,
+                    };
+                };
+                const result = await runPikkuFunc(PikkuWiringTypes.workflow, workflowMeta.workflowName, workflowMeta.pikkuFuncName, {
+                    singletonServices: this.singletonServices,
+                    interaction: { workflow: workflowInteraction },
+                    getAllServices,
+                    data: () => run.input,
+                });
+                // Workflow completed successfully
+                await this.updateRunStatus(runId, 'completed', result);
+            }
+            catch (error) {
+                // Check if it's a WorkflowAsyncException
+                if (error instanceof WorkflowAsyncException) {
+                    // Normal - workflow paused for step execution
+                    throw error;
+                }
+                // Check if it's a WorkflowCancelledException
+                if (error instanceof WorkflowCancelledException) {
+                    // Workflow was cancelled - status already updated, just rethrow
+                    throw error;
+                }
+                // Real error - mark as failed
+                await this.updateRunStatus(runId, 'failed', undefined, {
+                    message: error.message,
+                    stack: error.stack,
+                    code: error.code,
+                });
+                throw error;
+            }
+        });
+    }
+    /**
+     * Execute a single workflow step (called by worker)
+     * Handles idempotency, RPC execution, result storage, retry logic, and orchestrator triggering
+     */
+    async executeWorkflowStep(runId, stepName, rpcName, data, rpcService) {
+        // Get step state
+        let stepState = await this.getStepState(runId, stepName);
+        // Idempotency - if already succeeded, resume orchestrator and return
+        if (stepState.status === 'succeeded') {
+            await this.resumeWorkflow(runId);
+            return;
+        }
+        // Log warning if already running (race condition)
+        if (stepState.status === 'running') {
+            return;
+        }
+        // If status is 'failed', this is a retry - create new attempt history
+        if (stepState.status === 'failed') {
+            stepState = await this.createRetryAttempt(stepState.stepId, 'running');
+        }
+        if (stepState.status !== 'pending') {
+            // Mark step as running (pending or failed status)
+            await this.setStepRunning(stepState.stepId);
+        }
+        try {
+            // Execute RPC with workflow step context
+            const result = await rpcService.rpcWithInteraction(rpcName, data, {
+                workflowStep: {
+                    runId,
+                    stepId: stepState.stepId,
+                    attemptCount: stepState.attemptCount,
+                },
+            });
+            // Store result and mark succeeded
+            await this.setStepResult(stepState.stepId, result);
+            // Resume orchestrator to continue workflow
+            try {
+                await this.resumeWorkflow(runId);
+            }
+            catch (resumeError) {
+                throw resumeError;
+            }
+        }
+        catch (error) {
+            // Store error and mark failed
+            await this.setStepError(stepState.stepId, error);
+            const maxAttempts = (stepState.retries ?? 0) + 1;
+            const retriesExhausted = stepState.attemptCount >= maxAttempts;
+            if (retriesExhausted) {
+                // No more retries - resume orchestrator to mark workflow as failed
+                await this.resumeWorkflow(runId);
+            }
+            // Always throw so queue knows the job failed and can retry if needed
+            throw error;
+        }
+    }
+    /**
+     * Orchestrate workflow execution (called by orchestrator)
+     * Runs workflow job and handles async exceptions
+     */
+    async orchestrateWorkflow(runId, rpcService) {
+        try {
+            // Run workflow job (replays with caching)
+            await this.runWorkflowJob(runId, rpcService);
+        }
+        catch (error) {
+            if (error.name === 'WorkflowAsyncException' ||
+                error.name === 'WorkflowCancelledException') {
+                return;
+            }
+            await this.updateRunStatus(runId, 'failed', undefined, {
+                message: error.message,
+                stack: error.stack,
+                code: error.code,
+            });
+            throw error;
+        }
+    }
+    verifyQueueService() {
+        if (!this.singletonServices?.queueService) {
+            throw new Error('QueueService not configured. Remote workflows require a queue service.');
+        }
+        return this.singletonServices.queueService;
+    }
+    async rpcStep(runId, stepName, rpcName, data, rpcService, stepOptions) {
+        // Check if step already exists
+        let stepState;
+        try {
+            stepState = await this.getStepState(runId, stepName);
+        }
+        catch (error) {
+            // Step doesn't exist - create it
+            stepState = await this.insertStepState(runId, stepName, rpcName, data, stepOptions);
+        }
+        if (stepState.status === 'succeeded') {
+            // Return cached result
+            return stepState.result;
+        }
+        if (stepState.status === 'failed') {
+            // Step failed with retries exhausted - throw error to fail the workflow
+            const error = new Error(stepState.error?.message ||
+                `Step '${stepName}' failed after exhausting all retries`);
+            // Preserve original error properties if available
+            if (stepState.error) {
+                Object.assign(error, stepState.error);
+            }
+            throw error;
+        }
+        if (stepState.status === 'scheduled') {
+            // Step is already scheduled, pause workflow
+            throw new WorkflowAsyncException(runId, stepName);
+        }
+        // Step is pending - schedule it
+        await this.setStepScheduled(stepState.stepId);
+        // Enqueue step worker
+        if (this.singletonServices.queueService) {
+            // Map step retry options to queue job options
+            const retries = stepOptions?.retries ?? 0;
+            const retryDelay = stepOptions?.retryDelay;
+            await this.singletonServices.queueService.add(this.getConfig().stepWorkerQueueName, {
+                runId,
+                stepName,
+                rpcName,
+                data,
+            }, {
+                // attempts includes initial attempt, retries doesn't
+                attempts: retries + 1,
+                // Map retry delay to backoff
+                backoff: typeof retryDelay === 'number'
+                    ? { type: 'fixed', delay: retryDelay }
+                    : retryDelay === 'exponential'
+                        ? 'exponential'
+                        : undefined,
+            });
+        }
+        else {
+            // No queue service - execute locally
+            const retries = stepOptions?.retries ?? 0;
+            const retryDelay = stepOptions?.retryDelay;
+            const attemptCount = stepState.attemptCount;
+            try {
+                const result = await rpcService.rpcWithInteraction(rpcName, data, {
+                    workflowStep: {
+                        runId,
+                        stepId: stepState.stepId,
+                        attemptCount: stepState.attemptCount,
+                    },
+                });
+                await this.setStepResult(stepState.stepId, result);
+                return result;
+            }
+            catch (error) {
+                // Record the error (marks step as failed)
+                await this.setStepError(stepState.stepId, error);
+                // Check if we should retry
+                if (attemptCount < retries) {
+                    // Create a new pending retry attempt (copies metadata from failed step)
+                    await this.createRetryAttempt(stepState.stepId, 'pending');
+                    // Schedule orchestrator to retry after delay
+                    await this.scheduleOrchestratorRetry(runId, retryDelay);
+                    // Pause workflow - orchestrator will replay and pick up new attempt
+                    throw new WorkflowAsyncException(runId, stepName);
+                }
+                // No more retries, fail the workflow
+                throw error;
+            }
+        }
+        // Pause workflow - step will callback when done
+        throw new WorkflowAsyncException(runId, stepName);
+    }
+    async inlineStep(runId, stepName, fn, stepOptions) {
+        // Check if step already exists
+        let stepState;
+        try {
+            stepState = await this.getStepState(runId, stepName);
+        }
+        catch (error) {
+            // Step doesn't exist - create it (inline, no RPC)
+            stepState = await this.insertStepState(runId, stepName, null, // No RPC for inline steps
+            null, // No data for inline steps
+            stepOptions);
+        }
+        if (stepState.status === 'succeeded') {
+            // Return cached result
+            return stepState.result;
+        }
+        // Execute inline function
+        const retries = stepOptions?.retries ?? this.getConfig().retries;
+        const retryDelay = stepOptions?.retryDelay ?? this.getConfig().retryDelay;
+        const attemptCount = stepState.attemptCount;
+        try {
+            const result = await fn();
+            await this.setStepResult(stepState.stepId, result);
+            return result;
+        }
+        catch (error) {
+            // Record the error (marks step as failed)
+            await this.setStepError(stepState.stepId, error);
+            // Check if we should retry
+            if (attemptCount < retries) {
+                // Create a new pending retry attempt (copies metadata from failed step)
+                await this.createRetryAttempt(stepState.stepId, 'pending');
+                // Schedule orchestrator to retry after delay
+                await this.scheduleOrchestratorRetry(runId, retryDelay);
+                // Pause workflow - orchestrator will replay and pick up new attempt
+                throw new WorkflowAsyncException(runId, stepName);
+            }
+            // No more retries, fail the workflow
+            throw error;
+        }
+    }
+    async sleepStep(runId, stepName, duration) {
+        // Check if step already exists
+        let stepState;
+        try {
+            stepState = await this.getStepState(runId, stepName);
+        }
+        catch (error) {
+            // Step doesn't exist - create it (sleep step, no RPC)
+            stepState = await this.insertStepState(runId, stepName, null, // No RPC for sleep steps
+            { duration }, // Store duration as data
+            undefined // No retry options for sleep
+            );
+        }
+        if (stepState.status === 'succeeded') {
+            // Sleep already completed, return immediately
+            return;
+        }
+        if (stepState.status === 'scheduled') {
+            // Sleep is already scheduled, pause workflow
+            throw new WorkflowAsyncException(runId, stepName);
+        }
+        // Step is pending - schedule it
+        await this.setStepScheduled(stepState.stepId);
+        // Check if we have queue service (remote mode) or inline mode
+        if (this.singletonServices.schedulerService) {
+            // Remote mode - enqueue sleep worker with delay
+            await this.singletonServices.schedulerService.scheduleRPC(duration, this.getConfig().sleeperRPCName, {
+                runId,
+                stepId: stepState.stepId,
+            });
+        }
+        else {
+            // Inline mode - use setTimeout with actual duration
+            await new Promise((resolve) => setTimeout(resolve, getDurationInMilliseconds(duration)));
+            await this.setStepResult(stepState.stepId, null);
+            return;
+        }
+        // Pause workflow - sleep will callback when done
+        throw new WorkflowAsyncException(runId, stepName);
+    }
+    async cancelWorkflow(runId, reason) {
+        // Update workflow run status to cancelled
+        await this.updateRunStatus(runId, 'cancelled', undefined, {
+            message: reason || 'Workflow cancelled by user',
+            stack: '',
+            code: 'WORKFLOW_CANCELLED',
+        });
+        // Throw cancellation exception to stop workflow execution
+        throw new WorkflowCancelledException(runId, reason);
+    }
+    createWorkflowInteraction(workflowName, runId, rpcService) {
+        const workflowInteraction = {
+            workflowName,
+            runId,
+            getRun: async () => (await this.getRun(runId)),
+            // Implement workflow.do() - RPC form
+            do: async (stepName, rpcNameOrFn, dataOrOptions, options) => {
+                this.verifyStepName(stepName);
+                if (typeof rpcNameOrFn === 'string') {
+                    return await this.rpcStep(runId, stepName, rpcNameOrFn, dataOrOptions, rpcService, options);
+                }
+                else {
+                    return await this.inlineStep(runId, stepName, rpcNameOrFn, dataOrOptions);
+                }
+            },
+            // Implement workflow.sleep()
+            sleep: async (stepName, duration) => {
+                this.verifyStepName(stepName);
+                await this.sleepStep(runId, stepName, getDurationInMilliseconds(duration));
+            },
+            // Implement workflow.cancel()
+            cancel: async (reason) => {
+                await this.cancelWorkflow(runId, reason);
+            },
+        };
+        return workflowInteraction;
+    }
+    verifyStepName(stepName) {
+        if (typeof stepName !== 'string') {
+            throw new WorkflowStepNameNotString(stepName);
+        }
+    }
+    getConfig() {
+        if (!this.singletonServices || !this.config) {
+            throw new WorkflowServiceNotInitialized();
+        }
+        return this.config;
+    }
+}

package/dist/wirings/workflow/workflow-runner.d.ts

@@ -0,0 +1,35 @@
+import type { CoreWorkflow } from './workflow.types.js';
+import type { CorePikkuFunctionConfig } from '../../function/functions.types.js';
+import { PikkuError } from '../../errors/error-handler.js';
+/**
+ * Exception thrown when workflow needs to pause for async step
+ */
+export declare class WorkflowAsyncException extends Error {
+    readonly runId: string;
+    readonly stepName: string;
+    constructor(runId: string, stepName: string);
+}
+/**
+ * Exception thrown when workflow is cancelled
+ */
+export declare class WorkflowCancelledException extends Error {
+    readonly runId: string;
+    readonly reason?: string | undefined;
+    constructor(runId: string, reason?: string | undefined);
+}
+/**
+ * Error class for workflow not found
+ */
+export declare class WorkflowNotFoundError extends PikkuError {
+    constructor(name: string);
+}
+/**
+ * Error class for workflow not found
+ */
+export declare class WorkflowRunNotFound extends PikkuError {
+    constructor(runId: string);
+}
+/**
+ * Register a workflow with the system
+ */
+export declare const wireWorkflow: <PikkuFunctionConfig extends CorePikkuFunctionConfig<any, any, any> = CorePikkuFunctionConfig<any, any, any>>(workflow: CoreWorkflow<PikkuFunctionConfig>) => void;