@dotsetlabs/bellwether 0.10.0

package/dist/workflow/executor.js

@@ -0,0 +1,611 @@
+/**
+ * Workflow executor - runs workflows and tracks data flow.
+ */
+import { StateTracker } from './state-tracker.js';
+import { buildWorkflowStepAnalysisPrompt, buildWorkflowSummaryPrompt, COMPLETION_OPTIONS, } from '../prompts/templates.js';
+import { getLogger, startTiming } from '../logging/logger.js';
+import { withTimeout, DEFAULT_TIMEOUTS, checkAborted } from '../utils/timeout.js';
+/**
+ * Default executor options (excluding callbacks and signal).
+ */
+const DEFAULT_OPTIONS = {
+    continueOnError: false,
+    stepTimeout: DEFAULT_TIMEOUTS.toolCall,
+    analyzeSteps: true,
+    generateSummary: true,
+    requireSuccessfulDependencies: true,
+    timeouts: {
+        toolCall: DEFAULT_TIMEOUTS.toolCall,
+        stateSnapshot: DEFAULT_TIMEOUTS.stateSnapshot,
+        probeTool: DEFAULT_TIMEOUTS.probeTool,
+        llmAnalysis: DEFAULT_TIMEOUTS.responseAnalysis,
+        llmSummary: DEFAULT_TIMEOUTS.profileSynthesis,
+    },
+};
+/**
+ * Executes workflows against an MCP server.
+ */
+export class WorkflowExecutor {
+    client;
+    llm;
+    tools;
+    options;
+    stepResults = [];
+    onProgress;
+    logger = getLogger('workflow-executor');
+    stateTracker;
+    timeouts;
+    constructor(client, llm, tools, options = {}) {
+        this.client = client;
+        this.llm = llm;
+        this.tools = tools;
+        this.options = options;
+        // Merge options with defaults, including nested timeouts
+        this.options = {
+            ...DEFAULT_OPTIONS,
+            ...options,
+            timeouts: { ...DEFAULT_OPTIONS.timeouts, ...options.timeouts },
+        };
+        this.timeouts = this.options.timeouts;
+        this.onProgress = options.onProgress;
+        // Initialize state tracker if enabled and LLM is available
+        // State tracking requires LLM for analysis
+        if (this.options.stateTracking?.enabled && llm) {
+            this.stateTracker = new StateTracker(client, tools, llm, this.options.stateTracking, {
+                snapshotTimeout: this.timeouts.stateSnapshot,
+                probeTimeout: this.timeouts.probeTool,
+            });
+        }
+    }
+    /**
+     * Emit progress event if callback is registered.
+     */
+    emitProgress(workflow, phase, currentStep, startTime, currentStepInfo) {
+        if (!this.onProgress)
+            return;
+        const stepsFailed = this.stepResults.filter(r => !r.success).length;
+        this.onProgress({
+            phase,
+            workflow,
+            currentStep,
+            totalSteps: workflow.steps.length,
+            currentStepInfo,
+            stepsCompleted: this.stepResults.length,
+            stepsFailed,
+            elapsedMs: Date.now() - startTime,
+        });
+    }
+    /**
+     * Execute a workflow.
+     *
+     * @param workflow - The workflow to execute
+     * @throws AbortError if the operation is aborted via signal
+     */
+    async execute(workflow) {
+        const signal = this.options.signal;
+        const done = startTiming(this.logger, `workflow:${workflow.id}`);
+        const startTime = Date.now();
+        this.stepResults = [];
+        // Check if already aborted before starting
+        checkAborted(signal, `Workflow '${workflow.id}'`);
+        this.logger.info({
+            workflowId: workflow.id,
+            workflowName: workflow.name,
+            stepCount: workflow.steps.length,
+            stateTrackingEnabled: !!this.stateTracker,
+        }, 'Starting workflow execution');
+        let success = true;
+        let failureReason;
+        let failedStepIndex;
+        // State tracking data
+        const snapshots = [];
+        const changes = [];
+        // Emit starting progress
+        this.emitProgress(workflow, 'starting', 0, startTime);
+        // Take initial state snapshot if enabled
+        if (this.stateTracker && this.options.stateTracking?.snapshotBefore) {
+            checkAborted(signal, 'Initial state snapshot');
+            this.logger.debug('Taking initial state snapshot');
+            const initialSnapshot = await this.stateTracker.takeSnapshot(-1);
+            snapshots.push(initialSnapshot);
+        }
+        for (let i = 0; i < workflow.steps.length; i++) {
+            const step = workflow.steps[i];
+            // Check for abort before each step
+            checkAborted(signal, `Workflow step ${i + 1}/${workflow.steps.length}`);
+            // Check if any dependency steps have failed (when requireSuccessfulDependencies is enabled)
+            const requireSuccessfulDeps = this.options.requireSuccessfulDependencies ?? DEFAULT_OPTIONS.requireSuccessfulDependencies;
+            if (requireSuccessfulDeps) {
+                const failedDependencies = this.getFailedDependencies(step, i);
+                if (failedDependencies.length > 0) {
+                    const failedStepNames = failedDependencies.map((idx) => `step ${idx + 1} (${workflow.steps[idx]?.tool ?? 'unknown'})`).join(', ');
+                    this.logger.debug({
+                        stepIndex: i,
+                        tool: step.tool,
+                        failedDependencies,
+                    }, 'Skipping step due to failed dependencies');
+                    const skipResult = {
+                        step,
+                        stepIndex: i,
+                        success: false,
+                        response: null,
+                        error: `Skipped: depends on failed ${failedStepNames}`,
+                        resolvedArgs: step.args ?? {},
+                        durationMs: 0,
+                    };
+                    this.stepResults.push(skipResult);
+                    // Continue to next step (don't break, let the workflow continue)
+                    // The step failure will be handled by the existing logic
+                    if (!step.optional && !this.options.continueOnError) {
+                        success = false;
+                        failureReason = skipResult.error;
+                        failedStepIndex = i;
+                        break;
+                    }
+                    continue;
+                }
+            }
+            // Emit executing progress before step
+            this.emitProgress(workflow, 'executing', i, startTime, step);
+            const stepResult = await this.executeStep(step, i, workflow);
+            this.stepResults.push(stepResult);
+            // Take snapshot after each step if enabled
+            if (this.stateTracker && this.options.stateTracking?.snapshotAfterEachStep) {
+                checkAborted(signal, `State snapshot after step ${i + 1}`);
+                const snapshot = await this.stateTracker.takeSnapshot(i);
+                snapshots.push(snapshot);
+                // Compare with previous snapshot to detect changes
+                if (snapshots.length >= 2) {
+                    const prevSnapshot = snapshots[snapshots.length - 2];
+                    const stepChanges = this.stateTracker.compareSnapshots(prevSnapshot, snapshot, i);
+                    changes.push(...stepChanges);
+                }
+            }
+            if (!stepResult.success) {
+                if (!step.optional && !this.options.continueOnError) {
+                    success = false;
+                    failureReason = stepResult.error ?? 'Step failed';
+                    failedStepIndex = i;
+                    break;
+                }
+            }
+        }
+        // Take final state snapshot if enabled
+        if (this.stateTracker && this.options.stateTracking?.snapshotAfter) {
+            checkAborted(signal, 'Final state snapshot');
+            this.logger.debug('Taking final state snapshot');
+            const finalSnapshot = await this.stateTracker.takeSnapshot(this.stepResults.length - 1);
+            // If we haven't been taking per-step snapshots, compare initial to final
+            if (!this.options.stateTracking?.snapshotAfterEachStep && snapshots.length > 0) {
+                const initialSnapshot = snapshots[0];
+                const overallChanges = this.stateTracker.compareSnapshots(initialSnapshot, finalSnapshot, this.stepResults.length - 1);
+                changes.push(...overallChanges);
+            }
+            snapshots.push(finalSnapshot);
+        }
+        // Build data flow graph
+        const dataFlow = this.buildDataFlowGraph(workflow);
+        // Build state tracking result
+        let stateTracking;
+        if (this.stateTracker) {
+            checkAborted(signal, 'State tracking summary');
+            const toolRoles = this.stateTracker.getAllToolInfo();
+            let dependencies = this.stateTracker.inferDependencies(this.stepResults);
+            // Verify dependencies if we have snapshots
+            if (snapshots.length > 0 && changes.length > 0) {
+                dependencies = this.stateTracker.verifyDependencies(dependencies, snapshots, changes);
+            }
+            const summary = await this.stateTracker.generateSummary({
+                snapshots,
+                changes,
+                dependencies,
+                toolRoles,
+            });
+            stateTracking = {
+                snapshots,
+                changes,
+                dependencies,
+                toolRoles,
+                summary,
+            };
+            this.logger.debug({
+                snapshotCount: snapshots.length,
+                changeCount: changes.length,
+                dependencyCount: dependencies.length,
+            }, 'State tracking complete');
+        }
+        // Generate summary if requested
+        let summary;
+        if (this.options.generateSummary) {
+            checkAborted(signal, 'Workflow summary generation');
+            // Emit summarizing progress
+            this.emitProgress(workflow, 'summarizing', workflow.steps.length, startTime);
+            summary = await this.generateWorkflowSummary(workflow, this.stepResults, success);
+        }
+        // Emit complete progress
+        this.emitProgress(workflow, 'complete', workflow.steps.length, startTime);
+        const durationMs = Date.now() - startTime;
+        this.logger.info({
+            workflowId: workflow.id,
+            success,
+            stepsCompleted: this.stepResults.length,
+            stepsFailed: this.stepResults.filter(r => !r.success).length,
+            durationMs,
+        }, 'Workflow execution complete');
+        done();
+        return {
+            workflow,
+            steps: this.stepResults,
+            success,
+            failureReason,
+            failedStepIndex,
+            durationMs,
+            dataFlow,
+            summary,
+            stateTracking,
+        };
+    }
+    /**
+     * Execute a single workflow step.
+     */
+    async executeStep(step, stepIndex, workflow) {
+        const startTime = Date.now();
+        // Verify tool exists
+        const tool = this.tools.find(t => t.name === step.tool);
+        if (!tool) {
+            return {
+                step,
+                stepIndex,
+                success: false,
+                response: null,
+                error: `Tool not found: ${step.tool}`,
+                resolvedArgs: {},
+                durationMs: Date.now() - startTime,
+            };
+        }
+        // Resolve arguments (apply mapping from previous steps)
+        let resolvedArgs;
+        try {
+            resolvedArgs = this.resolveArguments(step, stepIndex);
+        }
+        catch (error) {
+            return {
+                step,
+                stepIndex,
+                success: false,
+                response: null,
+                error: `Failed to resolve arguments: ${error instanceof Error ? error.message : String(error)}`,
+                resolvedArgs: step.args ?? {},
+                durationMs: Date.now() - startTime,
+            };
+        }
+        // Execute the tool call with timeout
+        let response = null;
+        let error;
+        const stepTimeout = this.options.stepTimeout ?? DEFAULT_OPTIONS.stepTimeout;
+        try {
+            response = await withTimeout(this.client.callTool(step.tool, resolvedArgs), stepTimeout, `Tool call '${step.tool}'`);
+            if (response.isError) {
+                error = this.extractErrorMessage(response);
+            }
+        }
+        catch (e) {
+            error = e instanceof Error ? e.message : String(e);
+        }
+        // Run assertions
+        const assertionResults = step.assertions
+            ? this.runAssertions(step.assertions, response)
+            : undefined;
+        const assertionsFailed = assertionResults?.some(r => !r.passed) ?? false;
+        const success = !error && !assertionsFailed;
+        // Generate analysis if requested
+        let analysis;
+        if (this.options.analyzeSteps) {
+            analysis = await this.analyzeStep(step, stepIndex, workflow, response, error);
+        }
+        return {
+            step,
+            stepIndex,
+            success,
+            response,
+            error,
+            resolvedArgs,
+            assertionResults,
+            durationMs: Date.now() - startTime,
+            analysis,
+        };
+    }
+    /**
+     * Resolve arguments for a step, applying mappings from previous step outputs.
+     */
+    resolveArguments(step, stepIndex) {
+        const args = { ...(step.args ?? {}) };
+        if (!step.argMapping) {
+            return args;
+        }
+        for (const [paramName, pathExpr] of Object.entries(step.argMapping)) {
+            const value = this.resolvePath(pathExpr, stepIndex);
+            if (value !== undefined) {
+                args[paramName] = value;
+            }
+        }
+        return args;
+    }
+    /**
+     * Resolve a JSONPath-like expression against previous step results.
+     * Supports: $steps[n].result.path.to.value
+     */
+    resolvePath(pathExpr, currentStepIndex) {
+        // Parse the path expression
+        const match = pathExpr.match(/^\$steps\[(\d+)\]\.(.+)$/);
+        if (!match) {
+            throw new Error(`Invalid path expression: ${pathExpr}. Expected format: $steps[n].path.to.value`);
+        }
+        const stepIndex = parseInt(match[1], 10);
+        const propertyPath = match[2];
+        if (stepIndex >= currentStepIndex) {
+            throw new Error(`Cannot reference step ${stepIndex} from step ${currentStepIndex} (can only reference earlier steps)`);
+        }
+        const stepResult = this.stepResults[stepIndex];
+        if (!stepResult) {
+            throw new Error(`Step ${stepIndex} has not been executed yet`);
+        }
+        if (!stepResult.success || !stepResult.response) {
+            throw new Error(`Step ${stepIndex} failed or has no response`);
+        }
+        // Navigate the property path
+        // First check if we're accessing 'result' (the raw content) or 'response' (the full response)
+        let target;
+        if (propertyPath.startsWith('result.') || propertyPath === 'result') {
+            // Extract text content from the response
+            const content = stepResult.response.content;
+            const textContent = content.find(c => c.type === 'text' && c.text !== undefined);
+            if (!textContent || textContent.text === undefined) {
+                throw new Error(`Step ${stepIndex} response has no text content`);
+            }
+            const textValue = textContent.text;
+            // Try to parse as JSON
+            try {
+                target = JSON.parse(textValue);
+            }
+            catch {
+                target = textValue;
+            }
+            // Navigate the remaining path after 'result.'
+            const remainingPath = propertyPath.replace(/^result\.?/, '');
+            if (remainingPath) {
+                target = this.navigatePath(target, remainingPath);
+            }
+        }
+        else if (propertyPath.startsWith('response.')) {
+            target = this.navigatePath(stepResult.response, propertyPath.replace(/^response\./, ''));
+        }
+        else {
+            throw new Error(`Path must start with 'result.' or 'response.': ${pathExpr}`);
+        }
+        return target;
+    }
+    /**
+     * Navigate a dot-separated path in an object.
+     */
+    navigatePath(obj, path) {
+        const parts = path.split('.');
+        let current = obj;
+        for (const part of parts) {
+            if (current === null || current === undefined) {
+                return undefined;
+            }
+            // Handle array access: field[0]
+            const arrayMatch = part.match(/^(\w+)\[(\d+)\]$/);
+            if (arrayMatch) {
+                const [, fieldName, indexStr] = arrayMatch;
+                const index = parseInt(indexStr, 10);
+                current = current[fieldName];
+                if (Array.isArray(current)) {
+                    current = current[index];
+                }
+                else {
+                    return undefined;
+                }
+            }
+            else {
+                current = current[part];
+            }
+        }
+        return current;
+    }
+    /**
+     * Run assertions against a step response.
+     */
+    runAssertions(assertions, response) {
+        return assertions.map(assertion => this.runAssertion(assertion, response));
+    }
+    /**
+     * Run a single assertion.
+     */
+    runAssertion(assertion, response) {
+        if (!response) {
+            return {
+                assertion,
+                passed: false,
+                message: assertion.message ?? 'No response to assert against',
+            };
+        }
+        // Extract the value at the path
+        let actualValue;
+        try {
+            // Parse the response content as JSON
+            const textContent = response.content.find(c => c.type === 'text' && c.text !== undefined);
+            if (!textContent || textContent.text === undefined) {
+                throw new Error('No text content in response');
+            }
+            const textValue = textContent.text;
+            let parsed;
+            try {
+                parsed = JSON.parse(textValue);
+            }
+            catch {
+                parsed = textValue;
+            }
+            actualValue = this.navigatePath(parsed, assertion.path);
+        }
+        catch (error) {
+            return {
+                assertion,
+                passed: false,
+                message: `Failed to extract path ${assertion.path}: ${error instanceof Error ? error.message : String(error)}`,
+            };
+        }
+        // Check the condition
+        let passed = false;
+        switch (assertion.condition) {
+            case 'exists':
+                passed = actualValue !== undefined && actualValue !== null;
+                break;
+            case 'truthy':
+                passed = Boolean(actualValue);
+                break;
+            case 'equals':
+                passed = actualValue === assertion.value;
+                break;
+            case 'contains':
+                if (typeof actualValue === 'string' && typeof assertion.value === 'string') {
+                    passed = actualValue.includes(assertion.value);
+                }
+                else if (Array.isArray(actualValue)) {
+                    passed = actualValue.includes(assertion.value);
+                }
+                break;
+            case 'type':
+                passed = typeof actualValue === assertion.value;
+                break;
+        }
+        return {
+            assertion,
+            passed,
+            actualValue,
+            message: passed ? undefined : (assertion.message ?? `Assertion failed: ${assertion.condition}`),
+        };
+    }
+    /**
+     * Extract error message from a tool response.
+     */
+    extractErrorMessage(response) {
+        const textContent = response.content.find(c => c.type === 'text');
+        if (textContent && 'text' in textContent) {
+            return String(textContent.text);
+        }
+        return 'Unknown error';
+    }
+    /**
+     * Get indices of dependency steps that have failed.
+     * Dependencies are steps referenced in argMapping.
+     */
+    getFailedDependencies(step, stepIndex) {
+        if (!step.argMapping) {
+            return [];
+        }
+        const failedDeps = [];
+        const seenIndices = new Set();
+        for (const pathExpr of Object.values(step.argMapping)) {
+            const match = pathExpr.match(/^\$steps\[(\d+)\]\.(.+)$/);
+            if (!match)
+                continue;
+            const depIndex = parseInt(match[1], 10);
+            // Skip if we already checked this dependency
+            if (seenIndices.has(depIndex))
+                continue;
+            seenIndices.add(depIndex);
+            // Check if the dependency step exists and has failed
+            if (depIndex >= 0 && depIndex < stepIndex) {
+                const depResult = this.stepResults[depIndex];
+                if (depResult && !depResult.success) {
+                    failedDeps.push(depIndex);
+                }
+            }
+        }
+        return failedDeps;
+    }
+    /**
+     * Build a data flow graph showing how data moves between steps.
+     */
+    buildDataFlowGraph(workflow) {
+        const edges = [];
+        for (let i = 0; i < workflow.steps.length; i++) {
+            const step = workflow.steps[i];
+            if (!step.argMapping)
+                continue;
+            for (const [targetParam, pathExpr] of Object.entries(step.argMapping)) {
+                const match = pathExpr.match(/^\$steps\[(\d+)\]\.(.+)$/);
+                if (!match)
+                    continue;
+                const fromStep = parseInt(match[1], 10);
+                const sourcePath = match[2];
+                // Try to get sample value from executed results
+                let sampleValue;
+                try {
+                    const stepResult = this.stepResults[fromStep];
+                    if (stepResult?.response) {
+                        sampleValue = this.resolvePath(pathExpr, i);
+                    }
+                }
+                catch {
+                    // Ignore resolution errors for graph building
+                }
+                edges.push({
+                    fromStep,
+                    toStep: i,
+                    sourcePath,
+                    targetParam,
+                    sampleValue,
+                });
+            }
+        }
+        return edges;
+    }
+    /**
+     * Generate LLM analysis for a step.
+     * Returns a fallback message if LLM is not available (check mode).
+     */
+    async analyzeStep(step, stepIndex, workflow, response, error) {
+        // Skip LLM analysis if no LLM is available (check mode)
+        if (!this.llm) {
+            return error ? `Step failed: ${error}` : 'Step completed.';
+        }
+        const prompt = buildWorkflowStepAnalysisPrompt({
+            workflow,
+            step,
+            stepIndex,
+            response,
+            error,
+        });
+        try {
+            return await this.llm.complete(prompt, COMPLETION_OPTIONS.workflowStepAnalysis);
+        }
+        catch {
+            return error ? `Step failed: ${error}` : 'Step completed.';
+        }
+    }
+    /**
+     * Generate a summary of the workflow execution.
+     * Returns a fallback message if LLM is not available (check mode).
+     */
+    async generateWorkflowSummary(workflow, stepResults, success) {
+        // Skip LLM summary if no LLM is available (check mode)
+        if (!this.llm) {
+            return success
+                ? `Workflow "${workflow.name}" completed successfully with ${stepResults.length} steps.`
+                : `Workflow "${workflow.name}" failed at step ${stepResults.findIndex(r => !r.success) + 1}.`;
+        }
+        const prompt = buildWorkflowSummaryPrompt({ workflow, stepResults, success });
+        try {
+            return await this.llm.complete(prompt, COMPLETION_OPTIONS.workflowSummary);
+        }
+        catch {
+            return success
+                ? `Workflow "${workflow.name}" completed successfully with ${stepResults.length} steps.`
+                : `Workflow "${workflow.name}" failed at step ${stepResults.findIndex(r => !r.success) + 1}.`;
+        }
+    }
+}
+//# sourceMappingURL=executor.js.map

package/dist/workflow/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Workflow module - chained tool testing.
+ */
+export * from './types.js';
+export * from './executor.js';
+export * from './discovery.js';
+export * from './loader.js';
+export * from './state-tracker.js';
+export * from './auto-generator.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/workflow/index.js

@@ -0,0 +1,10 @@
+/**
+ * Workflow module - chained tool testing.
+ */
+export * from './types.js';
+export * from './executor.js';
+export * from './discovery.js';
+export * from './loader.js';
+export * from './state-tracker.js';
+export * from './auto-generator.js';
+//# sourceMappingURL=index.js.map

package/dist/workflow/loader.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Workflow loader - loads workflows from YAML files.
+ */
+import type { Workflow } from './types.js';
+/** Default file name for workflow definitions */
+export declare const DEFAULT_WORKFLOWS_FILE: "bellwether-workflows.yaml";
+/**
+ * Load workflows from a YAML file.
+ * Supports both single-document and multi-document YAML (separated by ---).
+ */
+export declare function loadWorkflowsFromFile(path: string): Workflow[];
+/**
+ * Try to load workflows from the default file in a directory.
+ * Returns null if file doesn't exist.
+ *
+ * This enables auto-discovery of workflow files similar to how
+ * scenarios are auto-loaded from bellwether-tests.yaml.
+ */
+export declare function tryLoadDefaultWorkflows(directory: string): Workflow[] | null;
+/**
+ * Generate a sample workflow YAML template.
+ */
+export declare function generateSampleWorkflowYaml(): string;
+//# sourceMappingURL=loader.d.ts.map