keystone-cli 1.0.3 → 1.1.0

package/src/runner/{foreach-executor.ts → executors/foreach-executor.ts}

@@ -1,12 +1,13 @@
 import { randomUUID } from 'node:crypto';
-import type { WorkflowDb } from '../db/workflow-db.ts';
-import { type ExpressionContext, ExpressionEvaluator } from '../expression/evaluator.ts';
-import type { Step } from '../parser/schema.ts';
-import { StepStatus, WorkflowStatus } from '../types/status.ts';
-import type { Logger } from '../utils/logger.ts';
-import type { ResourcePoolManager } from './resource-pool.ts';
-import { WorkflowSuspendedError } from './step-executor.ts';
-import type { ForeachStepContext, StepContext } from './workflow-runner.ts';
+import type { WorkflowDb } from '../../db/workflow-db.ts';
+import { type ExpressionContext, ExpressionEvaluator } from '../../expression/evaluator.ts';
+import type { Step } from '../../parser/schema.ts';
+import { StepStatus, type StepStatusType, WorkflowStatus } from '../../types/status.ts';
+import { LIMITS } from '../../utils/constants.ts';
+import type { Logger } from '../../utils/logger.ts';
+import type { ResourcePoolManager } from '../resource-pool.ts';
+import type { ForeachStepContext, StepContext } from '../workflow-state.ts';
+import { WorkflowSuspendedError } from './types.ts';
 
 export type ExecuteStepCallback = (
   step: Step,
@@ -15,7 +16,6 @@ export type ExecuteStepCallback = (
 ) => Promise<StepContext>;
 
 export class ForeachExecutor {
-  private static readonly MEMORY_WARNING_THRESHOLD = 1000;
   private hasWarnedMemory = false;
 
   constructor(
@@ -77,14 +77,34 @@ export class ForeachExecutor {
       throw new Error('Step is not a foreach step');
     }
 
-    const items = ExpressionEvaluator.evaluate(step.foreach, baseContext);
-    if (!Array.isArray(items)) {
-      throw new Error(`foreach expression must evaluate to an array: ${step.foreach}`);
+    let items: unknown[];
+    const persistedItems = existingContext?.foreachItems;
+    if (Array.isArray(persistedItems)) {
+      items = persistedItems;
+    } else {
+      if (persistedItems !== undefined) {
+        this.logger.warn(
+          `  ⚠️  Warning: Persisted foreach items for step ${step.id} are invalid. Re-evaluating expression.`
+        );
+      }
+      const evaluatedItems = ExpressionEvaluator.evaluate(step.foreach, baseContext);
+      if (!Array.isArray(evaluatedItems)) {
+        throw new Error(`foreach expression must evaluate to an array: ${step.foreach}`);
+      }
+      items = evaluatedItems;
+    }
+
+    // Validate iteration count to prevent memory exhaustion
+    if (items.length > LIMITS.MAX_FOREACH_ITERATIONS) {
+      throw new Error(
+        `Foreach step "${step.id}" exceeds maximum iteration limit of ${LIMITS.MAX_FOREACH_ITERATIONS}. ` +
+          `Got ${items.length} items. Consider batching or reducing the dataset.`
+      );
     }
 
     this.logger.log(`  ⤷ Executing step ${step.id} for ${items.length} items`);
 
-    if (items.length > ForeachExecutor.MEMORY_WARNING_THRESHOLD && !this.hasWarnedMemory) {
+    if (items.length > LIMITS.FOREACH_MEMORY_WARNING_THRESHOLD && !this.hasWarnedMemory) {
       this.logger.warn(
         `  ⚠️  Warning: Large foreach loop detected (${items.length} items). This may consume significant memory and lead to instability.`
       );
@@ -92,21 +112,32 @@ export class ForeachExecutor {
     }
 
     // Evaluate concurrency
-    let concurrencyLimit = items.length;
+    // Default to a safe limit (50) to prevent resource exhaustion/DoS, unless explicitly overridden.
+    const DEFAULT_MAX_CONCURRENCY = 50;
+    let concurrencyLimit = Math.min(items.length, DEFAULT_MAX_CONCURRENCY);
+
     if (step.concurrency !== undefined) {
+      let explicitConcurrency: number;
+
       if (typeof step.concurrency === 'string') {
-        concurrencyLimit = Number(ExpressionEvaluator.evaluate(step.concurrency, baseContext));
-        if (!Number.isInteger(concurrencyLimit) || concurrencyLimit <= 0) {
-          throw new Error(
-            `concurrency must evaluate to a positive integer, got: ${concurrencyLimit}`
-          );
-        }
+        explicitConcurrency = Number(ExpressionEvaluator.evaluate(step.concurrency, baseContext));
       } else {
-        concurrencyLimit = step.concurrency;
-        if (!Number.isInteger(concurrencyLimit) || concurrencyLimit <= 0) {
-          throw new Error(`concurrency must be a positive integer, got: ${concurrencyLimit}`);
-        }
+        explicitConcurrency = step.concurrency;
+      }
+
+      if (!Number.isInteger(explicitConcurrency) || explicitConcurrency <= 0) {
+        throw new Error(
+          `concurrency must evaluate to a positive integer, got: ${explicitConcurrency}`
+        );
+      }
+
+      // If user explicitly sets a higher concurrency, we respect it but log a debug message
+      if (explicitConcurrency > DEFAULT_MAX_CONCURRENCY) {
+        this.logger.debug(
+          `Step ${step.id} has explicit concurrency ${explicitConcurrency} > default ${DEFAULT_MAX_CONCURRENCY}. Proceeding.`
+        );
       }
+      concurrencyLimit = explicitConcurrency;
     }
 
     // Create parent step record in DB
@@ -121,12 +152,91 @@ export class ForeachExecutor {
       // Initialize results array
       const itemResults: StepContext[] = existingContext?.items || new Array(items.length);
       const shouldCheckDb = !!existingContext;
+      // Track estimated result size to prevent memory exhaustion
+      let estimatedResultsBytes = 0;
 
       // Ensure array is correct length
       if (itemResults.length !== items.length) {
         itemResults.length = items.length;
       }
 
+      // Optimization: Fetch all existing iterations in one query
+      // This avoids N queries in the loop
+      const existingIterations = new Map<number, any>();
+      if (shouldCheckDb) {
+        try {
+          // Use getStepIterations(runId, stepId) for optimized fetch
+          const iterations = await this.db.getStepIterations(runId, step.id);
+          for (const s of iterations) {
+            if (typeof s.iteration_index === 'number') {
+              existingIterations.set(s.iteration_index, s);
+            }
+          }
+        } catch (e) {
+          /* ignore */
+        }
+      }
+
+      // Pre-generate IDs and batch-create step records for all pending iterations
+      const iterationIds = new Map<number, string>();
+      const toCreate: Array<{
+        id: string;
+        runId: string;
+        stepId: string;
+        iterationIndex: number;
+      }> = [];
+
+      for (let i = 0; i < items.length; i++) {
+        // Skip if already in results (from existingContext)
+        if (
+          itemResults[i] &&
+          (itemResults[i].status === StepStatus.SUCCESS ||
+            itemResults[i].status === StepStatus.SKIPPED)
+        ) {
+          continue;
+        }
+
+        // Check DB for resume if needed
+        if (shouldCheckDb) {
+          const existingExec = existingIterations.get(i);
+          if (
+            existingExec &&
+            (existingExec.status === StepStatus.SUCCESS ||
+              existingExec.status === StepStatus.SKIPPED)
+          ) {
+            // Hydrate result from DB
+            let output: unknown = null;
+            try {
+              output = existingExec.output ? JSON.parse(existingExec.output) : null;
+            } catch (error) {
+              this.logger.warn(
+                `Failed to parse output for step ${step.id} iteration ${i}: ${error}`
+              );
+            }
+            itemResults[i] = {
+              output,
+              outputs:
+                typeof output === 'object' && output !== null && !Array.isArray(output)
+                  ? (output as Record<string, unknown>)
+                  : {},
+              status: existingExec.status as StepStatusType,
+              error: existingExec.error || undefined,
+            } as StepContext;
+            continue;
+          }
+        }
+
+        // Needs execution
+        const id = randomUUID();
+        iterationIds.set(i, id);
+        toCreate.push({ id, runId, stepId: step.id, iterationIndex: i });
+      }
+
+      // Batch create all pending iterations
+      if (toCreate.length > 0) {
+        await this.db.batchCreateSteps(toCreate);
+      }
+
       // Worker pool implementation
       let currentIndex = 0;
       let aborted = false;
@@ -149,7 +259,7 @@ export class ForeachExecutor {
 
             const item = items[i];
 
-            // Skip if already successful or skipped
+            // Skip if already successful or skipped (either from memory or just hydrated above)
             if (
               itemResults[i] &&
               (itemResults[i].status === StepStatus.SUCCESS ||
@@ -165,61 +275,10 @@ export class ForeachExecutor {
               index: i,
             };
 
-            // Check DB again for robustness (resume flows only)
-            const existingExec = shouldCheckDb
-              ? await this.db.getStepByIteration(runId, step.id, i)
-              : undefined;
-            if (
-              existingExec &&
-              (existingExec.status === StepStatus.SUCCESS ||
-                existingExec.status === StepStatus.SKIPPED)
-            ) {
-              let output: unknown = null;
-              let itemStatus = existingExec.status as
-                | typeof StepStatus.SUCCESS
-                | typeof StepStatus.SKIPPED
-                | typeof StepStatus.FAILED;
-              let itemError: string | undefined = existingExec.error || undefined;
-
-              try {
-                output = existingExec.output ? JSON.parse(existingExec.output) : null;
-              } catch (error) {
-                this.logger.warn(
-                  `Failed to parse output for step ${step.id} iteration ${i}: ${error}`
-                );
-                output = { error: 'Failed to parse output' };
-                itemStatus = StepStatus.FAILED;
-                itemError = 'Failed to parse output';
-                aborted = true; // Fail fast if we find corrupted data
-                try {
-                  await this.db.completeStep(
-                    existingExec.id,
-                    StepStatus.FAILED,
-                    output,
-                    'Failed to parse output'
-                  );
-                } catch (dbError) {
-                  this.logger.warn(
-                    `Failed to update DB for corrupted output on step ${step.id} iteration ${i}: ${dbError}`
-                  );
-                }
-              }
-              itemResults[i] = {
-                output,
-                outputs:
-                  typeof output === 'object' && output !== null && !Array.isArray(output)
-                    ? (output as Record<string, unknown>)
-                    : {},
-                status: itemStatus,
-                error: itemError,
-              } as StepContext;
-              continue;
-            }
-
             if (aborted || this.abortSignal?.aborted) break;
 
-            const stepExecId = randomUUID();
-            await this.db.createStep(stepExecId, runId, step.id, i);
+            const stepExecId = iterationIds.get(i);
+            if (!stepExecId) continue; // Should not happen
 
             // Execute and store result
             try {
@@ -235,6 +294,23 @@ export class ForeachExecutor {
 
                 this.logger.log(`  ⤷ [${i + 1}/${items.length}] Executing iteration...`);
                 itemResults[i] = await this.executeStepFn(step, itemContext, stepExecId);
+
+                // Track result size to prevent memory exhaustion
+                if (itemResults[i]?.output !== undefined) {
+                  try {
+                    estimatedResultsBytes += JSON.stringify(itemResults[i].output).length;
+                  } catch {
+                    // If serialization fails, estimate based on type
+                    estimatedResultsBytes += 1024;
+                  }
+                  if (estimatedResultsBytes > LIMITS.MAX_FOREACH_RESULTS_BYTES) {
+                    throw new Error(
+                      `Foreach step "${step.id}" accumulated results exceed maximum size of ` +
+                        `${LIMITS.MAX_FOREACH_RESULTS_BYTES} bytes. Consider reducing output size or batching.`
+                    );
+                  }
+                }
+
                 if (
                   itemResults[i].status === StepStatus.FAILED ||
                   itemResults[i].status === StepStatus.SUSPENDED
@@ -245,6 +321,16 @@ export class ForeachExecutor {
                 release?.();
               }
             } catch (error) {
+              if (error instanceof WorkflowSuspendedError) {
+                itemResults[i] = {
+                  status: StepStatus.SUSPENDED,
+                  output: null,
+                  outputs: {},
+                  error: error.message,
+                };
+                aborted = true;
+                return;
+              }
               aborted = true;
               throw error;
             }
@@ -257,8 +343,10 @@ export class ForeachExecutor {
       const firstError = workerResults.find((r) => r.status === 'rejected') as
         | PromiseRejectedResult
         | undefined;
-      if (firstError) {
-        throw firstError.reason;
+      const error = firstError?.reason;
+
+      if (error && !(error instanceof WorkflowSuspendedError)) {
+        throw error;
       }
 
       // Aggregate results

package/src/runner/executors/human-executor.ts

@@ -0,0 +1,144 @@
+import * as readlinePromises from 'node:readline/promises';
+import type { ExpressionContext } from '../../expression/evaluator.ts';
+import { ExpressionEvaluator } from '../../expression/evaluator.ts';
+import type { HumanStep, SleepStep } from '../../parser/schema.ts';
+import type { Logger } from '../../utils/logger.ts';
+import { type StepResult, WorkflowSuspendedError, WorkflowWaitingError } from './types.ts';
+
+/**
+ * Execute a human input step
+ */
+export async function executeHumanStep(
+  step: HumanStep,
+  context: ExpressionContext,
+  logger: Logger,
+  abortSignal?: AbortSignal
+): Promise<StepResult> {
+  if (abortSignal?.aborted) {
+    throw new Error('Step canceled');
+  }
+  const message = ExpressionEvaluator.evaluateString(step.message, context);
+
+  // Check if we already have input for this step in context.inputs (from previous suspension)
+  const stepInputs = (context.inputs as Record<string, any>)?.[step.id];
+  if (stepInputs && stepInputs.__answer !== undefined) {
+    logger.log(`  ✓ Received human input: ${stepInputs.__answer}`);
+    return {
+      status: 'success',
+      output: stepInputs.__answer,
+    };
+  }
+
+  const inputType = step.inputType || 'text';
+  if (!process.stdin.isTTY) {
+    logger.log(`  ⏳ Suspending for human input: ${message}`);
+    throw new WorkflowSuspendedError(message, step.id, inputType);
+  }
+
+  const rl = readlinePromises.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  try {
+    const prompt = inputType === 'confirm' ? `${message} [Y/n] ` : `${message} `;
+    const answer = await rl.question(prompt);
+
+    if (inputType === 'confirm') {
+      const normalized = answer.trim().toLowerCase();
+      if (normalized === '') {
+        return { status: 'success', output: true };
+      }
+      if (['y', 'yes', 'true', '1'].includes(normalized)) {
+        return { status: 'success', output: true };
+      }
+      if (['n', 'no', 'false', '0'].includes(normalized)) {
+        return { status: 'success', output: false };
+      }
+      return { status: 'success', output: answer.trim() };
+    }
+
+    return { status: 'success', output: answer };
+  } finally {
+    rl.close();
+  }
+}
+
+/**
+ * Execute a sleep step
+ */
+export async function executeSleepStep(
+  step: SleepStep,
+  context: ExpressionContext,
+  logger: Logger,
+  abortSignal?: AbortSignal
+): Promise<StepResult> {
+  if (abortSignal?.aborted) {
+    throw new Error('Step canceled');
+  }
+
+  let durationMs = 0;
+  let wakeAt: string | undefined;
+
+  // Check if this is a resume of a durable sleep
+  const previousOutput = (context.steps as Record<string, any>)?.[step.id]?.output;
+  if (
+    step.durable &&
+    previousOutput &&
+    typeof previousOutput === 'object' &&
+    'wakeAt' in previousOutput
+  ) {
+    wakeAt = previousOutput.wakeAt as string;
+    const untilDate = new Date(wakeAt);
+    if (!Number.isNaN(untilDate.getTime())) {
+      durationMs = untilDate.getTime() - Date.now();
+    }
+  }
+
+  if (wakeAt) {
+    // Already handled by resume logic above
+  } else if (step.until) {
+    const untilStr = ExpressionEvaluator.evaluateString(step.until, context);
+    const untilDate = new Date(untilStr);
+    if (Number.isNaN(untilDate.getTime())) {
+      throw new Error(`Invalid 'until' date format: ${untilStr}`);
+    }
+    wakeAt = untilDate.toISOString();
+    durationMs = untilDate.getTime() - Date.now();
+  } else if (step.duration) {
+    let duration: string;
+    if (typeof step.duration === 'number') {
+      duration = `${step.duration}ms`;
+    } else {
+      duration = ExpressionEvaluator.evaluateString(step.duration, context);
+    }
+
+    // Parse duration (e.g., "10s", "1m", "1h", "50ms")
+    const match = duration.match(/^(\d+)([smh]|ms)$/);
+    if (!match) {
+      throw new Error(
+        `Invalid duration format: ${duration}. Expected e.g. "10s", "1m", "1h", "50ms"`
+      );
+    }
+    const val = Number.parseInt(match[1], 10);
+    const unit = match[2];
+    durationMs = val * (unit === 'ms' ? 1 : unit === 's' ? 1000 : unit === 'm' ? 60000 : 3600000);
+    wakeAt = new Date(Date.now() + durationMs).toISOString();
+  } else {
+    throw new Error("Sleep step requires either 'duration' or 'until'");
+  }
+
+  if (durationMs <= 0) {
+    logger.log('  ✓ Sleep duration already passed or is zero');
+    return { status: 'success', output: 'slept' };
+  }
+
+  logger.log(`  ⏳ Sleeping until ${wakeAt} (${Math.round(durationMs / 1000)}s remaining)`);
+
+  if (step.durable) {
+    throw new WorkflowWaitingError(`Sleeping until ${wakeAt}`, step.id, wakeAt);
+  }
+
+  await Bun.sleep(durationMs);
+  return { status: 'success', output: 'slept' };
+}

package/src/runner/executors/join-executor.ts

@@ -0,0 +1,75 @@
+import type { ExpressionContext } from '../../expression/evaluator.ts';
+import type { JoinStep } from '../../parser/schema.ts';
+import type { Logger } from '../../utils/logger.ts';
+import type { StepResult } from './types.ts';
+
+/**
+ * Execute a join step
+ */
+export async function executeJoinStep(
+  step: JoinStep,
+  context: ExpressionContext,
+  _logger: Logger,
+  abortSignal?: AbortSignal
+): Promise<StepResult> {
+  if (abortSignal?.aborted) {
+    throw new Error('Join operation aborted');
+  }
+  // Join step logic:
+  // It aggregates outputs from its 'needs'.
+  // Since the runner ensures dependencies are met (or processed),
+  // we just need to collect the results from context.steps.
+
+  const inputs: Record<string, unknown> = {};
+  const statusMap: Record<string, string> = {};
+  const realStatusMap: Record<string, 'success' | 'failed'> = {}; // Status considering allowFailure errors
+  const errors: string[] = [];
+
+  for (const depId of step.needs) {
+    const depContext = (context.steps as Record<string, any>)?.[depId];
+    if (depContext) {
+      inputs[depId] = depContext.output;
+      if (depContext.status) {
+        statusMap[depId] = depContext.status;
+      }
+
+      // Determine effective status:
+      // If status is success but error exists (allowFailure), treat as failed for the join condition
+      const isRealSuccess = depContext.status === 'success' && !depContext.error;
+      realStatusMap[depId] = isRealSuccess ? 'success' : 'failed';
+
+      if (depContext.error) {
+        errors.push(`Dependency ${depId} failed: ${depContext.error}`);
+      }
+    }
+  }
+
+  // Validate condition
+  const condition = step.condition || 'all';
+  const total = step.needs.length;
+  // Use realStatusMap to count successes/failures
+  const successCount = Object.values(realStatusMap).filter((s) => s === 'success').length;
+
+  let passed = false;
+
+  if (condition === 'all') {
+    passed = successCount === total;
+  } else if (condition === 'any') {
+    passed = successCount > 0;
+  } else if (typeof condition === 'number') {
+    passed = successCount >= condition;
+  }
+
+  if (!passed) {
+    return {
+      output: { inputs, status: statusMap },
+      status: 'failed',
+      error: `Join condition '${condition}' not met. Success: ${successCount}/${total}. Errors: ${errors.join('; ')}`,
+    };
+  }
+
+  return {
+    output: { inputs, status: statusMap },
+    status: 'success',
+  };
+}