keystone-cli 1.0.2 → 1.1.0

package/src/runner/stream-utils.ts

@@ -26,6 +26,8 @@ export async function processOpenAIStream(
   const toolCalls: LLMToolCall[] = [];
   let buffer = '';
 
+  let usage: LLMResponse['usage'];
+
   try {
     while (true) {
       const { done, value } = await reader.read();
@@ -48,9 +50,10 @@ export async function processOpenAIStream(
         try {
           const data = JSON.parse(trimmedLine.slice(6));
 
-          // Handle Copilot's occasional 'choices' missing or different structure if needed,
-          // but generally they match OpenAI.
-          // Some proxies might return null delta.
+          if (data.usage) {
+            usage = data.usage;
+          }
+
           const delta = data.choices?.[0]?.delta;
           if (!delta) continue;
 
@@ -182,5 +185,6 @@ export async function processOpenAIStream(
       content: fullContent || null,
       tool_calls: toolCalls.length > 0 ? toolCalls.filter(Boolean) : undefined,
     },
+    usage,
   };
 }

package/src/runner/test-harness.ts

@@ -32,6 +32,10 @@ export interface TestSnapshot {
   outputs: Record<string, unknown>;
 }
 
+export interface TestOptions {
+  allowSideEffects?: boolean;
+}
+
 export class TestHarness {
   private stepResults: Map<string, { status: string; output: unknown; error?: string }> = new Map();
   private mockResponses: Map<string, unknown> = new Map();
@@ -39,7 +43,8 @@ export class TestHarness {
 
   constructor(
     private workflow: Workflow,
-    private fixture: TestFixture = {}
+    private fixture: TestFixture = {},
+    private options: TestOptions = {}
   ) {
     if (fixture.mocks) {
       for (const mock of fixture.mocks) {
@@ -116,6 +121,12 @@ export class TestHarness {
       return result;
     }
 
+    if (!this.options.allowSideEffects && this.isSideEffectStep(step)) {
+      throw new Error(
+        `🛑 Safety Violation: Step "${step.id}" of type "${step.type}" attempts to execute a side-effect.\nTo allow this, set 'options.allowSideEffects: true' in your test file.\nOtherwise, provide a mock response in 'fixture.mocks'.`
+      );
+    }
+
     // Default to real execution but capture snapshot
     const result = await executeStep(step, context, logger, {
       ...options,
@@ -132,6 +143,14 @@ export class TestHarness {
     return result;
   }
 
+  private isSideEffectStep(step: Step): boolean {
+    if (['shell', 'script', 'engine', 'request', 'artifact'].includes(step.type)) return true;
+    if (step.type === 'file' && (step as any).op !== 'read') return true;
+    // LLM is generally considered "safe" (no system modification) but costly.
+    // For now we allow LLM unless mocked, as users might want to test prompt logic.
+    return false;
+  }
+
   private getMockAdapter(model: string): { adapter: LLMAdapter; resolvedModel: string } {
     return {
       resolvedModel: model,

package/src/runner/timeout.test.ts

@@ -17,4 +17,14 @@ describe('timeout', () => {
     const promise = new Promise((resolve) => setTimeout(() => resolve('ok'), 100));
     await expect(withTimeout(promise, 10, 'MyStep')).rejects.toThrow(/MyStep timed out/);
   });
+
+  it('should abort the controller when the timeout triggers', async () => {
+    const controller = new AbortController();
+    const promise = new Promise(() => {});
+
+    await expect(
+      withTimeout(promise, 10, 'SlowOp', { abortController: controller })
+    ).rejects.toThrow(TimeoutError);
+    expect(controller.signal.aborted).toBe(true);
+  });
 });

package/src/runner/timeout.ts

@@ -9,16 +9,25 @@ export class TimeoutError extends Error {
   }
 }
 
+export interface TimeoutOptions {
+  abortController?: AbortController;
+}
+
 export async function withTimeout<T>(
   promise: Promise<T>,
   timeoutMs: number,
-  operation = 'Operation'
+  operation = 'Operation',
+  options: TimeoutOptions = {}
 ): Promise<T> {
   let timeoutId: Timer | undefined;
 
   const timeoutPromise = new Promise<never>((_, reject) => {
     timeoutId = setTimeout(() => {
-      reject(new TimeoutError(`${operation} timed out after ${timeoutMs}ms`));
+      const timeoutError = new TimeoutError(`${operation} timed out after ${timeoutMs}ms`);
+      if (options.abortController && !options.abortController.signal.aborted) {
+        options.abortController.abort(timeoutError);
+      }
+      reject(timeoutError);
     }, timeoutMs);
   });
 

package/src/runner/tool-integration.test.ts

@@ -3,8 +3,8 @@ import { mkdirSync, unlinkSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import type { ExpressionContext } from '../expression/evaluator';
 import type { LlmStep, Step } from '../parser/schema';
+import { executeLlmStep } from './executors/llm-executor.ts';
 import type { LLMAdapter } from './llm-adapter';
-import { executeLlmStep } from './llm-executor';
 import type { StepResult } from './step-executor';
 
 interface MockToolCall {

package/src/runner/wait-step.test.ts

@@ -0,0 +1,102 @@
+import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test';
+import { WorkflowDb } from '../db/workflow-db';
+import type { WaitStep } from '../parser/schema';
+import { container } from '../utils/container';
+import { ConsoleLogger } from '../utils/logger';
+import { executeStep } from './step-executor';
+
+describe('Wait Step', () => {
+  let db: WorkflowDb;
+  const logger = new ConsoleLogger();
+  const context = { inputs: {}, steps: {} };
+
+  beforeEach(() => {
+    db = new WorkflowDb(':memory:');
+    container.register('db', db);
+    container.register('logger', logger);
+  });
+
+  afterEach(() => {
+    db.close();
+  });
+
+  it('should succeed when event exists and consume it by default (oneShot: true)', async () => {
+    const eventName = 'test-event';
+    const eventData = { foo: 'bar' };
+    await db.storeEvent(eventName, eventData);
+
+    const step: WaitStep = {
+      id: 'wait1',
+      type: 'wait',
+      event: eventName,
+      needs: [],
+    };
+
+    const result = await executeStep(step, context, logger, { db });
+    expect(result.status).toBe('success');
+    expect(result.output).toEqual(eventData);
+
+    // Verify event is consumed
+    const eventAfter = await db.getEvent(eventName);
+    expect(eventAfter).toBeNull();
+  });
+
+  it('should suspend when event does not exist', async () => {
+    const eventName = 'non-existent';
+    const step: WaitStep = {
+      id: 'wait1',
+      type: 'wait',
+      event: eventName,
+      needs: [],
+    };
+
+    const result = await executeStep(step, context, logger, { db });
+    expect(result.status).toBe('suspended');
+    expect(result.output).toEqual({ event: eventName });
+  });
+
+  it('should NOT consume event when oneShot is false', async () => {
+    const eventName = 'persistent-event';
+    const eventData = { hello: 'world' };
+    await db.storeEvent(eventName, eventData);
+
+    const step: WaitStep = {
+      id: 'wait1',
+      type: 'wait',
+      event: eventName,
+      oneShot: false,
+      needs: [],
+    };
+
+    const result = await executeStep(step, context, logger, { db });
+    expect(result.status).toBe('success');
+    expect(result.output).toEqual(eventData);
+
+    // Verify event STILL exists
+    const eventAfter = await db.getEvent(eventName);
+    expect(eventAfter).not.toBeNull();
+    if (eventAfter) {
+      expect(JSON.parse(eventAfter.data)).toEqual(eventData);
+    }
+  });
+
+  it('should handle sequential wait steps for the same one-shot event', async () => {
+    const eventName = 'seq-event';
+    await db.storeEvent(eventName, { count: 1 });
+
+    const step: WaitStep = {
+      id: 'wait1',
+      type: 'wait',
+      event: eventName,
+      needs: [],
+    };
+
+    // First wait succeeds and consumes
+    const result1 = await executeStep(step, context, logger, { db });
+    expect(result1.status).toBe('success');
+
+    // Second wait suspends because event is gone
+    const result2 = await executeStep(step, context, logger, { db });
+    expect(result2.status).toBe('suspended');
+  });
+});

package/src/runner/workflow-runner.test.ts

@@ -1,14 +1,23 @@
 import { afterAll, afterEach, describe, expect, it, mock, spyOn } from 'bun:test';
+import { randomUUID } from 'node:crypto';
 import { existsSync, rmSync } from 'node:fs';
+import { MemoryDb } from '../db/memory-db';
 import { WorkflowDb } from '../db/workflow-db';
 import type { Workflow } from '../parser/schema';
 import { WorkflowParser } from '../parser/workflow-parser';
 import { ConfigLoader } from '../utils/config-loader';
+import { container } from '../utils/container';
+import { ConsoleLogger } from '../utils/logger';
 import { WorkflowRegistry } from '../utils/workflow-registry';
 import { WorkflowRunner } from './workflow-runner';
 
 describe('WorkflowRunner', () => {
   const dbPath = ':memory:';
+
+  // Setup DI container for tests
+  container.register('logger', new ConsoleLogger());
+  container.register('db', new WorkflowDb(dbPath));
+  container.register('memoryDb', new MemoryDb());
   const activeSpies: Array<{ mockRestore: () => void }> = [];
   const trackSpy = <T extends { mockRestore: () => void }>(spy: T): T => {
     activeSpies.push(spy);
@@ -45,6 +54,7 @@ describe('WorkflowRunner', () => {
         id: 'step2',
         type: 'shell',
         run: 'echo "${{ steps.step1.output.stdout.trim() }} world"',
+        allowInsecure: true,
         needs: ['step1'],
       },
     ],
@@ -73,6 +83,7 @@ describe('WorkflowRunner', () => {
           id: 'print',
           type: 'shell',
           run: 'echo $TOKEN',
+          allowInsecure: true,
           needs: [],
         },
       ],
@@ -124,6 +135,7 @@ describe('WorkflowRunner', () => {
           id: 'gen',
           type: 'shell',
           run: 'echo "[1, 2, 3]"',
+          allowInsecure: true,
           transform: 'JSON.parse(output.stdout)',
           needs: [],
         },
@@ -131,6 +143,7 @@ describe('WorkflowRunner', () => {
           id: 'process',
           type: 'shell',
           run: 'echo "item-${{ item }}"',
+          allowInsecure: true,
           foreach: '${{ steps.gen.output }}',
           needs: ['gen'],
         },
@@ -209,6 +222,7 @@ describe('WorkflowRunner', () => {
           id: 's1',
           type: 'shell',
           run: 'echo "${{ inputs.name }} ${{ inputs.count }}"',
+          allowInsecure: true,
           needs: [],
         },
       ],
@@ -223,7 +237,7 @@ describe('WorkflowRunner', () => {
   });
 
   it('should validate step input schema', async () => {
-    const schemaDbPath = 'test-step-input-schema.db';
+    const schemaDbPath = `test-step-input-schema-${randomUUID()}.db`;
     const workflowWithInputSchema: Workflow = {
       name: 'step-input-schema-wf',
       steps: [
@@ -254,7 +268,7 @@ describe('WorkflowRunner', () => {
   });
 
   it('should validate step output schema', async () => {
-    const schemaDbPath = 'test-step-output-schema.db';
+    const schemaDbPath = `test-step-output-schema-${randomUUID()}.db`;
     const workflowWithOutputSchema: Workflow = {
       name: 'step-output-schema-wf',
       steps: [
@@ -368,7 +382,7 @@ describe('WorkflowRunner', () => {
   });
 
   it('should deduplicate steps using idempotencyKey within a run', async () => {
-    const idempotencyDbPath = 'test-idempotency.db';
+    const idempotencyDbPath = `test-idempotency-${randomUUID()}.db`;
     if (existsSync(idempotencyDbPath)) rmSync(idempotencyDbPath);
 
     let idempotencyHitCount = 0;
@@ -421,7 +435,7 @@ describe('WorkflowRunner', () => {
   });
 
   it('should allow disabling idempotency deduplication', async () => {
-    const idempotencyDbPath = 'test-idempotency-disabled.db';
+    const idempotencyDbPath = `test-idempotency-disabled-${randomUUID()}.db`;
     if (existsSync(idempotencyDbPath)) rmSync(idempotencyDbPath);
 
     const idempotencyWorkflow: Workflow = {
@@ -460,7 +474,7 @@ describe('WorkflowRunner', () => {
   });
 
   it('should detect in-flight idempotency keys', async () => {
-    const idempotencyDbPath = 'test-idempotency-inflight.db';
+    const idempotencyDbPath = `test-idempotency-inflight-${randomUUID()}.db`;
     if (existsSync(idempotencyDbPath)) rmSync(idempotencyDbPath);
 
     const idempotencyWorkflow: Workflow = {
@@ -469,14 +483,14 @@ describe('WorkflowRunner', () => {
         {
           id: 's1',
           type: 'sleep',
-          duration: 50,
+          duration: '50ms',
           needs: [],
           idempotencyKey: '"same-key"',
         },
         {
           id: 's2',
           type: 'sleep',
-          duration: 50,
+          duration: '50ms',
           needs: [],
           idempotencyKey: '"same-key"',
         },
@@ -489,12 +503,92 @@ describe('WorkflowRunner', () => {
     if (existsSync(idempotencyDbPath)) rmSync(idempotencyDbPath);
   });
 
+  it('should memoize deterministic steps across runs', async () => {
+    const memoizeDbPath = `test-memoize-${randomUUID()}.db`;
+    if (existsSync(memoizeDbPath)) rmSync(memoizeDbPath);
+
+    const memoizeWorkflow: Workflow = {
+      name: 'memoize-wf',
+      steps: [
+        {
+          id: 's1',
+          type: 'shell',
+          run: 'bun -e "console.log(Date.now())"',
+          allowInsecure: true,
+          memoize: true,
+          needs: [],
+        },
+      ],
+      outputs: {
+        out: '${{ steps.s1.output.stdout.trim() }}',
+      },
+    } as unknown as Workflow;
+
+    const runner1 = new WorkflowRunner(memoizeWorkflow, { dbPath: memoizeDbPath });
+    const outputs1 = await runner1.run();
+    await Bun.sleep(5);
+
+    const runner2 = new WorkflowRunner(memoizeWorkflow, { dbPath: memoizeDbPath });
+    const outputs2 = await runner2.run();
+
+    expect(outputs2.out).toBe(outputs1.out);
+
+    if (existsSync(memoizeDbPath)) rmSync(memoizeDbPath);
+  });
+
+  it('should redact memoized outputs at rest', async () => {
+    const memoizeDbPath = `test-memoize-redact-${randomUUID()}.db`;
+    if (existsSync(memoizeDbPath)) rmSync(memoizeDbPath);
+
+    const secret = 'supersecret';
+    const memoizeWorkflow: Workflow = {
+      name: 'memoize-redact-wf',
+      steps: [
+        {
+          id: 's1',
+          type: 'shell',
+          run: `echo "${secret}"`,
+          memoize: true,
+          needs: [],
+        },
+      ],
+      outputs: {
+        out: '${{ steps.s1.output.stdout.trim() }}',
+      },
+    } as unknown as Workflow;
+
+    const runner = new WorkflowRunner(memoizeWorkflow, {
+      dbPath: memoizeDbPath,
+      secrets: { TOKEN: secret },
+    });
+    await runner.run();
+
+    const db = new WorkflowDb(memoizeDbPath);
+    const step = memoizeWorkflow.steps[0] as Workflow['steps'][number];
+    const stepInputs = { run: (step as { run: string }).run };
+    const cacheKey = Bun.hash(
+      JSON.stringify({
+        type: step.type,
+        inputs: stepInputs, // shell steps put 'run' in inputs
+        env: (step as { env?: Record<string, string> }).env,
+        version: 2,
+      })
+    ).toString(16);
+    const cached = await db.getStepCache(cacheKey);
+    expect(cached).not.toBeNull();
+    expect(cached?.output).not.toContain(secret);
+    expect(JSON.parse(cached?.output).stdout).toContain('***REDACTED***');
+    db.close();
+
+    if (existsSync(memoizeDbPath)) rmSync(memoizeDbPath);
+  });
+
   it('should execute steps in parallel', async () => {
     const parallelWorkflow: Workflow = {
       name: 'parallel-wf',
       steps: [
-        { id: 's1', type: 'sleep', duration: 100, needs: [] },
-        { id: 's2', type: 'sleep', duration: 100, needs: [] },
+        { id: 's1', type: 'sleep', duration: '100ms', needs: [] },
+        { id: 's2', type: 'sleep', duration: '100ms', needs: [] },
       ],
       outputs: {
         done: 'true',
@@ -518,7 +612,15 @@ describe('WorkflowRunner', () => {
       inputs: {
         val: { type: 'string' },
       },
-      steps: [{ id: 'cs1', type: 'shell', run: 'echo "child-${{ inputs.val }}"', needs: [] }],
+      steps: [
+        {
+          id: 'cs1',
+          type: 'shell',
+          run: 'echo "child-${{ inputs.val }}"',
+          allowInsecure: true,
+          needs: [],
+        },
+      ],
       outputs: {
         out: '${{ steps.cs1.output.stdout.trim() }}',
       },
@@ -549,7 +651,7 @@ describe('WorkflowRunner', () => {
   });
 
   it('should resume a failed workflow', async () => {
-    const resumeDbPath = 'test-resume.db';
+    const resumeDbPath = `test-resume-${randomUUID()}.db`;
     if (existsSync(resumeDbPath)) rmSync(resumeDbPath);
 
     const workflow: Workflow = {
@@ -590,6 +692,8 @@ describe('WorkflowRunner', () => {
       },
       error: () => {},
       warn: () => {},
+      info: () => {},
+      debug: () => {},
     };
 
     const runner2 = new WorkflowRunner(fixedWorkflow, {
@@ -605,7 +709,7 @@ describe('WorkflowRunner', () => {
   });
 
   it('should merge resumeInputs with stored inputs on resume', async () => {
-    const resumeDbPath = 'test-merge-inputs.db';
+    const resumeDbPath = `test-merge-inputs-${randomUUID()}.db`;
     if (existsSync(resumeDbPath)) rmSync(resumeDbPath);
 
     const workflow: Workflow = {
@@ -669,7 +773,7 @@ describe('WorkflowRunner', () => {
   });
 
   it('should redact secret inputs at rest', async () => {
-    const dbFile = 'test-secret-at-rest.db';
+    const dbFile = `test-secret-at-rest-${randomUUID()}.db`;
     const workflow: Workflow = {
       name: 'secret-input-wf',
       inputs: {
@@ -688,6 +792,7 @@ describe('WorkflowRunner', () => {
       mcp_servers: {},
       engines: { allowlist: {}, denylist: [] },
       concurrency: { default: 10, pools: { llm: 2, shell: 5, http: 10, engine: 2 } },
+      expression: { strict: false },
     });
 
     const runner = new WorkflowRunner(workflow, {
@@ -724,6 +829,8 @@ describe('WorkflowRunner', () => {
         }
       },
       warn: () => {},
+      info: () => {},
+      debug: () => {},
     };
 
     const failFinallyWorkflow: Workflow = {
@@ -747,6 +854,8 @@ describe('WorkflowRunner', () => {
       },
       error: () => {},
       warn: () => {},
+      info: () => {},
+      debug: () => {},
     };
 
     const retryWorkflow: Workflow = {
@@ -782,6 +891,7 @@ describe('WorkflowRunner', () => {
           id: 'gen',
           type: 'shell',
           run: 'echo "[1, 2]"',
+          allowInsecure: true,
           transform: 'JSON.parse(output.stdout)',
           needs: [],
         },
@@ -854,6 +964,64 @@ describe('WorkflowRunner', () => {
     if (existsSync(resumeDbPath)) rmSync(resumeDbPath);
   });
 
+  it('should reuse persisted foreach items on resume even if inputs change', async () => {
+    const resumeDbPath = 'test-foreach-resume-items.db';
+    if (existsSync(resumeDbPath)) rmSync(resumeDbPath);
+
+    const workflow: Workflow = {
+      name: 'foreach-resume-items',
+      steps: [
+        {
+          id: 'process',
+          type: 'human',
+          message: 'Item ${{ item }}',
+          foreach: '${{ inputs.items }}',
+          needs: [],
+        },
+      ],
+      outputs: {
+        results: '${{ steps.process.output }}',
+      },
+    } as unknown as Workflow;
+
+    const originalIsTTY = process.stdin.isTTY;
+    process.stdin.isTTY = false;
+
+    const runner1 = new WorkflowRunner(workflow, {
+      dbPath: resumeDbPath,
+      inputs: { items: [1, 2] },
+    });
+    let suspendedError: unknown;
+    try {
+      await runner1.run();
+    } catch (e) {
+      suspendedError = e;
+    } finally {
+      process.stdin.isTTY = originalIsTTY;
+    }
+
+    expect(suspendedError).toBeDefined();
+    expect(
+      typeof suspendedError === 'object' && suspendedError !== null && 'name' in suspendedError
+        ? (suspendedError as { name: string }).name
+        : undefined
+    ).toBe('WorkflowSuspendedError');
+
+    const runner2 = new WorkflowRunner(workflow, {
+      dbPath: resumeDbPath,
+      resumeRunId: runner1.runId,
+      resumeInputs: {
+        process: { __answer: 'ok' },
+        items: [1, 2, 3],
+      },
+    });
+
+    const outputs = await runner2.run();
+    expect(outputs.results).toEqual(['ok', 'ok']);
+
+    if (existsSync(resumeDbPath)) rmSync(resumeDbPath);
+  });
+
   it('should resume a workflow marked as running (crashed process)', async () => {
     const resumeDbPath = 'test-running-resume.db';
     if (existsSync(resumeDbPath)) rmSync(resumeDbPath);
@@ -943,8 +1111,8 @@ describe('WorkflowRunner', () => {
     const workflow: Workflow = {
       name: 'cancel-wf',
       steps: [
-        { id: 's1', type: 'sleep', duration: 10, needs: [] },
-        { id: 's2', type: 'sleep', duration: 10, needs: ['s1'] },
+        { id: 's1', type: 'sleep', duration: '10ms', needs: [] },
+        { id: 's2', type: 'sleep', duration: '10ms', needs: ['s1'] },
       ],
     } as unknown as Workflow;
 
@@ -1021,4 +1189,29 @@ describe('WorkflowRunner', () => {
 
     if (existsSync(resumeDbPath)) rmSync(resumeDbPath);
   });
+
+  it('should support safe direct shell execution via args', async () => {
+    const argsWorkflow: Workflow = {
+      name: 'args-wf',
+      inputs: {
+        val: { type: 'string', default: 'foo "bar" baz' },
+      },
+      steps: [
+        {
+          id: 's1',
+          type: 'shell',
+          args: ['echo', '${{ inputs.val }}'],
+          needs: [],
+        },
+      ],
+      outputs: {
+        out: '${{ steps.s1.output.stdout.trim() }}',
+      },
+    } as unknown as Workflow;
+
+    const runner = new WorkflowRunner(argsWorkflow, { dbPath });
+    const outputs = await runner.run();
+    // Bun.spawn with args array should preserve quotes and spaces without needing escape()
+    expect(outputs.out).toBe('foo "bar" baz');
+  });
 });