keystone-cli 0.2.0 → 0.3.0

package/src/runner/mcp-client.ts

@@ -57,12 +57,16 @@ class StdConfigTransport implements MCPTransport {
         const response = JSON.parse(line) as MCPResponse;
         callback(response);
       } catch (e) {
-        // Ignore non-JSON lines
+        // Log non-JSON lines to stderr so they show up in the terminal
+        if (line.trim()) {
+          process.stderr.write(`[MCP Server Output] ${line}\n`);
+        }
       }
     });
   }
 
   close(): void {
+    this.rl.close();
     this.process.kill();
   }
 }
@@ -70,49 +74,168 @@ class StdConfigTransport implements MCPTransport {
 class SSETransport implements MCPTransport {
   private url: string;
   private headers: Record<string, string>;
-  private eventSource: EventSource | null = null;
   private endpoint?: string;
   private onMessageCallback?: (message: MCPResponse) => void;
+  private abortController: AbortController | null = null;
+  private sessionId?: string;
 
   constructor(url: string, headers: Record<string, string> = {}) {
     this.url = url;
     this.headers = headers;
   }
 
-  async connect(): Promise<void> {
-    return new Promise((resolve, reject) => {
-      // @ts-ignore - Bun supports EventSource
-      this.eventSource = new EventSource(this.url, { headers: this.headers });
+  async connect(timeout = 60000): Promise<void> {
+    this.abortController = new AbortController();
 
-      if (!this.eventSource) {
-        reject(new Error('Failed to create EventSource'));
-        return;
-      }
+    return new Promise((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        this.close();
+        reject(new Error(`SSE connection timeout: ${this.url}`));
+      }, timeout);
+
+      (async () => {
+        try {
+          let response = await fetch(this.url, {
+            headers: {
+              Accept: 'application/json, text/event-stream',
+              ...this.headers,
+            },
+            signal: this.abortController?.signal,
+          });
+
+          if (response.status === 405) {
+            // Some MCP servers (like GitHub) require POST to start a session
+            response = await fetch(this.url, {
+              method: 'POST',
+              headers: {
+                Accept: 'application/json, text/event-stream',
+                'Content-Type': 'application/json',
+                ...this.headers,
+              },
+              body: JSON.stringify({
+                jsonrpc: '2.0',
+                id: 'ping',
+                method: 'ping',
+              }),
+              signal: this.abortController?.signal,
+            });
+          }
 
-      this.eventSource.addEventListener('endpoint', (event: MessageEvent) => {
-        this.endpoint = event.data;
-        if (this.endpoint?.startsWith('/')) {
-          const urlObj = new URL(this.url);
-          this.endpoint = `${urlObj.origin}${this.endpoint}`;
-        }
-        resolve();
-      });
+          if (!response.ok) {
+            clearTimeout(timeoutId);
+            reject(new Error(`SSE connection failed: ${response.status} ${response.statusText}`));
+            return;
+          }
 
-      this.eventSource.addEventListener('message', (event: MessageEvent) => {
-        if (this.onMessageCallback) {
-          try {
-            const response = JSON.parse(event.data) as MCPResponse;
-            this.onMessageCallback(response);
-          } catch (e) {
-            // Ignore
+          // Check for session ID in headers
+          this.sessionId =
+            response.headers.get('mcp-session-id') ||
+            response.headers.get('Mcp-Session-Id') ||
+            undefined;
+
+          const reader = response.body?.getReader();
+          if (!reader) {
+            clearTimeout(timeoutId);
+            reject(new Error('Failed to get response body reader'));
+            return;
           }
-        }
-      });
 
-      this.eventSource.onerror = (err) => {
-        const error = err as ErrorEvent;
-        reject(new Error(`SSE connection failed: ${error?.message || 'Unknown error'}`));
-      };
+          // Process the stream in the background
+          (async () => {
+            let buffer = '';
+            const decoder = new TextDecoder();
+            let currentEvent: { event?: string; data?: string } = {};
+            let isResolved = false;
+
+            const dispatchEvent = () => {
+              if (currentEvent.data) {
+                if (currentEvent.event === 'endpoint') {
+                  this.endpoint = currentEvent.data;
+                  if (this.endpoint) {
+                    this.endpoint = new URL(this.endpoint, this.url).href;
+                  }
+                  if (!isResolved) {
+                    isResolved = true;
+                    clearTimeout(timeoutId);
+                    resolve();
+                  }
+                } else if (!currentEvent.event || currentEvent.event === 'message') {
+                  // If we get a message before an endpoint, assume the URL itself is the endpoint
+                  // (Common in some MCP over SSE implementations like GitHub's)
+                  if (!this.endpoint) {
+                    this.endpoint = this.url;
+                    if (!isResolved) {
+                      isResolved = true;
+                      clearTimeout(timeoutId);
+                      resolve();
+                    }
+                  }
+
+                  if (this.onMessageCallback && currentEvent.data) {
+                    try {
+                      const message = JSON.parse(currentEvent.data) as MCPResponse;
+                      this.onMessageCallback(message);
+                    } catch (e) {
+                      // Ignore parse errors
+                    }
+                  }
+                }
+              }
+              currentEvent = {};
+            };
+
+            try {
+              while (true) {
+                const { done, value } = await reader.read();
+                if (done) {
+                  // Dispatch any remaining data
+                  dispatchEvent();
+                  break;
+                }
+
+                buffer += decoder.decode(value, { stream: true });
+                const lines = buffer.split(/\r\n|\r|\n/);
+                buffer = lines.pop() || '';
+
+                for (const line of lines) {
+                  if (line.trim() === '') {
+                    dispatchEvent();
+                    continue;
+                  }
+
+                  if (line.startsWith('event:')) {
+                    currentEvent.event = line.substring(6).trim();
+                  } else if (line.startsWith('data:')) {
+                    const data = line.substring(5).trim();
+                    currentEvent.data = currentEvent.data ? `${currentEvent.data}\n${data}` : data;
+                  }
+                }
+              }
+
+              if (!isResolved) {
+                // If the stream ended before we resolved, but we have a session ID, we can try to resolve
+                if (this.sessionId && !this.endpoint) {
+                  this.endpoint = this.url;
+                  isResolved = true;
+                  clearTimeout(timeoutId);
+                  resolve();
+                } else {
+                  clearTimeout(timeoutId);
+                  reject(new Error('SSE stream ended before connection established'));
+                }
+              }
+            } catch (err) {
+              if ((err as Error).name !== 'AbortError' && !isResolved) {
+                clearTimeout(timeoutId);
+                reject(err);
+              }
+            }
+          })();
+        } catch (err) {
+          clearTimeout(timeoutId);
+          reject(err);
+        }
+      })();
     });
   }
 
@@ -121,17 +244,87 @@ class SSETransport implements MCPTransport {
       throw new Error('SSE transport not connected or endpoint not received');
     }
 
+    const headers = {
+      'Content-Type': 'application/json',
+      ...this.headers,
+    };
+
+    if (this.sessionId) {
+      headers['mcp-session-id'] = this.sessionId;
+    }
+
     const response = await fetch(this.endpoint, {
       method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-        ...this.headers,
-      },
+      headers,
       body: JSON.stringify(message),
     });
 
     if (!response.ok) {
-      throw new Error(`Failed to send message to MCP server: ${response.statusText}`);
+      const text = await response.text();
+      throw new Error(
+        `Failed to send message to MCP server: ${response.status} ${response.statusText}${
+          text ? ` - ${text}` : ''
+        }`
+      );
+    }
+
+    // Some MCP servers (like GitHub) send the response directly in the POST response as SSE
+    const contentType = response.headers.get('content-type');
+    if (contentType?.includes('text/event-stream')) {
+      const reader = response.body?.getReader();
+      if (reader) {
+        (async () => {
+          let buffer = '';
+          const decoder = new TextDecoder();
+          let currentEvent: { event?: string; data?: string } = {};
+
+          const dispatchEvent = () => {
+            if (
+              this.onMessageCallback &&
+              currentEvent.data &&
+              (!currentEvent.event || currentEvent.event === 'message')
+            ) {
+              try {
+                const message = JSON.parse(currentEvent.data) as MCPResponse;
+                this.onMessageCallback(message);
+              } catch (e) {
+                // Ignore parse errors
+              }
+            }
+            currentEvent = {};
+          };
+
+          try {
+            while (true) {
+              const { done, value } = await reader.read();
+              if (done) {
+                dispatchEvent();
+                break;
+              }
+
+              buffer += decoder.decode(value, { stream: true });
+              const lines = buffer.split(/\r\n|\r|\n/);
+              buffer = lines.pop() || '';
+
+              for (const line of lines) {
+                if (line.trim() === '') {
+                  dispatchEvent();
+                  continue;
+                }
+
+                if (line.startsWith('event:')) {
+                  currentEvent.event = line.substring(6).trim();
+                } else if (line.startsWith('data:')) {
+                  const data = line.substring(5).trim();
+                  currentEvent.data = currentEvent.data ? `${currentEvent.data}\n${data}` : data;
+                }
+              }
+            }
+          } catch (e) {
+            // Ignore stream errors
+          }
+        })();
+      }
     }
   }
 
@@ -140,7 +333,7 @@ class SSETransport implements MCPTransport {
   }
 
   close(): void {
-    this.eventSource?.close();
+    this.abortController?.abort();
   }
 }
 
@@ -154,14 +347,14 @@ export class MCPClient {
     transportOrCommand: MCPTransport | string,
     timeoutOrArgs: number | string[] = [],
     env: Record<string, string> = {},
-    timeout = 30000
+    timeout = 60000
   ) {
     if (typeof transportOrCommand === 'string') {
       this.transport = new StdConfigTransport(transportOrCommand, timeoutOrArgs as string[], env);
       this.timeout = timeout;
     } else {
       this.transport = transportOrCommand;
-      this.timeout = (timeoutOrArgs as number) || 30000;
+      this.timeout = (timeoutOrArgs as number) || 60000;
     }
 
     this.transport.onMessage((response) => {
@@ -179,7 +372,7 @@ export class MCPClient {
     command: string,
     args: string[] = [],
     env: Record<string, string> = {},
-    timeout = 30000
+    timeout = 60000
   ): Promise<MCPClient> {
     const transport = new StdConfigTransport(command, args, env);
     return new MCPClient(transport, timeout);
@@ -188,10 +381,10 @@ export class MCPClient {
   static async createRemote(
     url: string,
     headers: Record<string, string> = {},
-    timeout = 30000
+    timeout = 60000
   ): Promise<MCPClient> {
     const transport = new SSETransport(url, headers);
-    await transport.connect();
+    await transport.connect(timeout);
     return new MCPClient(transport, timeout);
   }
 

package/src/runner/mcp-manager.ts

@@ -10,6 +10,9 @@ export interface MCPServerConfig {
   env?: Record<string, string>;
   url?: string;
   headers?: Record<string, string>;
+  oauth?: {
+    scope?: string;
+  };
 }
 
 export class MCPManager {
@@ -70,10 +73,47 @@ export class MCPManager {
       try {
         if (config.type === 'remote') {
           if (!config.url) throw new Error('Remote MCP server missing URL');
-          client = await MCPClient.createRemote(config.url, config.headers || {});
+
+          const headers = { ...(config.headers || {}) };
+
+          if (config.oauth) {
+            const { AuthManager } = await import('../utils/auth-manager');
+            const auth = AuthManager.load();
+            const token = auth.mcp_tokens?.[config.name]?.access_token;
+
+            if (!token) {
+              throw new Error(
+                `MCP server ${config.name} requires OAuth. Please run "keystone mcp login ${config.name}" first.`
+              );
+            }
+
+            headers.Authorization = `Bearer ${token}`;
+          }
+
+          client = await MCPClient.createRemote(config.url, headers);
         } else {
           if (!config.command) throw new Error('Local MCP server missing command');
-          client = await MCPClient.createLocal(config.command, config.args || [], config.env || {});
+
+          const env = { ...(config.env || {}) };
+
+          if (config.oauth) {
+            const { AuthManager } = await import('../utils/auth-manager');
+            const auth = AuthManager.load();
+            const token = auth.mcp_tokens?.[config.name]?.access_token;
+
+            if (!token) {
+              throw new Error(
+                `MCP server ${config.name} requires OAuth. Please run "keystone mcp login ${config.name}" first.`
+              );
+            }
+
+            // Pass token to the local proxy via environment variables
+            // Most proxies expect AUTHORIZATION or MCP_TOKEN
+            env.AUTHORIZATION = `Bearer ${token}`;
+            env.MCP_TOKEN = token;
+          }
+
+          client = await MCPClient.createLocal(config.command, config.args || [], env);
         }
 
         await client.initialize();

package/src/runner/step-executor.test.ts

@@ -374,7 +374,7 @@ describe('step-executor', () => {
       const step: WorkflowStep = {
         id: 'w1',
         type: 'workflow',
-        workflow: 'child.yaml',
+        path: 'child.yaml',
       };
       // @ts-ignore
       const executeWorkflowFn = mock(() =>
@@ -392,7 +392,7 @@ describe('step-executor', () => {
       const step: WorkflowStep = {
         id: 'w1',
         type: 'workflow',
-        workflow: 'child.yaml',
+        path: 'child.yaml',
       };
       const result = await executeStep(step, context);
       expect(result.status).toBe('failed');

package/src/runner/step-executor.ts

@@ -42,7 +42,8 @@ export async function executeStep(
   context: ExpressionContext,
   logger: Logger = console,
   executeWorkflowFn?: (step: WorkflowStep, context: ExpressionContext) => Promise<StepResult>,
-  mcpManager?: MCPManager
+  mcpManager?: MCPManager,
+  workflowDir?: string
 ): Promise<StepResult> {
   try {
     let result: StepResult;
@@ -66,9 +67,10 @@ export async function executeStep(
         result = await executeLlmStep(
           step,
           context,
-          (s, c) => executeStep(s, c, logger, executeWorkflowFn, mcpManager),
+          (s, c) => executeStep(s, c, logger, executeWorkflowFn, mcpManager, workflowDir),
           logger,
-          mcpManager
+          mcpManager,
+          workflowDir
         );
         break;
       case 'workflow':
@@ -180,6 +182,13 @@ async function executeFileStep(
         throw new Error('Content is required for write operation');
       }
       const content = ExpressionEvaluator.evaluateString(step.content, context);
+
+      // Ensure parent directory exists
+      const fs = await import('node:fs/promises');
+      const pathModule = await import('node:path');
+      const dir = pathModule.dirname(path);
+      await fs.mkdir(dir, { recursive: true });
+
       const bytes = await Bun.write(path, content);
       return {
         output: { path, bytes },
@@ -193,8 +202,13 @@ async function executeFileStep(
       }
       const content = ExpressionEvaluator.evaluateString(step.content, context);
 
-      // Use Node.js fs for efficient append operation
+      // Ensure parent directory exists
       const fs = await import('node:fs/promises');
+      const pathModule = await import('node:path');
+      const dir = pathModule.dirname(path);
+      await fs.mkdir(dir, { recursive: true });
+
+      // Use Node.js fs for efficient append operation
       await fs.appendFile(path, content, 'utf-8');
 
       return {
@@ -310,10 +324,10 @@ async function executeHumanStep(
   try {
     if (step.inputType === 'confirm') {
       logger.log(`\n❓ ${message}`);
-      logger.log('Press Enter to continue, or Ctrl+C to cancel...');
-      await rl.question('');
+      const answer = await rl.question('Confirm? (Y/n): ');
+      const isConfirmed = answer.toLowerCase() !== 'n';
       return {
-        output: true,
+        output: isConfirmed,
         status: 'success',
       };
     }

package/src/runner/workflow-runner.ts

@@ -1,4 +1,5 @@
 import { randomUUID } from 'node:crypto';
+import { dirname } from 'node:path';
 import { WorkflowDb } from '../db/workflow-db.ts';
 import type { ExpressionContext } from '../expression/evaluator.ts';
 import { ExpressionEvaluator } from '../expression/evaluator.ts';
@@ -46,6 +47,7 @@ export interface RunOptions {
   logger?: Logger;
   mcpManager?: MCPManager;
   preventExit?: boolean; // Defaults to false
+  workflowDir?: string;
 }
 
 export interface StepContext {
@@ -456,7 +458,8 @@ export class WorkflowRunner {
         context,
         this.logger,
         this.executeSubWorkflow.bind(this),
-        this.mcpManager
+        this.mcpManager,
+        this.options.workflowDir
       );
       if (result.status === 'failed') {
         throw new Error(result.error || 'Step failed');
@@ -700,6 +703,7 @@ export class WorkflowRunner {
   ): Promise<StepResult> {
     const workflowPath = WorkflowRegistry.resolvePath(step.path);
     const workflow = WorkflowParser.loadWorkflow(workflowPath);
+    const subWorkflowDir = dirname(workflowPath);
 
     // Evaluate inputs for the sub-workflow
     const inputs: Record<string, unknown> = {};
@@ -716,6 +720,7 @@ export class WorkflowRunner {
       dbPath: this.db.dbPath,
       logger: this.logger,
       mcpManager: this.mcpManager,
+      workflowDir: subWorkflowDir,
     });
 
     try {
@@ -874,7 +879,9 @@ export class WorkflowRunner {
     } finally {
       this.removeSignalHandlers();
       await this.runFinally();
-      await this.mcpManager.stopAll();
+      if (!this.options.mcpManager) {
+        await this.mcpManager.stopAll();
+      }
       this.db.close();
     }
   }

package/src/utils/auth-manager.test.ts

@@ -149,4 +149,90 @@ describe('AuthManager', () => {
       consoleSpy.mockRestore();
     });
   });
+
+  describe('Device Login', () => {
+    it('initGitHubDeviceLogin should return device code data', async () => {
+      const mockFetch = mock(() =>
+        Promise.resolve(
+          new Response(
+            JSON.stringify({
+              device_code: 'dev_code',
+              user_code: 'USER-CODE',
+              verification_uri: 'https://github.com/login/device',
+              expires_in: 900,
+              interval: 5,
+            }),
+            { status: 200 }
+          )
+        )
+      );
+      // @ts-ignore
+      global.fetch = mockFetch;
+
+      const result = await AuthManager.initGitHubDeviceLogin();
+      expect(result.device_code).toBe('dev_code');
+      expect(result.user_code).toBe('USER-CODE');
+      expect(mockFetch).toHaveBeenCalled();
+    });
+
+    it('pollGitHubDeviceLogin should return token when successful', async () => {
+      let callCount = 0;
+      const mockFetch = mock(() => {
+        callCount++;
+        if (callCount === 1) {
+          return Promise.resolve(
+            new Response(
+              JSON.stringify({
+                error: 'authorization_pending',
+              }),
+              { status: 200 }
+            )
+          );
+        }
+        return Promise.resolve(
+          new Response(
+            JSON.stringify({
+              access_token: 'gh_access_token',
+            }),
+            { status: 200 }
+          )
+        );
+      });
+      // @ts-ignore
+      global.fetch = mockFetch;
+
+      // Mock setTimeout to resolve immediately
+      const originalTimeout = global.setTimeout;
+      // @ts-ignore
+      global.setTimeout = (fn) => fn();
+
+      try {
+        const token = await AuthManager.pollGitHubDeviceLogin('dev_code');
+        expect(token).toBe('gh_access_token');
+        expect(callCount).toBe(2);
+      } finally {
+        global.setTimeout = originalTimeout;
+      }
+    });
+
+    it('pollGitHubDeviceLogin should throw on other errors', async () => {
+      const mockFetch = mock(() =>
+        Promise.resolve(
+          new Response(
+            JSON.stringify({
+              error: 'expired_token',
+              error_description: 'The device code has expired',
+            }),
+            { status: 200 }
+          )
+        )
+      );
+      // @ts-ignore
+      global.fetch = mockFetch;
+
+      await expect(AuthManager.pollGitHubDeviceLogin('dev_code')).rejects.toThrow(
+        'The device code has expired'
+      );
+    });
+  });
 });