keystone-cli 0.1.0 → 0.2.0

package/src/cli.ts

@@ -2,18 +2,27 @@
 import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { Command } from 'commander';
+
+import exploreAgent from './templates/agents/explore.md' with { type: 'text' };
+import generalAgent from './templates/agents/general.md' with { type: 'text' };
+import architectAgent from './templates/agents/keystone-architect.md' with { type: 'text' };
+// Default templates
+import scaffoldWorkflow from './templates/scaffold-feature.yaml' with { type: 'text' };
+
 import { WorkflowDb } from './db/workflow-db.ts';
 import { WorkflowParser } from './parser/workflow-parser.ts';
 import { ConfigLoader } from './utils/config-loader.ts';
 import { generateMermaidGraph, renderMermaidAsAscii } from './utils/mermaid.ts';
 import { WorkflowRegistry } from './utils/workflow-registry.ts';
 
+import pkg from '../package.json' with { type: 'json' };
+
 const program = new Command();
 
 program
   .name('keystone')
   .description('A local-first, declarative, agentic workflow orchestrator')
-  .version('0.1.0');
+  .version(pkg.version);
 
 // ===== keystone init =====
 program
@@ -62,6 +71,11 @@ model_mappings:
   "o1-*": openai
   "llama-*": groq
 
+# mcp_servers:
+#   filesystem:
+#     command: npx
+#     args: ["-y", "@modelcontextprotocol/server-filesystem", "."]
+
 storage:
   retention_days: 30
 workflows_directory: workflows
@@ -85,6 +99,35 @@ workflows_directory: workflows
       console.log(`⊘ ${envPath} already exists`);
     }
 
+    // Seed default workflows and agents
+    const seeds = [
+      {
+        path: '.keystone/workflows/scaffold-feature.yaml',
+        content: scaffoldWorkflow,
+      },
+      {
+        path: '.keystone/workflows/agents/keystone-architect.md',
+        content: architectAgent,
+      },
+      {
+        path: '.keystone/workflows/agents/general.md',
+        content: generalAgent,
+      },
+      {
+        path: '.keystone/workflows/agents/explore.md',
+        content: exploreAgent,
+      },
+    ];
+
+    for (const seed of seeds) {
+      if (!existsSync(seed.path)) {
+        writeFileSync(seed.path, seed.content);
+        console.log(`✓ Seeded ${seed.path}`);
+      } else {
+        console.log(`⊘ ${seed.path} already exists`);
+      }
+    }
+
     console.log('\n✨ Keystone project initialized!');
     console.log('\nNext steps:');
     console.log('  1. Add your API keys to .env');
@@ -499,90 +542,51 @@ auth
   .command('login')
   .description('Login to an authentication provider')
   .option('-p, --provider <provider>', 'Authentication provider', 'github')
+  .option('-t, --token <token>', 'Personal Access Token (if not using interactive mode)')
   .action(async (options) => {
     const { AuthManager } = await import('./utils/auth-manager.ts');
     const provider = options.provider.toLowerCase();
 
-    if (provider !== 'github' && provider !== 'copilot') {
-      console.error(`✗ Unsupported provider: ${provider}`);
-      process.exit(1);
-    }
-
-    console.log(`🏛️  ${provider === 'copilot' ? 'GitHub Copilot' : 'GitHub'} Login\n`);
+    if (provider === 'github') {
+      let token = options.token;
 
-    try {
-      // Step 1: Request device code
-      const deviceCodeResponse = await fetch('https://github.com/login/device/code', {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-          Accept: 'application/json',
-        },
-        body: JSON.stringify({
-          client_id: '01ab8ac9400c4e429b23',
-          scope: 'read:user',
-        }),
-      });
-
-      if (!deviceCodeResponse.ok) {
-        throw new Error(`GitHub API error: ${deviceCodeResponse.statusText}`);
+      if (!token) {
+        console.log('\nTo login with GitHub:');
+        console.log(
+          '1. Generate a Personal Access Token (Classic) with "copilot" scope (or full repo access).'
+        );
+        console.log('   https://github.com/settings/tokens/new');
+        console.log('2. Paste the token below:\n');
+
+        const prompt = 'Token: ';
+        process.stdout.write(prompt);
+        for await (const line of console) {
+          token = line.trim();
+          break;
+        }
       }
 
-      const { device_code, user_code, verification_uri, interval } =
-        (await deviceCodeResponse.json()) as {
-          device_code: string;
-          user_code: string;
-          verification_uri: string;
-          interval: number;
-        };
-
-      console.log(`1. Visit: ${verification_uri}`);
-      console.log(`2. Enter code: ${user_code}\n`);
-      console.log('Waiting for authorization...');
-
-      // Step 3: Poll for access token
-      const poll = async (): Promise<string> => {
-        while (true) {
-          await new Promise((resolve) => setTimeout(resolve, interval * 1000));
-
-          const response = await fetch('https://github.com/login/oauth/access_token', {
-            method: 'POST',
-            headers: {
-              'Content-Type': 'application/json',
-              Accept: 'application/json',
-            },
-            body: JSON.stringify({
-              client_id: '01ab8ac9400c4e429b23',
-              device_code,
-              grant_type: 'urn:ietf:params:oauth:grant-type:device_code',
-            }),
-          });
-
-          const data = (await response.json()) as {
-            access_token?: string;
-            error?: string;
-          };
-
-          if (data.access_token) {
-            return data.access_token;
-          }
-
-          if (data.error === 'authorization_pending') {
-            continue;
+      if (token) {
+        AuthManager.save({ github_token: token });
+        // Force refresh of Copilot token to verify
+        try {
+          const copilotToken = await AuthManager.getCopilotToken();
+          if (copilotToken) {
+            console.log('\n✓ Successfully logged in to GitHub and retrieved Copilot token.');
+          } else {
+            console.error(
+              '\n✗ Saved GitHub token, but failed to retrieve Copilot token. Please check scopes.'
+            );
           }
-
-          throw new Error(`GitHub error: ${data.error}`);
+        } catch (e) {
+          console.error('\n✗ Failed to verify token:', e instanceof Error ? e.message : e);
         }
-      };
-
-      const accessToken = await poll();
-      AuthManager.save({ github_token: accessToken });
-
-      console.log(
-        `\n✨ Successfully logged into ${provider === 'copilot' ? 'GitHub Copilot' : 'GitHub'}!`
-      );
-    } catch (error) {
-      console.error('\n✗ Login failed:', error instanceof Error ? error.message : error);
+      } else {
+        console.error('✗ No token provided.');
+        process.exit(1);
+      }
+    } else {
+      console.error(`✗ Unsupported provider: ${provider}`);
       process.exit(1);
     }
   });
@@ -590,11 +594,12 @@ auth
 auth
   .command('status')
   .description('Show authentication status')
+  .argument('[provider]', 'Authentication provider')
   .option('-p, --provider <provider>', 'Authentication provider')
-  .action(async (options) => {
+  .action(async (providerArg, options) => {
     const { AuthManager } = await import('./utils/auth-manager.ts');
     const auth = AuthManager.load();
-    const provider = options.provider?.toLowerCase();
+    const provider = (options.provider || providerArg)?.toLowerCase();
 
     console.log('\n🏛️  Authentication Status:');
 
@@ -620,10 +625,14 @@ auth
 auth
   .command('logout')
   .description('Logout and clear authentication tokens')
-  .option('-p, --provider <provider>', 'Authentication provider')
-  .action(async (options) => {
+  .argument('[provider]', 'Authentication provider')
+  .option(
+    '-p, --provider <provider>',
+    'Authentication provider (deprecated, use positional argument)'
+  )
+  .action(async (providerArg, options) => {
     const { AuthManager } = await import('./utils/auth-manager.ts');
-    const provider = options.provider?.toLowerCase();
+    const provider = (options.provider || providerArg)?.toLowerCase();
 
     if (!provider || provider === 'github' || provider === 'copilot') {
       AuthManager.save({

package/src/db/workflow-db.ts

@@ -99,13 +99,6 @@ export class WorkflowDb {
       CREATE INDEX IF NOT EXISTS idx_steps_status ON step_executions(status);
       CREATE INDEX IF NOT EXISTS idx_steps_iteration ON step_executions(run_id, step_id, iteration_index);
     `);
-
-    // Migration: Add iteration_index if it doesn't exist
-    try {
-      this.db.exec('ALTER TABLE step_executions ADD COLUMN iteration_index INTEGER;');
-    } catch (e) {
-      // Ignore if column already exists
-    }
   }
 
   // ===== Workflow Runs =====

package/src/expression/evaluator.test.ts

@@ -238,10 +238,52 @@ describe('ExpressionEvaluator', () => {
     expect(ExpressionEvaluator.evaluate('${{ runFn(x => x + 5) }}', contextWithFunc)).toBe(15);
   });
 
+  test('should handle multiple expressions and fallback values', () => {
+    // line 83: multiple expressions returning null/undefined
+    const contextWithNull = { ...context, nullVal: null };
+    expect(ExpressionEvaluator.evaluate('Val: ${{ nullVal }}', contextWithNull)).toBe('Val: ');
+
+    // line 87: multiple expressions returning objects
+    expect(ExpressionEvaluator.evaluate('Data: ${{ steps.step1.outputs.data }}', context)).toBe(
+      'Data: {\n  "id": 1\n}'
+    );
+  });
+
+  test('should handle evaluateString fallback for null/undefined', () => {
+    // line 103: evaluateString returning null/undefined
+    const contextWithNull = { ...context, nullVal: null };
+    expect(ExpressionEvaluator.evaluateString('${{ nullVal }}', contextWithNull)).toBe('');
+  });
+
   test('should throw error for unsupported unary operator', () => {
     // '~' is a unary operator jsep supports but we don't
     expect(() => ExpressionEvaluator.evaluate('${{ ~1 }}', context)).toThrow(
       /Unsupported unary operator: ~/
     );
   });
+
+  test('should throw error when calling non-function method', () => {
+    // Calling map on a string (should hit line 391 fallback)
+    expect(() => ExpressionEvaluator.evaluate("${{ 'abc'.map(i => i) }}", context)).toThrow(
+      /Cannot call method map on string/
+    );
+  });
+
+  test('should throw error for unsupported call expression', () => {
+    // Triggering line 417: Only method calls and safe function calls are supported
+    // We need something that jsep parses as CallExpression but callee is not MemberExpression or Identifier
+    // Hard to do with jsep as it usually parses callee as one of those.
+    // But we can try to mock an AST if we really wanted to.
+  });
+
+  test('should handle evaluateString with object result', () => {
+    expect(ExpressionEvaluator.evaluateString('${{ inputs.items }}', context)).toBe(
+      '[\n  "a",\n  "b",\n  "c"\n]'
+    );
+  });
+
+  test('should handle evaluate with template string containing only null/undefined expression', () => {
+    const contextWithNull = { ...context, nullVal: null };
+    expect(ExpressionEvaluator.evaluate('${{ nullVal }}', contextWithNull)).toBe(null);
+  });
 });

package/src/expression/evaluator.ts

@@ -78,10 +78,38 @@ export class ExpressionEvaluator {
       // Extract the expression content between ${{ and }}
       const expr = match.replace(/^\$\{\{\s*|\s*\}\}$/g, '');
       const result = ExpressionEvaluator.evaluateExpression(expr, context);
+
+      if (result === null || result === undefined) {
+        return '';
+      }
+
+      if (typeof result === 'object') {
+        return JSON.stringify(result, null, 2);
+      }
+
       return String(result);
     });
   }
 
+  /**
+   * Evaluate a string and ensure the result is a string.
+   * Objects and arrays are stringified to JSON.
+   * null and undefined return an empty string.
+   */
+  static evaluateString(template: string, context: ExpressionContext): string {
+    const result = ExpressionEvaluator.evaluate(template, context);
+
+    if (result === null || result === undefined) {
+      return '';
+    }
+
+    if (typeof result === 'string') {
+      return result;
+    }
+
+    return JSON.stringify(result, null, 2);
+  }
+
   /**
    * Evaluate a single expression (without the ${{ }} wrapper)
    * This is public to support transform expressions in shell steps

package/src/parser/agent-parser.test.ts

@@ -63,6 +63,16 @@ tools:
       expect(agent.tools[0].execution.id).toBe('tool-tool-without-id');
     });
 
+    it('should parse single-line frontmatter', () => {
+      const agentContent = '---name: single-line---\nPrompt';
+      const filePath = join(tempDir, 'single-line.md');
+      writeFileSync(filePath, agentContent);
+
+      const agent = parseAgent(filePath);
+      expect(agent.name).toBe('single-line');
+      expect(agent.systemPrompt).toBe('Prompt');
+    });
+
     it('should throw error for missing frontmatter', () => {
       const agentContent = 'Just some content without frontmatter';
       const filePath = join(tempDir, 'invalid-format.md');

package/src/parser/agent-parser.ts

@@ -6,7 +6,8 @@ import { type Agent, AgentSchema } from './schema';
 
 export function parseAgent(filePath: string): Agent {
   const content = readFileSync(filePath, 'utf8');
-  const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n([\s\S]*))?$/);
+  // Flexible regex to handle both standard and single-line frontmatter
+  const match = content.match(/^---[\r\n]*([\s\S]*?)[\r\n]*---(?:\r?\n?([\s\S]*))?$/);
 
   if (!match) {
     throw new Error(`Invalid agent format in ${filePath}. Missing frontmatter.`);

package/src/parser/config-schema.ts

@@ -42,11 +42,19 @@ export const ConfigSchema = z.object({
   workflows_directory: z.string().default('workflows'),
   mcp_servers: z
     .record(
-      z.object({
-        command: z.string(),
-        args: z.array(z.string()).optional(),
-        env: z.record(z.string()).optional(),
-      })
+      z.discriminatedUnion('type', [
+        z.object({
+          type: z.literal('local').default('local'),
+          command: z.string(),
+          args: z.array(z.string()).optional(),
+          env: z.record(z.string()).optional(),
+        }),
+        z.object({
+          type: z.literal('remote'),
+          url: z.string().url(),
+          headers: z.record(z.string()).optional(),
+        }),
+      ])
     )
     .default({}),
 });

package/src/parser/workflow-parser.ts

@@ -180,11 +180,6 @@ export class WorkflowParser {
       }
     }
 
-    // Initialize in-degree
-    for (const step of workflow.steps) {
-      inDegree.set(step.id, 0);
-    }
-
     // Calculate in-degree
     // In-degree = number of dependencies a step has
     for (const step of workflow.steps) {

package/src/runner/llm-adapter.test.ts

@@ -268,14 +268,6 @@ describe('CopilotAdapter', () => {
     await expect(adapter.chat([])).rejects.toThrow(/GitHub Copilot token not found/);
     spy.mockRestore();
   });
-
-  it('should throw error if token not found (duplicated)', async () => {
-    const spy = spyOn(AuthManager, 'getCopilotToken').mockResolvedValue(undefined);
-
-    const adapter = new CopilotAdapter();
-    await expect(adapter.chat([])).rejects.toThrow(/GitHub Copilot token not found/);
-    spy.mockRestore();
-  });
 });
 
 describe('getAdapter', () => {

package/src/runner/llm-adapter.ts

@@ -141,19 +141,42 @@ export class AnthropicAdapter implements LLMAdapter {
           role: 'assistant',
           content: [
             ...(m.content ? [{ type: 'text' as const, text: m.content }] : []),
-            ...m.tool_calls.map((tc) => ({
-              type: 'tool_use' as const,
-              id: tc.id,
-              name: tc.function.name,
-              input: JSON.parse(tc.function.arguments),
-            })),
+            ...m.tool_calls.map((tc) => {
+              let input = {};
+              try {
+                input =
+                  typeof tc.function.arguments === 'string'
+                    ? JSON.parse(tc.function.arguments)
+                    : tc.function.arguments;
+              } catch (e) {
+                console.error(`Failed to parse tool arguments: ${tc.function.arguments}`);
+              }
+              return {
+                type: 'tool_use' as const,
+                id: tc.id,
+                name: tc.function.name,
+                input,
+              };
+            }),
           ],
         });
       } else {
-        anthropicMessages.push({
-          role: m.role as 'user' | 'assistant',
-          content: m.content || '',
-        });
+        const role = m.role as 'user' | 'assistant';
+        const lastMsg = anthropicMessages[anthropicMessages.length - 1];
+
+        if (
+          lastMsg &&
+          lastMsg.role === role &&
+          typeof lastMsg.content === 'string' &&
+          typeof m.content === 'string'
+        ) {
+          lastMsg.content += `\n\n${m.content}`;
+        } else {
+          anthropicMessages.push({
+            role,
+            content: m.content || '',
+          });
+        }
       }
     }
 

package/src/runner/llm-executor.test.ts

@@ -3,11 +3,18 @@ import { mkdirSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import type { ExpressionContext } from '../expression/evaluator';
 import type { LlmStep, Step } from '../parser/schema';
-import { AnthropicAdapter, CopilotAdapter, OpenAIAdapter } from './llm-adapter';
-import { MCPClient } from './mcp-client';
+import { ConfigLoader } from '../utils/config-loader';
+import {
+  AnthropicAdapter,
+  CopilotAdapter,
+  type LLMMessage,
+  type LLMResponse,
+  type LLMTool,
+  OpenAIAdapter,
+} from './llm-adapter';
 import { executeLlmStep } from './llm-executor';
+import { MCPClient, type MCPResponse } from './mcp-client';
 import { MCPManager } from './mcp-manager';
-import { ConfigLoader } from '../utils/config-loader';
 import type { StepResult } from './step-executor';
 
 // Mock adapters
@@ -302,9 +309,7 @@ You are a test agent.`;
     const context: ExpressionContext = { inputs: {}, steps: {} };
     const executeStepFn = mock(async () => ({ status: 'success' as const, output: 'ok' }));
 
-    const initSpy = spyOn(MCPClient.prototype, 'initialize').mockResolvedValue(
-      {} as unknown as any
-    );
+    const initSpy = spyOn(MCPClient.prototype, 'initialize').mockResolvedValue({} as MCPResponse);
     const listSpy = spyOn(MCPClient.prototype, 'listTools').mockResolvedValue([
       { name: 'mcp-tool', inputSchema: {} },
     ]);
@@ -317,7 +322,7 @@ You are a test agent.`;
     const originalAnthropicChatInner = AnthropicAdapter.prototype.chat;
     let toolErrorCaptured = false;
 
-    const mockChat = mock(async (messages: any[]) => {
+    const mockChat = mock(async (messages: LLMMessage[]) => {
       const toolResultMessage = messages.find((m) => m.role === 'tool');
       if (toolResultMessage?.content?.includes('Error: Tool failed')) {
         toolErrorCaptured = true;
@@ -331,7 +336,7 @@ You are a test agent.`;
           ],
         },
       };
-    }) as any;
+    }) as unknown as typeof originalOpenAIChat;
 
     OpenAIAdapter.prototype.chat = mockChat;
     CopilotAdapter.prototype.chat = mockChat;
@@ -377,21 +382,19 @@ You are a test agent.`;
     const context: ExpressionContext = { inputs: {}, steps: {} };
     const executeStepFn = mock(async () => ({ status: 'success' as const, output: 'ok' }));
 
-    const initSpy = spyOn(MCPClient.prototype, 'initialize').mockResolvedValue(
-      {} as unknown as any
-    );
+    const initSpy = spyOn(MCPClient.prototype, 'initialize').mockResolvedValue({} as MCPResponse);
     const listSpy = spyOn(MCPClient.prototype, 'listTools').mockResolvedValue([
       { name: 'global-tool', description: 'A global tool', inputSchema: {} },
     ]);
 
     let toolFound = false;
     const originalOpenAIChatInner = OpenAIAdapter.prototype.chat;
-    const mockChat = mock(async (_messages: any[], options: any) => {
-      if (options.tools?.some((t: any) => t.function.name === 'global-tool')) {
+    const mockChat = mock(async (_messages: LLMMessage[], options: { tools?: LLMTool[] }) => {
+      if (options.tools?.some((t: LLMTool) => t.function.name === 'global-tool')) {
         toolFound = true;
       }
       return { message: { role: 'assistant', content: 'hello' } };
-    }) as any;
+    }) as unknown as typeof originalOpenAIChat;
 
     OpenAIAdapter.prototype.chat = mockChat;
 
@@ -499,15 +502,13 @@ You are a test agent.`;
     const context: ExpressionContext = { inputs: {}, steps: {} };
     const executeStepFn = mock(async () => ({ status: 'success' as const, output: 'ok' }));
 
-    const initSpy = spyOn(MCPClient.prototype, 'initialize').mockResolvedValue(
-      {} as unknown as any
-    );
+    const initSpy = spyOn(MCPClient.prototype, 'initialize').mockResolvedValue({} as MCPResponse);
     const listSpy = spyOn(MCPClient.prototype, 'listTools').mockResolvedValue([]);
 
     const originalOpenAIChatInner = OpenAIAdapter.prototype.chat;
     const mockChat = mock(async () => ({
       message: { role: 'assistant', content: 'hello' },
-    })) as any;
+    })) as unknown as typeof originalOpenAIChat;
     OpenAIAdapter.prototype.chat = mockChat;
 
     const managerSpy = spyOn(manager, 'getGlobalServers');
@@ -534,4 +535,44 @@ You are a test agent.`;
     managerSpy.mockRestore();
     ConfigLoader.clear();
   });
+
+  it('should handle object prompts by stringifying them', async () => {
+    const step: LlmStep = {
+      id: 'l1',
+      type: 'llm',
+      agent: 'test-agent',
+      prompt: '${{ steps.prev.output }}' as unknown as string,
+      needs: [],
+    };
+    const context: ExpressionContext = {
+      inputs: {},
+      steps: {
+        prev: { output: { key: 'value' }, status: 'success' },
+      },
+    };
+
+    let capturedPrompt = '';
+    const originalOpenAIChatInner = OpenAIAdapter.prototype.chat;
+    const mockChat = mock(async (messages: LLMMessage[]) => {
+      // console.log('MESSAGES:', JSON.stringify(messages, null, 2));
+      capturedPrompt = messages.find((m) => m.role === 'user')?.content || '';
+      return { message: { role: 'assistant', content: 'Response' } };
+    }) as unknown as typeof originalOpenAIChat;
+    OpenAIAdapter.prototype.chat = mockChat;
+    CopilotAdapter.prototype.chat = mockChat;
+    AnthropicAdapter.prototype.chat = mockChat;
+
+    const executeStepFn = mock(async () => ({ status: 'success' as const, output: 'ok' }));
+
+    await executeLlmStep(
+      step,
+      context,
+      executeStepFn as unknown as (step: Step, context: ExpressionContext) => Promise<StepResult>
+    );
+
+    expect(capturedPrompt).toContain('"key": "value"');
+    expect(capturedPrompt).not.toContain('[object Object]');
+
+    OpenAIAdapter.prototype.chat = originalOpenAIChatInner;
+  });
 });

package/src/runner/llm-executor.ts

@@ -30,7 +30,7 @@ export async function executeLlmStep(
 
   const provider = step.provider || agent.provider;
   const model = step.model || agent.model || 'gpt-4o';
-  const prompt = ExpressionEvaluator.evaluate(step.prompt, context) as string;
+  const prompt = ExpressionEvaluator.evaluateString(step.prompt, context);
 
   const fullModelString = provider ? `${provider}:${model}` : model;
   const { adapter, resolvedModel } = getAdapter(fullModelString);