@marktoflow/gui 2.0.0-alpha.1 → 2.0.0-alpha.12

package/src/server/services/WorkflowService.ts

@@ -1,9 +1,11 @@
 import { readdir, readFile, writeFile, unlink, mkdir } from 'fs/promises';
-import { join, relative, dirname } from 'path';
+import { join, relative, dirname, basename, extname } from 'path';
 import { existsSync } from 'fs';
-import { stringify as yamlStringify } from 'yaml';
+import { stringify as yamlStringify, parse as yamlParse } from 'yaml';
+import AdmZip from 'adm-zip';
 // Import from @marktoflow/core for proper parsing
-import { parseFile as coreParseFile } from '@marktoflow/core';
+import { parseFile as coreParseFile, type ExecutionRecord } from '@marktoflow/core';
+import { getStateStore } from '../index.js';
 
 interface WorkflowListItem {
   path: string;
@@ -44,6 +46,7 @@ interface Workflow {
   tools?: Record<string, unknown>;
   inputs?: Record<string, unknown>;
   triggers?: unknown[];
+  markdown?: string;
 }
 
 export class WorkflowService {
@@ -75,17 +78,41 @@ export class WorkflowService {
             entry.isFile() &&
             (entry.name.endsWith('.md') || entry.name.endsWith('.yaml') || entry.name.endsWith('.yml'))
           ) {
-            // Check if it's a workflow file by looking for frontmatter
+            // Skip README files (they're documentation, not workflows)
+            if (entry.name.toLowerCase() === 'readme.md') {
+              continue;
+            }
+
+            // Check if it's a workflow file by verifying YAML frontmatter
             try {
               const content = await readFile(fullPath, 'utf-8');
-              if (content.includes('workflow:') || content.includes('steps:')) {
-                const relativePath = relative(baseDir, fullPath);
-                const workflowInfo = extractWorkflowInfo(content, entry.name);
-                workflows.push({
-                  path: relativePath,
-                  ...workflowInfo,
-                });
+
+              // For markdown files, verify it has YAML frontmatter
+              if (entry.name.endsWith('.md')) {
+                if (!content.trimStart().startsWith('---\n')) {
+                  continue;
+                }
+                const frontmatterMatch = content.match(/^---\n[\s\S]*?\n---/);
+                if (!frontmatterMatch) {
+                  continue;
+                }
+                const frontmatter = frontmatterMatch[0];
+                if (!frontmatter.includes('workflow:') && !frontmatter.includes('steps:')) {
+                  continue;
+                }
+              } else {
+                // For YAML files, check if they contain workflow structure
+                if (!content.includes('workflow:') && !content.includes('steps:')) {
+                  continue;
+                }
               }
+
+              const relativePath = relative(baseDir, fullPath);
+              const workflowInfo = extractWorkflowInfo(content, entry.name);
+              workflows.push({
+                path: relativePath,
+                ...workflowInfo,
+              });
             } catch {
               // Skip files that can't be read
             }
@@ -108,6 +135,9 @@ export class WorkflowService {
     }
 
     try {
+      // Read the raw markdown content for GUI parser
+      const rawContent = await readFile(fullPath, 'utf-8');
+
       // Use core parser for proper workflow parsing
       const result = await coreParseFile(fullPath);
       if (result.warnings && result.warnings.length > 0) {
@@ -115,7 +145,13 @@ export class WorkflowService {
       }
 
       // Convert core Workflow type to GUI Workflow type
-      return this.convertCoreWorkflow(result.workflow, workflowPath);
+      const workflow = this.convertCoreWorkflow(result.workflow, workflowPath);
+
+      // Add raw markdown for GUI control flow parsing
+      return {
+        ...workflow,
+        markdown: rawContent,
+      };
     } catch (error) {
       console.error(`Error parsing workflow ${workflowPath}:`, error);
       // Fallback to local parsing if core parser fails
@@ -197,10 +233,134 @@ export class WorkflowService {
     }
   }
 
-  async getExecutionHistory(_workflowPath: string): Promise<unknown[]> {
-    // TODO: Query state store for execution history
-    // This will integrate with @marktoflow/core StateStore
-    return [];
+  async getExecutionHistory(workflowPath: string): Promise<ExecutionRecord[]> {
+    try {
+      const stateStore = getStateStore();
+      // Use the workflow path as the workflow ID for querying
+      const executions = stateStore.listExecutions({
+        workflowId: workflowPath,
+        limit: 50,
+      });
+      return executions;
+    } catch (error) {
+      console.error('Error getting execution history:', error);
+      return [];
+    }
+  }
+
+  async importWorkflow(
+    fileBuffer: Buffer,
+    originalFilename: string
+  ): Promise<{ success: boolean; results?: Array<{ success: boolean; filename: string; message?: string }>; filename?: string; message?: string }> {
+    const ext = extname(originalFilename).toLowerCase();
+
+    try {
+      // Handle ZIP files - extract and import all workflow files
+      if (ext === '.zip') {
+        return await this.importZipBundle(fileBuffer);
+      }
+
+      // Handle single workflow files (.md, .yaml, .yml)
+      if (['.md', '.yaml', '.yml'].includes(ext)) {
+        const content = fileBuffer.toString('utf-8');
+        await this.importSingleWorkflow(content, originalFilename);
+
+        return {
+          success: true,
+          filename: originalFilename,
+          message: 'Workflow imported successfully',
+        };
+      }
+
+      throw new Error('Unsupported file type');
+    } catch (error) {
+      console.error('Import error:', error);
+      throw error;
+    }
+  }
+
+  private async importSingleWorkflow(content: string, filename: string): Promise<void> {
+    // Validate workflow content
+    try {
+      // Basic validation - try to parse YAML frontmatter or direct YAML
+      if (filename.endsWith('.md')) {
+        const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---/);
+        if (frontmatterMatch) {
+          yamlParse(frontmatterMatch[1]);
+        } else {
+          throw new Error('Invalid markdown workflow: missing YAML frontmatter');
+        }
+      } else {
+        yamlParse(content);
+      }
+    } catch (error) {
+      throw new Error(`Invalid workflow format: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+
+    // Check for filename conflicts
+    let targetFilename = filename;
+    let fullPath = this.resolvePath(targetFilename);
+    let counter = 1;
+
+    while (existsSync(fullPath)) {
+      const nameWithoutExt = basename(filename, extname(filename));
+      targetFilename = `${nameWithoutExt}-${counter}${extname(filename)}`;
+      fullPath = this.resolvePath(targetFilename);
+      counter++;
+    }
+
+    // Write file
+    await writeFile(fullPath, content, 'utf-8');
+  }
+
+  private async importZipBundle(zipBuffer: Buffer): Promise<{ success: boolean; results: Array<{ success: boolean; filename: string; message?: string }> }> {
+    const results: Array<{ success: boolean; filename: string; message?: string }> = [];
+
+    try {
+      const zip = new AdmZip(zipBuffer);
+      const zipEntries = zip.getEntries();
+
+      for (const entry of zipEntries) {
+        // Skip directories and non-workflow files
+        if (entry.isDirectory) continue;
+
+        const filename = basename(entry.entryName);
+        const ext = extname(filename).toLowerCase();
+
+        if (!['.md', '.yaml', '.yml'].includes(ext)) {
+          results.push({
+            success: false,
+            filename,
+            message: 'Skipped: not a workflow file',
+          });
+          continue;
+        }
+
+        try {
+          const content = entry.getData().toString('utf-8');
+          await this.importSingleWorkflow(content, filename);
+
+          results.push({
+            success: true,
+            filename,
+            message: 'Imported successfully',
+          });
+        } catch (error) {
+          results.push({
+            success: false,
+            filename,
+            message: error instanceof Error ? error.message : 'Unknown error',
+          });
+        }
+      }
+
+      return {
+        success: results.some(r => r.success),
+        results,
+      };
+    } catch (error) {
+      throw new Error(`Failed to extract ZIP file: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
   }
 
   private resolvePath(workflowPath: string): string {

package/src/server/services/agents/codex-provider.ts

@@ -0,0 +1,398 @@
+/**
+ * OpenAI Codex AI agent provider
+ * Uses the @openai/codex-sdk for AI capabilities
+ */
+
+import { parse as yamlParse } from 'yaml';
+import type {
+  AgentProvider,
+  AgentCapabilities,
+  AgentConfig,
+  PromptResult,
+  Workflow,
+} from './types.js';
+import { buildPrompt, generateSuggestions } from './prompts.js';
+
+// Dynamic import types for Codex SDK
+interface CodexSDK {
+  Codex: new (options?: CodexOptions) => CodexInstance;
+}
+
+interface CodexOptions {
+  codexPathOverride?: string;
+  baseUrl?: string;
+  apiKey?: string;
+  env?: Record<string, string>;
+}
+
+interface ThreadOptions {
+  model?: string;
+  sandboxMode?: 'read-only' | 'workspace-write' | 'danger-full-access';
+  workingDirectory?: string;
+  skipGitRepoCheck?: boolean;
+  modelReasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
+  networkAccessEnabled?: boolean;
+  webSearchMode?: 'disabled' | 'cached' | 'live';
+  approvalPolicy?: 'never' | 'on-request' | 'on-failure' | 'untrusted';
+  additionalDirectories?: string[];
+}
+
+interface ThreadItem {
+  id: string;
+  type: string;
+  text?: string;
+  command?: string;
+  aggregated_output?: string;
+  status?: string;
+  changes?: unknown[];
+}
+
+interface Usage {
+  input_tokens: number;
+  cached_input_tokens: number;
+  output_tokens: number;
+}
+
+interface TurnResult {
+  items: ThreadItem[];
+  finalResponse: string;
+  usage: Usage | null;
+}
+
+interface ThreadEvent {
+  type: string;
+  thread_id?: string;
+  item?: ThreadItem;
+  usage?: Usage;
+  message?: string;
+  error?: { message: string };
+}
+
+interface StreamedTurnResult {
+  events: AsyncGenerator<ThreadEvent>;
+}
+
+interface Thread {
+  id: string | null;
+  run(input: string, options?: unknown): Promise<TurnResult>;
+  runStreamed(input: string, options?: unknown): Promise<StreamedTurnResult>;
+}
+
+interface CodexInstance {
+  startThread(options?: ThreadOptions): Thread;
+  resumeThread(id: string, options?: ThreadOptions): Thread;
+}
+
+export class CodexProvider implements AgentProvider {
+  readonly id = 'codex';
+  readonly name = 'OpenAI Codex';
+  readonly capabilities: AgentCapabilities = {
+    streaming: true,
+    toolUse: true,
+    codeExecution: true,
+    systemPrompts: true,
+    models: [
+      'codex-1',
+      'o3',
+      'o3-mini',
+      'o4-mini',
+      'gpt-4.1',
+    ],
+  };
+
+  private codex: CodexInstance | null = null;
+  private model: string = 'codex-1';
+  private ready: boolean = false;
+  private error: string | undefined;
+  private workingDirectory: string = process.cwd();
+  private lastThreadId: string | null = null;
+
+  async initialize(config: AgentConfig): Promise<void> {
+    try {
+      // Try to import the Codex SDK (dynamic import with webpackIgnore to avoid bundling issues)
+      const sdkModule = (await import(
+        /* webpackIgnore: true */ '@openai/codex-sdk'
+      ).catch(() => null)) as CodexSDK | null;
+
+      if (!sdkModule || !sdkModule.Codex) {
+        this.ready = false;
+        this.error = 'OpenAI Codex SDK not installed. Run: npm install @openai/codex-sdk';
+        return;
+      }
+
+      const { Codex } = sdkModule;
+
+      const codexOptions: CodexOptions = {};
+
+      // API key from config or environment
+      if (config.apiKey) {
+        codexOptions.apiKey = config.apiKey;
+      } else if (process.env.OPENAI_API_KEY) {
+        codexOptions.apiKey = process.env.OPENAI_API_KEY;
+      }
+
+      // Base URL
+      if (config.baseUrl) {
+        codexOptions.baseUrl = config.baseUrl;
+      }
+
+      // Codex path override
+      if (config.options?.codexPath) {
+        codexOptions.codexPathOverride = config.options.codexPath as string;
+      }
+
+      // Environment variables
+      if (config.options?.env) {
+        codexOptions.env = config.options.env as Record<string, string>;
+      }
+
+      this.codex = new Codex(codexOptions);
+
+      if (config.model) {
+        this.model = config.model;
+      }
+
+      // Working directory
+      if (config.options?.workingDirectory) {
+        this.workingDirectory = config.options.workingDirectory as string;
+      } else if (config.options?.cwd) {
+        this.workingDirectory = config.options.cwd as string;
+      }
+
+      // Test connectivity by starting a simple thread
+      try {
+        const thread = this.codex.startThread({
+          skipGitRepoCheck: true,
+          sandboxMode: 'read-only',
+          workingDirectory: this.workingDirectory,
+        });
+        // Just verify it can be created
+        if (thread) {
+          this.ready = true;
+          this.error = undefined;
+        }
+      } catch (testError) {
+        this.ready = false;
+        this.error = `Cannot initialize Codex: ${testError instanceof Error ? testError.message : 'Unknown error'}`;
+      }
+    } catch (err) {
+      this.ready = false;
+      this.error = err instanceof Error ? err.message : 'Unknown error initializing Codex';
+    }
+  }
+
+  isReady(): boolean {
+    return this.ready;
+  }
+
+  getStatus(): { ready: boolean; model?: string; error?: string } {
+    return {
+      ready: this.ready,
+      model: this.model,
+      error: this.error,
+    };
+  }
+
+  async processPrompt(
+    prompt: string,
+    workflow: Workflow,
+    context?: { selectedStepId?: string; recentHistory?: string[] }
+  ): Promise<PromptResult> {
+    if (!this.codex || !this.ready) {
+      return {
+        explanation: 'OpenAI Codex provider not available.',
+        error: this.error || 'Provider not initialized',
+      };
+    }
+
+    try {
+      // Build context-aware prompts
+      const { systemPrompt, userPrompt } = buildPrompt(prompt, workflow, context);
+
+      // Combine system and user prompts for Codex
+      const fullPrompt = `${systemPrompt}\n\n---\n\n${userPrompt}`;
+
+      const thread = this.codex.startThread({
+        model: this.model,
+        skipGitRepoCheck: true,
+        sandboxMode: 'read-only', // Safe for workflow modifications
+        workingDirectory: this.workingDirectory,
+        modelReasoningEffort: 'medium',
+      });
+
+      const result = await thread.run(fullPrompt);
+      this.lastThreadId = thread.id;
+
+      // Extract the response text
+      const responseText = result.finalResponse ||
+        result.items.find((item) => item.type === 'agent_message')?.text || '';
+
+      return this.parseAIResponse(responseText, workflow);
+    } catch (err) {
+      return {
+        explanation: '',
+        error: err instanceof Error ? err.message : 'Unknown error',
+      };
+    }
+  }
+
+  async getSuggestions(workflow: Workflow, selectedStepId?: string): Promise<string[]> {
+    return generateSuggestions(workflow, selectedStepId);
+  }
+
+  async streamPrompt(
+    prompt: string,
+    workflow: Workflow,
+    onChunk: (chunk: string) => void,
+    context?: { selectedStepId?: string; recentHistory?: string[] }
+  ): Promise<PromptResult> {
+    if (!this.codex || !this.ready) {
+      return this.processPrompt(prompt, workflow, context);
+    }
+
+    const { systemPrompt, userPrompt } = buildPrompt(prompt, workflow, context);
+    const fullPrompt = `${systemPrompt}\n\n---\n\n${userPrompt}`;
+    let fullResponse = '';
+
+    try {
+      const thread = this.codex.startThread({
+        model: this.model,
+        skipGitRepoCheck: true,
+        sandboxMode: 'read-only',
+        workingDirectory: this.workingDirectory,
+        modelReasoningEffort: 'medium',
+      });
+
+      const streamResult = await thread.runStreamed(fullPrompt);
+
+      for await (const event of streamResult.events) {
+        if (event.type === 'thread.started') {
+          this.lastThreadId = event.thread_id || null;
+        } else if (event.type === 'item.completed' && event.item?.type === 'agent_message') {
+          const text = event.item.text || '';
+          fullResponse = text;
+          onChunk(text);
+        } else if (event.type === 'error') {
+          throw new Error(event.message || 'Stream error');
+        } else if (event.type === 'turn.failed') {
+          throw new Error(event.error?.message || 'Turn failed');
+        }
+      }
+
+      return this.parseAIResponse(fullResponse, workflow);
+    } catch (err) {
+      return {
+        explanation: '',
+        error: err instanceof Error ? err.message : 'Unknown error',
+      };
+    }
+  }
+
+  async cancel(): Promise<void> {
+    // Codex SDK doesn't have explicit cancellation, but we can reset state
+    this.lastThreadId = null;
+  }
+
+  /**
+   * Get the last thread ID for resumption
+   */
+  getLastThreadId(): string | null {
+    return this.lastThreadId;
+  }
+
+  /**
+   * Resume a previous thread
+   */
+  async resumePrompt(
+    threadId: string,
+    prompt: string,
+    workflow: Workflow,
+    context?: { selectedStepId?: string; recentHistory?: string[] }
+  ): Promise<PromptResult> {
+    if (!this.codex || !this.ready) {
+      return {
+        explanation: 'OpenAI Codex provider not available.',
+        error: this.error || 'Provider not initialized',
+      };
+    }
+
+    try {
+      const { systemPrompt, userPrompt } = buildPrompt(prompt, workflow, context);
+      const fullPrompt = `${systemPrompt}\n\n---\n\n${userPrompt}`;
+
+      const thread = this.codex.resumeThread(threadId, {
+        model: this.model,
+        skipGitRepoCheck: true,
+        sandboxMode: 'read-only',
+        workingDirectory: this.workingDirectory,
+      });
+
+      const result = await thread.run(fullPrompt);
+      this.lastThreadId = thread.id;
+
+      const responseText = result.finalResponse ||
+        result.items.find((item) => item.type === 'agent_message')?.text || '';
+
+      return this.parseAIResponse(responseText, workflow);
+    } catch (err) {
+      return {
+        explanation: '',
+        error: err instanceof Error ? err.message : 'Unknown error',
+      };
+    }
+  }
+
+  private parseAIResponse(responseText: string, originalWorkflow: Workflow): PromptResult {
+    const yamlMatch = responseText.match(/```yaml\n([\s\S]*?)\n```/);
+    let modifiedWorkflow: Workflow | undefined;
+    let explanation = responseText;
+
+    if (yamlMatch) {
+      try {
+        const parsedYaml = yamlParse(yamlMatch[1]);
+        if (parsedYaml && (parsedYaml.steps || parsedYaml.metadata)) {
+          modifiedWorkflow = parsedYaml as Workflow;
+          const explanationMatch = responseText.match(/^([\s\S]*?)```yaml/);
+          if (explanationMatch) {
+            explanation = explanationMatch[1].trim();
+          }
+        }
+      } catch {
+        // Failed to parse YAML
+      }
+    }
+
+    let diff: string | undefined;
+    if (modifiedWorkflow) {
+      diff = this.generateDiff(originalWorkflow, modifiedWorkflow);
+    }
+
+    return { explanation, workflow: modifiedWorkflow, diff };
+  }
+
+  private generateDiff(original: Workflow, modified: Workflow): string {
+    const originalStepIds = new Set(original.steps?.map((s) => s.id) || []);
+    const modifiedStepIds = new Set(modified.steps?.map((s) => s.id) || []);
+
+    const added = modified.steps?.filter((s) => !originalStepIds.has(s.id)) || [];
+    const removed = original.steps?.filter((s) => !modifiedStepIds.has(s.id)) || [];
+
+    let diff = '';
+    if (added.length > 0) {
+      diff += `+ Added ${added.length} step(s): ${added.map((s) => s.name || s.id).join(', ')}\n`;
+    }
+    if (removed.length > 0) {
+      diff += `- Removed ${removed.length} step(s): ${removed.map((s) => s.name || s.id).join(', ')}\n`;
+    }
+
+    return diff || 'No structural changes detected';
+  }
+}
+
+export function createCodexProvider(config?: AgentConfig): CodexProvider {
+  const provider = new CodexProvider();
+  if (config) {
+    provider.initialize(config);
+  }
+  return provider;
+}

package/src/server/services/agents/prompts.ts

@@ -142,6 +142,33 @@ export const AVAILABLE_SERVICES = {
     description: 'Local Ollama LLM for AI tasks',
     commonActions: ['generate', 'chat', 'embeddings'],
   },
+  codex: {
+    sdk: '@openai/codex-sdk',
+    description: 'OpenAI Codex for AI-powered coding workflows',
+    commonActions: [
+      'chat',
+      'codeModify',
+      'codeAnalyze',
+      'codeReview',
+      'webSearch',
+      'execute',
+      'structured',
+      'resume',
+      'withImages',
+    ],
+  },
+  copilot: {
+    sdk: '@github/copilot-sdk',
+    description: 'GitHub Copilot for AI code assistance',
+    commonActions: [
+      'chat',
+      'codeReview',
+      'codeModify',
+      'withTools',
+      'withAgents',
+      'withMcp',
+    ],
+  },
 } as const;
 
 // Base system prompt with comprehensive context

package/src/server/services/agents/registry.ts

@@ -14,6 +14,7 @@ import type {
 import { createClaudeProvider } from './claude-provider.js';
 import { createClaudeCodeProvider } from './claude-code-provider.js';
 import { createCopilotProvider } from './copilot-provider.js';
+import { createCodexProvider } from './codex-provider.js';
 import { createDemoProvider } from './demo-provider.js';
 import { createOllamaProvider } from './ollama-provider.js';
 
@@ -38,6 +39,11 @@ export class AgentRegistry {
       name: 'GitHub Copilot (SDK)',
       factory: createCopilotProvider,
     });
+    this.registerProvider({
+      id: 'codex',
+      name: 'OpenAI Codex (SDK)',
+      factory: createCodexProvider,
+    });
     // API key-based providers
     this.registerProvider({
       id: 'claude',
@@ -153,6 +159,21 @@ export class AgentRegistry {
       }
     }
 
+    // Try OpenAI Codex SDK (uses OPENAI_API_KEY)
+    const codexProvider = this.providers.get('codex');
+    if (codexProvider) {
+      await codexProvider.initialize({
+        apiKey: process.env.OPENAI_API_KEY,
+        options: {
+          cwd: process.cwd(),
+        },
+      });
+      if (codexProvider.isReady()) {
+        this.activeProviderId = 'codex';
+        return 'codex';
+      }
+    }
+
     // Try Claude with API key (requires ANTHROPIC_API_KEY)
     const claudeProvider = this.providers.get('claude');
     if (claudeProvider) {