@synergenius/flow-weaver-pack-weaver 0.8.3 → 0.9.3

package/src/bot/step-executor.ts

@@ -1,27 +1,335 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
+import { execSync } from 'node:child_process';
 import { runCommand } from '@synergenius/flow-weaver';
 
+// ---------------------------------------------------------------------------
+// Safety thresholds
+// ---------------------------------------------------------------------------
+
+/** Refuse writes that shrink an existing file by more than this ratio (0-1). */
+const MAX_SHRINK_RATIO = 0.5;
+
+/** Minimum file size (bytes) before shrink guard kicks in. */
+const SHRINK_GUARD_MIN_SIZE = 100;
+
+/** Maximum number of files that can be written in a single plan. */
+const MAX_FILES_PER_PLAN = 50;
+
+/** Maximum shell command execution time (ms). */
+const SHELL_TIMEOUT = 60_000;
+
+/** Maximum file size for read/patch operations (1MB). */
+const MAX_READ_SIZE = 1_048_576;
+
+/** Maximum files returned by list-files. */
+const MAX_LIST_FILES = 1000;
+
+/** Shell commands that are NEVER allowed (destructive operations). */
+const BLOCKED_SHELL_PATTERNS = [
+  /\brm\s+(-[a-z]*r|-[a-z]*f)[a-z]*\s/i,  // rm with -r or -f flags
+  /\bgit\s+push\b/i,              // git push (no remote ops)
+  /\bgit\s+reset\s+--hard\b/i,    // git reset --hard
+  /\bnpm\s+publish\b/i,           // npm publish
+  /\bcurl\b.*\|\s*(sh|bash)\b/i,  // curl | sh/bash
+  /\bwget\b.*\|\s*(sh|bash)\b/i,  // wget | sh/bash
+  /\bsudo\b/i,                    // sudo
+  /\bchmod\s+777\b/i,             // chmod 777
+  /\bkill\s+-9\b/i,               // kill -9
+  /\bmkfs\b/i,                    // format disk
+  /\bdd\s+if=/i,                  // dd (disk destroyer)
+  />\s*\/dev\/sd/i,               // write to raw disk
+];
+
+/** Track files written in this process to enforce the per-plan cap. */
+let filesWrittenThisPlan = 0;
+
+/** Reset the per-plan counter between plans. */
+export function resetPlanFileCounter(): void {
+  filesWrittenThisPlan = 0;
+}
+
+// ---------------------------------------------------------------------------
+// Path safety
+// ---------------------------------------------------------------------------
+
+function assertSafePath(filePath: string, projectDir: string): void {
+  const resolved = path.resolve(projectDir, filePath);
+  if (!resolved.startsWith(path.resolve(projectDir))) {
+    throw new Error(
+      `Path traversal blocked: "${filePath}" resolves outside project directory.`,
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Write safety
+// ---------------------------------------------------------------------------
+
+interface WriteGuardResult {
+  allowed: boolean;
+  reason?: string;
+}
+
+function checkWriteSafety(filePath: string, content: string): WriteGuardResult {
+  // Guard 1: Empty content
+  if (!content || content.trim().length === 0) {
+    return {
+      allowed: false,
+      reason:
+        `Refusing to write empty content to ${path.basename(filePath)}. ` +
+        `Use read-file first, then write the complete modified file back.`,
+    };
+  }
+
+  // Guard 2: Per-plan file cap
+  if (filesWrittenThisPlan >= MAX_FILES_PER_PLAN) {
+    return {
+      allowed: false,
+      reason: `File write limit reached (${MAX_FILES_PER_PLAN} files per plan).`,
+    };
+  }
+
+  // Guard 3: Shrink detection
+  if (fs.existsSync(filePath)) {
+    const existingSize = fs.statSync(filePath).size;
+    const newSize = Buffer.byteLength(content, 'utf-8');
+    if (existingSize > SHRINK_GUARD_MIN_SIZE && newSize < existingSize * MAX_SHRINK_RATIO) {
+      const shrinkPct = Math.round((1 - newSize / existingSize) * 100);
+      return {
+        allowed: false,
+        reason:
+          `Refusing to write ${path.basename(filePath)}: new content (${newSize}B) ` +
+          `is ${shrinkPct}% smaller than existing (${existingSize}B). ` +
+          `Use read-file first, make targeted changes, write complete file back.`,
+      };
+    }
+  }
+
+  return { allowed: true };
+}
+
+// ---------------------------------------------------------------------------
+// Shell safety
+// ---------------------------------------------------------------------------
+
+function checkShellSafety(command: string): WriteGuardResult {
+  for (const pattern of BLOCKED_SHELL_PATTERNS) {
+    if (pattern.test(command)) {
+      return {
+        allowed: false,
+        reason: `Shell command blocked by safety policy: matches "${pattern.source}"`,
+      };
+    }
+  }
+  return { allowed: true };
+}
+
+// ---------------------------------------------------------------------------
+// Main executor
+// ---------------------------------------------------------------------------
+
+export interface StepResult {
+  file?: string;
+  files?: string[];
+  created?: boolean;
+  output?: string;
+  blocked?: boolean;
+  blockReason?: string;
+}
+
 export async function executeStep(
   step: { operation: string; args: Record<string, unknown> },
   projectDir: string,
-): Promise<{ file?: string; files?: string[]; created?: boolean; output?: string }> {
+): Promise<StepResult> {
   const args = step.args;
   const file = args.file as string | undefined;
 
   switch (step.operation) {
+    // -----------------------------------------------------------------
+    // File write operations (with safety guards)
+    // -----------------------------------------------------------------
     case 'write-file':
     case 'create-workflow':
     case 'modify-source':
     case 'implement-node': {
-      const filePath = path.resolve(projectDir, file!);
+      if (!file) {
+        return { blocked: true, blockReason: `${step.operation} requires a "file" argument.` };
+      }
+      assertSafePath(file, projectDir);
+      const filePath = path.resolve(projectDir, file);
+      const content = (args.content as string) ?? (args.body as string) ?? '';
+
+      const guard = checkWriteSafety(filePath, content);
+      if (!guard.allowed) {
+        return { file: filePath, blocked: true, blockReason: guard.reason };
+      }
+
       fs.mkdirSync(path.dirname(filePath), { recursive: true });
       const existed = fs.existsSync(filePath);
-      fs.writeFileSync(filePath, (args.content as string) ?? (args.body as string) ?? '', 'utf-8');
+      fs.writeFileSync(filePath, content, 'utf-8');
+      filesWrittenThisPlan++;
       return { file: filePath, created: !existed };
     }
-    case 'read-file':
-      return {};
+
+    // -----------------------------------------------------------------
+    // Patch file: surgical find-and-replace (no full rewrite needed)
+    // -----------------------------------------------------------------
+    case 'patch-file': {
+      if (!file) {
+        return { blocked: true, blockReason: 'patch-file requires a "file" argument.' };
+      }
+      assertSafePath(file, projectDir);
+      const filePath = path.resolve(projectDir, file);
+
+      if (!fs.existsSync(filePath)) {
+        return { blocked: true, blockReason: `File not found: ${file}` };
+      }
+
+      // File size guard
+      const fileSize = fs.statSync(filePath).size;
+      if (fileSize > MAX_READ_SIZE) {
+        return { blocked: true, blockReason: `File too large for patch-file (${fileSize} bytes, max ${MAX_READ_SIZE}).` };
+      }
+
+      let content = fs.readFileSync(filePath, 'utf-8');
+      const patches = (args.patches as Array<{ find: string; replace: string }>) ?? [];
+
+      if (!patches.length && args.find && args.replace !== undefined) {
+        patches.push({ find: args.find as string, replace: args.replace as string });
+      }
+
+      if (!patches.length) {
+        return { blocked: true, blockReason: 'patch-file requires "patches" array or "find"+"replace" args.' };
+      }
+
+      let applied = 0;
+      const notFound: string[] = [];
+
+      for (const patch of patches) {
+        if (content.includes(patch.find)) {
+          content = content.split(patch.find).join(patch.replace); // replaceAll without regex
+          applied++;
+        } else {
+          notFound.push(patch.find.substring(0, 60));
+        }
+      }
+
+      if (applied === 0) {
+        return {
+          file: filePath,
+          output: `No patches applied. Search strings not found: ${notFound.join('; ')}`,
+        };
+      }
+
+      fs.writeFileSync(filePath, content, 'utf-8');
+      filesWrittenThisPlan++;
+
+      const summary = `Applied ${applied}/${patches.length} patches` +
+        (notFound.length ? `. Not found: ${notFound.join('; ')}` : '');
+      return { file: filePath, output: summary };
+    }
+
+    // -----------------------------------------------------------------
+    // Read file: return content for AI context
+    // -----------------------------------------------------------------
+    case 'read-file': {
+      if (!file) {
+        return { output: '' };
+      }
+      assertSafePath(file, projectDir);
+      const filePath = path.resolve(projectDir, file);
+      if (!fs.existsSync(filePath)) {
+        return { output: `File not found: ${file}` };
+      }
+      if (fs.statSync(filePath).isDirectory()) {
+        const entries = fs.readdirSync(filePath, { encoding: 'utf-8' });
+        return { output: `"${file}" is a directory. Contents:\n${entries.join('\n')}` };
+      }
+      const fileSize = fs.statSync(filePath).size;
+      if (fileSize > MAX_READ_SIZE) {
+        return { output: `File too large to read (${fileSize} bytes, max ${MAX_READ_SIZE}). Use run-shell with head/tail instead.` };
+      }
+      const content = fs.readFileSync(filePath, 'utf-8');
+      return { file: filePath, output: content };
+    }
+
+    // -----------------------------------------------------------------
+    // Shell command: run arbitrary command with safety guards
+    // -----------------------------------------------------------------
+    case 'run-shell': {
+      const command = (args.command as string) ?? '';
+      if (!command.trim()) {
+        return { blocked: true, blockReason: 'run-shell requires a "command" argument.' };
+      }
+
+      const shellGuard = checkShellSafety(command);
+      if (!shellGuard.allowed) {
+        return { blocked: true, blockReason: shellGuard.reason };
+      }
+
+      try {
+        const output = execSync(command, {
+          cwd: projectDir,
+          encoding: 'utf-8',
+          timeout: SHELL_TIMEOUT,
+          maxBuffer: 1024 * 1024, // 1MB output cap
+          stdio: ['pipe', 'pipe', 'pipe'],
+        });
+        return { output: output.trim() };
+      } catch (err: unknown) {
+        // Shell commands that exit non-zero still return useful output
+        const execErr = err as { stdout?: string; stderr?: string; status?: number };
+        const stdout = (execErr.stdout ?? '').trim();
+        const stderr = (execErr.stderr ?? '').trim();
+        const combined = [stdout, stderr].filter(Boolean).join('\n');
+        return {
+          output: combined || (err instanceof Error ? err.message : String(err)),
+        };
+      }
+    }
+
+    // -----------------------------------------------------------------
+    // List files: glob-like directory listing
+    // -----------------------------------------------------------------
+    case 'list-files': {
+      const dir = (args.directory as string) ?? (args.dir as string) ?? '.';
+      const pattern = (args.pattern as string) ?? '';
+      assertSafePath(dir, projectDir);
+      const targetDir = path.resolve(projectDir, dir);
+
+      if (!fs.existsSync(targetDir)) {
+        return { output: `Directory not found: ${dir}` };
+      }
+
+      const entries = fs.readdirSync(targetDir, { recursive: true, encoding: 'utf-8' }) as string[];
+      let files = entries
+        .filter(e => {
+          if (e.includes('node_modules') || e.includes('.git')) return false;
+          const full = path.join(targetDir, e);
+          try { return fs.statSync(full).isFile(); } catch { return false; }
+        })
+        .sort();
+
+      if (pattern) {
+        try {
+          const regex = new RegExp(pattern);
+          files = files.filter(f => regex.test(f));
+        } catch {
+          return { output: `Invalid regex pattern: ${pattern}` };
+        }
+      }
+
+      if (files.length > MAX_LIST_FILES) {
+        files = files.slice(0, MAX_LIST_FILES);
+      }
+
+      return { files: files.map(f => path.join(dir, f)), output: files.join('\n') };
+    }
+
+    // -----------------------------------------------------------------
+    // Flow-weaver CLI commands (via programmatic API)
+    // -----------------------------------------------------------------
     default: {
       const result = await runCommand(step.operation, { ...args, cwd: projectDir });
       return {

package/src/bot/system-prompt.ts

@@ -177,9 +177,38 @@ Genesis is a 17-step self-evolving workflow engine:
 
 When stabilize mode is active, only fix-up operations are allowed: removeNode, removeConnection, implementNode. No new nodes or connections.
 
-## Response Format
+## Tool Use
 
-Return ONLY valid JSON. No markdown, no code fences, no explanation outside the JSON structure. Your response must parse with JSON.parse() directly.`;
+You have tools available: validate, read_file, patch_file, run_shell, list_files, write_file.
+
+USE TOOLS to complete tasks. Do NOT describe what you would do — actually do it by calling tools. You can see tool results and decide your next action dynamically.
+
+Workflow for fixing validation errors:
+1. Call validate(file) to see exact errors
+2. Call read_file(file) to see the code
+3. Call patch_file(file, patches) with exact find/replace strings
+4. Call validate(file) again to confirm fixes
+5. Repeat if errors remain
+
+Rules:
+- Always validate BEFORE and AFTER patching
+- Always read a file before patching it (you need exact strings for find/replace)
+- Use patch_file for modifications, write_file only for new files
+- Be concise in your text responses — let tool results speak
+
+Flow Weaver workflows are TypeScript. You can also help create supporting files in other formats (JSON configs, shell scripts, Markdown docs).
+
+Before starting a task on a file, call recall(filename) to check if there is stored knowledge about known issues or patterns for that file.
+After discovering something important (a pattern, a common fix, a gotcha), call learn(key, value) to store it for future tasks.
+
+## Teaching
+
+When creating or modifying workflows, briefly explain your decisions:
+- Why you chose a particular template or pattern (1 line)
+- What each node does and why it is @expression vs standard (1 line)
+- What the data flow looks like (1 line)
+Do NOT lecture. Keep explanations short. The user is learning Flow Weaver by watching you work.
+Example: "Using sequential template — best for linear pipelines. The validator is @expression (pure, no side effects). Data flows: input -> validate -> transform -> output."`;
 }
 
 export async function buildSystemPrompt(): Promise<string> {
@@ -205,11 +234,25 @@ export async function buildSystemPrompt(): Promise<string> {
 
 function formatBotOperations(cliCommands: CliCommandDoc[]): string {
   const packOps = [
+    '## File Operations',
     '- create-workflow: Create a new workflow file. args: { file, content }',
     '- implement-node: Write a node type implementation. args: { file, content }',
-    '- modify-source: Direct source file modification. args: { file, content }',
-    '- read-file: Read a file for context. args: { file }',
-    '- write-file: Write a file. args: { file, content }',
+    '- write-file: Write a file. args: { file, content }. Content must be the COMPLETE file.',
+    '- read-file: Read a file and return its content. args: { file }',
+    '- patch-file: Surgical find-and-replace edits. args: { file, patches: [{ find: "old text", replace: "new text" }] }. PREFERRED for modifying existing files — no need to rewrite the entire file.',
+    '- list-files: List files in a directory. args: { directory, pattern? } (pattern is regex)',
+    '',
+    '## Shell Commands',
+    '- run-shell: Execute a shell command and return output. args: { command }. Use for: npx vitest, git status, grep, find, etc.',
+    '  Examples: { "command": "npx vitest run --reporter verbose" }, { "command": "npx flow-weaver validate src/workflow.ts --json" }',
+    '  Blocked: rm -rf, git push, npm publish, sudo, curl|sh (safety policy).',
+    '',
+    '## Best Practices',
+    'PREFER patch-file over write-file for modifying existing files (surgical edits, no truncation risk).',
+    'Use run-shell for running tests (npx vitest), validation (flow-weaver validate), and inspecting output.',
+    'Use read-file to understand a file before modifying it.',
+    'Use list-files to discover project structure.',
+    'Writes that shrink a file by >50% or write empty content are automatically BLOCKED.',
   ];
 
   const fwOps = cliCommands
@@ -229,48 +272,28 @@ function formatBotOperations(cliCommands: CliCommandDoc[]): string {
   return [...packOps, ...fwOps].join('\n');
 }
 
-export function buildBotSystemPrompt(contextBundle?: string, cliCommands?: CliCommandDoc[]): string {
-  const operationsList = formatBotOperations(cliCommands ?? []);
-
-  const planSchema = `## Bot Plan Schema
-
-When asked to plan a task, return a JSON object with this structure:
-{
-  "steps": [
-    {
-      "id": "step-1",
-      "operation": "<operation>",
-      "description": "What this step does",
-      "args": { ... operation-specific arguments ... }
-    }
-  ],
-  "summary": "Brief description of the overall plan"
-}
-
-Available operations:
-${operationsList}`;
-
-  const botInstructions = `## Bot Mode Instructions
-
-You are operating in autonomous bot mode. Your job is to plan and execute workflow creation or modification tasks.
-
-When planning:
-1. Break the task into concrete, ordered steps using the plan schema above
-2. For new workflows, plan: scaffold/create -> implement nodes -> compile -> validate
-3. For modifications, plan: read current state -> modify -> compile -> validate
-4. Each step is executed via the flow-weaver programmatic API
-5. Use templates when they match the task
-6. Prefer @expression nodes for deterministic operations
-7. Use proper JSDoc annotations on all node types and workflows
-8. Include visualization metadata (colors, icons, positions) on workflow nodes
-
-When fixing validation errors:
-1. Read the error messages carefully
-2. Map each error to a specific fix operation
-3. Common fixes: add missing connections, fix port names, resolve type mismatches
-4. Return a new plan with only the fix steps`;
-
-  let prompt = planSchema + '\n\n' + botInstructions;
+export function buildBotSystemPrompt(contextBundle?: string, _cliCommands?: CliCommandDoc[], projectDir?: string): string {
+  let prompt = `## Safety Policy
+
+Writes that shrink a file by >50% or write empty content are automatically BLOCKED.
+Blocked shell commands: rm -rf, git push, npm publish, sudo, curl|sh.
+Always validate BEFORE and AFTER patching.
+Always read a file before patching it (you need exact strings for find/replace).
+Use patch_file for modifications, write_file only for new files.
+Be concise in your text responses — let tool results speak.`;
+
+  // Load project plan file if it exists — this is the vision spec that guides all work
+  if (projectDir) {
+    try {
+      const fs = require('node:fs');
+      const path = require('node:path');
+      const planPath = path.resolve(projectDir, '.weaver-plan.md');
+      if (fs.existsSync(planPath)) {
+        const plan = fs.readFileSync(planPath, 'utf-8').trim();
+        prompt += '\n\n## Project Plan & Vision\n\nIMPORTANT: All work MUST align with this plan. If a task contradicts the plan, skip it and explain why.\n\n' + plan;
+      }
+    } catch { /* plan file not available */ }
+  }
 
   if (contextBundle) {
     prompt += '\n\n## Project Context\n\n' + contextBundle;

package/src/bot/task-decomposer.ts

@@ -0,0 +1,100 @@
+/**
+ * Auto task decomposition — splits broad tasks into per-file tasks.
+ *
+ * When a task says "fix all templates" or "validate everything",
+ * decompose it into one task per file. This gives the AI focused
+ * context and prevents one failure from blocking all files.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+export interface DecomposableTask {
+  id: string;
+  instruction: string;
+  mode?: string;
+  targets?: string[];
+  priority?: number;
+}
+
+export interface DecomposedResult {
+  decomposed: boolean;
+  tasks: DecomposableTask[];
+}
+
+// Patterns that suggest a broad task targeting multiple files
+const BROAD_PATTERNS = [
+  /\b(all|every|each)\b.*\b(template|workflow|file|node.?type)s?\b/i,
+  /\bfix\b.*\bin\s+src\/(templates|node-types|workflows)\/?$/i,
+  /\bvalidat(e|ion)\b.*\b(all|every|each|src\/)\b/i,
+];
+
+/**
+ * Check if a task should be decomposed into per-file tasks.
+ * Returns the original task unchanged if no decomposition is needed.
+ */
+export function decomposeTask(
+  task: DecomposableTask,
+  projectDir: string,
+): DecomposedResult {
+  const instruction = task.instruction;
+
+  // Already has specific targets — don't decompose
+  if (task.targets && task.targets.length === 1) {
+    return { decomposed: false, tasks: [task] };
+  }
+
+  // Check if instruction matches broad patterns
+  const isBroad = BROAD_PATTERNS.some(p => p.test(instruction));
+  if (!isBroad) {
+    return { decomposed: false, tasks: [task] };
+  }
+
+  // Determine which directory to scan
+  let targetDir: string | undefined;
+  if (instruction.match(/template/i)) targetDir = 'src/templates';
+  else if (instruction.match(/node.?type/i)) targetDir = 'src/node-types';
+  else if (instruction.match(/workflow/i)) targetDir = 'src/workflows';
+
+  if (!targetDir) {
+    return { decomposed: false, tasks: [task] };
+  }
+
+  const absDir = path.resolve(projectDir, targetDir);
+  if (!fs.existsSync(absDir)) {
+    return { decomposed: false, tasks: [task] };
+  }
+
+  // List .ts files in the directory — only files that actually exist
+  let files: string[];
+  try {
+    files = fs.readdirSync(absDir)
+      .filter(f => f.endsWith('.ts') && !f.startsWith('index'))
+      .filter(f => fs.existsSync(path.resolve(absDir, f))) // verify file exists
+      .sort();
+  } catch {
+    return { decomposed: false, tasks: [task] };
+  }
+
+  if (files.length === 0 || files.length > 50) {
+    return { decomposed: false, tasks: [task] };
+  }
+
+  // Extract the verb from the original instruction for clean per-file instructions
+  const verb = instruction.match(/^(Fix|Validate|Add|Update|Review|Run|Check|Test|Improve)/i)?.[0] ?? 'Process';
+
+  // Create per-file tasks with clean, grammatical instructions
+  const tasks: DecomposableTask[] = files.map((file, i) => {
+    const filePath = path.join(targetDir!, file);
+
+    return {
+      id: `${task.id}-${i + 1}`,
+      instruction: `${verb} ${filePath}`,
+      mode: task.mode ?? 'modify',
+      targets: [filePath],
+      priority: task.priority ?? 0,
+    };
+  });
+
+  return { decomposed: true, tasks };
+}