@synergenius/flow-weaver-pack-weaver 0.9.0 → 0.9.4

package/src/bot/system-prompt.ts

@@ -177,9 +177,38 @@ Genesis is a 17-step self-evolving workflow engine:
 
 When stabilize mode is active, only fix-up operations are allowed: removeNode, removeConnection, implementNode. No new nodes or connections.
 
-## Response Format
+## Tool Use
 
-Return ONLY valid JSON. No markdown, no code fences, no explanation outside the JSON structure. Your response must parse with JSON.parse() directly.`;
+You have tools available: validate, read_file, patch_file, run_shell, list_files, write_file.
+
+USE TOOLS to complete tasks. Do NOT describe what you would do — actually do it by calling tools. You can see tool results and decide your next action dynamically.
+
+Workflow for fixing validation errors:
+1. Call validate(file) to see exact errors
+2. Call read_file(file) to see the code
+3. Call patch_file(file, patches) with exact find/replace strings
+4. Call validate(file) again to confirm fixes
+5. Repeat if errors remain
+
+Rules:
+- Always validate BEFORE and AFTER patching
+- Always read a file before patching it (you need exact strings for find/replace)
+- Use patch_file for modifications, write_file only for new files
+- Be concise in your text responses — let tool results speak
+
+Flow Weaver workflows are TypeScript. You can also help create supporting files in other formats (JSON configs, shell scripts, Markdown docs).
+
+Before starting a task on a file, call recall(filename) to check if there is stored knowledge about known issues or patterns for that file.
+After discovering something important (a pattern, a common fix, a gotcha), call learn(key, value) to store it for future tasks.
+
+## Teaching
+
+When creating or modifying workflows, briefly explain your decisions:
+- Why you chose a particular template or pattern (1 line)
+- What each node does and why it is @expression vs standard (1 line)
+- What the data flow looks like (1 line)
+Do NOT lecture. Keep explanations short. The user is learning Flow Weaver by watching you work.
+Example: "Using sequential template — best for linear pipelines. The validator is @expression (pure, no side effects). Data flows: input -> validate -> transform -> output."`;
 }
 
 export async function buildSystemPrompt(): Promise<string> {
@@ -205,11 +234,25 @@ export async function buildSystemPrompt(): Promise<string> {
 
 function formatBotOperations(cliCommands: CliCommandDoc[]): string {
   const packOps = [
+    '## File Operations',
     '- create-workflow: Create a new workflow file. args: { file, content }',
     '- implement-node: Write a node type implementation. args: { file, content }',
-    '- modify-source: Direct source file modification. args: { file, content }',
-    '- read-file: Read a file for context. args: { file }',
-    '- write-file: Write a file. args: { file, content }',
+    '- write-file: Write a file. args: { file, content }. Content must be the COMPLETE file.',
+    '- read-file: Read a file and return its content. args: { file }',
+    '- patch-file: Surgical find-and-replace edits. args: { file, patches: [{ find: "old text", replace: "new text" }] }. PREFERRED for modifying existing files — no need to rewrite the entire file.',
+    '- list-files: List files in a directory. args: { directory, pattern? } (pattern is regex)',
+    '',
+    '## Shell Commands',
+    '- run-shell: Execute a shell command and return output. args: { command }. Use for: npx vitest, git status, grep, find, etc.',
+    '  Examples: { "command": "npx vitest run --reporter verbose" }, { "command": "npx flow-weaver validate src/workflow.ts --json" }',
+    '  Blocked: rm -rf, git push, npm publish, sudo, curl|sh (safety policy).',
+    '',
+    '## Best Practices',
+    'PREFER patch-file over write-file for modifying existing files (surgical edits, no truncation risk).',
+    'Use run-shell for running tests (npx vitest), validation (flow-weaver validate), and inspecting output.',
+    'Use read-file to understand a file before modifying it.',
+    'Use list-files to discover project structure.',
+    'Writes that shrink a file by >50% or write empty content are automatically BLOCKED.',
   ];
 
   const fwOps = cliCommands
@@ -229,48 +272,28 @@ function formatBotOperations(cliCommands: CliCommandDoc[]): string {
   return [...packOps, ...fwOps].join('\n');
 }
 
-export function buildBotSystemPrompt(contextBundle?: string, cliCommands?: CliCommandDoc[]): string {
-  const operationsList = formatBotOperations(cliCommands ?? []);
-
-  const planSchema = `## Bot Plan Schema
-
-When asked to plan a task, return a JSON object with this structure:
-{
-  "steps": [
-    {
-      "id": "step-1",
-      "operation": "<operation>",
-      "description": "What this step does",
-      "args": { ... operation-specific arguments ... }
-    }
-  ],
-  "summary": "Brief description of the overall plan"
-}
-
-Available operations:
-${operationsList}`;
-
-  const botInstructions = `## Bot Mode Instructions
-
-You are operating in autonomous bot mode. Your job is to plan and execute workflow creation or modification tasks.
-
-When planning:
-1. Break the task into concrete, ordered steps using the plan schema above
-2. For new workflows, plan: scaffold/create -> implement nodes -> compile -> validate
-3. For modifications, plan: read current state -> modify -> compile -> validate
-4. Each step is executed via the flow-weaver programmatic API
-5. Use templates when they match the task
-6. Prefer @expression nodes for deterministic operations
-7. Use proper JSDoc annotations on all node types and workflows
-8. Include visualization metadata (colors, icons, positions) on workflow nodes
-
-When fixing validation errors:
-1. Read the error messages carefully
-2. Map each error to a specific fix operation
-3. Common fixes: add missing connections, fix port names, resolve type mismatches
-4. Return a new plan with only the fix steps`;
-
-  let prompt = planSchema + '\n\n' + botInstructions;
+export function buildBotSystemPrompt(contextBundle?: string, _cliCommands?: CliCommandDoc[], projectDir?: string): string {
+  let prompt = `## Safety Policy
+
+Writes that shrink a file by >50% or write empty content are automatically BLOCKED.
+Blocked shell commands: rm -rf, git push, npm publish, sudo, curl|sh.
+Always validate BEFORE and AFTER patching.
+Always read a file before patching it (you need exact strings for find/replace).
+Use patch_file for modifications, write_file only for new files.
+Be concise in your text responses — let tool results speak.`;
+
+  // Load project plan file if it exists — this is the vision spec that guides all work
+  if (projectDir) {
+    try {
+      const fs = require('node:fs');
+      const path = require('node:path');
+      const planPath = path.resolve(projectDir, '.weaver-plan.md');
+      if (fs.existsSync(planPath)) {
+        const plan = fs.readFileSync(planPath, 'utf-8').trim();
+        prompt += '\n\n## Project Plan & Vision\n\nIMPORTANT: All work MUST align with this plan. If a task contradicts the plan, skip it and explain why.\n\n' + plan;
+      }
+    } catch { /* plan file not available */ }
+  }
 
   if (contextBundle) {
     prompt += '\n\n## Project Context\n\n' + contextBundle;

package/src/bot/task-decomposer.ts

@@ -0,0 +1,100 @@
+/**
+ * Auto task decomposition — splits broad tasks into per-file tasks.
+ *
+ * When a task says "fix all templates" or "validate everything",
+ * decompose it into one task per file. This gives the AI focused
+ * context and prevents one failure from blocking all files.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+export interface DecomposableTask {
+  id: string;
+  instruction: string;
+  mode?: string;
+  targets?: string[];
+  priority?: number;
+}
+
+export interface DecomposedResult {
+  decomposed: boolean;
+  tasks: DecomposableTask[];
+}
+
+// Patterns that suggest a broad task targeting multiple files
+const BROAD_PATTERNS = [
+  /\b(all|every|each)\b.*\b(template|workflow|file|node.?type)s?\b/i,
+  /\bfix\b.*\bin\s+src\/(templates|node-types|workflows)\/?$/i,
+  /\bvalidat(e|ion)\b.*\b(all|every|each|src\/)\b/i,
+];
+
+/**
+ * Check if a task should be decomposed into per-file tasks.
+ * Returns the original task unchanged if no decomposition is needed.
+ */
+export function decomposeTask(
+  task: DecomposableTask,
+  projectDir: string,
+): DecomposedResult {
+  const instruction = task.instruction;
+
+  // Already has specific targets — don't decompose
+  if (task.targets && task.targets.length === 1) {
+    return { decomposed: false, tasks: [task] };
+  }
+
+  // Check if instruction matches broad patterns
+  const isBroad = BROAD_PATTERNS.some(p => p.test(instruction));
+  if (!isBroad) {
+    return { decomposed: false, tasks: [task] };
+  }
+
+  // Determine which directory to scan
+  let targetDir: string | undefined;
+  if (instruction.match(/template/i)) targetDir = 'src/templates';
+  else if (instruction.match(/node.?type/i)) targetDir = 'src/node-types';
+  else if (instruction.match(/workflow/i)) targetDir = 'src/workflows';
+
+  if (!targetDir) {
+    return { decomposed: false, tasks: [task] };
+  }
+
+  const absDir = path.resolve(projectDir, targetDir);
+  if (!fs.existsSync(absDir)) {
+    return { decomposed: false, tasks: [task] };
+  }
+
+  // List .ts files in the directory — only files that actually exist
+  let files: string[];
+  try {
+    files = fs.readdirSync(absDir)
+      .filter(f => f.endsWith('.ts') && !f.startsWith('index'))
+      .filter(f => fs.existsSync(path.resolve(absDir, f))) // verify file exists
+      .sort();
+  } catch {
+    return { decomposed: false, tasks: [task] };
+  }
+
+  if (files.length === 0 || files.length > 50) {
+    return { decomposed: false, tasks: [task] };
+  }
+
+  // Extract the verb from the original instruction for clean per-file instructions
+  const verb = instruction.match(/^(Fix|Validate|Add|Update|Review|Run|Check|Test|Improve)/i)?.[0] ?? 'Process';
+
+  // Create per-file tasks with clean, grammatical instructions
+  const tasks: DecomposableTask[] = files.map((file, i) => {
+    const filePath = path.join(targetDir!, file);
+
+    return {
+      id: `${task.id}-${i + 1}`,
+      instruction: `${verb} ${filePath}`,
+      mode: task.mode ?? 'modify',
+      targets: [filePath],
+      priority: task.priority ?? 0,
+    };
+  });
+
+  return { decomposed: true, tasks };
+}

package/src/bot/task-queue.ts

@@ -1,9 +1,9 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
-import * as os from 'node:os';
 import * as crypto from 'node:crypto';
 import { withFileLock } from './file-lock.js';
 import { parseNdjson } from './safe-json.js';
+import { resolveWeaverDir } from './paths.js';
 
 export interface QueuedTask {
   id: string;
@@ -13,19 +13,53 @@ export interface QueuedTask {
   options?: Record<string, unknown>;
   priority: number;
   addedAt: number;
-  status: 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
+  status: 'pending' | 'running' | 'completed' | 'no-op' | 'failed' | 'cancelled';
+  /** Error reason (set on failure) */
+  failureReason?: string;
 }
 
+export interface AddResult {
+  id: string;
+  duplicate: boolean;
+}
+
+/** Max pending tasks before queue rejects new additions. */
+const MAX_PENDING = 200;
+/** Don't re-queue tasks completed within this window (ms). */
+const CYCLE_DEDUP_WINDOW = 3600_000; // 1 hour
+
 export class TaskQueue {
-  private filePath: string;
+  readonly filePath: string;
 
   constructor(dir?: string) {
-    const base = dir ?? path.join(os.homedir(), '.weaver');
+    const base = dir ?? resolveWeaverDir();
     this.filePath = path.join(base, 'task-queue.ndjson');
   }
 
-  async add(task: Omit<QueuedTask, 'id' | 'addedAt' | 'status'>): Promise<string> {
+  async add(task: Omit<QueuedTask, 'id' | 'addedAt' | 'status'>): Promise<AddResult> {
     return withFileLock(this.filePath, () => {
+      const existing = this.readAll();
+
+      // Dedup: skip if a pending task with the same instruction exists
+      const pendingDup = existing.find(
+        t => t.status === 'pending' && t.instruction === task.instruction,
+      );
+      if (pendingDup) return { id: pendingDup.id, duplicate: true };
+
+      // Cycle-aware dedup: skip if same instruction was completed recently
+      const recentDup = existing.find(
+        t => (t.status === 'completed' || t.status === 'no-op')
+          && t.instruction === task.instruction
+          && Date.now() - t.addedAt < CYCLE_DEDUP_WINDOW,
+      );
+      if (recentDup) return { id: recentDup.id, duplicate: true };
+
+      // Queue size cap
+      const pendingCount = existing.filter(t => t.status === 'pending').length;
+      if (pendingCount >= MAX_PENDING) {
+        throw new Error(`Queue full (${MAX_PENDING} pending tasks). Clear or process existing tasks first.`);
+      }
+
       const entry: QueuedTask = {
         ...task,
         id: crypto.randomUUID().slice(0, 8),
@@ -35,7 +69,7 @@ export class TaskQueue {
       const dir = path.dirname(this.filePath);
       if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
       fs.appendFileSync(this.filePath, JSON.stringify(entry) + '\n', 'utf-8');
-      return entry.id;
+      return { id: entry.id, duplicate: false };
     });
   }
 
@@ -78,8 +112,66 @@ export class TaskQueue {
     await this.updateStatus(id, 'completed');
   }
 
-  async markFailed(id: string): Promise<void> {
-    await this.updateStatus(id, 'failed');
+  async markNoOp(id: string): Promise<void> {
+    await this.updateStatus(id, 'no-op');
+  }
+
+  async markFailed(id: string, reason?: string): Promise<void> {
+    return withFileLock(this.filePath, () => {
+      const tasks = this.readAll();
+      const task = tasks.find(t => t.id === id);
+      if (task) {
+        task.status = 'failed';
+        if (reason) task.failureReason = reason.slice(0, 500);
+        this.writeAll(tasks);
+      }
+    });
+  }
+
+  /** Reset a failed or running task back to pending. */
+  async retry(id: string): Promise<boolean> {
+    return withFileLock(this.filePath, () => {
+      const tasks = this.readAll();
+      const task = tasks.find(t => t.id === id && (t.status === 'failed' || t.status === 'running'));
+      if (!task) return false;
+      task.status = 'pending';
+      task.failureReason = undefined;
+      this.writeAll(tasks);
+      return true;
+    });
+  }
+
+  /** Reset ALL failed tasks back to pending. Returns count reset. */
+  async retryAll(): Promise<number> {
+    return withFileLock(this.filePath, () => {
+      const tasks = this.readAll();
+      let count = 0;
+      for (const t of tasks) {
+        if (t.status === 'failed') {
+          t.status = 'pending';
+          t.failureReason = undefined;
+          count++;
+        }
+      }
+      if (count > 0) this.writeAll(tasks);
+      return count;
+    });
+  }
+
+  /** Reset orphaned "running" tasks to pending (crash recovery). */
+  async recoverOrphans(): Promise<number> {
+    return withFileLock(this.filePath, () => {
+      const tasks = this.readAll();
+      let count = 0;
+      for (const t of tasks) {
+        if (t.status === 'running') {
+          t.status = 'pending';
+          count++;
+        }
+      }
+      if (count > 0) this.writeAll(tasks);
+      return count;
+    });
   }
 
   private async updateStatus(id: string, status: QueuedTask['status']): Promise<void> {

package/src/bot/terminal-renderer.ts

@@ -0,0 +1,238 @@
+/**
+ * Terminal renderer — centralizes all weaver CLI output through
+ * a consistent visual grammar. Pipe-safe (stderr only for decoration).
+ *
+ * Icons: ✓ success · ✗ error · ⚠ warning · ◆ action · ● running
+ * Colors: green/red/yellow/cyan/dim/bold — strict assignments
+ */
+
+import type { StreamEvent, ToolEvent } from '@synergenius/flow-weaver/agent';
+import { c } from './ansi.js';
+import { VERBOSE_TOOL_NAMES } from './tool-registry.js';
+
+export interface RendererOptions {
+  verbose?: boolean;
+  quiet?: boolean;
+  noColor?: boolean;
+  /** Override stderr writer (for testing) */
+  write?: (s: string) => void;
+}
+
+export interface TaskEndStats {
+  toolCalls: number;
+  inputTokens: number;
+  outputTokens: number;
+  estimatedCost: number;
+  filesModified: number;
+  elapsed: number;
+  gitMessage?: string;
+}
+
+export interface SessionEndStats {
+  tasks: number;
+  completed: number;
+  failed: number;
+  totalInputTokens: number;
+  totalOutputTokens: number;
+  totalCost: number;
+  elapsed: number;
+}
+
+export class TerminalRenderer {
+  private verbose: boolean;
+  private quiet: boolean;
+  private out: (s: string) => void;
+  private taskStartTime = 0;
+  private lastToolStartTime = 0;
+  private textBuffer = '';
+  private hasActiveText = false;
+
+  constructor(opts: RendererOptions = {}) {
+    this.verbose = opts.verbose ?? false;
+    this.quiet = opts.quiet ?? false;
+    if (opts.noColor) {
+      // Strip all color functions
+      for (const key of Object.keys(c) as (keyof typeof c)[]) {
+        (c as Record<string, (s: string) => string>)[key] = (s: string) => s;
+      }
+    }
+    this.out = opts.write ?? ((s: string) => process.stderr.write(s));
+  }
+
+  // --- Session lifecycle ---
+
+  sessionStart(info: { provider: string; parallel?: number; deadline?: string }): void {
+    if (this.quiet) return;
+    this.out(`${c.bold('[weaver]')} Session started ${c.dim('(Ctrl+C to stop)')}\n`);
+    const parts = [`Provider: ${info.provider}`];
+    if (info.parallel && info.parallel > 1) parts.push(`Parallel: ${info.parallel}`);
+    if (info.deadline) parts.push(`Deadline: ${info.deadline}`);
+    this.out(`${c.bold('[weaver]')} ${c.dim(parts.join(' · '))}\n`);
+  }
+
+  sessionEnd(stats: SessionEndStats): void {
+    if (this.quiet) return;
+    this.out('\n');
+    const parts: string[] = [];
+    parts.push(`${stats.tasks} task${stats.tasks === 1 ? '' : 's'}`);
+    if (stats.completed > 0) parts.push(c.green(`${stats.completed} completed`));
+    if (stats.failed > 0) parts.push(c.red(`${stats.failed} failed`));
+    const skipped = stats.tasks - stats.completed - stats.failed;
+    if (skipped > 0) parts.push(c.yellow(`${skipped} skipped`));
+    this.out(`${c.bold('[weaver]')} Session complete: ${parts.join(' · ')}\n`);
+
+    const totalTokens = stats.totalInputTokens + stats.totalOutputTokens;
+    if (totalTokens > 0) {
+      this.out(`${c.bold('[weaver]')} ${c.dim(`Total: ${formatTokens(totalTokens)} tokens · $${stats.totalCost.toFixed(3)} · ${formatElapsed(stats.elapsed)}`)}\n`);
+    }
+  }
+
+  // --- Task lifecycle ---
+
+  taskStart(index: number, instruction: string): void {
+    if (this.quiet) return;
+    this.taskStartTime = Date.now();
+    this.hasActiveText = false;
+    this.textBuffer = '';
+    const label = instruction.length > 70 ? instruction.slice(0, 67) + '...' : instruction;
+    this.out(`\n${c.cyan('◆')} ${c.bold(`Task ${index}:`)} ${label}\n`);
+  }
+
+  taskEnd(success: boolean, stats: TaskEndStats): void {
+    if (this.quiet) return;
+    // Flush any remaining text
+    this.flushText();
+
+    const elapsed = formatElapsed(stats.elapsed);
+    const icon = success ? c.green('✓') : c.red('✗');
+    const status = success ? 'completed' : 'failed';
+    this.out(`${icon} Task ${status} ${c.dim(elapsed)}\n`);
+
+    // Summary line
+    const parts: string[] = [];
+    if (stats.toolCalls > 0) parts.push(`${stats.toolCalls} tool calls`);
+    const totalTokens = stats.inputTokens + stats.outputTokens;
+    if (totalTokens > 0) parts.push(`${formatTokens(totalTokens)} tokens`);
+    if (stats.estimatedCost > 0) parts.push(`$${stats.estimatedCost.toFixed(3)}`);
+    if (stats.filesModified > 0) parts.push(`${stats.filesModified} file${stats.filesModified === 1 ? '' : 's'} modified`);
+    if (parts.length > 0) {
+      this.out(`  ${c.dim(parts.join(' · '))}\n`);
+    }
+
+    if (stats.gitMessage) {
+      this.out(`  ${c.dim('→ Git: ' + stats.gitMessage)}\n`);
+    }
+  }
+
+  // --- Stream event handling ---
+
+  onStreamEvent(event: StreamEvent): void {
+    if (this.quiet) return;
+
+    switch (event.type) {
+      case 'thinking_delta':
+        if (this.verbose) {
+          // In verbose mode, stream thinking as dim text
+          this.flushText();
+          this.out(`  ${c.dim(event.text.replace(/\n/g, '\n  '))}`);
+        }
+        // In normal mode, thinking is completely hidden
+        break;
+
+      case 'text_delta':
+        if (this.verbose) {
+          this.flushText();
+          this.out(event.text);
+          this.hasActiveText = true;
+        }
+        // In normal mode, AI text is hidden (tool calls tell the story)
+        break;
+
+      case 'tool_result':
+        // CLI internal tool result — show as result line
+        break;
+
+      default:
+        break;
+    }
+  }
+
+  onToolEvent(event: ToolEvent): void {
+    if (this.quiet) return;
+
+    if (event.type === 'tool_call_start') {
+      this.flushText();
+      this.lastToolStartTime = Date.now();
+      const args = event.args ?? {};
+      const preview = toolPreview(event.name, args);
+      this.out(`  ${c.cyan('◆')} ${event.name}${preview ? c.dim(`(${preview})`) : ''}\n`);
+    }
+
+    if (event.type === 'tool_call_result') {
+      const elapsed = Date.now() - this.lastToolStartTime;
+      const raw = event.result ?? '';
+      const icon = event.isError ? c.red('✗') : c.dim('→');
+      // Show full multiline output for verbose tools; one-line summary for others
+      const isVerboseTool = VERBOSE_TOOL_NAMES.has(event.name);
+      if (isVerboseTool && raw.includes('\n') && raw.length > 120) {
+        this.out(`  ${icon} ${c.dim(formatElapsed(elapsed))}\n${raw}\n`);
+      } else {
+        const result = raw.replace(/\n/g, ' ').slice(0, 200);
+        this.out(`  ${icon} ${result} ${c.dim(formatElapsed(elapsed))}\n`);
+      }
+    }
+  }
+
+  // --- Direct messages ---
+
+  info(msg: string): void {
+    if (!this.quiet) this.out(`${c.bold('[weaver]')} ${msg}\n`);
+  }
+
+  warn(msg: string): void {
+    if (!this.quiet) this.out(`${c.yellow('⚠')} ${msg}\n`);
+  }
+
+  error(title: string, detail?: string): void {
+    this.out(`${c.redBold('✗')} ${c.red(title)}\n`);
+    if (detail) this.out(`  ${c.red(detail)}\n`);
+  }
+
+  // --- Private ---
+
+  private flushText(): void {
+    if (this.hasActiveText) {
+      this.out('\n');
+      this.hasActiveText = false;
+    }
+  }
+}
+
+// --- Formatting helpers ---
+
+export function formatTokens(n: number): string {
+  if (n < 1000) return String(n);
+  if (n < 1_000_000) return (n / 1000).toFixed(1).replace(/\.0$/, '') + 'k';
+  return (n / 1_000_000).toFixed(1).replace(/\.0$/, '') + 'M';
+}
+
+export function formatElapsed(ms: number): string {
+  if (ms < 1000) return `${ms}ms`;
+  if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`;
+  const m = Math.floor(ms / 60_000);
+  const s = Math.round((ms % 60_000) / 1000);
+  return s > 0 ? `${m}m ${s}s` : `${m}m`;
+}
+
+function toolPreview(name: string, args: Record<string, unknown>): string {
+  if (args.file) return String(args.file).split('/').pop() ?? '';
+  if (args.command) return String(args.command).slice(0, 50);
+  if (args.directory) return String(args.directory).split('/').pop() ?? '';
+  // For patch_file, show file + patch count
+  if (name === 'patch_file' && args.patches) {
+    const file = String(args.file ?? '').split('/').pop() ?? '';
+    const count = Array.isArray(args.patches) ? args.patches.length : '?';
+    return `${file}, ${count} patches`;
+  }
+  return '';
+}