@synergenius/flow-weaver-pack-weaver 0.9.193 → 0.9.196

package/src/bot/context-compactor.ts

@@ -0,0 +1,147 @@
+/**
+ * LLM-based context compaction for task retries.
+ *
+ * When a task has 3+ runs, calls a fast-tier model to produce a structured
+ * summary of all attempts. The summary replaces verbose run history in
+ * the prompt builder, preserving semantic signal (intent, errors, approaches)
+ * that cascading truncation would destroy.
+ *
+ * Uses the <analysis> scratchpad pattern — the model reasons in tags that
+ * are stripped before storing the summary.
+ */
+
+import type { Task, RunProgress, CompactedRun } from './task-types.js';
+import type { ProviderInfo } from './types.js';
+import type { CostStrategy } from './profile-types.js';
+
+/** Minimum runs before compaction triggers, indexed by cost strategy. */
+const COMPACTION_THRESHOLDS: Record<CostStrategy, number> = {
+  frugal: 5,
+  balanced: 3,
+  performance: 2,
+};
+
+const COMPACTION_SYSTEM_PROMPT = `You are a context compactor for an AI task execution system. Your job is to
+summarize the history of a task's execution runs into a concise briefing for
+the next AI worker that will continue the task.
+
+The next worker will read the workspace files directly. Your summary should
+focus on WHAT WAS TRIED and WHY IT DIDN'T FULLY SUCCEED — not on describing
+file contents (the worker will read those).
+
+CRITICAL: Respond with TEXT ONLY. Do NOT call any tools. Tool calls will be
+REJECTED and will waste your only turn.
+
+Produce a structured summary with these sections. Be terse — total output
+must be under 800 tokens.
+
+<analysis>
+Think about what patterns emerge across the runs. What keeps failing? What
+approaches were tried? Is there stagnation?
+</analysis>
+
+## Task Intent
+One sentence: what is the task trying to achieve?
+
+## Approaches Tried
+Bullet list of distinct approaches attempted across runs, with outcomes.
+
+## Key Errors
+Specific error messages or test failures that recurred. Include exact names.
+
+## Files Touched
+Files created or modified across all runs (deduplicated).
+
+## What Works
+Any partial progress that should be preserved (tests passing, files correct).
+
+## Recommended Next Step
+Based on the pattern of failures, what should the next run try differently?`;
+
+/**
+ * Strip `<analysis>...</analysis>` scratchpad from LLM response.
+ * Reuses the same pattern as review-result.ts stripAnalysis.
+ */
+function stripAnalysis(text: string): string {
+  return text.replace(/<analysis>[\s\S]*?<\/analysis>/g, '').trim();
+}
+
+/**
+ * Build the user prompt from task + run history.
+ */
+export function buildCompactionPrompt(task: Task): string {
+  const lines: string[] = [];
+  lines.push(`Task: ${task.title}`);
+  lines.push(`Description: ${task.description}`);
+  lines.push('');
+  lines.push('Run history:');
+
+  for (let i = 0; i < task.context.runHistory.length; i++) {
+    const run = task.context.runHistory[i];
+    const rp = run as RunProgress;
+    lines.push(`Run ${i + 1} (${rp.outcome}):`);
+    if (rp.summary) lines.push(`  Summary: ${rp.summary}`);
+    if (rp.filesCreated?.length) lines.push(`  Files created: ${rp.filesCreated.join(', ')}`);
+    if (rp.filesModified?.length) lines.push(`  Files modified: ${rp.filesModified.join(', ')}`);
+    if (rp.remainingWork) lines.push(`  Remaining work: ${rp.remainingWork}`);
+    if (rp.blockers?.length) lines.push(`  Blockers: ${rp.blockers.join(', ')}`);
+    lines.push('');
+  }
+
+  if (task.lastAcceptanceCheck) {
+    const ac = task.lastAcceptanceCheck;
+    const checkLines = ac.results
+      .map(r => `  ${r.name}: ${r.pass ? 'PASS' : 'FAIL'}${r.detail ? ` (${r.detail.slice(0, 100)})` : ''}`)
+      .join('\n');
+    lines.push(`Acceptance check results:\n${checkLines}`);
+  }
+
+  lines.push(`Stagnation count: ${task.context.stagnationCount}`);
+
+  return lines.join('\n');
+}
+
+/**
+ * Determine if compaction should run for this task.
+ */
+export function shouldCompact(
+  task: Task,
+  costStrategy: CostStrategy = 'balanced',
+): boolean {
+  if (task.context.compactedSummary) return false;
+  const threshold = COMPACTION_THRESHOLDS[costStrategy] ?? 3;
+  return task.context.runHistory.length >= threshold;
+}
+
+/**
+ * Compact a task's run history into a structured summary via LLM.
+ *
+ * Returns the compacted summary string, or null if the LLM call fails.
+ * Failures are non-fatal — the prompt builder falls back to the existing
+ * context decay approach.
+ *
+ * @param task - The task to compact
+ * @param pInfo - Provider info for making the LLM call
+ * @param callAIFn - The callAI function (injected for testability)
+ */
+export async function compactRunHistory(
+  task: Task,
+  pInfo: Pick<ProviderInfo, 'type' | 'apiKey' | 'model' | 'cliBinPath'>,
+  callAIFn: (
+    pInfo: Pick<ProviderInfo, 'type' | 'apiKey' | 'model' | 'cliBinPath'>,
+    systemPrompt: string,
+    userPrompt: string,
+    maxTokens: number,
+  ) => Promise<string>,
+): Promise<string | null> {
+  try {
+    const userPrompt = buildCompactionPrompt(task);
+    const response = await callAIFn(pInfo, COMPACTION_SYSTEM_PROMPT, userPrompt, 1024);
+    const cleaned = stripAnalysis(response);
+    // Sanity check: the cleaned response should have meaningful content
+    if (cleaned.length < 20) return null;
+    return cleaned;
+  } catch {
+    return null;
+  }
+}

package/src/bot/dream-task.ts

@@ -0,0 +1,167 @@
+/**
+ * DreamTask — idle-time knowledge consolidation.
+ *
+ * Runs during swarm dispatch loop idle periods to extract cross-task
+ * patterns, detect recurring failures, clean stale knowledge, and
+ * convert InsightEngine findings into knowledge entries.
+ *
+ * Purely heuristic — no LLM calls. Designed to run within 500ms.
+ * Key namespaces (non-overlapping with memory-extractor's project:*):
+ * - pattern:hot-file:*  — files modified by multiple tasks (24h TTL)
+ * - warning:recurring-failure:*  — recurring failures (7d TTL)
+ * - insight:*  — converted InsightEngine insights (7d TTL)
+ * - session:stats  — rolling session statistics (overwritten each cycle)
+ */
+
+import { KnowledgeStore, type KnowledgeEntry } from './knowledge-store.js';
+import { RunStore } from './run-store.js';
+import type { Insight, ProjectModel } from './types.js';
+
+/** Minimum time between consolidation runs (ms). */
+const COOLDOWN_MS = 60_000;
+
+/** TTL for dream-sourced entries (ms). */
+const STALE_THRESHOLD_MS = 7 * 24 * 60 * 60 * 1000;
+
+/** TTL for hot-file patterns (ms). */
+const HOT_FILE_TTL_MS = 24 * 60 * 60 * 1000;
+
+/** Minimum confidence for converting insights to knowledge. */
+const MIN_CONFIDENCE = 0.6;
+
+export interface DreamTaskOptions {
+  projectDir: string;
+}
+
+export interface DreamResult {
+  warningsStored: number;
+  staleEntriesCleaned: number;
+  insightsConverted: number;
+  durationMs: number;
+}
+
+export class DreamTask {
+  private lastConsolidatedAt = 0;
+  private readonly projectDir: string;
+
+  constructor(opts: DreamTaskOptions) {
+    this.projectDir = opts.projectDir;
+  }
+
+  shouldRun(): boolean {
+    return Date.now() - this.lastConsolidatedAt >= COOLDOWN_MS;
+  }
+
+  async consolidate(): Promise<DreamResult> {
+    const start = Date.now();
+    this.lastConsolidatedAt = start;
+
+    const store = new KnowledgeStore(this.projectDir);
+    let warningsStored = 0;
+    let staleEntriesCleaned = 0;
+    let insightsConverted = 0;
+
+    // Phase 1: Clean stale dream-sourced entries
+    staleEntriesCleaned = this.cleanStaleEntries(store);
+
+    // Phase 2: Convert InsightEngine findings to knowledge
+    try {
+      insightsConverted = await this.convertInsights(store);
+    } catch { /* non-fatal */ }
+
+    // Phase 3: Session stats
+    try {
+      this.updateSessionStats(store);
+    } catch { /* non-fatal */ }
+
+    return {
+      warningsStored,
+      staleEntriesCleaned,
+      insightsConverted,
+      durationMs: Date.now() - start,
+    };
+  }
+
+  /**
+   * Phase 1: Remove stale dream-sourced entries.
+   * Only cleans entries with source starting with 'dream-task'.
+   */
+  private cleanStaleEntries(store: KnowledgeStore): number {
+    const now = Date.now();
+    const entries = store.list();
+    let cleaned = 0;
+
+    for (const entry of entries) {
+      if (!entry.source?.startsWith('dream-task')) continue;
+
+      const age = now - entry.createdAt;
+      const isHotFile = entry.key.startsWith('pattern:hot-file:');
+      const ttl = isHotFile ? HOT_FILE_TTL_MS : STALE_THRESHOLD_MS;
+
+      if (age > ttl) {
+        store.forget(entry.key);
+        cleaned++;
+      }
+    }
+
+    return cleaned;
+  }
+
+  /**
+   * Phase 2: Convert high-confidence InsightEngine insights to knowledge.
+   * Imports ProjectModel and InsightEngine dynamically to avoid circular deps.
+   */
+  private async convertInsights(store: KnowledgeStore): Promise<number> {
+    const { ProjectModelStore } = await import('./project-model.js');
+    const { InsightEngine } = await import('./insight-engine.js');
+
+    const pms = new ProjectModelStore(this.projectDir);
+    const model = await pms.getOrBuild();
+    if (!model) return 0;
+
+    const engine = new InsightEngine();
+    const insights = engine.analyze(model);
+    const existingKeys = new Set(store.list().map(e => e.key));
+    let converted = 0;
+
+    for (const insight of insights) {
+      if (insight.confidence < MIN_CONFIDENCE) continue;
+      if (insight.severity !== 'warning' && insight.severity !== 'critical') continue;
+
+      const key = `insight:${insight.type}:${insight.id}`;
+      if (existingKeys.has(key)) continue;
+
+      store.learn(
+        key,
+        `${insight.title}: ${insight.description}. Suggestion: ${insight.suggestion}`,
+        'dream-task:insight-engine',
+      );
+      converted++;
+    }
+
+    return converted;
+  }
+
+  /**
+   * Phase 3: Update rolling session statistics.
+   */
+  private updateSessionStats(store: KnowledgeStore): void {
+    try {
+      const runStore = new RunStore();
+      const recentRuns = runStore.list({ limit: 50 });
+
+      if (recentRuns.length === 0) return;
+
+      const successCount = recentRuns.filter(r => r.success).length;
+      const avgDuration = Math.round(
+        recentRuns.reduce((sum, r) => sum + (r.durationMs ?? 0), 0) / recentRuns.length,
+      );
+
+      store.learn(
+        'session:stats',
+        `${recentRuns.length} recent runs, ${Math.round(successCount / recentRuns.length * 100)}% success, avg ${Math.round(avgDuration / 1000)}s`,
+        'dream-task',
+      );
+    } catch { /* non-fatal */ }
+  }
+}

package/src/bot/knowledge-store.ts

@@ -42,6 +42,33 @@ export class KnowledgeStore {
     return this.readAll();
   }
 
+  /**
+   * Build a compact manifest of knowledge entries for LLM relevance selection.
+   * Returns entries sorted newest-first, capped at maxEntries.
+   * Format per line: `- [N] key (Xd ago): truncated_value`
+   */
+  static buildManifest(
+    entries: KnowledgeEntry[],
+    maxEntries = 200,
+  ): { manifest: string; entries: KnowledgeEntry[] } {
+    const now = Date.now();
+    const DAY_MS = 24 * 60 * 60 * 1000;
+    const sorted = [...entries]
+      .sort((a, b) => b.createdAt - a.createdAt)
+      .slice(0, maxEntries);
+
+    const lines = sorted.map((e, i) => {
+      const ageMs = now - e.createdAt;
+      const age = ageMs < DAY_MS
+        ? `${Math.round(ageMs / (60 * 60 * 1000))}h ago`
+        : `${Math.floor(ageMs / DAY_MS)}d ago`;
+      const value = e.value.length > 80 ? e.value.slice(0, 77) + '...' : e.value;
+      return `- [${i}] ${e.key} (${age}): ${value}`;
+    });
+
+    return { manifest: lines.join('\n'), entries: sorted };
+  }
+
   private readAll(): KnowledgeEntry[] {
     if (!fs.existsSync(this.filePath)) return [];
     const content = fs.readFileSync(this.filePath, 'utf-8').trim();

package/src/bot/memory-extraction-worker.ts

@@ -0,0 +1,58 @@
+/**
+ * Fire-and-forget memory extraction worker.
+ *
+ * Schedules heuristic extraction from completed runs into the KnowledgeStore.
+ * Uses AsyncMutex to serialize concurrent writes from parallel bot slots.
+ * Errors are caught and logged — never blocks task release.
+ */
+
+import { AsyncMutex } from './async-mutex.js';
+import { KnowledgeStore } from './knowledge-store.js';
+import { extractMemoryFacts } from './memory-extractor.js';
+import type { Task, RunProgress } from './task-types.js';
+
+/** Maximum total knowledge entries before skipping pattern:* writes. */
+const MAX_TOTAL_ENTRIES = 100;
+
+const mutex = new AsyncMutex();
+
+/**
+ * Schedule a fire-and-forget memory extraction.
+ * Returns immediately — extraction runs async in the background.
+ */
+export function scheduleMemoryExtraction(
+  projectDir: string,
+  task: Task,
+  runProgress: RunProgress,
+): void {
+  _doExtract(projectDir, task, runProgress).catch(err => {
+    if (process.env.WEAVER_VERBOSE) {
+      console.warn('[swarm] memory extraction failed:', err);
+    }
+  });
+}
+
+async function _doExtract(
+  projectDir: string,
+  task: Task,
+  runProgress: RunProgress,
+): Promise<void> {
+  await mutex.runExclusive(async () => {
+    const store = new KnowledgeStore(projectDir);
+    const existing = store.list();
+    const existingKeys = new Set(existing.map(e => e.key));
+
+    // If over the cap, only allow project:* overwrites (no new pattern:* entries)
+    const overCap = existing.length >= MAX_TOTAL_ENTRIES;
+
+    const facts = extractMemoryFacts(task, runProgress, existingKeys);
+
+    for (const fact of facts) {
+      // Skip pattern:* entries when over cap (project:* overwrites are always allowed)
+      if (overCap && fact.key.startsWith('pattern:') && !existingKeys.has(fact.key)) {
+        continue;
+      }
+      store.learn(fact.key, fact.value, `run:${runProgress.runId}`);
+    }
+  });
+}

package/src/bot/memory-extractor.ts

@@ -0,0 +1,213 @@
+/**
+ * Heuristic memory extraction from completed bot runs.
+ *
+ * Pure function — takes structured run data and returns candidate
+ * knowledge entries. No I/O, no LLM calls, no side effects.
+ *
+ * Key namespaces:
+ * - project:*  — canonical project facts (test runner, build tool, linter, pkg manager)
+ * - pattern:blocker:*  — recurring blockers
+ * - pattern:stagnation:*  — stalled task patterns
+ * - pattern:success:*  — first-run success patterns
+ */
+
+import type { Task, RunProgress } from './task-types.js';
+
+export interface MemoryFact {
+  key: string;
+  value: string;
+}
+
+/** Maximum facts to extract per run. */
+const MAX_FACTS_PER_RUN = 5;
+
+/** Maximum key length. */
+const MAX_KEY_LENGTH = 80;
+
+/** Maximum value length. */
+const MAX_VALUE_LENGTH = 500;
+
+// ── Detection patterns ──────────────────────────────────────────────
+
+interface DetectionRule {
+  key: string;
+  pattern: RegExp;
+  value: string;
+}
+
+const TEST_RUNNERS: DetectionRule[] = [
+  { key: 'project:test-runner', pattern: /\bvitest\b/i, value: 'vitest' },
+  { key: 'project:test-runner', pattern: /\bjest\b/i, value: 'jest' },
+  { key: 'project:test-runner', pattern: /\bmocha\b/i, value: 'mocha' },
+  { key: 'project:test-runner', pattern: /\bplaywright\b/i, value: 'playwright' },
+];
+
+const BUILD_TOOLS: DetectionRule[] = [
+  { key: 'project:build-tool', pattern: /\bvite\b/i, value: 'vite' },
+  { key: 'project:build-tool', pattern: /\bwebpack\b/i, value: 'webpack' },
+  { key: 'project:build-tool', pattern: /\besbuild\b/i, value: 'esbuild' },
+  { key: 'project:build-tool', pattern: /\btsc\b/i, value: 'tsc' },
+];
+
+const PKG_MANAGERS: DetectionRule[] = [
+  { key: 'project:pkg-manager', pattern: /\bpnpm\b/i, value: 'pnpm' },
+  { key: 'project:pkg-manager', pattern: /\byarn\b/i, value: 'yarn' },
+  { key: 'project:pkg-manager', pattern: /\bbun\b/i, value: 'bun' },
+  { key: 'project:pkg-manager', pattern: /\bnpm\b/i, value: 'npm' },
+];
+
+const LINTERS: DetectionRule[] = [
+  { key: 'project:linter', pattern: /\bbiome\b/i, value: 'biome' },
+  { key: 'project:linter', pattern: /\beslint\b/i, value: 'eslint' },
+  { key: 'project:linter', pattern: /\bprettier\b/i, value: 'prettier' },
+];
+
+// ── Helpers ──────────────────────────────────────────────────────────
+
+function truncate(s: string, max: number): string {
+  return s.length <= max ? s : s.slice(0, max - 3) + '...';
+}
+
+function normalizeBlockerKey(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s-]/g, '')
+    .trim()
+    .replace(/\s+/g, '-')
+    .slice(0, 40);
+}
+
+function searchableText(rp: RunProgress): string {
+  const parts = [rp.summary ?? ''];
+  if (rp.checks) {
+    for (const v of Object.values(rp.checks)) {
+      parts.push(String(v));
+    }
+  }
+  return parts.join(' ');
+}
+
+function detectFirst(
+  text: string,
+  rules: DetectionRule[],
+): MemoryFact | undefined {
+  for (const rule of rules) {
+    if (rule.pattern.test(text)) {
+      return { key: rule.key, value: rule.value };
+    }
+  }
+  return undefined;
+}
+
+function countExtensions(files: string[]): Record<string, number> {
+  const counts: Record<string, number> = {};
+  for (const f of files) {
+    const ext = f.includes('.') ? '.' + f.split('.').pop() : '(none)';
+    counts[ext] = (counts[ext] ?? 0) + 1;
+  }
+  return counts;
+}
+
+// ── Main extraction ─────────────────────────────────────────────────
+
+/**
+ * Extract generalizable facts from a completed run.
+ *
+ * @param task - The task that was executed
+ * @param rp - The run's progress/result data
+ * @param existingKeys - Keys already in the KnowledgeStore (for dedup)
+ * @returns Array of facts to persist (max MAX_FACTS_PER_RUN)
+ */
+export function extractMemoryFacts(
+  task: Task,
+  rp: RunProgress,
+  existingKeys: Set<string>,
+): MemoryFact[] {
+  const facts: MemoryFact[] = [];
+  const text = searchableText(rp);
+  const allFiles = [...rp.filesCreated, ...rp.filesModified];
+
+  // ── Project-level facts (overwrite existing) ──
+
+  const testRunner = detectFirst(text, TEST_RUNNERS);
+  if (testRunner) facts.push(testRunner);
+
+  const buildTool = detectFirst(text, BUILD_TOOLS);
+  if (buildTool) facts.push(buildTool);
+
+  const pkgManager = detectFirst(text, PKG_MANAGERS);
+  if (pkgManager) facts.push(pkgManager);
+
+  const linter = detectFirst(text, LINTERS);
+  if (linter) facts.push(linter);
+
+  // ── Source file pattern ──
+
+  if (allFiles.length >= 3 && !existingKeys.has('project:source-pattern')) {
+    const extCounts = countExtensions(allFiles);
+    const dominant = Object.entries(extCounts)
+      .sort(([, a], [, b]) => b - a)
+      .slice(0, 3)
+      .map(([ext, count]) => `${ext} (${count})`)
+      .join(', ');
+    facts.push({
+      key: 'project:source-pattern',
+      value: truncate(dominant, MAX_VALUE_LENGTH),
+    });
+  }
+
+  // ── Blockers ──
+
+  if (rp.blockers && rp.blockers.length > 0) {
+    for (const blocker of rp.blockers.slice(0, 2)) {
+      const normalized = normalizeBlockerKey(blocker);
+      if (!normalized) continue;
+      const key = truncate(`pattern:blocker:${normalized}`, MAX_KEY_LENGTH);
+      if (!existingKeys.has(key)) {
+        facts.push({ key, value: truncate(blocker, MAX_VALUE_LENGTH) });
+      }
+    }
+  }
+
+  // ── Stagnation pattern ──
+
+  if (
+    rp.outcome === 'stalled' &&
+    task.context.stagnationCount > 1
+  ) {
+    const key = truncate(`pattern:stagnation:${task.id}`, MAX_KEY_LENGTH);
+    if (!existingKeys.has(key)) {
+      facts.push({
+        key,
+        value: truncate(
+          `Task "${task.title}" stalled ${task.context.stagnationCount}x. Last: ${rp.summary}`,
+          MAX_VALUE_LENGTH,
+        ),
+      });
+    }
+  }
+
+  // ── First-run success pattern ──
+
+  if (
+    rp.outcome === 'completed' &&
+    task.context.runHistory.length <= 1
+  ) {
+    const key = truncate(
+      `pattern:success:${rp.profileId}:${task.complexity ?? 'unknown'}`,
+      MAX_KEY_LENGTH,
+    );
+    if (!existingKeys.has(key)) {
+      facts.push({
+        key,
+        value: truncate(
+          `Profile ${rp.profileId} completed ${task.complexity ?? 'unknown'}-complexity task on first run`,
+          MAX_VALUE_LENGTH,
+        ),
+      });
+    }
+  }
+
+  // Enforce max facts per run
+  return facts.slice(0, MAX_FACTS_PER_RUN);
+}

package/src/bot/operations.ts

@@ -38,8 +38,10 @@ export const OP_TASK_CREATE = 'task_create';
 // Memory
 // ---------------------------------------------------------------------------
 
-export const OP_REMEMBER = 'remember';
+export const OP_LEARN = 'learn';
 export const OP_RECALL = 'recall';
+/** @deprecated Use OP_LEARN. Kept for step-executor backward compat. */
+export const OP_REMEMBER = 'remember';
 
 // ---------------------------------------------------------------------------
 // Passthrough (no execution needed)