open-agents-ai 0.39.0 → 0.40.0

package/README.md

@@ -109,7 +109,7 @@ Long conversations consume context window tokens. Open Agents uses progressive c
 
 ### How It Works
 
-Compaction triggers automatically when estimated token usage reaches 75% of the model's context window. The system:
+Compaction triggers automatically when estimated token usage reaches a tier-proportional threshold of the model's context window. The system:
 
 1. **Preserves** the system prompt and initial user task (head messages)
 2. **Summarizes** middle messages (tool calls, results, exploration) into a structured digest
@@ -131,13 +131,36 @@ Six strategies are available via `/compact <strategy>`:
 
 ### Automatic Compaction
 
-Compaction thresholds scale dynamically with model size:
+Compaction thresholds scale **proportionally** with the model's actual context window size:
 
-| Model Tier | Threshold | Recent Messages Kept |
-|------------|-----------|---------------------|
-| Large (30B+) | 40,000 tokens (or 75% of context) | 12 messages |
-| Medium (8-29B) | 24,000 tokens (or 75% of context) | 8 messages |
-| Small (≤7B) | 12,000 tokens (or 75% of context) | 4-6 messages |
+| Model Tier | Normal Mode | Deep Context Mode | Recent Messages Kept |
+|------------|-------------|-------------------|---------------------|
+| Large (30B+) | 75% of context window | 85% of context window | 4-12 (normal) / 4-24 (deep) |
+| Medium (8-29B) | 70% of context window | 85% of context window | 4-12 (normal) / 4-24 (deep) |
+| Small (≤7B) | 65% of context window | 85% of context window | 4-12 (normal) / 4-24 (deep) |
+
+For example, a 128K-context large model compacts at ~96K tokens in normal mode (75%) or ~109K tokens in deep mode (85%) — instead of the previous fixed 40K threshold that wasted 69% of available context.
+
+### Deep Context Mode (`/deep`)
+
+Toggle with `/deep` — relaxes compaction so large models leverage more of their context window for complex multi-step reasoning.
+
+When deep context is active:
+- **Compaction fires at 85%** of context instead of 65-75% — the model retains much more working memory
+- **Double the recent messages** (up to 24 instead of 12) preserved after compaction
+- **Richer summaries** — compression budget increased from 20% to 30% of context
+- **Larger tool outputs** — cap raised from 8K to 16K chars per tool result
+- **Relaxed output folding** — more head/tail lines preserved (50/25 instead of 20/10 for large models)
+
+This mirrors how human cognition works during deep problem-solving: situationally-relevant memories are transiently activated to occupy a larger portion of working memory, with the most relevant details in high-attention positions while supporting context backs them up. LLM attention mechanisms work similarly — earlier relevant context still influences generation even at lower positional weight.
+
+Use deep context for:
+- Complex multi-file refactoring or debugging
+- Architecture analysis across many files
+- Long debugging sessions where error context from earlier is critical
+- Tasks where the agent needs to reason about patterns across many files
+
+The setting persists to `.oa/settings.json`. Deep context is particularly valuable for models with 64K+ context windows (Qwen3.5-122B, Llama 3.1 70B, etc.) where the default thresholds were leaving significant capacity unused.
 
 ### Status Bar Context Tracking (`Ctx:`)
 

package/dist/index.js

@@ -13180,6 +13180,7 @@ Rules:
           taskTimeoutMs: options?.taskTimeoutMs ?? 36e5,
           // 60 min
           compactionThreshold: options?.compactionThreshold ?? 4e4,
+          deepContext: options?.deepContext ?? false,
           dynamicContext: options?.dynamicContext ?? "",
           streamEnabled: options?.streamEnabled ?? false,
           bruteForce: options?.bruteForce ?? true,
@@ -13225,19 +13226,47 @@ Rules:
       /**
        * Compute all context-dependent limits from the current contextWindowSize
        * and modelTier. Returns sensible defaults when contextWindowSize is 0.
+       *
+       * Key insight: LLM attention is most effective when the most relevant content
+       * sits in high-attention positions (start and end of context). Rather than
+       * compacting aggressively and losing detail, we scale thresholds proportionally
+       * to the model's actual context window so larger models can leverage more of
+       * their capacity. The deepContext flag further relaxes limits for complex
+       * multi-step reasoning where earlier context carries critical signal.
+       *
+       * Context utilization targets:
+       * - Small models:  compact at ~65% (limited attention span, aggressive helps)
+       * - Medium models: compact at ~70% (balanced)
+       * - Large models:  compact at ~75% (plenty of room, let attention work)
+       * - Deep context:  compact at ~85% (trust the model's full attention window)
        */
       contextLimits() {
         const ctx = this.options.contextWindowSize;
         const tier = this.options.modelTier ?? "large";
-        const compactionThreshold = ctx > 0 ? Math.min(this.options.compactionThreshold, Math.floor(ctx * 0.75)) : this.options.compactionThreshold;
-        const keepRecentDivisor = tier === "small" ? 2e3 : tier === "medium" ? 3e3 : 4e3;
-        const keepRecent = ctx > 0 ? Math.max(4, Math.min(12, Math.floor(ctx / keepRecentDivisor))) : 12;
+        const deep = this.options.deepContext ?? false;
+        let compactionThreshold;
+        if (ctx > 0) {
+          const factor = deep ? 0.85 : tier === "small" ? 0.65 : tier === "medium" ? 0.7 : 0.75;
+          compactionThreshold = Math.floor(ctx * factor);
+        } else {
+          compactionThreshold = deep ? 8e4 : this.options.compactionThreshold;
+        }
+        const keepRecentMax = deep ? 24 : 12;
+        let keepRecent;
+        if (ctx > 0) {
+          const keepRecentDivisor = tier === "small" ? 2e3 : tier === "medium" ? 3e3 : 4e3;
+          keepRecent = Math.max(4, Math.min(keepRecentMax, Math.floor(ctx / keepRecentDivisor)));
+        } else {
+          keepRecent = deep ? 20 : 12;
+        }
         const maxOutputTokens = ctx > 0 ? Math.min(this.options.maxTokens, Math.max(2048, Math.floor(ctx * 0.25))) : this.options.maxTokens;
-        const toolOutputMaxChars = ctx > 0 ? Math.max(2e3, Math.min(8e3, Math.floor(ctx * 0.5))) : 8e3;
-        const foldLineThreshold = tier === "small" ? 30 : tier === "medium" ? 35 : 40;
-        const foldHeadLines = tier === "small" ? 12 : tier === "medium" ? 16 : 20;
-        const foldTailLines = tier === "small" ? 5 : tier === "medium" ? 8 : 10;
-        const maxSummaryChars = ctx > 0 ? Math.max(2e3, Math.min(8e3, Math.floor(ctx * 0.2))) : 4e3;
+        const toolOutputCeiling = deep ? 16e3 : 8e3;
+        const toolOutputMaxChars = ctx > 0 ? Math.max(2e3, Math.min(toolOutputCeiling, Math.floor(ctx * (deep ? 0.8 : 0.5)))) : toolOutputCeiling;
+        const foldLineThreshold = deep ? tier === "small" ? 50 : tier === "medium" ? 80 : 120 : tier === "small" ? 30 : tier === "medium" ? 35 : 40;
+        const foldHeadLines = deep ? tier === "small" ? 20 : tier === "medium" ? 30 : 50 : tier === "small" ? 12 : tier === "medium" ? 16 : 20;
+        const foldTailLines = deep ? tier === "small" ? 10 : tier === "medium" ? 15 : 25 : tier === "small" ? 5 : tier === "medium" ? 8 : 10;
+        const maxSummaryCeiling = deep ? 16e3 : 8e3;
+        const maxSummaryChars = ctx > 0 ? Math.max(2e3, Math.min(maxSummaryCeiling, Math.floor(ctx * (deep ? 0.3 : 0.2)))) : deep ? 8e3 : 4e3;
         const repetitionWindow = tier === "small" ? 6 : tier === "medium" ? 8 : 10;
         return {
           compactionThreshold,
@@ -16914,6 +16943,7 @@ function renderSlashHelp() {
     ["/compact", "Force context compaction now (default strategy)"],
     ["/compact <strategy>", "Compact with strategy: aggressive, decisions, errors, summary, structured"],
     ["/bruteforce", "Toggle brute-force mode (auto re-engage on turn limit)"],
+    ["/deep", "Toggle deep context \u2014 relaxes compaction so large models use 85% of context"],
     ["/tools", "List agent-created custom tools"],
     ["/skills", "List available AIWG skills"],
     ["/skills <keyword>", "Filter skills by name or trigger"],
@@ -19451,6 +19481,18 @@ async function handleSlashCommand(input, ctx) {
       renderInfo(`Brute-force mode: ${isOn ? "on" : "off"}${hasLocal ? " (project-local)" : ""}` + (isOn ? " \u2014 agent will auto re-engage when turn limit is hit, reassess and try creative strategies" : ""));
       return "handled";
     }
+    case "deep":
+    case "deep-context": {
+      if (!ctx.deepContextToggle) {
+        renderWarning("Deep context not available");
+        return "handled";
+      }
+      const isOn = ctx.deepContextToggle();
+      const save = hasLocal ? ctx.saveLocalSettings.bind(ctx) : ctx.saveSettings.bind(ctx);
+      save({ deepContext: isOn });
+      renderInfo(`Deep context mode: ${isOn ? "ON" : "OFF"}${hasLocal ? " (project-local)" : ""}` + (isOn ? " \u2014 compaction relaxed to 85% of context window. Large models will retain more working memory for complex reasoning." : " \u2014 compaction restored to default thresholds."));
+      return "handled";
+    }
     case "emojis":
     case "emoji": {
       const current = ctx.getEmojis?.() ?? true;
@@ -23580,7 +23622,12 @@ var init_bless_engine = __esm({
 // packages/cli/dist/tui/dmn-engine.js
 import { existsSync as existsSync26, readFileSync as readFileSync19, writeFileSync as writeFileSync11, mkdirSync as mkdirSync12, readdirSync as readdirSync11, unlinkSync as unlinkSync5 } from "node:fs";
 import { join as join36, basename as basename13 } from "node:path";
-function buildDMNGatherPrompt(recentTaskSummaries, dueReminders, attentionItems, memoryTopics, capabilities) {
+function buildDMNGatherPrompt(recentTaskSummaries, dueReminders, attentionItems, memoryTopics, capabilities, competence, reflectionBuffer) {
+  const competenceReport = competence.length > 0 ? competence.map((c3) => {
+    const rate = c3.attempts > 0 ? Math.round(c3.successes / c3.attempts * 100) : 0;
+    return `  - ${c3.taskType}: ${c3.attempts} attempts, ${rate}% success`;
+  }).join("\n") : "  (no data yet \u2014 this is a fresh start)";
+  const reflectionsText = reflectionBuffer.length > 0 ? reflectionBuffer.map((r, i) => `  ${i + 1}. ${r}`).join("\n") : "  (none)";
   return `DEFAULT MODE NETWORK \u2014 SELF-REFLECTION CYCLE
 
 You are the agent's Default Mode Network. You activate between tasks to reflect,
@@ -23594,16 +23641,28 @@ task to pursue next. Think like a brain at rest: consolidating, planning, connec
 
 PHASE 1: GATHER CONTEXT
 
-Use memory_search and memory_read to explore all stored knowledge. Look for:
+Use memory_search and memory_read to explore ALL stored knowledge. Be thorough.
+Look for:
 - Standing directives ("always do X", "seek Y", "monitor Z")
 - Unfinished goals or partially completed missions
 - Knowledge gaps that could be filled
 - Patterns in recent task history that suggest momentum direction
 - Environmental signals (reminders, attention items)
 
+IMPORTANT: Start by searching memory broadly. Use memory_search with terms like
+"goal", "directive", "plan", "todo", "important" to find standing orders. Then
+read specific topics that seem relevant. The richness of your reasoning depends
+on how well you explore what you already know.
+
 Recent task history (last completed tasks):
 ${recentTaskSummaries.length > 0 ? recentTaskSummaries.map((s, i) => `  ${i + 1}. ${s}`).join("\n") : "  (no recent tasks)"}
 
+Recent failures (Reflexion buffer \u2014 learn from these):
+${reflectionsText}
+
+Competence tracker (attempts and success rate by category):
+${competenceReport}
+
 Due reminders:
 ${dueReminders.length > 0 ? dueReminders.map((r) => `  - ${r}`).join("\n") : "  (none)"}
 
@@ -23618,36 +23677,53 @@ ${capabilities.map((c3) => `  - ${c3}`).join("\n")}
 
 \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550
 
-PHASE 2: REFLECT & REASON
+PHASE 2: REFLECT, CONSOLIDATE & REASON (Generative Agents + Reflexion)
+
+After gathering context, consolidate and reason:
 
-After gathering context, reason through these questions:
+Memory consolidation:
+- Are there overlapping or redundant memories that should be merged?
+- Are there new insights from recent tasks worth writing to memory?
+- Write any new insights to memory NOW using memory_write.
+
+Self-evaluation (answer these questions in your reasoning):
 1. What directives or goals have been set that still need work?
 2. What was the momentum of recent tasks \u2014 what logically comes next?
 3. Are there capabilities I haven't exercised that could be valuable?
 4. What knowledge gaps exist that exploration could fill?
 5. Are there environmental signals (reminders, attention items) to address?
+6. What can I learn from recent failures? (Check the Reflexion buffer above)
+7. Where is my learning progress fastest? (Check competence tracker \u2014 pursue
+   categories where success rate is RISING, avoid categories where it's flat)
 
 \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550
 
-PHASE 3: GENERATE CANDIDATES
+PHASE 3: GENERATE CANDIDATES (Voyager-style curriculum)
+
+Propose 2-4 candidate next tasks. Use the "Goldilocks" principle from Voyager:
+propose tasks at the FRONTIER of current capabilities \u2014 neither too easy
+(already mastered) nor too hard (no chance of success). Check the competence
+tracker to calibrate difficulty.
 
-Propose 2-4 candidate next tasks. For each, specify:
-- The task description (specific, actionable)
+For each candidate, specify:
+- The task description (specific, actionable, measurable)
 - Rationale (why this task, what led you to it)
 - Provenance (which memories, directives, or signals informed this)
 - Category: directive | exploration | capability | maintenance | social
-- Confidence (0-1)
+- Confidence (0-1, calibrated against competence data)
 
 \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550
 
-PHASE 4: ADVERSARIAL CHALLENGE
+PHASE 4: ADVERSARIAL CHALLENGE (Self-Rewarding + Constitutional AI)
 
-For each candidate, challenge it:
+For each candidate, run a rigorous adversarial review:
 - Is this actually useful or just busywork?
 - Does it align with stored directives and goals?
-- Is it achievable with available tools?
+- Is it achievable with available tools and current competence?
 - Could it cause harm or waste resources?
 - Is there a higher-priority alternative?
+- Would this task help or hinder the agent's long-term growth?
+- Challenge your own confidence rating \u2014 are you overconfident?
 
 \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550
 
@@ -23677,9 +23753,9 @@ Or if nothing is worth doing:
   "reasoning": "why no task is worth pursuing right now"
 }
 
-IMPORTANT: You have access to memory_read, memory_write, and memory_search.
-Use them extensively. The richness of your reasoning depends on how well you
-explore what you already know. Write any new insights to memory for future cycles.
+REMEMBER: Use memory_read, memory_write, and memory_search EXTENSIVELY.
+Write consolidation insights and new reflections to memory before selecting a task.
+The next DMN cycle (and the main agent) will benefit from anything you store now.
 `;
 }
 function adaptTool2(tool) {
@@ -23746,7 +23822,9 @@ var init_dmn_engine = __esm({
         totalCycles: 0,
         lastCycleAt: null,
         tasksGenerated: 0,
-        consecutiveNulls: 0
+        consecutiveNulls: 0,
+        competence: [],
+        reflectionBuffer: []
       };
       stateDir;
       historyDir;
@@ -23766,11 +23844,41 @@ var init_dmn_engine = __esm({
         return { ...this.state };
       }
       /** Record a completed task summary for context in future DMN cycles */
-      recordTaskCompletion(summary) {
+      recordTaskCompletion(summary, category) {
         this.recentTaskSummaries.push(summary);
         if (this.recentTaskSummaries.length > 10) {
           this.recentTaskSummaries.shift();
         }
+        if (category) {
+          this.updateCompetence(category, true);
+        }
+        this.saveState();
+      }
+      /** Record a task failure for Reflexion-style learning */
+      recordTaskFailure(summary, errorContext, category) {
+        const reflection = errorContext ? `FAILED: ${summary.slice(0, 200)} \u2014 Error: ${errorContext.slice(0, 300)}` : `FAILED: ${summary.slice(0, 500)}`;
+        this.state.reflectionBuffer.push(reflection);
+        if (this.state.reflectionBuffer.length > 5) {
+          this.state.reflectionBuffer.shift();
+        }
+        if (category) {
+          this.updateCompetence(category, false);
+        }
+        this.saveState();
+      }
+      /** Update competence tracker for a task category */
+      updateCompetence(category, success) {
+        let entry = this.state.competence.find((c3) => c3.taskType === category);
+        if (!entry) {
+          entry = { taskType: category, attempts: 0, successes: 0, failures: 0, lastAttempt: "" };
+          this.state.competence.push(entry);
+        }
+        entry.attempts++;
+        if (success)
+          entry.successes++;
+        else
+          entry.failures++;
+        entry.lastAttempt = (/* @__PURE__ */ new Date()).toISOString();
       }
       /**
        * Run a DMN cycle — the core self-reflection loop.
@@ -23805,7 +23913,7 @@ var init_dmn_engine = __esm({
           "codebase_map, diagnostic, git_info \u2014 project analysis",
           "sub_agent \u2014 delegate subtasks to independent agents"
         ];
-        const prompt = buildDMNGatherPrompt(this.recentTaskSummaries, reminders, attention, memoryTopics, capabilities);
+        const prompt = buildDMNGatherPrompt(this.recentTaskSummaries, reminders, attention, memoryTopics, capabilities, this.state.competence, this.state.reflectionBuffer);
         const result = await this.runDMNAgent(prompt, onEvent);
         const durationMs = Date.now() - startMs;
         const proposal = this.parseProposal(result.summary);
@@ -25486,7 +25594,7 @@ Use task_status("${taskId}") or task_output("${taskId}") to check progress.`
     }
   };
 }
-function startTask(task, config, repoRoot, voice, stream, taskStores, bruteForce, statusBar, sudoCallback, costTracker, onComplete, taskType, contextWindowSize, modelCaps, personality) {
+function startTask(task, config, repoRoot, voice, stream, taskStores, bruteForce, statusBar, sudoCallback, costTracker, onComplete, taskType, contextWindowSize, modelCaps, personality, deepContext) {
   const voiceStyleMap = {
     concise: 1,
     balanced: 3,
@@ -25510,6 +25618,7 @@ function startTask(task, config, repoRoot, voice, stream, taskStores, bruteForce
     taskTimeoutMs: 36e5,
     // 60 minutes — never give up prematurely
     compactionThreshold,
+    deepContext: deepContext ?? false,
     dynamicContext,
     modelTier,
     streamEnabled: stream?.enabled ?? false,
@@ -25858,6 +25967,7 @@ async function startInteractive(config, repoPath) {
   let streamEnabled = savedSettings.stream ?? false;
   let bruteForceEnabled = savedSettings.bruteforce ?? true;
   let currentStyle = PRESET_NAMES.includes(savedSettings.style) ? savedSettings.style : "balanced";
+  let deepContextEnabled = savedSettings.deepContext ?? false;
   if (savedSettings.emojis !== void 0)
     setEmojisEnabled(savedSettings.emojis);
   if (savedSettings.colors !== void 0)
@@ -26109,6 +26219,7 @@ Rationale: ${proposal.rationale}${provenanceNote}`;
     "/telegram",
     "/compact",
     "/gc",
+    "/deep",
     "/style",
     "/personality"
   ];
@@ -26270,6 +26381,10 @@ Rationale: ${proposal.rationale}${provenanceNote}`;
       bruteForceEnabled = !bruteForceEnabled;
       return bruteForceEnabled;
     },
+    deepContextToggle() {
+      deepContextEnabled = !deepContextEnabled;
+      return deepContextEnabled;
+    },
     setStyle(preset) {
       currentStyle = preset;
     },
@@ -26809,7 +26924,7 @@ Execute this skill now. Follow the behavioral guidance above.`;
             toolPatternStore: toolPatternStore ?? void 0
           }, bruteForceEnabled, statusBar, handleSudoRequest, costTracker, (summary) => {
             lastCompletedSummary = summary;
-          }, currentTaskType, resolvedContextWindowSize, resolvedCaps, currentStyle);
+          }, currentTaskType, resolvedContextWindowSize, resolvedCaps, currentStyle, deepContextEnabled);
           activeTask = task;
           showPrompt();
           await task.promise;
@@ -26921,7 +27036,7 @@ NEW TASK: ${fullInput}`;
         toolPatternStore: toolPatternStore ?? void 0
       }, bruteForceEnabled, statusBar, handleSudoRequest, costTracker, (summary) => {
         lastCompletedSummary = summary;
-      }, currentTaskType, resolvedContextWindowSize, resolvedCaps, currentStyle);
+      }, currentTaskType, resolvedContextWindowSize, resolvedCaps, currentStyle, deepContextEnabled);
       activeTask = task;
       showPrompt();
       await task.promise;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.39.0",
+  "version": "0.40.0",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",