openclaw-node-harness 2.0.4 → 2.1.0

package/lib/memory-budget.mjs

@@ -0,0 +1,261 @@
+/**
+ * memory-budget.mjs — MEMORY.md Character Budget + Frozen Snapshot
+ *
+ * Enforces a hard character cap on MEMORY.md (~2,200 chars, matching Hermes's
+ * proven bound) and provides frozen-snapshot semantics per session.
+ *
+ * Key concepts:
+ *   - Session start: snapshot MEMORY.md and freeze it for the prompt
+ *   - Mid-session writes: persist to disk but DON'T mutate the active prompt
+ *   - On compression rebuild or new session: reload from disk
+ *   - Usage meter: tracks % used, emits warnings at thresholds
+ *
+ * EventEmitter events:
+ *   - 'add'     { entry, usedChars, totalBudget, pctUsed }
+ *   - 'warning' { pctUsed, message }
+ *   - 'trim'    { removed, reason }
+ *   - 'freeze'  { charCount, lineCount }
+ *   - 'reload'  { charCount, lineCount }
+ */
+
+import { EventEmitter } from 'events';
+import fs from 'fs';
+import path from 'path';
+
+const DEFAULT_CHAR_BUDGET = 2200;
+const WARNING_THRESHOLD = 0.80; // warn at 80%
+const CRITICAL_THRESHOLD = 0.95; // critical at 95%
+
+export class MemoryBudget extends EventEmitter {
+  #filePath;
+  #charBudget;
+  #frozenContent = null; // snapshot at session start
+  #frozenAt = null;
+
+  /**
+   * @param {string} filePath - Absolute path to MEMORY.md
+   * @param {Object} opts
+   * @param {number} opts.charBudget - Character budget (default 2200)
+   */
+  constructor(filePath, opts = {}) {
+    super();
+    this.#filePath = filePath;
+    this.#charBudget = opts.charBudget || DEFAULT_CHAR_BUDGET;
+  }
+
+  get filePath() { return this.#filePath; }
+  get charBudget() { return this.#charBudget; }
+  get isFrozen() { return this.#frozenContent !== null; }
+
+  // ── Snapshot Lifecycle ────────────────────────────────────
+
+  /**
+   * Freeze MEMORY.md content for this session.
+   * Call at session start (Phase 0 bootstrap).
+   * Returns the frozen content for prompt injection.
+   */
+  startSession() {
+    this.#frozenContent = this.#readFile();
+    this.#frozenAt = Date.now();
+
+    const stats = this.#computeStats(this.#frozenContent);
+    this.emit('freeze', {
+      charCount: this.#frozenContent.length,
+      lineCount: this.#frozenContent.split('\n').length,
+      ...stats,
+    });
+
+    return this.#frozenContent;
+  }
+
+  /**
+   * Get the frozen (session-start) content for prompt rendering.
+   * Returns the frozen snapshot — NOT the live disk content.
+   * This is the core of deterministic prompt content per session.
+   */
+  getRendered() {
+    if (this.#frozenContent === null) {
+      // Not frozen yet — return live content (pre-session or fallback)
+      return this.#readFile();
+    }
+    return this.#frozenContent;
+  }
+
+  /**
+   * Reload from disk and update the frozen snapshot.
+   * Call after compression rebuild or at new session start.
+   */
+  reload() {
+    this.#frozenContent = this.#readFile();
+    this.#frozenAt = Date.now();
+
+    const stats = this.#computeStats(this.#frozenContent);
+    this.emit('reload', {
+      charCount: this.#frozenContent.length,
+      lineCount: this.#frozenContent.split('\n').length,
+      ...stats,
+    });
+
+    return this.#frozenContent;
+  }
+
+  /**
+   * End the session — clear the frozen snapshot.
+   */
+  endSession() {
+    this.#frozenContent = null;
+    this.#frozenAt = null;
+  }
+
+  // ── Budget-Aware Write ────────────────────────────────────
+
+  /**
+   * Add an entry to MEMORY.md on disk (not the frozen prompt).
+   * Respects character budget. Returns false if over budget.
+   *
+   * @param {string} entry - The line to add (without leading "- ")
+   * @param {Object} opts
+   * @param {string} opts.section - Section to add under (default "Recent")
+   * @returns {{ added: boolean, pctUsed: number, charsRemaining: number }}
+   */
+  addEntry(entry, opts = {}) {
+    const { section = 'Recent' } = opts;
+    let content = this.#readFile();
+    const line = `- ${entry}`;
+    const newLength = content.length + line.length + 1; // +1 for newline
+
+    if (newLength > this.#charBudget) {
+      // Try trimming oldest entries first
+      const trimmed = this.#trimOldest(content, line.length + 1);
+      if (trimmed) {
+        content = trimmed;
+      } else {
+        return { added: false, pctUsed: this.#pctUsed(content), charsRemaining: 0 };
+      }
+    }
+
+    // Ensure section exists
+    if (!content.includes(`## ${section}`)) {
+      content = content.trimEnd() + `\n\n## ${section}\n`;
+    }
+
+    content = content.trimEnd() + `\n${line}\n`;
+    this.#writeFile(content);
+
+    const pctUsed = this.#pctUsed(content);
+    const charsRemaining = Math.max(0, this.#charBudget - content.length);
+
+    this.emit('add', {
+      entry,
+      usedChars: content.length,
+      totalBudget: this.#charBudget,
+      pctUsed,
+      charsRemaining,
+    });
+
+    // Threshold warnings
+    if (pctUsed >= CRITICAL_THRESHOLD * 100) {
+      this.emit('warning', {
+        pctUsed,
+        message: `MEMORY.md at ${pctUsed}% capacity (${charsRemaining} chars remaining)`,
+      });
+    } else if (pctUsed >= WARNING_THRESHOLD * 100) {
+      this.emit('warning', {
+        pctUsed,
+        message: `MEMORY.md approaching limit: ${pctUsed}% used (${charsRemaining} chars remaining)`,
+      });
+    }
+
+    return { added: true, pctUsed, charsRemaining };
+  }
+
+  // ── Usage Meter ────────────────────────────────────
+
+  /**
+   * Get current budget usage stats.
+   * @returns {{ usedChars: number, totalBudget: number, pctUsed: number, charsRemaining: number, lineCount: number, meterDisplay: string }}
+   */
+  getStats() {
+    const content = this.#readFile();
+    return this.#computeStats(content);
+  }
+
+  /**
+   * Render a usage meter string for logging/display.
+   * Example: "[67% — 1,474/2,200 chars]"
+   */
+  getMeterDisplay() {
+    return this.#computeStats(this.#readFile()).meterDisplay;
+  }
+
+  // ── Private Helpers ────────────────────────────────────
+
+  #readFile() {
+    if (!fs.existsSync(this.#filePath)) return '';
+    return fs.readFileSync(this.#filePath, 'utf-8');
+  }
+
+  #writeFile(content) {
+    const dir = path.dirname(this.#filePath);
+    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(this.#filePath, content);
+  }
+
+  #pctUsed(content) {
+    return Math.round((content.length / this.#charBudget) * 100);
+  }
+
+  #computeStats(content) {
+    const usedChars = content.length;
+    const pctUsed = this.#pctUsed(content);
+    const charsRemaining = Math.max(0, this.#charBudget - usedChars);
+    const lineCount = content.split('\n').length;
+    const meterDisplay = `[${pctUsed}% — ${usedChars.toLocaleString()}/${this.#charBudget.toLocaleString()} chars]`;
+
+    return { usedChars, totalBudget: this.#charBudget, pctUsed, charsRemaining, lineCount, meterDisplay };
+  }
+
+  /**
+   * Trim the oldest bullet entries to free up space.
+   * Returns the trimmed content, or null if can't free enough.
+   */
+  #trimOldest(content, bytesNeeded) {
+    const lines = content.split('\n');
+    const bulletIndices = [];
+
+    for (let i = 0; i < lines.length; i++) {
+      if (lines[i].startsWith('- ') || lines[i].startsWith('* ')) {
+        bulletIndices.push(i);
+      }
+    }
+
+    if (bulletIndices.length === 0) return null;
+
+    // Remove bullets from the top until we have enough space
+    let freed = 0;
+    const toRemove = new Set();
+
+    for (const idx of bulletIndices) {
+      if (freed >= bytesNeeded) break;
+      freed += lines[idx].length + 1;
+      toRemove.add(idx);
+
+      this.emit('trim', { removed: lines[idx], reason: 'budget overflow' });
+    }
+
+    if (freed < bytesNeeded) return null;
+
+    const trimmed = lines.filter((_, i) => !toRemove.has(i)).join('\n');
+    return trimmed;
+  }
+}
+
+/**
+ * Create a MemoryBudget instance with default OpenClaw paths.
+ * @param {string} workspace - OpenClaw workspace root
+ * @param {Object} opts - Options forwarded to MemoryBudget constructor
+ */
+export function createBudget(workspace, opts = {}) {
+  const filePath = path.join(workspace, 'MEMORY.md');
+  return new MemoryBudget(filePath, opts);
+}

package/lib/mesh-collab.js

@@ -29,6 +29,7 @@ const COLLAB_MODE = {
   PARALLEL: 'parallel',       // all nodes work simultaneously
   SEQUENTIAL: 'sequential',   // nodes take turns in order
   REVIEW: 'review',           // one leader + N reviewers
+  CIRCLING_STRATEGY: 'circling_strategy', // 1 worker + 2 reviewers, asymmetric directed rounds
 };
 
 // ── Convergence Strategies ──────────────────────────
@@ -57,7 +58,9 @@ function createSession(taskId, collabSpec) {
     status: COLLAB_STATUS.RECRUITING,
 
     // Node management
-    min_nodes: collabSpec.min_nodes || 2,
+    // Circling requires at least 3 nodes (1 worker + 2 reviewers).
+    // Default to 3 for circling, 2 for other modes.
+    min_nodes: collabSpec.min_nodes || (collabSpec.mode === COLLAB_MODE.CIRCLING_STRATEGY ? 3 : 2),
     max_nodes: collabSpec.max_nodes || null,  // null = unlimited
     join_window_s: collabSpec.join_window_s || 30,
     nodes: [],
@@ -85,10 +88,32 @@ function createSession(taskId, collabSpec) {
     // Scope strategy
     scope_strategy: collabSpec.scope_strategy || 'shared',
 
+    // Heterogeneous collab: per-node role/soul assignments (Phase E)
+    // Format: [{ soul: "blockchain-auditor", role: "solidity-dev" }, { soul: "identity-architect" }]
+    // When set, recruiting assigns specific souls to joining nodes in order.
+    // When null, all nodes run the same soul (homogeneous, backward compatible).
+    node_roles: collabSpec.node_roles || null,
+
     // Sequential mode: turn tracking
     turn_order: [],           // node_ids in execution order
     current_turn: null,       // node_id of active node (sequential mode)
 
+    // Circling Strategy mode: asymmetric directed rounds
+    // Only populated when mode === 'circling_strategy'. null for other modes.
+    circling: collabSpec.mode === COLLAB_MODE.CIRCLING_STRATEGY ? {
+      worker_node_id: null,           // assigned at recruiting close (node_roles[0])
+      reviewerA_node_id: null,        // assigned at recruiting close — first non-worker
+      reviewerB_node_id: null,        // assigned at recruiting close — second non-worker
+      max_subrounds: collabSpec.max_subrounds || 3,
+      current_subround: 0,
+      current_step: 0,                // 0 = init, 1 = review pass, 2 = integration
+      automation_tier: collabSpec.automation_tier || 2,
+      artifacts: {},                  // keyed: sr{N}_step{S}_{nodeRole}_{artifactType}
+      phase: 'init',                  // init | circling | finalization | complete
+      artifact_failures: {},          // { nodeId_step: count } — retry tracking per node per step
+      step_started_at: null,          // ISO timestamp — set by daemon at step start, used for timeout rehydration after restart
+    } : null,
+
     // Result (filled at completion)
     result: null,
 
@@ -374,6 +399,9 @@ class CollabStore {
       vote: reflection.vote || 'continue',
       parse_failed: reflection.parse_failed || false,
       submitted_at: new Date().toISOString(),
+      // Circling Strategy extensions (optional, backward compatible)
+      circling_step: reflection.circling_step ?? null,
+      circling_artifacts: reflection.circling_artifacts || [],  // [{ type, content }]
     });
 
     // Update node status
@@ -476,6 +504,278 @@ class CollabStore {
     return session.current_round >= session.max_rounds;
   }
 
+  // ── Circling Strategy: Artifact Store ──────────────
+
+  /**
+   * Store a typed artifact from a circling step.
+   * Key format: sr{N}_step{S}_{nodeRole}_{artifactType}
+   */
+  async storeArtifact(sessionId, key, content) {
+    const session = await this.get(sessionId);
+    if (!session || !session.circling) return null;
+    session.circling.artifacts[key] = content;
+
+    // Session blob size check — JetStream KV max is 1MB.
+    // Warn early so operators can plan external artifact store before hitting the wall.
+    const blobSize = Buffer.byteLength(JSON.stringify(session), 'utf8');
+    if (blobSize > 950_000) {
+      console.error(`[collab] CRITICAL: session ${sessionId} blob is ${(blobSize / 1024).toFixed(0)}KB — approaching JetStream KV 1MB limit`);
+    } else if (blobSize > 800_000) {
+      console.warn(`[collab] WARNING: session ${sessionId} blob is ${(blobSize / 1024).toFixed(0)}KB — consider external artifact store`);
+    }
+
+    try {
+      await this.put(session);
+    } catch (err) {
+      // JetStream KV write failure — likely blob exceeded 1MB limit.
+      // Remove the artifact that caused the overflow and re-persist without it.
+      console.error(`[collab] storeArtifact FAILED for ${sessionId}/${key}: ${err.message}. Removing artifact and persisting without it.`);
+      delete session.circling.artifacts[key];
+      await this.put(session);
+      return null;
+    }
+
+    return session;
+  }
+
+  /**
+   * Retrieve an artifact by key.
+   */
+  getArtifactByKey(session, key) {
+    if (!session || !session.circling) return null;
+    return session.circling.artifacts[key] || null;
+  }
+
+  /**
+   * Find the most recent version of an artifact type from a given role.
+   * Scans backward from (current_subround - 1) step 2 → sr0_step0 (init).
+   * Returns null if not found.
+   */
+  getLatestArtifact(session, nodeRole, artifactType) {
+    if (!session || !session.circling) return null;
+    const { current_subround, artifacts } = session.circling;
+
+    // Scan backward through sub-rounds
+    for (let sr = current_subround; sr >= 0; sr--) {
+      // Step 2 artifacts (produced during integration/refinement)
+      if (sr > 0) {
+        const key2 = `sr${sr}_step2_${nodeRole}_${artifactType}`;
+        if (artifacts[key2] !== undefined) return artifacts[key2];
+      }
+      // Step 1 artifacts (produced during review pass)
+      if (sr > 0) {
+        const key1 = `sr${sr}_step1_${nodeRole}_${artifactType}`;
+        if (artifacts[key1] !== undefined) return artifacts[key1];
+      }
+      // Init artifacts (sr0_step0)
+      if (sr === 0) {
+        const key0 = `sr0_step0_${nodeRole}_${artifactType}`;
+        if (artifacts[key0] !== undefined) return artifacts[key0];
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Build node-specific directed input for a circling step.
+   * Uses the information flow matrix from the Circling Strategy protocol spec (§5).
+   *
+   * Returns a string containing the directed artifacts for this node at this step.
+   */
+  compileDirectedInput(session, nodeId, taskDescription) {
+    if (!session || !session.circling) return '';
+    const { phase, current_subround, current_step } = session.circling;
+    const node = session.nodes.find(n => n.node_id === nodeId);
+    if (!node) return '';
+
+    const isWorker = nodeId === session.circling.worker_node_id;
+    const parts = [];
+
+    // Use stored reviewer IDs (assigned at recruiting close) for stable identity.
+    // Falls back to array-index computation if IDs aren't set (backward compat).
+    const reviewerLabel = (nId) => {
+      if (session.circling.reviewerA_node_id && session.circling.reviewerB_node_id) {
+        return nId === session.circling.reviewerA_node_id ? 'reviewerA' : 'reviewerB';
+      }
+      // Legacy fallback: compute from array position
+      const reviewerNodes = session.nodes.filter(n => n.node_id !== session.circling.worker_node_id);
+      const idx = reviewerNodes.findIndex(n => n.node_id === nId);
+      return idx === 0 ? 'reviewerA' : 'reviewerB';
+    };
+    const myReviewerRole = !isWorker ? reviewerLabel(nodeId) : null;
+
+    // Helper: add artifact to parts, handling null (required vs optional)
+    const addArtifact = (label, nodeRole, artifactType, required) => {
+      const content = this.getLatestArtifact(session, nodeRole, artifactType);
+      if (content !== null) {
+        parts.push(`## ${label}\n\n${content}`);
+      } else if (required) {
+        parts.push(`## ${label}\n\n[UNAVAILABLE: ${nodeRole}'s ${artifactType} was not produced — proceed with available inputs only]`);
+      }
+      // If not required and null, skip silently
+    };
+
+    switch (phase) {
+      case 'init':
+        parts.push(`## Task Plan\n\n${taskDescription || '(no description)'}`);
+        break;
+
+      case 'circling':
+        if (current_step === 1) {
+          // Step 1 — Review Pass
+          if (isWorker) {
+            // Worker receives: both reviewStrategies
+            addArtifact('Reviewer A Strategy', 'reviewerA', 'reviewStrategy', true);
+            addArtifact('Reviewer B Strategy', 'reviewerB', 'reviewStrategy', true);
+            // SR2+: also include reviewArtifacts so Worker can assess whether
+            // strategies are producing useful reviews (evidence alongside methodology)
+            if (current_subround > 1) {
+              addArtifact('Reviewer A — Review Findings', 'reviewerA', 'reviewArtifact', false);
+              addArtifact('Reviewer B — Review Findings', 'reviewerB', 'reviewArtifact', false);
+            }
+          } else {
+            // Reviewer receives: workArtifact + reconciliationDoc (optional in SR1)
+            addArtifact('Work Artifact', 'worker', 'workArtifact', true);
+            addArtifact('Reconciliation Document', 'worker', 'reconciliationDoc', current_subround > 1);
+          }
+        } else if (current_step === 2) {
+          // Step 2 — Integration + Refinement
+          if (isWorker) {
+            // Worker receives: both reviewArtifacts
+            addArtifact('Reviewer A Review', 'reviewerA', 'reviewArtifact', true);
+            addArtifact('Reviewer B Review', 'reviewerB', 'reviewArtifact', true);
+          } else {
+            // Reviewer receives: workerReviewsAnalysis + cross-review from the other reviewer.
+            // Cross-review enables inter-reviewer learning: "B caught something my
+            // methodology missed — I should incorporate that lens."
+            addArtifact('Worker Reviews Analysis', 'worker', 'workerReviewsAnalysis', true);
+            const otherReviewerRole = (myReviewerRole === 'reviewerA') ? 'reviewerB' : 'reviewerA';
+            addArtifact(`Cross-Review — ${otherReviewerRole} Findings`, otherReviewerRole, 'reviewArtifact', false);
+          }
+        }
+        break;
+
+      case 'finalization':
+        parts.push(`## Original Task Plan\n\n${taskDescription || '(no description)'}`);
+        addArtifact('Final Work Artifact', 'worker', 'workArtifact', true);
+        break;
+    }
+
+    return parts.join('\n\n---\n\n');
+  }
+
+  /**
+   * Check if all nodes have submitted for the current circling step.
+   */
+  isCirclingStepComplete(session) {
+    if (!session || !session.circling) return false;
+    const currentRound = session.rounds[session.rounds.length - 1];
+    if (!currentRound) return false;
+
+    const activeNodes = session.nodes.filter(n => n.status !== 'dead');
+    // For circling, reflections per step are tagged. Count reflections matching current step.
+    const stepReflections = currentRound.reflections.filter(
+      r => r.circling_step === session.circling.current_step
+    );
+    return stepReflections.length >= activeNodes.length;
+  }
+
+  /**
+   * Advance the circling state machine.
+   * Returns { phase, subround, step, needsGate } describing the new state.
+   *
+   * State machine transitions:
+   *   init/step0      → circling/SR1/step1
+   *   circling/step1  → circling/step2 (same SR)
+   *   circling/step2 (SR < max) → circling/SR+1/step1
+   *   circling/step2 (SR == max) → finalization/step0
+   */
+  async advanceCirclingStep(sessionId) {
+    const session = await this.get(sessionId);
+    if (!session || !session.circling) return null;
+
+    const c = session.circling;
+    let needsGate = false;
+
+    if (c.phase === 'init' && c.current_step === 0) {
+      // Init complete → start circling SR1/Step1
+      c.phase = 'circling';
+      c.current_subround = 1;
+      c.current_step = 1;
+    } else if (c.phase === 'circling' && c.current_step === 1) {
+      // Step 1 complete → Step 2 (same subround)
+      c.current_step = 2;
+    } else if (c.phase === 'circling' && c.current_step === 2) {
+      // Adaptive convergence: if all active nodes voted 'converged' after step 2,
+      // skip remaining sub-rounds and go directly to finalization.
+      const currentRound = session.rounds[session.rounds.length - 1];
+      const activeNodes = session.nodes.filter(n => n.status !== 'dead');
+      const step2Reflections = currentRound
+        ? currentRound.reflections.filter(r => r.circling_step === 2)
+        : [];
+      const allConverged = step2Reflections.length >= activeNodes.length &&
+                           step2Reflections.every(r => r.vote === 'converged');
+
+      if (allConverged && c.current_subround < c.max_subrounds) {
+        // Early exit — all nodes agree the work is ready
+        if (c.automation_tier >= 2) {
+          needsGate = true;
+        }
+        c.phase = 'finalization';
+        c.current_step = 0;
+      } else if (c.current_subround < c.max_subrounds) {
+        // Step 2 complete, more sub-rounds → next SR/Step1
+        // Check tier gate for Tier 3 (gates after every sub-round)
+        if (c.automation_tier === 3) {
+          needsGate = true;
+        }
+        c.current_subround++;
+        c.current_step = 1;
+      } else {
+        // Final sub-round complete → finalization
+        // Tier 2 gates on finalization entry
+        if (c.automation_tier >= 2) {
+          needsGate = true;
+        }
+        c.phase = 'finalization';
+        c.current_step = 0;
+      }
+    } else if (c.phase === 'finalization') {
+      // Finalization complete → done
+      c.phase = 'complete';
+    }
+
+    await this.put(session);
+    return {
+      phase: c.phase,
+      subround: c.current_subround,
+      step: c.current_step,
+      needsGate,
+    };
+  }
+
+  /**
+   * Record an artifact parse failure for a node at the current step.
+   * Returns the failure count.
+   */
+  async recordArtifactFailure(sessionId, nodeId) {
+    const session = await this.get(sessionId);
+    if (!session || !session.circling) return 0;
+    const key = `${nodeId}_sr${session.circling.current_subround}_step${session.circling.current_step}`;
+    session.circling.artifact_failures[key] = (session.circling.artifact_failures[key] || 0) + 1;
+    await this.put(session);
+    return session.circling.artifact_failures[key];
+  }
+
+  /**
+   * Get the artifact failure count for a node at the current step.
+   */
+  getArtifactFailureCount(session, nodeId) {
+    if (!session || !session.circling) return 0;
+    const key = `${nodeId}_sr${session.circling.current_subround}_step${session.circling.current_step}`;
+    return session.circling.artifact_failures[key] || 0;
+  }
+
   // ── Intel Compilation ──────────────────────────────
 
   /**