openclaw-node-harness 2.0.3 → 2.1.0

package/lib/mesh-collab.js

@@ -29,6 +29,7 @@ const COLLAB_MODE = {
   PARALLEL: 'parallel',       // all nodes work simultaneously
   SEQUENTIAL: 'sequential',   // nodes take turns in order
   REVIEW: 'review',           // one leader + N reviewers
+  CIRCLING_STRATEGY: 'circling_strategy', // 1 worker + 2 reviewers, asymmetric directed rounds
 };
 
 // ── Convergence Strategies ──────────────────────────
@@ -57,7 +58,9 @@ function createSession(taskId, collabSpec) {
     status: COLLAB_STATUS.RECRUITING,
 
     // Node management
-    min_nodes: collabSpec.min_nodes || 2,
+    // Circling requires at least 3 nodes (1 worker + 2 reviewers).
+    // Default to 3 for circling, 2 for other modes.
+    min_nodes: collabSpec.min_nodes || (collabSpec.mode === COLLAB_MODE.CIRCLING_STRATEGY ? 3 : 2),
     max_nodes: collabSpec.max_nodes || null,  // null = unlimited
     join_window_s: collabSpec.join_window_s || 30,
     nodes: [],
@@ -85,10 +88,32 @@ function createSession(taskId, collabSpec) {
     // Scope strategy
     scope_strategy: collabSpec.scope_strategy || 'shared',
 
+    // Heterogeneous collab: per-node role/soul assignments (Phase E)
+    // Format: [{ soul: "blockchain-auditor", role: "solidity-dev" }, { soul: "identity-architect" }]
+    // When set, recruiting assigns specific souls to joining nodes in order.
+    // When null, all nodes run the same soul (homogeneous, backward compatible).
+    node_roles: collabSpec.node_roles || null,
+
     // Sequential mode: turn tracking
     turn_order: [],           // node_ids in execution order
     current_turn: null,       // node_id of active node (sequential mode)
 
+    // Circling Strategy mode: asymmetric directed rounds
+    // Only populated when mode === 'circling_strategy'. null for other modes.
+    circling: collabSpec.mode === COLLAB_MODE.CIRCLING_STRATEGY ? {
+      worker_node_id: null,           // assigned at recruiting close (node_roles[0])
+      reviewerA_node_id: null,        // assigned at recruiting close — first non-worker
+      reviewerB_node_id: null,        // assigned at recruiting close — second non-worker
+      max_subrounds: collabSpec.max_subrounds || 3,
+      current_subround: 0,
+      current_step: 0,                // 0 = init, 1 = review pass, 2 = integration
+      automation_tier: collabSpec.automation_tier || 2,
+      artifacts: {},                  // keyed: sr{N}_step{S}_{nodeRole}_{artifactType}
+      phase: 'init',                  // init | circling | finalization | complete
+      artifact_failures: {},          // { nodeId_step: count } — retry tracking per node per step
+      step_started_at: null,          // ISO timestamp — set by daemon at step start, used for timeout rehydration after restart
+    } : null,
+
     // Result (filled at completion)
     result: null,
 
@@ -105,6 +130,10 @@ function createSession(taskId, collabSpec) {
 
 // ── CollabStore (KV-backed) ─────────────────────────
 
+// Rate-limit audit error logs: max 3 per session, then go silent
+const _auditErrorCounts = new Map();
+const AUDIT_ERROR_LOG_LIMIT = 3;
+
 class CollabStore {
   constructor(kv) {
     this.kv = kv;
@@ -139,7 +168,14 @@ class CollabStore {
         ...detail,
       });
       await this.put(session);
-    } catch { /* best-effort — never block on audit */ }
+    } catch (err) {
+      // Best-effort — never block on audit, but log first N failures per session
+      const count = (_auditErrorCounts.get(sessionId) || 0) + 1;
+      _auditErrorCounts.set(sessionId, count);
+      if (count <= AUDIT_ERROR_LOG_LIMIT) {
+        console.error(`[collab] audit append failed for ${sessionId}/${event}: ${err.message}${count === AUDIT_ERROR_LOG_LIMIT ? ' (suppressing further audit errors for this session)' : ''}`);
+      }
+    }
   }
 
   /**
@@ -176,6 +212,30 @@ class CollabStore {
     return sessions[0] || null;
   }
 
+  /**
+   * Find active sessions that contain a given node.
+   * O(sessions) single pass — avoids the O(sessions × nodes) scan
+   * that detectStalls() previously used with list() + inner find().
+   */
+  async findActiveSessionsByNode(nodeId) {
+    const results = [];
+    const allKeys = [];
+    const keys = await this.kv.keys();
+    for await (const key of keys) {
+      allKeys.push(key);
+    }
+    for (const key of allKeys) {
+      const entry = await this.kv.get(key);
+      if (!entry || !entry.value) continue;
+      const session = JSON.parse(sc.decode(entry.value));
+      if (session.status !== COLLAB_STATUS.ACTIVE) continue;
+      if (session.nodes.some(n => n.node_id === nodeId)) {
+        results.push(session);
+      }
+    }
+    return results;
+  }
+
   // ── Node Management ────────────────────────────────
 
   /**
@@ -190,7 +250,8 @@ class CollabStore {
     // Check max_nodes
     if (session.max_nodes && session.nodes.length >= session.max_nodes) return null;
 
-    // Check duplicate
+    // Check duplicate — single-threaded event loop prevents concurrent joins
+    // from interleaving between find() and push(). No mutex needed.
     if (session.nodes.find(n => n.node_id === nodeId)) return null;
 
     session.nodes.push({
@@ -320,6 +381,9 @@ class CollabStore {
     const session = await this.get(sessionId);
     if (!session) return null;
 
+    // Only accept reflections on active sessions
+    if (session.status !== COLLAB_STATUS.ACTIVE) return null;
+
     const currentRound = session.rounds[session.rounds.length - 1];
     if (!currentRound) return null;
 
@@ -335,6 +399,9 @@ class CollabStore {
       vote: reflection.vote || 'continue',
       parse_failed: reflection.parse_failed || false,
       submitted_at: new Date().toISOString(),
+      // Circling Strategy extensions (optional, backward compatible)
+      circling_step: reflection.circling_step ?? null,
+      circling_artifacts: reflection.circling_artifacts || [],  // [{ type, content }]
     });
 
     // Update node status
@@ -437,6 +504,278 @@ class CollabStore {
     return session.current_round >= session.max_rounds;
   }
 
+  // ── Circling Strategy: Artifact Store ──────────────
+
+  /**
+   * Store a typed artifact from a circling step.
+   * Key format: sr{N}_step{S}_{nodeRole}_{artifactType}
+   */
+  async storeArtifact(sessionId, key, content) {
+    const session = await this.get(sessionId);
+    if (!session || !session.circling) return null;
+    session.circling.artifacts[key] = content;
+
+    // Session blob size check — JetStream KV max is 1MB.
+    // Warn early so operators can plan external artifact store before hitting the wall.
+    const blobSize = Buffer.byteLength(JSON.stringify(session), 'utf8');
+    if (blobSize > 950_000) {
+      console.error(`[collab] CRITICAL: session ${sessionId} blob is ${(blobSize / 1024).toFixed(0)}KB — approaching JetStream KV 1MB limit`);
+    } else if (blobSize > 800_000) {
+      console.warn(`[collab] WARNING: session ${sessionId} blob is ${(blobSize / 1024).toFixed(0)}KB — consider external artifact store`);
+    }
+
+    try {
+      await this.put(session);
+    } catch (err) {
+      // JetStream KV write failure — likely blob exceeded 1MB limit.
+      // Remove the artifact that caused the overflow and re-persist without it.
+      console.error(`[collab] storeArtifact FAILED for ${sessionId}/${key}: ${err.message}. Removing artifact and persisting without it.`);
+      delete session.circling.artifacts[key];
+      await this.put(session);
+      return null;
+    }
+
+    return session;
+  }
+
+  /**
+   * Retrieve an artifact by key.
+   */
+  getArtifactByKey(session, key) {
+    if (!session || !session.circling) return null;
+    return session.circling.artifacts[key] || null;
+  }
+
+  /**
+   * Find the most recent version of an artifact type from a given role.
+   * Scans backward from (current_subround - 1) step 2 → sr0_step0 (init).
+   * Returns null if not found.
+   */
+  getLatestArtifact(session, nodeRole, artifactType) {
+    if (!session || !session.circling) return null;
+    const { current_subround, artifacts } = session.circling;
+
+    // Scan backward through sub-rounds
+    for (let sr = current_subround; sr >= 0; sr--) {
+      // Step 2 artifacts (produced during integration/refinement)
+      if (sr > 0) {
+        const key2 = `sr${sr}_step2_${nodeRole}_${artifactType}`;
+        if (artifacts[key2] !== undefined) return artifacts[key2];
+      }
+      // Step 1 artifacts (produced during review pass)
+      if (sr > 0) {
+        const key1 = `sr${sr}_step1_${nodeRole}_${artifactType}`;
+        if (artifacts[key1] !== undefined) return artifacts[key1];
+      }
+      // Init artifacts (sr0_step0)
+      if (sr === 0) {
+        const key0 = `sr0_step0_${nodeRole}_${artifactType}`;
+        if (artifacts[key0] !== undefined) return artifacts[key0];
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Build node-specific directed input for a circling step.
+   * Uses the information flow matrix from the Circling Strategy protocol spec (§5).
+   *
+   * Returns a string containing the directed artifacts for this node at this step.
+   */
+  compileDirectedInput(session, nodeId, taskDescription) {
+    if (!session || !session.circling) return '';
+    const { phase, current_subround, current_step } = session.circling;
+    const node = session.nodes.find(n => n.node_id === nodeId);
+    if (!node) return '';
+
+    const isWorker = nodeId === session.circling.worker_node_id;
+    const parts = [];
+
+    // Use stored reviewer IDs (assigned at recruiting close) for stable identity.
+    // Falls back to array-index computation if IDs aren't set (backward compat).
+    const reviewerLabel = (nId) => {
+      if (session.circling.reviewerA_node_id && session.circling.reviewerB_node_id) {
+        return nId === session.circling.reviewerA_node_id ? 'reviewerA' : 'reviewerB';
+      }
+      // Legacy fallback: compute from array position
+      const reviewerNodes = session.nodes.filter(n => n.node_id !== session.circling.worker_node_id);
+      const idx = reviewerNodes.findIndex(n => n.node_id === nId);
+      return idx === 0 ? 'reviewerA' : 'reviewerB';
+    };
+    const myReviewerRole = !isWorker ? reviewerLabel(nodeId) : null;
+
+    // Helper: add artifact to parts, handling null (required vs optional)
+    const addArtifact = (label, nodeRole, artifactType, required) => {
+      const content = this.getLatestArtifact(session, nodeRole, artifactType);
+      if (content !== null) {
+        parts.push(`## ${label}\n\n${content}`);
+      } else if (required) {
+        parts.push(`## ${label}\n\n[UNAVAILABLE: ${nodeRole}'s ${artifactType} was not produced — proceed with available inputs only]`);
+      }
+      // If not required and null, skip silently
+    };
+
+    switch (phase) {
+      case 'init':
+        parts.push(`## Task Plan\n\n${taskDescription || '(no description)'}`);
+        break;
+
+      case 'circling':
+        if (current_step === 1) {
+          // Step 1 — Review Pass
+          if (isWorker) {
+            // Worker receives: both reviewStrategies
+            addArtifact('Reviewer A Strategy', 'reviewerA', 'reviewStrategy', true);
+            addArtifact('Reviewer B Strategy', 'reviewerB', 'reviewStrategy', true);
+            // SR2+: also include reviewArtifacts so Worker can assess whether
+            // strategies are producing useful reviews (evidence alongside methodology)
+            if (current_subround > 1) {
+              addArtifact('Reviewer A — Review Findings', 'reviewerA', 'reviewArtifact', false);
+              addArtifact('Reviewer B — Review Findings', 'reviewerB', 'reviewArtifact', false);
+            }
+          } else {
+            // Reviewer receives: workArtifact + reconciliationDoc (optional in SR1)
+            addArtifact('Work Artifact', 'worker', 'workArtifact', true);
+            addArtifact('Reconciliation Document', 'worker', 'reconciliationDoc', current_subround > 1);
+          }
+        } else if (current_step === 2) {
+          // Step 2 — Integration + Refinement
+          if (isWorker) {
+            // Worker receives: both reviewArtifacts
+            addArtifact('Reviewer A Review', 'reviewerA', 'reviewArtifact', true);
+            addArtifact('Reviewer B Review', 'reviewerB', 'reviewArtifact', true);
+          } else {
+            // Reviewer receives: workerReviewsAnalysis + cross-review from the other reviewer.
+            // Cross-review enables inter-reviewer learning: "B caught something my
+            // methodology missed — I should incorporate that lens."
+            addArtifact('Worker Reviews Analysis', 'worker', 'workerReviewsAnalysis', true);
+            const otherReviewerRole = (myReviewerRole === 'reviewerA') ? 'reviewerB' : 'reviewerA';
+            addArtifact(`Cross-Review — ${otherReviewerRole} Findings`, otherReviewerRole, 'reviewArtifact', false);
+          }
+        }
+        break;
+
+      case 'finalization':
+        parts.push(`## Original Task Plan\n\n${taskDescription || '(no description)'}`);
+        addArtifact('Final Work Artifact', 'worker', 'workArtifact', true);
+        break;
+    }
+
+    return parts.join('\n\n---\n\n');
+  }
+
+  /**
+   * Check if all nodes have submitted for the current circling step.
+   */
+  isCirclingStepComplete(session) {
+    if (!session || !session.circling) return false;
+    const currentRound = session.rounds[session.rounds.length - 1];
+    if (!currentRound) return false;
+
+    const activeNodes = session.nodes.filter(n => n.status !== 'dead');
+    // For circling, reflections per step are tagged. Count reflections matching current step.
+    const stepReflections = currentRound.reflections.filter(
+      r => r.circling_step === session.circling.current_step
+    );
+    return stepReflections.length >= activeNodes.length;
+  }
+
+  /**
+   * Advance the circling state machine.
+   * Returns { phase, subround, step, needsGate } describing the new state.
+   *
+   * State machine transitions:
+   *   init/step0      → circling/SR1/step1
+   *   circling/step1  → circling/step2 (same SR)
+   *   circling/step2 (SR < max) → circling/SR+1/step1
+   *   circling/step2 (SR == max) → finalization/step0
+   */
+  async advanceCirclingStep(sessionId) {
+    const session = await this.get(sessionId);
+    if (!session || !session.circling) return null;
+
+    const c = session.circling;
+    let needsGate = false;
+
+    if (c.phase === 'init' && c.current_step === 0) {
+      // Init complete → start circling SR1/Step1
+      c.phase = 'circling';
+      c.current_subround = 1;
+      c.current_step = 1;
+    } else if (c.phase === 'circling' && c.current_step === 1) {
+      // Step 1 complete → Step 2 (same subround)
+      c.current_step = 2;
+    } else if (c.phase === 'circling' && c.current_step === 2) {
+      // Adaptive convergence: if all active nodes voted 'converged' after step 2,
+      // skip remaining sub-rounds and go directly to finalization.
+      const currentRound = session.rounds[session.rounds.length - 1];
+      const activeNodes = session.nodes.filter(n => n.status !== 'dead');
+      const step2Reflections = currentRound
+        ? currentRound.reflections.filter(r => r.circling_step === 2)
+        : [];
+      const allConverged = step2Reflections.length >= activeNodes.length &&
+                           step2Reflections.every(r => r.vote === 'converged');
+
+      if (allConverged && c.current_subround < c.max_subrounds) {
+        // Early exit — all nodes agree the work is ready
+        if (c.automation_tier >= 2) {
+          needsGate = true;
+        }
+        c.phase = 'finalization';
+        c.current_step = 0;
+      } else if (c.current_subround < c.max_subrounds) {
+        // Step 2 complete, more sub-rounds → next SR/Step1
+        // Check tier gate for Tier 3 (gates after every sub-round)
+        if (c.automation_tier === 3) {
+          needsGate = true;
+        }
+        c.current_subround++;
+        c.current_step = 1;
+      } else {
+        // Final sub-round complete → finalization
+        // Tier 2 gates on finalization entry
+        if (c.automation_tier >= 2) {
+          needsGate = true;
+        }
+        c.phase = 'finalization';
+        c.current_step = 0;
+      }
+    } else if (c.phase === 'finalization') {
+      // Finalization complete → done
+      c.phase = 'complete';
+    }
+
+    await this.put(session);
+    return {
+      phase: c.phase,
+      subround: c.current_subround,
+      step: c.current_step,
+      needsGate,
+    };
+  }
+
+  /**
+   * Record an artifact parse failure for a node at the current step.
+   * Returns the failure count.
+   */
+  async recordArtifactFailure(sessionId, nodeId) {
+    const session = await this.get(sessionId);
+    if (!session || !session.circling) return 0;
+    const key = `${nodeId}_sr${session.circling.current_subround}_step${session.circling.current_step}`;
+    session.circling.artifact_failures[key] = (session.circling.artifact_failures[key] || 0) + 1;
+    await this.put(session);
+    return session.circling.artifact_failures[key];
+  }
+
+  /**
+   * Get the artifact failure count for a node at the current step.
+   */
+  getArtifactFailureCount(session, nodeId) {
+    if (!session || !session.circling) return 0;
+    const key = `${nodeId}_sr${session.circling.current_subround}_step${session.circling.current_step}`;
+    return session.circling.artifact_failures[key] || 0;
+  }
+
   // ── Intel Compilation ──────────────────────────────
 
   /**
@@ -506,11 +845,14 @@ class CollabStore {
   }
 
   /**
-   * Mark session as aborted.
+   * Mark session as aborted. Returns null (no-op) if already completed or aborted.
+   * Callers can use truthiness to detect whether the abort actually happened.
    */
   async markAborted(sessionId, reason) {
     const session = await this.get(sessionId);
     if (!session) return null;
+    // Guard: don't corrupt completed/aborted sessions
+    if (['completed', 'aborted'].includes(session.status)) return null;
     session.status = COLLAB_STATUS.ABORTED;
     session.completed_at = new Date().toISOString();
     session.result = { success: false, summary: reason, aborted: true };
@@ -518,6 +860,14 @@ class CollabStore {
     return session;
   }
 
+  /**
+   * Clear the audit error rate-limit counter for a session.
+   * Call when a session is finalized (completed/aborted) to prevent Map leak.
+   */
+  clearAuditErrorCount(sessionId) {
+    _auditErrorCounts.delete(sessionId);
+  }
+
   /**
    * Get a summary of the session for reporting.
    */