openclaw-node-harness 2.1.0 → 2.2.0

package/lib/mesh-collab.js

@@ -150,6 +150,29 @@ class CollabStore {
     return JSON.parse(sc.decode(entry.value));
   }
 
+  /**
+   * Compare-and-swap helper: read → mutate → write with optimistic concurrency.
+   * Re-reads and retries on conflict (up to maxRetries).
+   * mutateFn receives the parsed data and must return the updated object, or falsy to skip.
+   */
+  async _updateWithCAS(key, mutateFn, maxRetries = 3) {
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+      const entry = await this.kv.get(key);
+      if (!entry) return null;
+      const data = JSON.parse(sc.decode(entry.value));
+      const updated = mutateFn(data);
+      if (!updated) return null;
+      try {
+        await this.kv.put(key, sc.encode(JSON.stringify(updated)), { previousSeq: entry.revision });
+        return updated;
+      } catch (err) {
+        const isCasConflict = err.code === '10071' || (err.message && err.message.includes('wrong last sequence'));
+        if (!isCasConflict || attempt === maxRetries - 1) throw err;
+        // CAS conflict — retry
+      }
+    }
+  }
+
   async delete(sessionId) {
     await this.kv.delete(sessionId);
   }
@@ -159,15 +182,15 @@ class CollabStore {
    */
   async appendAudit(sessionId, event, detail = {}) {
     try {
-      const session = await this.get(sessionId);
-      if (!session) return;
-      if (!session.audit_log) session.audit_log = [];
-      session.audit_log.push({
-        ts: new Date().toISOString(),
-        event,
-        ...detail,
+      await this._updateWithCAS(sessionId, (session) => {
+        if (!session.audit_log) session.audit_log = [];
+        session.audit_log.push({
+          ts: new Date().toISOString(),
+          event,
+          ...detail,
+        });
+        return session;
       });
-      await this.put(session);
     } catch (err) {
       // Best-effort — never block on audit, but log first N failures per session
       const count = (_auditErrorCounts.get(sessionId) || 0) + 1;
@@ -243,67 +266,59 @@ class CollabStore {
    * Returns the updated session or null if session full/closed.
    */
   async addNode(sessionId, nodeId, role = 'worker', scope = '*') {
-    const session = await this.get(sessionId);
-    if (!session) return null;
-    if (session.status !== COLLAB_STATUS.RECRUITING) return null;
-
-    // Check max_nodes
-    if (session.max_nodes && session.nodes.length >= session.max_nodes) return null;
-
-    // Check duplicate — single-threaded event loop prevents concurrent joins
-    // from interleaving between find() and push(). No mutex needed.
-    if (session.nodes.find(n => n.node_id === nodeId)) return null;
-
-    session.nodes.push({
-      node_id: nodeId,
-      role,
-      scope: Array.isArray(scope) ? scope : [scope],
-      joined_at: new Date().toISOString(),
-      status: 'active',
-    });
+    return this._updateWithCAS(sessionId, (session) => {
+      if (session.status !== COLLAB_STATUS.RECRUITING) return null;
 
-    // Set recruiting deadline on first join
-    if (session.nodes.length === 1) {
-      session.recruiting_deadline = new Date(
-        Date.now() + session.join_window_s * 1000
-      ).toISOString();
-    }
+      // Check max_nodes
+      if (session.max_nodes && session.nodes.length >= session.max_nodes) return null;
 
-    // For sequential mode, build turn order
-    if (session.mode === COLLAB_MODE.SEQUENTIAL) {
-      session.turn_order.push(nodeId);
-    }
+      // Check duplicate
+      if (session.nodes.find(n => n.node_id === nodeId)) return null;
 
-    await this.put(session);
-    return session;
+      session.nodes.push({
+        node_id: nodeId,
+        role,
+        scope: Array.isArray(scope) ? scope : [scope],
+        joined_at: new Date().toISOString(),
+        status: 'active',
+      });
+
+      // Set recruiting deadline on first join
+      if (session.nodes.length === 1) {
+        session.recruiting_deadline = new Date(
+          Date.now() + session.join_window_s * 1000
+        ).toISOString();
+      }
+
+      // For sequential mode, build turn order
+      if (session.mode === COLLAB_MODE.SEQUENTIAL) {
+        session.turn_order.push(nodeId);
+      }
+
+      return session;
+    });
   }
 
   /**
    * Remove a node from the session (graceful leave or kick).
    */
   async removeNode(sessionId, nodeId) {
-    const session = await this.get(sessionId);
-    if (!session) return null;
-
-    session.nodes = session.nodes.filter(n => n.node_id !== nodeId);
-    session.turn_order = session.turn_order.filter(id => id !== nodeId);
-
-    await this.put(session);
-    return session;
+    return this._updateWithCAS(sessionId, (session) => {
+      session.nodes = session.nodes.filter(n => n.node_id !== nodeId);
+      session.turn_order = session.turn_order.filter(id => id !== nodeId);
+      return session;
+    });
   }
 
   /**
    * Update a node's status within the session.
    */
   async setNodeStatus(sessionId, nodeId, status) {
-    const session = await this.get(sessionId);
-    if (!session) return null;
-
-    const node = session.nodes.find(n => n.node_id === nodeId);
-    if (node) node.status = status;
-
-    await this.put(session);
-    return session;
+    return this._updateWithCAS(sessionId, (session) => {
+      const node = session.nodes.find(n => n.node_id === nodeId);
+      if (node) node.status = status;
+      return session;
+    });
   }
 
   /**
@@ -323,54 +338,54 @@ class CollabStore {
    * Start a new round. Returns the round object with shared_intel.
    */
   async startRound(sessionId) {
-    const session = await this.get(sessionId);
-    if (!session) return null;
-
-    session.current_round++;
-    session.status = COLLAB_STATUS.ACTIVE;
+    let round = null;
+    const result = await this._updateWithCAS(sessionId, (session) => {
+      session.current_round++;
+      session.status = COLLAB_STATUS.ACTIVE;
+
+      // Snapshot recruited count on first round (immutable baseline for quorum)
+      if (session.current_round === 1) {
+        session.recruited_count = session.nodes.length;
+      }
 
-    // Snapshot recruited count on first round (immutable baseline for quorum)
-    if (session.current_round === 1) {
-      session.recruited_count = session.nodes.length;
-    }
+      // Per-round node health: prune nodes marked 'dead' before starting.
+      const deadNodes = session.nodes.filter(n => n.status === 'dead');
+      if (deadNodes.length > 0) {
+        session.nodes = session.nodes.filter(n => n.status !== 'dead');
+        session.turn_order = session.turn_order.filter(
+          id => !deadNodes.find(d => d.node_id === id)
+        );
+      }
 
-    // Per-round node health: prune nodes marked 'dead' before starting.
-    // This prevents hanging on reflections from nodes that will never respond.
-    const deadNodes = session.nodes.filter(n => n.status === 'dead');
-    if (deadNodes.length > 0) {
-      session.nodes = session.nodes.filter(n => n.status !== 'dead');
-      session.turn_order = session.turn_order.filter(
-        id => !deadNodes.find(d => d.node_id === id)
-      );
-    }
+      // Check if we still have enough nodes after pruning
+      if (session.nodes.length < session.min_nodes) {
+        session.status = COLLAB_STATUS.ABORTED;
+        round = null;
+        return session;
+      }
 
-    // Check if we still have enough nodes after pruning
-    if (session.nodes.length < session.min_nodes) {
-      // Not enough active nodes to continue — will be caught by caller
-      session.status = COLLAB_STATUS.ABORTED;
-      await this.put(session);
-      return null;
-    }
+      // Compile shared intel from previous round
+      const sharedIntel = this.compileSharedIntel(session);
 
-    // Compile shared intel from previous round
-    const sharedIntel = this.compileSharedIntel(session);
+      round = {
+        round_number: session.current_round,
+        started_at: new Date().toISOString(),
+        completed_at: null,
+        shared_intel: sharedIntel,
+        reflections: [],
+      };
 
-    const round = {
-      round_number: session.current_round,
-      started_at: new Date().toISOString(),
-      completed_at: null,
-      shared_intel: sharedIntel,
-      reflections: [],
-    };
+      session.rounds.push(round);
 
-    session.rounds.push(round);
+      // Sequential mode: set first turn
+      if (session.mode === COLLAB_MODE.SEQUENTIAL && session.turn_order.length > 0) {
+        session.current_turn = session.turn_order[0];
+      }
 
-    // Sequential mode: set first turn
-    if (session.mode === COLLAB_MODE.SEQUENTIAL && session.turn_order.length > 0) {
-      session.current_turn = session.turn_order[0];
-    }
+      return session;
+    });
 
-    await this.put(session);
+    if (!result) return null;
     return round;
   }
 
@@ -378,38 +393,35 @@ class CollabStore {
    * Submit a reflection from a node for the current round.
    */
   async submitReflection(sessionId, reflection) {
-    const session = await this.get(sessionId);
-    if (!session) return null;
+    return this._updateWithCAS(sessionId, (session) => {
+      // Only accept reflections on active sessions
+      if (session.status !== COLLAB_STATUS.ACTIVE) return null;
 
-    // Only accept reflections on active sessions
-    if (session.status !== COLLAB_STATUS.ACTIVE) return null;
-
-    const currentRound = session.rounds[session.rounds.length - 1];
-    if (!currentRound) return null;
-
-    // Prevent duplicate reflections from same node
-    if (currentRound.reflections.find(r => r.node_id === reflection.node_id)) return null;
-
-    currentRound.reflections.push({
-      node_id: reflection.node_id,
-      summary: reflection.summary || '',
-      learnings: reflection.learnings || '',
-      artifacts: reflection.artifacts || [],
-      confidence: reflection.confidence || 0.5,
-      vote: reflection.vote || 'continue',
-      parse_failed: reflection.parse_failed || false,
-      submitted_at: new Date().toISOString(),
-      // Circling Strategy extensions (optional, backward compatible)
-      circling_step: reflection.circling_step ?? null,
-      circling_artifacts: reflection.circling_artifacts || [],  // [{ type, content }]
-    });
+      const currentRound = session.rounds[session.rounds.length - 1];
+      if (!currentRound) return null;
+
+      // Prevent duplicate reflections from same node
+      if (currentRound.reflections.find(r => r.node_id === reflection.node_id)) return null;
+
+      currentRound.reflections.push({
+        node_id: reflection.node_id,
+        summary: reflection.summary || '',
+        learnings: reflection.learnings || '',
+        artifacts: reflection.artifacts || [],
+        confidence: reflection.confidence || 0.5,
+        vote: reflection.vote || 'continue',
+        parse_failed: reflection.parse_failed || false,
+        submitted_at: new Date().toISOString(),
+        circling_step: reflection.circling_step ?? null,
+        circling_artifacts: reflection.circling_artifacts || [],
+      });
 
-    // Update node status
-    const node = session.nodes.find(n => n.node_id === reflection.node_id);
-    if (node) node.status = 'reflecting';
+      // Update node status
+      const node = session.nodes.find(n => n.node_id === reflection.node_id);
+      if (node) node.status = 'reflecting';
 
-    await this.put(session);
-    return session;
+      return session;
+    });
   }
 
   /**
@@ -428,22 +440,25 @@ class CollabStore {
    * Returns the next node_id, or null if all turns done (round complete).
    */
   async advanceTurn(sessionId) {
-    const session = await this.get(sessionId);
-    if (!session || session.mode !== COLLAB_MODE.SEQUENTIAL) return null;
+    let nextTurn = null;
+    await this._updateWithCAS(sessionId, (session) => {
+      if (session.mode !== COLLAB_MODE.SEQUENTIAL) return null;
+      if (session.status !== 'active') return null;
 
-    const currentIdx = session.turn_order.indexOf(session.current_turn);
-    const nextIdx = currentIdx + 1;
+      const currentIdx = session.turn_order.indexOf(session.current_turn);
+      const nextIdx = currentIdx + 1;
 
-    if (nextIdx >= session.turn_order.length) {
-      // All nodes had their turn — round is complete
-      session.current_turn = null;
-      await this.put(session);
-      return null;
-    }
+      if (nextIdx >= session.turn_order.length) {
+        session.current_turn = null;
+        nextTurn = null;
+      } else {
+        session.current_turn = session.turn_order[nextIdx];
+        nextTurn = session.current_turn;
+      }
 
-    session.current_turn = session.turn_order[nextIdx];
-    await this.put(session);
-    return session.current_turn;
+      return session;
+    });
+    return nextTurn;
   }
 
   // ── Convergence ────────────────────────────────────
@@ -511,31 +526,42 @@ class CollabStore {
    * Key format: sr{N}_step{S}_{nodeRole}_{artifactType}
    */
   async storeArtifact(sessionId, key, content) {
-    const session = await this.get(sessionId);
-    if (!session || !session.circling) return null;
-    session.circling.artifacts[key] = content;
-
-    // Session blob size check — JetStream KV max is 1MB.
-    // Warn early so operators can plan external artifact store before hitting the wall.
-    const blobSize = Buffer.byteLength(JSON.stringify(session), 'utf8');
-    if (blobSize > 950_000) {
-      console.error(`[collab] CRITICAL: session ${sessionId} blob is ${(blobSize / 1024).toFixed(0)}KB — approaching JetStream KV 1MB limit`);
-    } else if (blobSize > 800_000) {
-      console.warn(`[collab] WARNING: session ${sessionId} blob is ${(blobSize / 1024).toFixed(0)}KB — consider external artifact store`);
-    }
+    for (let attempt = 0; attempt < 3; attempt++) {
+      const entry = await this.kv.get(sessionId);
+      if (!entry) return null;
+      const session = JSON.parse(sc.decode(entry.value));
+      if (!session.circling) return null;
+      session.circling.artifacts[key] = content;
+
+      // Session blob size check — JetStream KV max is 1MB.
+      const blobSize = Buffer.byteLength(JSON.stringify(session), 'utf8');
+      if (blobSize > 950_000) {
+        console.error(`[collab] CRITICAL: session ${sessionId} blob is ${(blobSize / 1024).toFixed(0)}KB — approaching JetStream KV 1MB limit`);
+      } else if (blobSize > 800_000) {
+        console.warn(`[collab] WARNING: session ${sessionId} blob is ${(blobSize / 1024).toFixed(0)}KB — consider external artifact store`);
+      }
 
-    try {
-      await this.put(session);
-    } catch (err) {
-      // JetStream KV write failure — likely blob exceeded 1MB limit.
-      // Remove the artifact that caused the overflow and re-persist without it.
-      console.error(`[collab] storeArtifact FAILED for ${sessionId}/${key}: ${err.message}. Removing artifact and persisting without it.`);
-      delete session.circling.artifacts[key];
-      await this.put(session);
-      return null;
+      try {
+        await this.kv.put(sessionId, sc.encode(JSON.stringify(session)), { previousSeq: entry.revision });
+        return session;
+      } catch (err) {
+        // Check if it's a blob-too-large error vs a CAS conflict
+        // CAS conflicts have specific error codes; blob failures are different
+        if (attempt === 2) {
+          // Final attempt — could be blob overflow or persistent conflict
+          console.error(`[collab] storeArtifact FAILED for ${sessionId}/${key}: ${err.message}. Removing artifact and persisting without it.`);
+          delete session.circling.artifacts[key];
+          try {
+            // Recovery write without CAS — acceptable because we're removing the artifact
+            // that caused the failure. Worst case: another concurrent write overwrites this,
+            // but that write also wouldn't have the problematic artifact.
+            await this.kv.put(sessionId, sc.encode(JSON.stringify(session)));
+          } catch (_) { /* best effort */ }
+          return null;
+        }
+        // conflict — retry
+      }
     }
-
-    return session;
   }
 
   /**
@@ -691,67 +717,63 @@ class CollabStore {
    *   circling/step2 (SR == max) → finalization/step0
    */
   async advanceCirclingStep(sessionId) {
-    const session = await this.get(sessionId);
-    if (!session || !session.circling) return null;
+    let stepResult = null;
+    const result = await this._updateWithCAS(sessionId, (session) => {
+      if (!session.circling) return null;
 
-    const c = session.circling;
-    let needsGate = false;
-
-    if (c.phase === 'init' && c.current_step === 0) {
-      // Init complete → start circling SR1/Step1
-      c.phase = 'circling';
-      c.current_subround = 1;
-      c.current_step = 1;
-    } else if (c.phase === 'circling' && c.current_step === 1) {
-      // Step 1 complete → Step 2 (same subround)
-      c.current_step = 2;
-    } else if (c.phase === 'circling' && c.current_step === 2) {
-      // Adaptive convergence: if all active nodes voted 'converged' after step 2,
-      // skip remaining sub-rounds and go directly to finalization.
-      const currentRound = session.rounds[session.rounds.length - 1];
-      const activeNodes = session.nodes.filter(n => n.status !== 'dead');
-      const step2Reflections = currentRound
-        ? currentRound.reflections.filter(r => r.circling_step === 2)
-        : [];
-      const allConverged = step2Reflections.length >= activeNodes.length &&
-                           step2Reflections.every(r => r.vote === 'converged');
-
-      if (allConverged && c.current_subround < c.max_subrounds) {
-        // Early exit — all nodes agree the work is ready
-        if (c.automation_tier >= 2) {
-          needsGate = true;
-        }
-        c.phase = 'finalization';
-        c.current_step = 0;
-      } else if (c.current_subround < c.max_subrounds) {
-        // Step 2 complete, more sub-rounds → next SR/Step1
-        // Check tier gate for Tier 3 (gates after every sub-round)
-        if (c.automation_tier === 3) {
-          needsGate = true;
-        }
-        c.current_subround++;
+      const c = session.circling;
+      let needsGate = false;
+
+      if (c.phase === 'init' && c.current_step === 0) {
+        c.phase = 'circling';
+        c.current_subround = 1;
         c.current_step = 1;
-      } else {
-        // Final sub-round complete → finalization
-        // Tier 2 gates on finalization entry
-        if (c.automation_tier >= 2) {
-          needsGate = true;
+      } else if (c.phase === 'circling' && c.current_step === 1) {
+        c.current_step = 2;
+      } else if (c.phase === 'circling' && c.current_step === 2) {
+        const currentRound = session.rounds[session.rounds.length - 1];
+        const activeNodes = session.nodes.filter(n => n.status !== 'dead');
+        const step2Reflections = currentRound
+          ? currentRound.reflections.filter(r => r.circling_step === 2)
+          : [];
+        const allConverged = step2Reflections.length >= activeNodes.length &&
+                             step2Reflections.every(r => r.vote === 'converged');
+
+        if (allConverged && c.current_subround < c.max_subrounds) {
+          if (c.automation_tier >= 2) {
+            needsGate = true;
+          }
+          c.phase = 'finalization';
+          c.current_step = 0;
+        } else if (c.current_subround < c.max_subrounds) {
+          if (c.automation_tier === 3) {
+            needsGate = true;
+          }
+          c.current_subround++;
+          c.current_step = 1;
+        } else {
+          if (c.automation_tier >= 2) {
+            needsGate = true;
+          }
+          c.phase = 'finalization';
+          c.current_step = 0;
         }
-        c.phase = 'finalization';
-        c.current_step = 0;
+      } else if (c.phase === 'finalization') {
+        c.phase = 'complete';
       }
-    } else if (c.phase === 'finalization') {
-      // Finalization complete → done
-      c.phase = 'complete';
-    }
 
-    await this.put(session);
-    return {
-      phase: c.phase,
-      subround: c.current_subround,
-      step: c.current_step,
-      needsGate,
-    };
+      stepResult = {
+        phase: c.phase,
+        subround: c.current_subround,
+        step: c.current_step,
+        needsGate,
+      };
+
+      return session;
+    });
+
+    if (!result) return null;
+    return stepResult;
   }
 
   /**
@@ -759,12 +781,16 @@ class CollabStore {
    * Returns the failure count.
    */
   async recordArtifactFailure(sessionId, nodeId) {
-    const session = await this.get(sessionId);
-    if (!session || !session.circling) return 0;
-    const key = `${nodeId}_sr${session.circling.current_subround}_step${session.circling.current_step}`;
-    session.circling.artifact_failures[key] = (session.circling.artifact_failures[key] || 0) + 1;
-    await this.put(session);
-    return session.circling.artifact_failures[key];
+    let failureCount = 0;
+    const result = await this._updateWithCAS(sessionId, (session) => {
+      if (!session.circling) return null;
+      const key = `${nodeId}_sr${session.circling.current_subround}_step${session.circling.current_step}`;
+      session.circling.artifact_failures[key] = (session.circling.artifact_failures[key] || 0) + 1;
+      failureCount = session.circling.artifact_failures[key];
+      return session;
+    });
+    if (!result) return 0;
+    return failureCount;
   }
 
   /**
@@ -814,34 +840,33 @@ class CollabStore {
    * Mark session as converged.
    */
   async markConverged(sessionId) {
-    const session = await this.get(sessionId);
-    if (!session) return null;
-    session.status = COLLAB_STATUS.CONVERGED;
+    return this._updateWithCAS(sessionId, (session) => {
+      session.status = COLLAB_STATUS.CONVERGED;
 
-    // Close current round
-    const currentRound = session.rounds[session.rounds.length - 1];
-    if (currentRound) currentRound.completed_at = new Date().toISOString();
+      // Close current round
+      const currentRound = session.rounds[session.rounds.length - 1];
+      if (currentRound) currentRound.completed_at = new Date().toISOString();
 
-    await this.put(session);
-    return session;
+      return session;
+    });
   }
 
   /**
    * Mark session as completed with final result.
    */
   async markCompleted(sessionId, result) {
-    const session = await this.get(sessionId);
-    if (!session) return null;
-    session.status = COLLAB_STATUS.COMPLETED;
-    session.completed_at = new Date().toISOString();
-    session.result = {
-      artifacts: result.artifacts || [],
-      summary: result.summary || '',
-      rounds_taken: session.current_round,
-      node_contributions: result.node_contributions || {},
-    };
-    await this.put(session);
-    return session;
+    return this._updateWithCAS(sessionId, (session) => {
+      if (['completed', 'aborted'].includes(session.status)) return null;
+      session.status = COLLAB_STATUS.COMPLETED;
+      session.completed_at = new Date().toISOString();
+      session.result = {
+        artifacts: result.artifacts || [],
+        summary: result.summary || '',
+        rounds_taken: session.current_round,
+        node_contributions: result.node_contributions || {},
+      };
+      return session;
+    });
   }
 
   /**
@@ -849,15 +874,14 @@ class CollabStore {
    * Callers can use truthiness to detect whether the abort actually happened.
    */
   async markAborted(sessionId, reason) {
-    const session = await this.get(sessionId);
-    if (!session) return null;
-    // Guard: don't corrupt completed/aborted sessions
-    if (['completed', 'aborted'].includes(session.status)) return null;
-    session.status = COLLAB_STATUS.ABORTED;
-    session.completed_at = new Date().toISOString();
-    session.result = { success: false, summary: reason, aborted: true };
-    await this.put(session);
-    return session;
+    return this._updateWithCAS(sessionId, (session) => {
+      // Guard: don't corrupt completed/aborted sessions
+      if (['completed', 'aborted'].includes(session.status)) return null;
+      session.status = COLLAB_STATUS.ABORTED;
+      session.completed_at = new Date().toISOString();
+      session.result = { success: false, summary: reason, aborted: true };
+      return session;
+    });
   }
 
   /**