openclaw-node-harness 2.0.4 → 2.1.1

package/lib/pre-compression-flush.mjs

@@ -0,0 +1,322 @@
+/**
+ * pre-compression-flush.mjs — Pre-compression memory extraction
+ *
+ * Detects when a session is approaching context compression
+ * (by JSONL size / estimated token count) and extracts durable facts from
+ * the conversation tail before they're lost.
+ *
+ * LLM-agnostic: uses transcript-parser.mjs to handle any JSONL format
+ * (Claude Code, OpenClaw Gateway, or future backends).
+ *
+ * Zero token cost — pure JSONL parsing + heuristic extraction.
+ * Writes to MEMORY.md with bigram-similarity dedup to prevent bloat.
+ *
+ * Adapted from Hermes's pre-compression flush pattern, fitted to
+ * OpenClaw's daemon architecture.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { parseJsonlFile, estimateFileTokens } from './transcript-parser.mjs';
+
+// ── Token Estimation ────────────────────────────────────
+
+const CHARS_PER_TOKEN = 4; // rough approximation across common LLM tokenizers
+
+/**
+ * Estimate token count from character length.
+ * Good enough for flush threshold — no tokenizer dependency needed.
+ */
+export function estimateTokens(text) {
+  return Math.ceil(text.length / CHARS_PER_TOKEN);
+}
+
+/**
+ * Estimate total conversation tokens from a JSONL session file.
+ * Format-agnostic — delegates to transcript-parser.
+ * Returns { totalTokens, messageCount, tailMessages }.
+ *
+ * @param {string} jsonlPath
+ * @param {number} tailCount
+ * @param {Object} opts
+ * @param {string} opts.format - Transcript format (auto-detected if omitted)
+ */
+export async function estimateSessionTokens(jsonlPath, tailCount = 40, opts = {}) {
+  if (!fs.existsSync(jsonlPath)) return { totalTokens: 0, messageCount: 0, tailMessages: [] };
+
+  const messages = await parseJsonlFile(jsonlPath, { format: opts.format });
+  let totalChars = 0;
+  for (const msg of messages) {
+    totalChars += msg.content.length;
+  }
+
+  const tailMessages = messages.slice(-tailCount);
+
+  return {
+    totalTokens: Math.ceil(totalChars / CHARS_PER_TOKEN),
+    messageCount: messages.length,
+    tailMessages,
+  };
+}
+
+// ── Flush Threshold ────────────────────────────────────
+
+/**
+ * Check if a session should trigger a pre-compression flush.
+ *
+ * @param {string} jsonlPath - Path to the session's JSONL file
+ * @param {Object} opts
+ * @param {number} opts.contextWindowTokens - Model's context window size in tokens (default: 200000)
+ * @param {number} opts.flushPct - Flush at this % of context window (default: 0.75)
+ * @returns {{ shouldFlush: boolean, estimatedTokens: number, pctUsed: number, threshold: number }}
+ */
+export async function shouldFlush(jsonlPath, opts = {}) {
+  const { contextWindowTokens = 200000, flushPct = 0.75 } = opts;
+  const threshold = Math.floor(contextWindowTokens * flushPct);
+
+  if (!fs.existsSync(jsonlPath)) return { shouldFlush: false, estimatedTokens: 0, pctUsed: 0, threshold };
+
+  const stat = fs.statSync(jsonlPath);
+  // Quick estimate from file size — ~4 chars/token, but JSONL has overhead (~2x)
+  const quickEstimate = Math.ceil(stat.size / (CHARS_PER_TOKEN * 2));
+
+  return {
+    shouldFlush: quickEstimate >= threshold,
+    estimatedTokens: quickEstimate,
+    pctUsed: Math.round((quickEstimate / contextWindowTokens) * 100),
+    threshold,
+  };
+}
+
+// ── Bigram Similarity ────────────────────────────────────
+
+/**
+ * Compute bigram similarity between two strings (0.0 - 1.0).
+ * Used for dedup when merging new facts into MEMORY.md.
+ */
+export function bigramSimilarity(a, b) {
+  if (!a || !b) return 0;
+
+  const norm = s => s.toLowerCase().replace(/[^a-z0-9\s]/g, '').trim();
+  const bigrams = s => {
+    const tokens = norm(s).split(/\s+/);
+    const bg = new Set();
+    for (let i = 0; i < tokens.length - 1; i++) {
+      bg.add(`${tokens[i]} ${tokens[i + 1]}`);
+    }
+    // Also add unigrams for short strings
+    for (const t of tokens) bg.add(t);
+    return bg;
+  };
+
+  const setA = bigrams(a);
+  const setB = bigrams(b);
+
+  if (setA.size === 0 || setB.size === 0) return 0;
+
+  let intersection = 0;
+  for (const bg of setA) {
+    if (setB.has(bg)) intersection++;
+  }
+
+  const union = new Set([...setA, ...setB]).size;
+  return union === 0 ? 0 : intersection / union;
+}
+
+// ── Fact Extraction ────────────────────────────────────
+
+/**
+ * Extract durable facts from conversation tail messages.
+ * Heuristic approach — looks for:
+ *   - User corrections / preferences ("don't...", "always...", "I prefer...")
+ *   - Decisions ("we decided...", "let's go with...")
+ *   - Environment discoveries ("the API is at...", "config is in...")
+ *   - Named entities + context (URLs, file paths, project names)
+ *
+ * Returns array of { fact, category, confidence } objects.
+ */
+export function extractFacts(tailMessages) {
+  const facts = [];
+  const seen = new Set();
+
+  const patterns = [
+    // User corrections / preferences
+    { re: /(?:don'?t|never|always|stop|prefer|please)\s+(.{10,80})/i, category: 'preference', confidence: 85 },
+    // Decisions
+    { re: /(?:decided|let'?s go with|we'?ll use|switching to|going with)\s+(.{10,80})/i, category: 'decision', confidence: 80 },
+    // Environment / config
+    { re: /(?:api|endpoint|url|port|config|database|db)\s+(?:is|at|on|in)\s+(.{5,80})/i, category: 'environment', confidence: 75 },
+    // File paths
+    { re: /((?:\/[\w.-]+){3,})/g, category: 'reference', confidence: 60 },
+    // URLs
+    { re: /(https?:\/\/\S{10,80})/g, category: 'reference', confidence: 65 },
+  ];
+
+  for (const msg of tailMessages) {
+    if (msg.role !== 'user') continue; // focus on user statements
+    const content = msg.content;
+
+    for (const { re, category, confidence } of patterns) {
+      const flags = re.flags.includes('g') ? re.flags : re.flags + 'g';
+      const matches = content.matchAll(new RegExp(re.source, flags.includes('i') ? flags : flags + 'i'));
+      for (const match of matches) {
+        const factText = match[0].trim().slice(0, 120);
+
+        // Dedup within extraction
+        const key = factText.toLowerCase().replace(/\s+/g, ' ');
+        if (seen.has(key)) continue;
+        seen.add(key);
+
+        facts.push({ fact: factText, category, confidence });
+      }
+    }
+  }
+
+  return facts;
+}
+
+// ── MEMORY.md Merge ────────────────────────────────────
+
+/**
+ * Parse MEMORY.md into structured entries.
+ * Each entry is a markdown line (typically a "- " bullet under a section).
+ */
+export function parseMemoryMd(content) {
+  const lines = content.split('\n');
+  const entries = [];
+  let currentSection = '';
+
+  for (const line of lines) {
+    if (line.startsWith('##')) {
+      currentSection = line.replace(/^#+\s*/, '').trim();
+    } else if (line.startsWith('- ') || line.startsWith('* ')) {
+      entries.push({
+        section: currentSection,
+        text: line.replace(/^[-*]\s*/, '').trim(),
+        raw: line,
+      });
+    }
+  }
+
+  return entries;
+}
+
+/**
+ * Merge new facts into MEMORY.md content with dedup.
+ *
+ * Strategy:
+ *   - >90% similarity to existing entry → skip (already known)
+ *   - >70% similarity → merge (append new info to existing entry)
+ *   - <70% similarity → append as new entry under appropriate section
+ *
+ * @param {string} memoryContent - Current MEMORY.md content
+ * @param {Array} facts - Array of { fact, category, confidence }
+ * @param {number} charBudget - Max character budget (default 2200)
+ * @returns {{ content: string, added: number, merged: number, skipped: number }}
+ */
+export function mergeFacts(memoryContent, facts, charBudget = 2200) {
+  const entries = parseMemoryMd(memoryContent);
+  let content = memoryContent;
+  let added = 0, merged = 0, skipped = 0;
+
+  for (const { fact, category } of facts) {
+    // Check against existing entries
+    let bestSim = 0;
+    let bestEntry = null;
+
+    for (const entry of entries) {
+      const sim = bigramSimilarity(fact, entry.text);
+      if (sim > bestSim) {
+        bestSim = sim;
+        bestEntry = entry;
+      }
+    }
+
+    if (bestSim > 0.9) {
+      skipped++;
+      continue;
+    }
+
+    if (bestSim > 0.7 && bestEntry) {
+      // Merge: replace the existing line with a combined version
+      const combined = `${bestEntry.text} (updated: ${fact.slice(0, 60)})`;
+      content = content.replace(bestEntry.raw, `- ${combined}`);
+      merged++;
+      continue;
+    }
+
+    // Budget check before appending
+    if (content.length + fact.length + 10 > charBudget) {
+      break; // respect character budget
+    }
+
+    // Append under "## Recent" section (create if missing)
+    if (!content.includes('## Recent')) {
+      content = content.trimEnd() + '\n\n## Recent\n';
+    }
+    content = content.trimEnd() + `\n- ${fact}`;
+    added++;
+    entries.push({ section: 'Recent', text: fact, raw: `- ${fact}` });
+  }
+
+  return { content: content.trimEnd() + '\n', added, merged, skipped };
+}
+
+// ── Main Flush Pipeline ────────────────────────────────────
+
+/**
+ * Run the pre-compression flush pipeline.
+ *
+ * 1. Read tail of JSONL conversation
+ * 2. Extract durable facts
+ * 3. Merge into MEMORY.md with dedup
+ * 4. Return stats
+ *
+ * @param {string} jsonlPath - Path to current session JSONL
+ * @param {string} memoryMdPath - Path to MEMORY.md
+ * @param {Object} opts
+ * @param {number} opts.tailCount - Number of tail messages to scan (default 40)
+ * @param {number} opts.charBudget - MEMORY.md character budget (default 2200)
+ * @param {string} opts.format - Transcript format (auto-detected if omitted)
+ * @returns {Promise<{ flushed: boolean, facts: number, added: number, merged: number, skipped: number }>}
+ */
+export async function runFlush(jsonlPath, memoryMdPath, opts = {}) {
+  const { tailCount = 40, charBudget = 2200, format } = opts;
+
+  if (!fs.existsSync(jsonlPath)) {
+    return { flushed: false, facts: 0, added: 0, merged: 0, skipped: 0 };
+  }
+
+  // 1. Get tail messages (format-agnostic via transcript-parser)
+  const { tailMessages } = await estimateSessionTokens(jsonlPath, tailCount, { format });
+
+  if (tailMessages.length === 0) {
+    return { flushed: false, facts: 0, added: 0, merged: 0, skipped: 0 };
+  }
+
+  // 2. Extract facts
+  const facts = extractFacts(tailMessages);
+
+  if (facts.length === 0) {
+    return { flushed: true, facts: 0, added: 0, merged: 0, skipped: 0 };
+  }
+
+  // 3. Read and merge into MEMORY.md
+  let memoryContent = '';
+  if (fs.existsSync(memoryMdPath)) {
+    memoryContent = fs.readFileSync(memoryMdPath, 'utf-8');
+  }
+
+  const result = mergeFacts(memoryContent, facts, charBudget);
+
+  // 4. Write back
+  fs.writeFileSync(memoryMdPath, result.content);
+
+  return {
+    flushed: true,
+    facts: facts.length,
+    added: result.added,
+    merged: result.merged,
+    skipped: result.skipped,
+  };
+}

package/lib/role-loader.js

@@ -0,0 +1,292 @@
+/**
+ * role-loader.js — Load, validate, and format role profiles for mesh tasks.
+ *
+ * Role profiles define:
+ *   - responsibilities: what the agent SHOULD do (prompt injection)
+ *   - must_not: what the agent must NOT do (prompt + post-validation)
+ *   - framework: structured thinking scaffold (prompt injection)
+ *   - required_outputs: post-completion structural validation
+ *   - forbidden_patterns: post-completion negative validation
+ *   - scope_paths: default scope if task doesn't specify one
+ *   - escalation: failure routing map
+ *
+ * Roles live in config/roles/*.yaml (shipped) and ~/.openclaw/roles/ (user).
+ * Uses js-yaml for parsing (already a dependency via plan-templates).
+ */
+
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+// ── Role Loading ─────────────────────────────────────
+
+/**
+ * Load a single role profile from a YAML file.
+ */
+function loadRole(rolePath) {
+  const content = fs.readFileSync(rolePath, 'utf-8');
+  const role = yaml.load(content);
+  if (!role.id) role.id = path.basename(rolePath, '.yaml');
+  return role;
+}
+
+/**
+ * Find and load a role by ID, searching user dir first then shipped config.
+ * @param {string} roleId — e.g. "solidity-dev"
+ * @param {string[]} searchDirs — directories to search (first match wins)
+ * @returns {object|null} — role profile or null if not found
+ */
+function findRole(roleId, searchDirs) {
+  for (const dir of searchDirs) {
+    for (const ext of ['.yaml', '.yml']) {
+      const candidate = path.join(dir, `${roleId}${ext}`);
+      if (fs.existsSync(candidate)) {
+        try {
+          return loadRole(candidate);
+        } catch (err) {
+          console.error(`[role-loader] Failed to load ${candidate}: ${err.message}`);
+        }
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * List all available roles across search directories.
+ * @returns {Array<{id, name, description, file}>}
+ */
+function listRoles(searchDirs) {
+  const seen = new Set();
+  const roles = [];
+
+  for (const dir of searchDirs) {
+    if (!fs.existsSync(dir)) continue;
+    const files = fs.readdirSync(dir).filter(f => f.endsWith('.yaml') || f.endsWith('.yml'));
+    for (const file of files) {
+      try {
+        const role = loadRole(path.join(dir, file));
+        if (!seen.has(role.id)) {
+          seen.add(role.id);
+          roles.push({
+            id: role.id,
+            name: role.name || role.id,
+            description: role.description || '',
+            file,
+          });
+        }
+      } catch { /* skip malformed */ }
+    }
+  }
+
+  return roles;
+}
+
+// ── Role Validation ──────────────────────────────────
+
+/**
+ * Validate a role profile for structural correctness.
+ * @returns {{ valid: boolean, errors: string[] }}
+ */
+function validateRole(role) {
+  const errors = [];
+  if (!role.id) errors.push('Missing role id');
+  if (role.responsibilities && !Array.isArray(role.responsibilities)) {
+    errors.push('responsibilities must be an array');
+  }
+  if (role.must_not && !Array.isArray(role.must_not)) {
+    errors.push('must_not must be an array');
+  }
+  if (role.required_outputs && !Array.isArray(role.required_outputs)) {
+    errors.push('required_outputs must be an array');
+  }
+  if (role.forbidden_patterns && !Array.isArray(role.forbidden_patterns)) {
+    errors.push('forbidden_patterns must be an array');
+  }
+  if (role.escalation && typeof role.escalation !== 'object') {
+    errors.push('escalation must be an object');
+  }
+  return { valid: errors.length === 0, errors };
+}
+
+// ── Prompt Formatting ────────────────────────────────
+
+/**
+ * Format a role profile into markdown for prompt injection.
+ * Injected between Scope and Instructions in the agent prompt.
+ * LLM-agnostic: standard markdown that any LLM can consume.
+ */
+function formatRoleForPrompt(role) {
+  if (!role) return '';
+  const parts = [];
+
+  parts.push(`## Role: ${role.name || role.id}`);
+  parts.push('');
+
+  if (role.responsibilities && role.responsibilities.length > 0) {
+    parts.push('### Responsibilities');
+    for (const r of role.responsibilities) {
+      parts.push(`- ${r}`);
+    }
+    parts.push('');
+  }
+
+  if (role.must_not && role.must_not.length > 0) {
+    parts.push('### Boundaries (Must NOT Do)');
+    for (const m of role.must_not) {
+      parts.push(`- ❌ ${m}`);
+    }
+    parts.push('');
+  }
+
+  if (role.framework) {
+    parts.push(`### Framework: ${role.framework.name}`);
+    parts.push(role.framework.prompt);
+    parts.push('');
+  }
+
+  return parts.join('\n');
+}
+
+// ── Post-Completion Validation ───────────────────────
+
+/**
+ * Validate task output against role's required_outputs.
+ * @param {object} role — role profile
+ * @param {string[]} outputFiles — files created/modified by the task
+ * @param {string} worktreePath — path to task worktree
+ * @returns {{ passed: boolean, failures: Array<{type, description, detail}> }}
+ */
+function validateRequiredOutputs(role, outputFiles, worktreePath) {
+  if (!role || !role.required_outputs) return { passed: true, failures: [] };
+
+  const failures = [];
+
+  for (const req of role.required_outputs) {
+    if (req.type === 'file_match') {
+      // Check if any output file matches the pattern
+      const { globMatch } = require('./rule-loader');
+      const matched = outputFiles.some(f => globMatch(req.pattern, f));
+      if (!matched) {
+        failures.push({
+          type: 'file_match',
+          description: req.description,
+          detail: `No output file matches pattern: ${req.pattern}`,
+        });
+      }
+    } else if (req.type === 'content_check') {
+      // Check if files matching pattern contain required content
+      const { globMatch } = require('./rule-loader');
+      const matchingFiles = outputFiles.filter(f => globMatch(req.pattern, f));
+      if (matchingFiles.length > 0 && worktreePath) {
+        let found = false;
+        for (const file of matchingFiles) {
+          try {
+            const content = fs.readFileSync(path.join(worktreePath, file), 'utf-8');
+            if (content.includes(req.check)) {
+              found = true;
+              break;
+            }
+          } catch { /* file not readable */ }
+        }
+        if (!found) {
+          failures.push({
+            type: 'content_check',
+            description: req.description,
+            detail: `Required content "${req.check}" not found in ${req.pattern} files`,
+          });
+        }
+      }
+    }
+  }
+
+  return { passed: failures.length === 0, failures };
+}
+
+/**
+ * Check output against role's forbidden_patterns.
+ * @param {object} role — role profile
+ * @param {string[]} outputFiles — files created/modified
+ * @param {string} worktreePath — path to task worktree
+ * @returns {{ passed: boolean, violations: Array<{pattern, in, description, file, match}> }}
+ */
+function checkForbiddenPatterns(role, outputFiles, worktreePath) {
+  if (!role || !role.forbidden_patterns) return { passed: true, violations: [] };
+
+  const { globMatch } = require('./rule-loader');
+  const violations = [];
+
+  for (const fp of role.forbidden_patterns) {
+    const regex = new RegExp(fp.pattern, 'gm');
+    const scopeFiles = fp.in
+      ? outputFiles.filter(f => globMatch(fp.in, f))
+      : outputFiles;
+
+    for (const file of scopeFiles) {
+      if (!worktreePath) continue;
+      try {
+        const content = fs.readFileSync(path.join(worktreePath, file), 'utf-8');
+        const matches = content.match(regex);
+        if (matches) {
+          violations.push({
+            pattern: fp.pattern,
+            in: fp.in,
+            description: fp.description,
+            file,
+            match: matches[0].slice(0, 100),
+          });
+        }
+      } catch { /* skip unreadable */ }
+    }
+  }
+
+  return { passed: violations.length === 0, violations };
+}
+
+/**
+ * Find the best-matching role for a set of task scope paths.
+ * Matches scope paths against each role's scope_paths field.
+ * Returns the role with the most scope path matches, or null.
+ */
+function findRoleByScope(scopePaths, searchDirs) {
+  if (!scopePaths || scopePaths.length === 0) return null;
+
+  const { globMatch } = require('./rule-loader');
+  const allRoles = listRoles(searchDirs);
+  let bestRole = null;
+  let bestScore = 0;
+
+  for (const roleSummary of allRoles) {
+    const role = findRole(roleSummary.id, searchDirs);
+    if (!role || !role.scope_paths) continue;
+
+    // Score: how many of the task's scope paths match this role's scope_paths?
+    let score = 0;
+    for (const taskPath of scopePaths) {
+      for (const rolePattern of role.scope_paths) {
+        if (globMatch(rolePattern, taskPath)) {
+          score++;
+          break; // one match per task path is enough
+        }
+      }
+    }
+
+    if (score > bestScore) {
+      bestScore = score;
+      bestRole = role;
+    }
+  }
+
+  return bestRole;
+}
+
+module.exports = {
+  loadRole,
+  findRole,
+  findRoleByScope,
+  listRoles,
+  validateRole,
+  formatRoleForPrompt,
+  validateRequiredOutputs,
+  checkForbiddenPatterns,
+};