openclaw-node-harness 2.0.3 → 2.1.0

package/workspace-bin/memory-daemon.mjs

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 /**
- * memory-daemon.mjs — OpenClaw platform-level memory lifecycle daemon (v2)
+ * memory-daemon.mjs — OpenClaw platform-level memory lifecycle daemon (v3)
  *
  * Long-running Node.js process that detects activity from ANY frontend
  * by polling JSONL transcript mtimes. No touchfiles. No hooks required.
@@ -9,9 +9,16 @@
  *
  * Phases:
  *   0. Session-start bootstrap (once per new session)
+ *      - Freezes MEMORY.md snapshot (memory-budget)
+ *      - Imports sessions into SQLite archive (session-store)
  *   1. Status sync (every tick when active, ~5ms)
  *   2. Throttled background work (recap 10min, maintenance 30min,
- *      obsidian-sync 30min, trust-health 30min)
+ *      obsidian-sync 30min, trust-health 30min, session-import 10min)
+ *
+ * v3 additions (Hermes-inspired):
+ *   - Pre-compression memory flush (durable fact extraction before context loss)
+ *   - MEMORY.md character budget with frozen session snapshots
+ *   - SQLite session archive with FTS5 for episodic recall
  *
  * Install: bin/install-daemon (detects OS, sets up launchd/systemd/pm2)
  * Manual:  node bin/memory-daemon.mjs [--test] [--verbose]
@@ -24,6 +31,38 @@ import os from 'os';
 import { execFile, spawn } from 'child_process';
 import { promisify } from 'util';
 
+// --- Hermes-inspired modules ---
+import { shouldFlush, runFlush } from '../lib/pre-compression-flush.mjs';
+import { createBudget } from '../lib/memory-budget.mjs';
+
+// Session store loaded lazily (requires better-sqlite3)
+let _sessionStore = null;
+async function getSessionStore() {
+  if (_sessionStore) return _sessionStore;
+  try {
+    const { SessionStore } = await import('../lib/session-store.mjs');
+    _sessionStore = new SessionStore();
+    return _sessionStore;
+  } catch (err) {
+    log(`session-store unavailable: ${err.message}`);
+    return null;
+  }
+}
+
+// HyperAgent store loaded lazily (requires better-sqlite3)
+let _haStore = null;
+async function getHyperAgentStore() {
+  if (_haStore) return _haStore;
+  try {
+    const { createHyperAgentStore } = await import('../lib/hyperagent-store.mjs');
+    _haStore = createHyperAgentStore();
+    return _haStore;
+  } catch (err) {
+    log(`hyperagent-store unavailable: ${err.message}`);
+    return null;
+  }
+}
+
 const execFileAsync = promisify(execFile);
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -51,6 +90,8 @@ function loadConfig() {
       maintenanceMs: 1800000,     // 30 min
       obsidianSyncMs: 1800000,    // 30 min
     },
+    contextWindowTokens: 200000,     // active model's context window (override per LLM)
+    memoryCharBudget: 2200,           // MEMORY.md character cap
     clawvaultBin: 'bin/clawvault-local',
     obsidianVault: 'projects/arcane-vault',
   };
@@ -141,6 +182,7 @@ function detectActivity(sources, activityWindowMs) {
   let newestSession = null;
   let newestMtime = 0;
   let newestSource = null;
+  let newestFormat = null;
 
   for (const source of sources) {
     if (!fs.existsSync(source.path)) continue;
@@ -160,12 +202,13 @@ function detectActivity(sources, activityWindowMs) {
           newestMtime = mtime;
           newestSession = path.basename(f, '.jsonl');
           newestSource = source.name;
+          newestFormat = source.format || null;
         }
       } catch { continue; }
     }
   }
 
-  return { active, newestSession, newestMtime, newestSource };
+  return { active, newestSession, newestMtime, newestSource, newestFormat };
 }
 
 // ============================================================
@@ -275,6 +318,31 @@ class SessionStateMachine {
   }
 }
 
+// ============================================================
+// MEMORY BUDGET (Hermes-inspired frozen snapshot)
+// ============================================================
+
+let memoryBudget = null;
+
+function initMemoryBudget(config) {
+  if (memoryBudget) return memoryBudget;
+  memoryBudget = createBudget(config.workspace || WORKSPACE, {
+    charBudget: config.memoryCharBudget || 2200,
+  });
+
+  memoryBudget.on('add', ({ entry, pctUsed, charsRemaining }) => {
+    log(`  [memory] +added (${pctUsed}% used, ${charsRemaining} chars free)`);
+  });
+  memoryBudget.on('warning', ({ pctUsed, message }) => {
+    log(`  [memory] WARNING: ${message}`);
+  });
+  memoryBudget.on('trim', ({ removed }) => {
+    log(`  [memory] trimmed: ${removed.slice(0, 60)}...`);
+  });
+
+  return memoryBudget;
+}
+
 // ============================================================
 // PHASE 0: SESSION-START BOOTSTRAP
 // ============================================================
@@ -361,6 +429,29 @@ async function runPhase0Bootstrap(sessionId, config) {
     } catch (e) { log(`  clawvault doctor failed: ${e.message}`); }
   }
 
+  // 9. Freeze MEMORY.md snapshot for deterministic prompt content
+  try {
+    const budget = initMemoryBudget(config);
+    budget.startSession();
+    const stats = budget.getStats();
+    log(`  memory-budget frozen ${stats.meterDisplay}`);
+  } catch (e) { log(`  memory-budget freeze failed: ${e.message}`); }
+
+  // 10. Import recent sessions into SQLite archive
+  try {
+    const store = await getSessionStore();
+    if (store) {
+      const sources = loadTranscriptSources();
+      let totalImported = 0;
+      for (const source of sources) {
+        if (!fs.existsSync(source.path)) continue;
+        const result = await store.importDirectory(source.path, { source: source.name, format: source.format });
+        totalImported += result.imported;
+      }
+      if (totalImported > 0) log(`  session-store: imported ${totalImported} sessions`);
+    }
+  } catch (e) { log(`  session-store import failed: ${e.message}`); }
+
   log('Phase 0: Bootstrap complete');
 }
 
@@ -499,6 +590,7 @@ function loadThrottleState() {
   return {
     lastRecap: 0, lastMaintenance: 0, lastObsidianSync: 0, lastTrustHealth: 0,
     lastClawvaultReflect: 0, lastClawvaultArchive: 0, lastClawvaultObserve: 0,
+    lastSessionImport: 0, lastHyperagentReflect: 0,
   };
 }
 
@@ -553,6 +645,48 @@ async function runPhase2ThrottledWork(config) {
     }
   }
 
+  // Session archive import — incremental (every 10min, aligned with recap)
+  if (now - throttle.lastSessionImport >= config.intervals.sessionRecapMs) {
+    throttle.lastSessionImport = now;
+    stage1.push(
+      (async () => {
+        try {
+          const store = await getSessionStore();
+          if (!store) return;
+          const sources = loadTranscriptSources();
+          let totalImported = 0;
+          for (const source of sources) {
+            if (!fs.existsSync(source.path)) continue;
+            const result = await store.importDirectory(source.path, { source: source.name, format: source.format });
+            totalImported += result.imported;
+          }
+          if (totalImported > 0) log(`  Phase 2: session-store imported ${totalImported} sessions`);
+        } catch (e) { log(`  Phase 2: session-import failed: ${e.message}`); }
+      })()
+    );
+  }
+
+  // HyperAgent: reflection check + shadow window expiry (every 30min)
+  if (now - throttle.lastHyperagentReflect >= (config.intervals.hyperagentReflectMs || 1800000)) {
+    throttle.lastHyperagentReflect = now;
+    stage1.push(
+      (async () => {
+        try {
+          const ha = await getHyperAgentStore();
+          if (!ha) return;
+          const unreflected = ha.getUnreflectedCount(); // sync — better-sqlite3
+          if (unreflected >= 5) {
+            await runSubprocess('node', [
+              path.join(HOME, '.openclaw/bin/hyperagent.mjs'), 'reflect'
+            ], 30000);
+            log(`  Phase 2: hyperagent reflect (${unreflected} entries)`);
+          }
+          ha.checkShadowWindows(); // sync
+        } catch (e) { log(`  Phase 2: hyperagent failed: ${e.message}`); }
+      })()
+    );
+  }
+
   const clawvault = path.join(WORKSPACE, config.clawvaultBin);
   if (fs.existsSync(clawvault)) {
     // ClawVault observe — incremental session compression (every 10min)
@@ -650,12 +784,32 @@ async function handleTransitions(transitions, config) {
       return STATES.ACTIVE; // signal to complete boot
     }
 
-    // ACTIVE → IDLE: Observe + recap + checkpoint
+    // ACTIVE → IDLE: Observe + recap + checkpoint + pre-compression flush
     if (t.from === STATES.ACTIVE && t.to === STATES.IDLE) {
-      log('Entering idle — running observe + recap + checkpoint');
+      log('Entering idle — running observe + recap + checkpoint + flush');
       const recap = path.join(WORKSPACE, 'bin/session-recap');
       const clawvault = path.join(WORKSPACE, config.clawvaultBin);
 
+      // Pre-compression flush: extract durable facts before context may be lost
+      const sources = loadTranscriptSources();
+      const currentJsonl = findCurrentJsonl(sources);
+      if (currentJsonl) {
+        try {
+          const flushCheck = await shouldFlush(currentJsonl, {
+            contextWindowTokens: config.contextWindowTokens || 200000,
+          });
+          if (flushCheck.shouldFlush) {
+            log(`  pre-compression flush triggered (${flushCheck.pctUsed}% of ${flushCheck.threshold} token threshold)`);
+            const memoryMd = path.join(WORKSPACE, 'MEMORY.md');
+            const budget = initMemoryBudget(config);
+            const result = await runFlush(currentJsonl, memoryMd, {
+              charBudget: budget.charBudget,
+            });
+            log(`  flush: ${result.facts} facts found, ${result.added} added, ${result.merged} merged, ${result.skipped} skipped`);
+          }
+        } catch (e) { log(`  pre-compression flush failed: ${e.message}`); }
+      }
+
       const tasks = [];
       if (fs.existsSync(recap)) {
         tasks.push(runSubprocess('node', [recap], 30000).catch(() => {}));
@@ -674,6 +828,46 @@ async function handleTransitions(transitions, config) {
       const obsSync = path.join(WORKSPACE, 'bin/obsidian-sync.mjs');
       const subagentAudit = path.join(WORKSPACE, 'bin/subagent-audit.mjs');
 
+      // 0. Pre-compression flush (final chance to capture facts)
+      const sources = loadTranscriptSources();
+      const currentJsonl = findCurrentJsonl(sources);
+      if (currentJsonl) {
+        try {
+          const memoryMd = path.join(WORKSPACE, 'MEMORY.md');
+          const budget = initMemoryBudget(config);
+          const result = await runFlush(currentJsonl, memoryMd, {
+            charBudget: budget.charBudget,
+          });
+          if (result.added > 0 || result.merged > 0) {
+            log(`  end-of-session flush: ${result.added} added, ${result.merged} merged`);
+          }
+        } catch (e) { log(`  end-of-session flush failed: ${e.message}`); }
+      }
+
+      // 0b. Archive current session to SQLite
+      if (currentJsonl) {
+        try {
+          const store = await getSessionStore();
+          if (store) {
+            // Detect which transcript source this JSONL came from
+            const activity = detectActivity(loadTranscriptSources(), config.intervals.activityWindowMs);
+            const result = await store.importSession(currentJsonl, {
+              source: activity.newestSource || 'unknown',
+              format: activity.newestFormat,
+            });
+            if (result.imported) {
+              log(`  session-store: archived ${result.sessionId.slice(0, 8)} (${result.messageCount} msgs)`);
+            }
+          }
+        } catch (e) { log(`  session-store archive failed: ${e.message}`); }
+      }
+
+      // 0c. Release frozen MEMORY.md snapshot
+      if (memoryBudget) {
+        memoryBudget.endSession();
+        log('  memory-budget: snapshot released');
+      }
+
       // 1. ClawVault: final observe + reflect + archive → persist all learnings
       if (fs.existsSync(clawvault)) {
         await runSubprocess(clawvault, ['observe', '--cron'], 60000).catch(() => {});

package/workspace-bin/session-search.mjs

@@ -0,0 +1,204 @@
+#!/usr/bin/env node
+/**
+ * session-search.mjs — FTS5 Session Search CLI
+ *
+ * Searches the SQLite session archive for episodic recall.
+ * Auto-detects piped output: JSON to stdout for tool consumption,
+ * human-readable to stderr for interactive use.
+ *
+ * Usage:
+ *   node workspace-bin/session-search.mjs "NATS URL fix"
+ *   node workspace-bin/session-search.mjs --role user "API key"
+ *   node workspace-bin/session-search.mjs --limit 5 --json "memory daemon"
+ *   node workspace-bin/session-search.mjs --import           # import all sessions
+ *   node workspace-bin/session-search.mjs --stats            # show db stats
+ *
+ * Registered as an OpenClaw skill for model-callable search.
+ */
+
+import { fileURLToPath } from 'url';
+import path from 'path';
+import fs from 'fs';
+import os from 'os';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const WORKSPACE = process.env.OPENCLAW_WORKSPACE || path.dirname(__dirname);
+const HOME = os.homedir();
+
+// ── Args ────────────────────────────────────
+
+const args = process.argv.slice(2);
+const flags = {
+  limit: 10,
+  role: null,
+  json: false,
+  import: false,
+  stats: false,
+  help: false,
+};
+const positional = [];
+
+for (let i = 0; i < args.length; i++) {
+  const arg = args[i];
+  if (arg === '--limit' && args[i + 1]) { flags.limit = parseInt(args[++i], 10); }
+  else if (arg === '--role' && args[i + 1]) { flags.role = args[++i]; }
+  else if (arg === '--json') { flags.json = true; }
+  else if (arg === '--import') { flags.import = true; }
+  else if (arg === '--stats') { flags.stats = true; }
+  else if (arg === '--help' || arg === '-h') { flags.help = true; }
+  else { positional.push(arg); }
+}
+
+const query = positional.join(' ');
+const isPiped = !process.stdout.isTTY;
+const useJson = flags.json || isPiped;
+
+// ── Help ────────────────────────────────────
+
+if (flags.help) {
+  console.log(`
+session-search — FTS5 episodic recall for OpenClaw sessions
+
+Usage:
+  session-search <query>            Search transcripts
+  session-search --import           Import JSONL sessions into SQLite
+  session-search --stats            Show database stats
+
+Options:
+  --limit <n>     Max sessions to return (default: 10)
+  --role <role>   Filter by role: user, assistant
+  --json          Force JSON output (auto-detected when piped)
+  --help          Show this help
+
+Examples:
+  session-search "NATS URL fix"
+  session-search --role user "API key"
+  session-search --limit 5 "memory daemon"
+`);
+  process.exit(0);
+}
+
+// ── Dynamic Import ────────────────────────────────────
+
+async function main() {
+  // Import session-store (ESM)
+  const { SessionStore } = await import(path.join(WORKSPACE, 'lib/session-store.mjs'));
+  const store = new SessionStore();
+
+  // ── Import mode ────────────────────────────────────
+  if (flags.import) {
+    const transcriptRegistry = path.join(HOME, '.openclaw/config/transcript-sources.json');
+    let sources = []; // { path, name, format }
+
+    if (fs.existsSync(transcriptRegistry)) {
+      try {
+        const reg = JSON.parse(fs.readFileSync(transcriptRegistry, 'utf-8'));
+        sources = (reg.sources || [])
+          .filter(s => s.enabled !== false)
+          .map(s => ({
+            path: s.path.startsWith('~') ? path.join(HOME, s.path.slice(1)) : s.path,
+            name: s.name || 'unknown',
+            format: s.format || null,
+          }));
+      } catch { /* fall through */ }
+    }
+
+    if (sources.length === 0) {
+      // Fallback: known transcript locations with correct format tags
+      const wsAbs = fs.existsSync(WORKSPACE) ? fs.realpathSync(WORKSPACE) : WORKSPACE;
+      const slug = wsAbs.replace(/[/.]/g, '-');
+      sources = [
+        { path: path.join(HOME, '.claude/projects', slug), name: 'claude-code', format: 'claude-code' },
+        { path: path.join(HOME, '.claude/projects', '-' + path.basename(HOME)), name: 'claude-home', format: 'claude-code' },
+        { path: path.join(HOME, '.openclaw/agents/main/sessions'), name: 'gateway', format: 'openclaw-gateway' },
+      ];
+    }
+
+    let totalImported = 0, totalSkipped = 0;
+    for (const src of sources) {
+      if (!fs.existsSync(src.path)) continue;
+      const result = await store.importDirectory(src.path, { source: src.name, format: src.format });
+      totalImported += result.imported;
+      totalSkipped += result.skipped;
+      if (!useJson) {
+        process.stderr.write(`  ${src.path}: ${result.imported} imported, ${result.skipped} skipped\n`);
+      }
+    }
+
+    if (useJson) {
+      console.log(JSON.stringify({ imported: totalImported, skipped: totalSkipped }));
+    } else {
+      console.log(`\nTotal: ${totalImported} imported, ${totalSkipped} skipped`);
+    }
+
+    store.close();
+    return;
+  }
+
+  // ── Stats mode ────────────────────────────────────
+  if (flags.stats) {
+    const stats = store.getStats();
+    if (useJson) {
+      console.log(JSON.stringify(stats));
+    } else {
+      console.log(`Sessions:  ${stats.sessionCount}`);
+      console.log(`Messages:  ${stats.messageCount}`);
+      console.log(`DB size:   ${stats.dbSizeMb} MB`);
+    }
+    store.close();
+    return;
+  }
+
+  // ── Search mode ────────────────────────────────────
+  if (!query) {
+    console.error('Usage: session-search <query>');
+    console.error('       session-search --help for more options');
+    process.exit(1);
+  }
+
+  const results = store.search(query, {
+    limit: flags.limit,
+    role: flags.role,
+  });
+
+  if (useJson) {
+    console.log(JSON.stringify(results, null, 2));
+  } else {
+    if (results.length === 0) {
+      console.log('No matches found.');
+      store.close();
+      return;
+    }
+
+    for (const result of results) {
+      const date = result.startTime
+        ? new Date(result.startTime).toLocaleDateString('en-CA')
+        : 'unknown';
+
+      console.log(`\n${'─'.repeat(60)}`);
+      console.log(`Session: ${result.sessionId.slice(0, 12)} (${result.source})`);
+      console.log(`Date:    ${date}  |  Matches: ${result.matchCount}  |  Score: ${result.score}`);
+
+      for (const excerpt of result.excerpts) {
+        console.log(`  ┌ turns ${excerpt.startTurn}–${excerpt.endTurn}`);
+        for (const turn of excerpt.turns) {
+          const marker = turn.isMatch ? '►' : ' ';
+          const role = turn.role.padEnd(10);
+          const text = turn.content.replace(/\n/g, ' ').slice(0, 120);
+          console.log(`  ${marker} [${role}] ${text}`);
+        }
+        console.log(`  └`);
+      }
+    }
+    console.log(`\n${'─'.repeat(60)}`);
+    console.log(`${results.length} session(s) matched.`);
+  }
+
+  store.close();
+}
+
+main().catch(err => {
+  console.error(`Error: ${err.message}`);
+  process.exit(1);
+});

package/workspace-bin/web-fetch.mjs

@@ -0,0 +1,65 @@
+#!/usr/bin/env node
+/**
+ * web-fetch.mjs — Playwright-based web fetcher
+ *
+ * Renders JS-heavy pages and returns clean text/HTML.
+ * Fallback for when WebFetch gets blocked by anti-bot or JS rendering.
+ *
+ * Usage:
+ *   node bin/web-fetch.mjs <url>                    # returns text content
+ *   node bin/web-fetch.mjs <url> --html             # returns raw HTML
+ *   node bin/web-fetch.mjs <url> --selector "article"  # extract specific element
+ *   node bin/web-fetch.mjs <url> --wait 5000        # custom wait (ms)
+ *   node bin/web-fetch.mjs <url> --screenshot out.png  # save screenshot
+ */
+
+import { chromium } from 'playwright';
+
+const args = process.argv.slice(2);
+const url = args.find(a => !a.startsWith('--'));
+
+if (!url) {
+  console.error('Usage: web-fetch.mjs <url> [--html] [--selector "css"] [--wait ms] [--screenshot file]');
+  process.exit(1);
+}
+
+const flags = {
+  html: args.includes('--html'),
+  selector: null,
+  wait: 3000,
+  screenshot: null,
+};
+
+for (let i = 0; i < args.length; i++) {
+  if (args[i] === '--selector' && args[i + 1]) flags.selector = args[++i];
+  if (args[i] === '--wait' && args[i + 1]) flags.wait = parseInt(args[++i], 10);
+  if (args[i] === '--screenshot' && args[i + 1]) flags.screenshot = args[++i];
+}
+
+const browser = await chromium.launch({ headless: true });
+try {
+  const page = await browser.newPage();
+  await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
+
+  if (flags.wait > 0) {
+    await page.waitForTimeout(flags.wait);
+  }
+
+  if (flags.screenshot) {
+    await page.screenshot({ path: flags.screenshot, fullPage: true });
+    console.error(`Screenshot saved: ${flags.screenshot}`);
+  }
+
+  if (flags.selector) {
+    const el = await page.$(flags.selector);
+    if (!el) {
+      console.error(`Selector "${flags.selector}" not found`);
+      process.exit(1);
+    }
+    console.log(flags.html ? await el.innerHTML() : await el.innerText());
+  } else {
+    console.log(flags.html ? await page.content() : await page.innerText('body'));
+  }
+} finally {
+  await browser.close();
+}