neohive 6.0.0

package/lib/agents.js

@@ -0,0 +1,107 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { DATA_DIR, AGENTS_FILE, PROFILES_FILE, ACKS_FILE } = require('./config');
+const { cachedRead, invalidateCache, lockAgentsFile, unlockAgentsFile, withFileLock, readJsonFile } = require('./file-io');
+
+// Cache for isPidAlive results
+const _pidAliveCache = {};
+
+// isAutonomousMode is injected late to avoid circular dependency with workflows
+let _isAutonomousMode = () => false;
+function setAutonomousModeCheck(fn) { _isAutonomousMode = fn; }
+
+function isPidAlive(pid, lastActivity) {
+  const cacheKey = `${pid}_${lastActivity}`;
+  const cached = _pidAliveCache[cacheKey];
+  if (cached && Date.now() - cached.ts < 5000) return cached.alive;
+
+  const STALE_THRESHOLD = _isAutonomousMode() ? 30000 : 60000;
+  let alive = false;
+
+  if (lastActivity) {
+    const stale = Date.now() - new Date(lastActivity).getTime();
+    if (stale < STALE_THRESHOLD) alive = true;
+  }
+  if (!alive) {
+    try { process.kill(pid, 0); alive = true; } catch { alive = false; }
+  }
+  _pidAliveCache[cacheKey] = { alive, ts: Date.now() };
+  const keys = Object.keys(_pidAliveCache);
+  if (keys.length > 200) {
+    const cutoff = Date.now() - 10000;
+    for (const k of keys) { if (_pidAliveCache[k].ts < cutoff) delete _pidAliveCache[k]; }
+  }
+  return alive;
+}
+
+function getAgents() {
+  return cachedRead('agents', () => {
+    if (!fs.existsSync(AGENTS_FILE)) return {};
+    let agents;
+    try { agents = JSON.parse(fs.readFileSync(AGENTS_FILE, 'utf8')); } catch { return {}; }
+    try {
+      const files = fs.readdirSync(DATA_DIR).filter(f => f.startsWith('heartbeat-') && f.endsWith('.json'));
+      for (const f of files) {
+        const name = f.slice(10, -5);
+        if (agents[name]) {
+          try {
+            const hb = JSON.parse(fs.readFileSync(path.join(DATA_DIR, f), 'utf8'));
+            if (hb.last_activity) agents[name].last_activity = hb.last_activity;
+            if (hb.pid) agents[name].pid = hb.pid;
+          } catch {}
+        }
+      }
+    } catch {}
+    return agents;
+  }, 1500);
+}
+
+function saveAgents(agents) {
+  const data = JSON.stringify(agents);
+  if (data && data.length > 2) {
+    fs.writeFileSync(AGENTS_FILE, data);
+  }
+  invalidateCache('agents');
+}
+
+function heartbeatFile(name) { return path.join(DATA_DIR, `heartbeat-${name}.json`); }
+
+function touchHeartbeat(name) {
+  if (!name) return;
+  try {
+    fs.writeFileSync(heartbeatFile(name), JSON.stringify({
+      last_activity: new Date().toISOString(),
+      pid: process.pid,
+    }));
+  } catch {}
+}
+
+function getAcks() {
+  if (!fs.existsSync(ACKS_FILE)) return {};
+  try { return JSON.parse(fs.readFileSync(ACKS_FILE, 'utf8')); } catch { return {}; }
+}
+
+function getProfiles() {
+  return cachedRead('profiles', () => {
+    if (!fs.existsSync(PROFILES_FILE)) return {};
+    try { return JSON.parse(fs.readFileSync(PROFILES_FILE, 'utf8')); } catch { return {}; }
+  }, 2000);
+}
+
+function saveProfiles(profiles) {
+  withFileLock(PROFILES_FILE, () => {
+    invalidateCache('profiles');
+    fs.writeFileSync(PROFILES_FILE, JSON.stringify(profiles));
+  });
+}
+
+module.exports = {
+  isPidAlive, setAutonomousModeCheck,
+  getAgents, saveAgents,
+  heartbeatFile, touchHeartbeat,
+  getAcks,
+  getProfiles, saveProfiles,
+  lockAgentsFile, unlockAgentsFile,
+};

package/lib/compact.js

@@ -0,0 +1,124 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const log = require('./logger');
+const state = require('./state');
+const { DATA_DIR, getMessagesFile, sanitizeName } = require('./config');
+const { getAgents, isPidAlive } = require('./agents');
+
+// --- Consumed ID tracking ---
+
+function consumedFile(agentName) {
+  sanitizeName(agentName);
+  return path.join(DATA_DIR, `consumed-${agentName}.json`);
+}
+
+function getConsumedIds(agentName) {
+  const file = consumedFile(agentName);
+  if (!fs.existsSync(file)) return new Set();
+  try {
+    return new Set(JSON.parse(fs.readFileSync(file, 'utf8')));
+  } catch {
+    return new Set();
+  }
+}
+
+function saveConsumedIds(agentName, ids) {
+  if (ids.size > 500) {
+    trimConsumedIds(agentName, ids);
+  }
+  fs.writeFileSync(consumedFile(agentName), JSON.stringify([...ids]));
+}
+
+function trimConsumedIds(agentName, ids) {
+  try {
+    const msgFile = getMessagesFile(state.currentBranch);
+    if (!fs.existsSync(msgFile)) { ids.clear(); return; }
+    const content = fs.readFileSync(msgFile, 'utf8').trim();
+    if (!content) { ids.clear(); return; }
+    const currentIds = new Set();
+    for (const line of content.split(/\r?\n/)) {
+      const match = line.match(/"id"\s*:\s*"([^"]+)"/);
+      if (match) currentIds.add(match[1]);
+    }
+    for (const id of ids) {
+      if (!currentIds.has(id)) ids.delete(id);
+    }
+  } catch {}
+}
+
+// --- Auto-compact ---
+
+function autoCompact() {
+  const msgFile = getMessagesFile(state.currentBranch);
+  if (!fs.existsSync(msgFile)) return;
+  try {
+    const content = fs.readFileSync(msgFile, 'utf8').trim();
+    if (!content) return;
+    const lines = content.split(/\r?\n/);
+    if (lines.length < 500) return;
+
+    const messages = lines.map(l => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
+
+    const agents = getAgents();
+    const aliveAgentNames = Object.keys(agents).filter(n => isPidAlive(agents[n].pid, agents[n].last_activity));
+    const allConsumed = new Set();
+    const perAgentConsumed = {};
+    if (fs.existsSync(DATA_DIR)) {
+      for (const f of fs.readdirSync(DATA_DIR)) {
+        if (f.startsWith('consumed-') && f.endsWith('.json')) {
+          const agentName = f.replace('consumed-', '').replace('.json', '');
+          try {
+            const ids = JSON.parse(fs.readFileSync(path.join(DATA_DIR, f), 'utf8'));
+            perAgentConsumed[agentName] = new Set(ids);
+            ids.forEach(id => allConsumed.add(id));
+          } catch {}
+        }
+      }
+    }
+
+    const active = messages.filter(m => {
+      if (m.to === '__group__') {
+        return !aliveAgentNames.every(n => n === m.from || (perAgentConsumed[n] && perAgentConsumed[n].has(m.id)));
+      }
+      if (!allConsumed.has(m.id)) return true;
+      return false;
+    });
+
+    const archived = messages.filter(m => !active.includes(m));
+    if (archived.length > 0) {
+      const dateStr = new Date().toISOString().slice(0, 10);
+      const archiveFile = path.join(DATA_DIR, `archive-${dateStr}.jsonl`);
+      const archiveContent = archived.map(m => JSON.stringify(m)).join('\n') + '\n';
+      try { fs.appendFileSync(archiveFile, archiveContent); } catch (e) { log.error('autoCompact archive write failed:', e.message); }
+    }
+
+    const newContent = active.map(m => JSON.stringify(m)).join('\n') + (active.length ? '\n' : '');
+    const tmpFile = msgFile + '.tmp';
+    fs.writeFileSync(tmpFile, newContent);
+    try {
+      fs.renameSync(tmpFile, msgFile);
+    } catch {
+      try { fs.unlinkSync(tmpFile); } catch {}
+      return;
+    }
+    state.lastReadOffset = Buffer.byteLength(newContent, 'utf8');
+
+    const activeIds = new Set(active.map(m => m.id));
+    for (const f of fs.readdirSync(DATA_DIR)) {
+      if (f.startsWith('consumed-') && f.endsWith('.json')) {
+        try {
+          const ids = JSON.parse(fs.readFileSync(path.join(DATA_DIR, f), 'utf8'));
+          const trimmed = ids.filter(id => activeIds.has(id));
+          fs.writeFileSync(path.join(DATA_DIR, f), JSON.stringify(trimmed));
+        } catch (e) { log.debug('consumed trim failed:', e.message); }
+      }
+    }
+  } catch (e) { log.warn('autoCompact failed:', e.message); }
+}
+
+module.exports = {
+  consumedFile, getConsumedIds, saveConsumedIds, trimConsumedIds,
+  autoCompact,
+};

package/lib/config.js

@@ -0,0 +1,127 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+// Data dir lives in the project where the CLI runs, not where the package is installed
+const DATA_DIR = process.env.NEOHIVE_DATA_DIR || path.join(process.cwd(), '.neohive');
+
+// File paths for all shared data
+const MESSAGES_FILE = path.join(DATA_DIR, 'messages.jsonl');
+const HISTORY_FILE = path.join(DATA_DIR, 'history.jsonl');
+const AGENTS_FILE = path.join(DATA_DIR, 'agents.json');
+const ACKS_FILE = path.join(DATA_DIR, 'acks.json');
+const TASKS_FILE = path.join(DATA_DIR, 'tasks.json');
+const PROFILES_FILE = path.join(DATA_DIR, 'profiles.json');
+const WORKFLOWS_FILE = path.join(DATA_DIR, 'workflows.json');
+const WORKSPACES_DIR = path.join(DATA_DIR, 'workspaces');
+const BRANCHES_FILE = path.join(DATA_DIR, 'branches.json');
+const DECISIONS_FILE = path.join(DATA_DIR, 'decisions.json');
+const KB_FILE = path.join(DATA_DIR, 'kb.json');
+const LOCKS_FILE = path.join(DATA_DIR, 'locks.json');
+const PROGRESS_FILE = path.join(DATA_DIR, 'progress.json');
+const VOTES_FILE = path.join(DATA_DIR, 'votes.json');
+const REVIEWS_FILE = path.join(DATA_DIR, 'reviews.json');
+const DEPS_FILE = path.join(DATA_DIR, 'dependencies.json');
+const REPUTATION_FILE = path.join(DATA_DIR, 'reputation.json');
+const COMPRESSED_FILE = path.join(DATA_DIR, 'compressed.json');
+const RULES_FILE = path.join(DATA_DIR, 'rules.json');
+const CONFIG_FILE = path.join(DATA_DIR, 'config.json');
+const PERMISSIONS_FILE = path.join(DATA_DIR, 'permissions.json');
+const READ_RECEIPTS_FILE = path.join(DATA_DIR, 'read_receipts.json');
+const DATA_VERSION_FILE = path.join(DATA_DIR, '.version');
+const CHANNELS_FILE_PATH = path.join(DATA_DIR, 'channels.json');
+
+// Constants
+const MAX_CONTENT_BYTES = 1000000; // 1 MB max message size
+const CURRENT_DATA_VERSION = 1;
+const RESERVED_NAMES = ['__system__', '__all__', '__open__', '__close__', 'system', 'dashboard', 'Dashboard'];
+
+// Config helpers
+function getConfig() {
+  if (!fs.existsSync(CONFIG_FILE)) return {};
+  try { return JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8')); } catch { return {}; }
+}
+
+function saveConfig(config) {
+  ensureDataDir();
+  fs.writeFileSync(CONFIG_FILE, JSON.stringify(config));
+}
+
+function isGroupMode() {
+  return getConfig().conversation_mode === 'group';
+}
+
+function isManagedMode() {
+  return getConfig().conversation_mode === 'managed';
+}
+
+function getManagedConfig() {
+  const config = getConfig();
+  return config.managed || {
+    manager: null,
+    phase: 'discussion',
+    floor: 'closed',
+    turn_queue: [],
+    turn_current: null,
+    phase_history: [],
+  };
+}
+
+function ensureDataDir() {
+  if (!fs.existsSync(DATA_DIR)) {
+    fs.mkdirSync(DATA_DIR, { recursive: true, mode: 0o700 });
+  }
+}
+
+function sanitizeName(name) {
+  if (typeof name !== 'string' || !/^[a-zA-Z0-9_-]{1,20}$/.test(name)) {
+    throw new Error(`Invalid name "${name}": must be 1-20 alphanumeric/underscore/hyphen chars`);
+  }
+  if (RESERVED_NAMES.includes(name.toLowerCase())) {
+    throw new Error(`Name "${name}" is reserved and cannot be used`);
+  }
+  return name;
+}
+
+function generateId() {
+  try { return Date.now().toString(36) + require('crypto').randomBytes(6).toString('hex'); }
+  catch { return Date.now().toString(36) + Math.random().toString(36).slice(2, 8); }
+}
+
+function generateToken() {
+  try { return require('crypto').randomBytes(16).toString('hex'); }
+  catch { return Math.random().toString(36).slice(2) + Math.random().toString(36).slice(2); }
+}
+
+function validateContentSize(content) {
+  if (typeof content !== 'string') return { error: 'content must be a string' };
+  if (Buffer.byteLength(content, 'utf8') > MAX_CONTENT_BYTES) {
+    return { error: 'Message content exceeds maximum size (1 MB)' };
+  }
+  return null;
+}
+
+function getMessagesFile(branch) {
+  if (!branch || branch === 'main') return MESSAGES_FILE;
+  return path.join(DATA_DIR, `branch-${sanitizeName(branch)}-messages.jsonl`);
+}
+
+function getHistoryFile(branch) {
+  if (!branch || branch === 'main') return HISTORY_FILE;
+  return path.join(DATA_DIR, `branch-${sanitizeName(branch)}-history.jsonl`);
+}
+
+module.exports = {
+  DATA_DIR,
+  MESSAGES_FILE, HISTORY_FILE, AGENTS_FILE, ACKS_FILE, TASKS_FILE,
+  PROFILES_FILE, WORKFLOWS_FILE, WORKSPACES_DIR, BRANCHES_FILE,
+  DECISIONS_FILE, KB_FILE, LOCKS_FILE, PROGRESS_FILE, VOTES_FILE,
+  REVIEWS_FILE, DEPS_FILE, REPUTATION_FILE, COMPRESSED_FILE, RULES_FILE,
+  CONFIG_FILE, PERMISSIONS_FILE, READ_RECEIPTS_FILE, DATA_VERSION_FILE,
+  CHANNELS_FILE_PATH,
+  MAX_CONTENT_BYTES, CURRENT_DATA_VERSION, RESERVED_NAMES,
+  getConfig, saveConfig, isGroupMode, isManagedMode, getManagedConfig,
+  ensureDataDir, sanitizeName, generateId, generateToken, validateContentSize,
+  getMessagesFile, getHistoryFile,
+};

package/lib/file-io.js

@@ -0,0 +1,166 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const log = require('./logger');
+const { DATA_DIR, AGENTS_FILE, CONFIG_FILE, ensureDataDir } = require('./config');
+
+// --- Read cache (eliminates 70%+ redundant disk I/O) ---
+const _cache = {};
+function cachedRead(key, readFn, ttlMs = 2000) {
+  const now = Date.now();
+  const entry = _cache[key];
+  if (entry && now - entry.ts < ttlMs) return entry.val;
+  const val = readFn();
+  _cache[key] = { val, ts: now };
+  return val;
+}
+function invalidateCache(key) { delete _cache[key]; }
+
+// --- JSONL readers ---
+
+function readJsonl(file) {
+  if (!fs.existsSync(file)) return [];
+  const content = fs.readFileSync(file, 'utf8').trim();
+  if (!content) return [];
+  return content.split(/\r?\n/).map(line => {
+    try { return JSON.parse(line); } catch { return null; }
+  }).filter(Boolean);
+}
+
+// Optimized: read only NEW lines from a JSONL file starting at byte offset
+function readJsonlFromOffset(file, offset) {
+  if (!fs.existsSync(file)) return { messages: [], newOffset: 0 };
+  const stat = fs.statSync(file);
+  if (stat.size <= offset) return { messages: [], newOffset: offset };
+  const fd = fs.openSync(file, 'r');
+  const buf = Buffer.alloc(stat.size - offset);
+  fs.readSync(fd, buf, 0, buf.length, offset);
+  fs.closeSync(fd);
+  const content = buf.toString('utf8').trim();
+  if (!content) return { messages: [], newOffset: stat.size };
+  const messages = content.split(/\r?\n/).map(line => {
+    try { return JSON.parse(line); } catch { return null; }
+  }).filter(Boolean);
+  return { messages, newOffset: stat.size };
+}
+
+// Read only last N lines of a JSONL file — O(N) instead of O(all)
+function tailReadJsonl(file, lineCount = 100) {
+  if (!fs.existsSync(file)) return [];
+  const stat = fs.statSync(file);
+  if (stat.size === 0) return [];
+  const readSize = Math.min(stat.size, lineCount * 300);
+  const offset = Math.max(0, stat.size - readSize);
+  const fd = fs.openSync(file, 'r');
+  const buf = Buffer.alloc(readSize);
+  fs.readSync(fd, buf, 0, readSize, offset);
+  fs.closeSync(fd);
+  const content = buf.toString('utf8');
+  const lines = content.split(/\r?\n/).filter(l => l.trim());
+  if (offset > 0 && lines.length > 0) lines.shift();
+  const messages = lines.map(line => {
+    try { return JSON.parse(line); } catch { return null; }
+  }).filter(Boolean);
+  return messages.slice(-lineCount);
+}
+
+// --- JSON file helpers ---
+
+function readJsonFile(file) {
+  if (!fs.existsSync(file)) return null;
+  try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return null; }
+}
+
+// File-to-cache-key map: writeJsonFile auto-invalidates the right cache entry
+const _fileCacheKeys = {};
+
+function registerFileCacheKey(file, cacheKey) {
+  _fileCacheKeys[file] = cacheKey;
+}
+
+function writeJsonFile(file, data) {
+  ensureDataDir();
+  const str = JSON.stringify(data);
+  if (str && str.length > 0) {
+    // Use file lock to prevent concurrent write corruption
+    const lockPath = file + '.lock';
+    let locked = false;
+    try { fs.writeFileSync(lockPath, String(process.pid), { flag: 'wx' }); locked = true; } catch {}
+    try {
+      fs.writeFileSync(file, str);
+    } finally {
+      if (locked) try { fs.unlinkSync(lockPath); } catch {}
+    }
+    const cacheKey = _fileCacheKeys[file];
+    if (cacheKey) invalidateCache(cacheKey);
+  }
+}
+
+// --- File locking ---
+
+// Dedicated lock for agents.json (exponential backoff)
+const AGENTS_LOCK = AGENTS_FILE + '.lock';
+function lockAgentsFile() {
+  const maxWait = 5000; const start = Date.now();
+  let backoff = 1;
+  while (Date.now() - start < maxWait) {
+    try { fs.writeFileSync(AGENTS_LOCK, String(process.pid), { flag: 'wx' }); return true; }
+    catch {}
+    const wait = Date.now(); while (Date.now() - wait < backoff) {}
+    backoff = Math.min(backoff * 2, 500);
+  }
+  try { fs.unlinkSync(AGENTS_LOCK); } catch {}
+  try { fs.writeFileSync(AGENTS_LOCK, String(process.pid), { flag: 'wx' }); return true; } catch {}
+  return false;
+}
+function unlockAgentsFile() { try { fs.unlinkSync(AGENTS_LOCK); } catch {} }
+
+// Dedicated lock for config.json
+const CONFIG_LOCK = CONFIG_FILE + '.lock';
+function lockConfigFile() {
+  const maxWait = 5000; const start = Date.now();
+  while (Date.now() - start < maxWait) {
+    try { fs.writeFileSync(CONFIG_LOCK, String(process.pid), { flag: 'wx' }); return true; }
+    catch {}
+    const wait = Date.now(); while (Date.now() - wait < 50) {}
+  }
+  try { fs.unlinkSync(CONFIG_LOCK); } catch {}
+  try { fs.writeFileSync(CONFIG_LOCK, String(process.pid), { flag: 'wx' }); return true; } catch {}
+  return false;
+}
+function unlockConfigFile() { try { fs.unlinkSync(CONFIG_LOCK); } catch {} }
+
+// Generic file lock for any JSON file
+function withFileLock(filePath, fn) {
+  const lockPath = filePath + '.lock';
+  const maxWait = 5000; const start = Date.now();
+  let backoff = 1;
+  while (Date.now() - start < maxWait) {
+    try { fs.writeFileSync(lockPath, String(process.pid), { flag: 'wx' }); break; }
+    catch {}
+    const wait = Date.now(); while (Date.now() - wait < backoff) {}
+    backoff = Math.min(backoff * 2, 500);
+    if (Date.now() - start >= maxWait) {
+      try {
+        const lockPid = parseInt(fs.readFileSync(lockPath, 'utf8').trim(), 10);
+        if (lockPid && lockPid !== process.pid) {
+          try { process.kill(lockPid, 0); return null; } catch {}
+        }
+      } catch {}
+      try { fs.unlinkSync(lockPath); } catch {}
+      try { fs.writeFileSync(lockPath, String(process.pid), { flag: 'wx' }); } catch { return fn(); }
+      break;
+    }
+  }
+  try { return fn(); } finally { try { fs.unlinkSync(lockPath); } catch {} }
+}
+
+module.exports = {
+  cachedRead, invalidateCache,
+  readJsonl, readJsonlFromOffset, tailReadJsonl,
+  readJsonFile, writeJsonFile, registerFileCacheKey,
+  lockAgentsFile, unlockAgentsFile,
+  lockConfigFile, unlockConfigFile,
+  withFileLock,
+};

package/lib/logger.js

@@ -0,0 +1,13 @@
+'use strict';
+
+const LOG_LEVEL = (process.env.NEOHIVE_LOG_LEVEL || 'warn').toLowerCase();
+const LOG_LEVELS = { error: 0, warn: 1, info: 2, debug: 3 };
+
+const log = {
+  error: (...args) => { if (LOG_LEVELS[LOG_LEVEL] >= 0) process.stderr.write('[NEOHIVE:ERROR] ' + args.map(String).join(' ') + '\n'); },
+  warn:  (...args) => { if (LOG_LEVELS[LOG_LEVEL] >= 1) process.stderr.write('[NEOHIVE:WARN] ' + args.map(String).join(' ') + '\n'); },
+  info:  (...args) => { if (LOG_LEVELS[LOG_LEVEL] >= 2) process.stderr.write('[NEOHIVE:INFO] ' + args.map(String).join(' ') + '\n'); },
+  debug: (...args) => { if (LOG_LEVELS[LOG_LEVEL] >= 3) process.stderr.write('[NEOHIVE:DEBUG] ' + args.map(String).join(' ') + '\n'); },
+};
+
+module.exports = log;

package/lib/messaging.js

@@ -0,0 +1,137 @@
+'use strict';
+
+const fs = require('fs');
+const state = require('./state');
+const { DATA_DIR, getMessagesFile, getHistoryFile, generateId, ensureDataDir } = require('./config');
+const { readJsonlFromOffset } = require('./file-io');
+const { getAgents, isPidAlive } = require('./agents');
+
+// Rate limiting constants
+const rateLimitWindow = 60000;
+const rateLimitMax = 30;
+
+function checkRateLimit(content, to) {
+  const now = Date.now();
+  state.rateLimitMessages = state.rateLimitMessages.filter(t => now - t < rateLimitWindow);
+  if (state.rateLimitMessages.length >= rateLimitMax) {
+    return { error: `Rate limit exceeded: max ${rateLimitMax} messages per minute. Wait before sending more.` };
+  }
+  state.recentSentMessages = state.recentSentMessages.filter(m => now - m.timestamp < 30000);
+  if (content && typeof content === 'string' && to) {
+    const contentKey = content.substring(0, 200);
+    const dup = state.recentSentMessages.find(m => m.to === to && m.content === contentKey);
+    if (dup) {
+      return { error: `Duplicate message detected — you already sent this to ${to} ${Math.round((now - dup.timestamp) / 1000)}s ago. Send a different message.` };
+    }
+    state.recentSentMessages.push({ content: contentKey, to, timestamp: now });
+    if (state.recentSentMessages.length > 50) state.recentSentMessages = state.recentSentMessages.slice(-30);
+  }
+  state.rateLimitMessages.push(now);
+  return null;
+}
+
+function sendSystemMessage(toAgent, content) {
+  state.messageSeq++;
+  const agents = getAgents();
+  const recipientBranch = (agents[toAgent] && agents[toAgent].branch) || state.currentBranch;
+  const msg = {
+    id: generateId(),
+    seq: state.messageSeq,
+    from: '__system__',
+    to: toAgent,
+    content,
+    timestamp: new Date().toISOString(),
+    system: true,
+  };
+  ensureDataDir();
+  fs.appendFileSync(getMessagesFile(recipientBranch), JSON.stringify(msg) + '\n');
+  fs.appendFileSync(getHistoryFile(recipientBranch), JSON.stringify(msg) + '\n');
+}
+
+function broadcastSystemMessage(content, excludeAgent = null) {
+  state.messageSeq++;
+  const msg = {
+    id: generateId(),
+    seq: state.messageSeq,
+    from: '__system__',
+    to: '__group__',
+    content,
+    timestamp: new Date().toISOString(),
+    system: true,
+  };
+  if (excludeAgent) msg.exclude_agent = excludeAgent;
+  ensureDataDir();
+  fs.appendFileSync(getMessagesFile(state.currentBranch), JSON.stringify(msg) + '\n');
+  fs.appendFileSync(getHistoryFile(state.currentBranch), JSON.stringify(msg) + '\n');
+}
+
+// Read new lines from messages.jsonl starting at a byte offset
+function readNewMessages(fromOffset, branch) {
+  const msgFile = getMessagesFile(branch || state.currentBranch);
+  return readNewMessagesFromFile(fromOffset, msgFile);
+}
+
+function readNewMessagesFromFile(fromOffset, filePath) {
+  if (!fs.existsSync(filePath)) return { messages: [], newOffset: 0 };
+  const stat = fs.statSync(filePath);
+  if (stat.size < fromOffset) return { messages: [], newOffset: 0 };
+  if (stat.size === fromOffset) return { messages: [], newOffset: fromOffset };
+  const fd = fs.openSync(filePath, 'r');
+  const buf = Buffer.alloc(stat.size - fromOffset);
+  fs.readSync(fd, buf, 0, buf.length, fromOffset);
+  fs.closeSync(fd);
+  const chunk = buf.toString('utf8').trim();
+  if (!chunk) return { messages: [], newOffset: stat.size };
+  const messages = chunk.split(/\r?\n/).map(line => {
+    try { return JSON.parse(line); } catch { return null; }
+  }).filter(Boolean);
+  return { messages, newOffset: stat.size };
+}
+
+// Build standard message delivery response
+function buildMessageResponse(msg, consumedIds) {
+  const log = require('./logger');
+  let pendingCount = 0;
+  try {
+    const msgFile = getMessagesFile(state.currentBranch);
+    if (fs.existsSync(msgFile)) {
+      const { messages: tail } = readNewMessages(state.lastReadOffset);
+      pendingCount = tail.filter(m => m.to === state.registeredName && m.id !== msg.id && !consumedIds.has(m.id)).length;
+    }
+  } catch (e) { log.debug('pending count failed:', e.message); }
+
+  const agents = getAgents();
+  const agentsOnline = Object.entries(agents).filter(([, info]) => isPidAlive(info.pid, info.last_activity)).length;
+
+  return {
+    success: true,
+    message: {
+      id: msg.id,
+      from: msg.from,
+      content: msg.content,
+      timestamp: msg.timestamp,
+      ...(msg.reply_to && { reply_to: msg.reply_to }),
+      ...(msg.thread_id && { thread_id: msg.thread_id }),
+    },
+    pending_count: pendingCount,
+    agents_online: agentsOnline,
+  };
+}
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+function adaptiveSleep(pollCount) {
+  if (pollCount < 10) return sleep(500);
+  if (pollCount < 30) return sleep(1000);
+  return sleep(2000);
+}
+
+module.exports = {
+  checkRateLimit,
+  sendSystemMessage, broadcastSystemMessage,
+  readNewMessages, readNewMessagesFromFile,
+  buildMessageResponse,
+  sleep, adaptiveSleep,
+};