@cgh567/agent 2.4.0 → 2.4.2

package/daemon/adapters/helios-rpc-adapter.js

@@ -33,11 +33,19 @@ const path = require('path');
 const fs   = require('fs');
 const { randomUUID } = require('crypto');
 
-const DEFAULT_TASK_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes
-const MAX_STDOUT_BUF = 10 * 1024 * 1024; // 10MB
-
 // Resolve helios-agent root relative to this file: daemon/adapters/ → two levels up
 const HELIOS_ROOT = path.resolve(__dirname, '..', '..');
+
+// Lazy hboStore for SQLite-first CostEvent writes (P2-7)
+let _hboStoreRpc = null;
+function _getHboStoreRpc() {
+  if (_hboStoreRpc) return _hboStoreRpc;
+  try { _hboStoreRpc = require(path.join(HELIOS_ROOT, 'lib', 'hbo-core-store')); } catch (_) {}
+  return _hboStoreRpc;
+}
+
+const DEFAULT_TASK_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes
+const MAX_STDOUT_BUF = 10 * 1024 * 1024; // 10MB
 const HELIOS_RPC_BIN = path.join(HELIOS_ROOT, 'bin', 'helios-rpc.js');
 
 class HRpcAdapter {
@@ -120,21 +128,22 @@ class HRpcAdapter {
         skill:       skill,
         wakeReason:  config.originKind ?? 'heartbeat_timer',
       }),
-      // ── Headroom compression proxy ──────────────────────────────────────────
-      // Route all pi subprocess LLM calls through the local Headroom proxy.
-      // CacheAligner stabilizes Anthropic/Bedrock KV-cache prefixes.
-      // SmartCrusher compresses tool outputs and HEMA recall payloads.
-      // The proxy URL is read here at call time so it picks up the correct
-      // address after an auto-restart (restart may change the port).
+      // ── Helios Compression Server ───────────────────────────────────────────
+      // Inject HEADROOM_PROXY_URL so the context-compaction.ts extension can
+      // call the compression server directly from the Pi subprocess.
+      //
+      // IMPORTANT: We do NOT set ANTHROPIC_BASE_URL — that would route all LLM
+      // calls through the compression server, which is not a full LLM proxy.
+      // Instead, compression happens in the context-compaction.ts extension hook
+      // BEFORE the LLM call (compress messages array → Pi sends compressed to Bedrock).
+      // This is the correct architecture: compress at assembly time, not at wire time.
       ...(() => {
         try {
           const { HeadroomProxyManager } = require('../lib/headroom-proxy-manager');
           const baseUrl = HeadroomProxyManager.getInstance().getBaseUrl();
           if (baseUrl) {
             return {
-              ANTHROPIC_BASE_URL:  baseUrl,
-              OPENAI_BASE_URL:     `${baseUrl}/v1`,
-              HEADROOM_PROXY_URL:  baseUrl,
+              HEADROOM_PROXY_URL: baseUrl,
             };
           }
         } catch (_) {}
@@ -395,20 +404,33 @@ class HRpcAdapter {
             onMeta?.({ type: 'agent_end', data: { taskId, textLength: lastAssistantText.length } });
             closeStdin(child);
             // Write real Bedrock spend as CostEvent so BudgetEnforcer can track it
-            if (totalCostCents > 0 && context.mgQuery && config.companyId) {
+            if (totalCostCents > 0 && config.companyId) {
               const ceId = 'ce:rpc:' + taskId + ':' + Date.now();
-              context.mgQuery(
-                `MERGE (ce:CostEvent {id: $id})
-                 ON CREATE SET
-                   ce.companyId   = $cid,
-                   ce.agentId     = $agentId,
-                   ce.costCents   = toInteger($costCents),
-                   ce.source      = 'helios_rpc',
-                   ce.taskId      = $taskId,
-                   ce.createdAt   = localdatetime()`,
-                { id: ceId, cid: config.companyId ?? '', agentId: config.agentId ?? agentId ?? '',
-                  costCents: totalCostCents, taskId }
-              ).catch(e => onLog?.({ stream: 'stderr', chunk: '[helios-rpc-adapter] CostEvent write failed: ' + e.message, ts: new Date().toISOString() }));
+              // SQLite-first CostEvent write (P2-7)
+              try {
+                const _store = _getHboStoreRpc();
+                if (_store && _store.createCostEvent) {
+                  _store.createCostEvent({
+                    id: ceId, companyId: config.companyId ?? '', agentId: config.agentId ?? agentId ?? '',
+                    costCents: totalCostCents, source: 'helios_rpc', taskId, createdAt: Date.now(),
+                  });
+                }
+              } catch (_) {}
+              // Non-blocking Memgraph projection (fire-and-forget)
+              if (context.mgQuery) {
+                setImmediate(() => context.mgQuery(
+                  `MERGE (ce:CostEvent {id: $id})
+                   ON CREATE SET
+                     ce.companyId   = $cid,
+                     ce.agentId     = $agentId,
+                     ce.costCents   = toInteger($costCents),
+                     ce.source      = 'helios_rpc',
+                     ce.taskId      = $taskId,
+                     ce.createdAt   = localdatetime()`,
+                  { id: ceId, cid: config.companyId ?? '', agentId: config.agentId ?? agentId ?? '',
+                    costCents: totalCostCents, taskId }
+                ).catch(e => onLog?.({ stream: 'stderr', chunk: '[helios-rpc-adapter] CostEvent write failed: ' + e.message, ts: new Date().toISOString() })));
+              }
             }
 
             // F1-3: if child doesn't exit within 30s after agent_end, settle now.

package/daemon/config/com.familiar.helios-daemon.plist

@@ -90,6 +90,11 @@
         <string>unified</string>
         <key>HELIOS_ALLOW_SEND</key>
         <string>0</string>
+        <!-- --experimental-sqlite enables node:sqlite built-in (hbo-core-store).
+             Stable in Node ≥ 23; required flag in Node 22.
+             --max-old-space-size=1024 matches PM2 ecosystem.config.js memory cap (SVC-2 fix). -->
+        <key>NODE_OPTIONS</key>
+        <string>--experimental-sqlite --max-old-space-size=1024</string>
         <!-- PATH extended to cover common Node.js install locations:
              nvm default, volta, homebrew, system. The launch script will
              source the user's shell profile which adds nvm to PATH. -->

package/daemon/config/helios-daemon.service

@@ -35,6 +35,10 @@ Environment=TZ=UTC
 Environment=ROUTING_AUTHORITY=unified
 Environment=AWS_REGION=us-east-1
 Environment=MEMGRAPH_BOLT_URL=bolt://127.0.0.1:7687
+# --experimental-sqlite enables node:sqlite built-in (hbo-core-store fallback store).
+# Stable in Node ≥ 23; required flag in Node 22.
+# --max-old-space-size=1024 matches the PM2 ecosystem.config.js memory cap (SVC-1 fix).
+Environment=NODE_OPTIONS=--experimental-sqlite --max-old-space-size=1024
 # HELIOS_ROOT is resolved by launch-daemon.sh — not hardcoded here
 # EnvironmentFile uses - prefix (dash) to not fail if file is missing
 EnvironmentFile=-%h/helios-agent/.env

package/daemon/context-enrichment.js

@@ -4,6 +4,14 @@
 
 const { HELIOS_ROOT: _HELIOS_ROOT_CE } = require('./lib/paths');
 
+// Lazy-require hbo-core-store for SQLite-first business entity reads (P2-11)
+let _hboStoreCE = null;
+function _getHboStoreCE() {
+  if (_hboStoreCE) return _hboStoreCE;
+  try { _hboStoreCE = require('../lib/hbo-core-store'); } catch (_) {}
+  return _hboStoreCE;
+}
+
 /** Marker appended when buildContextBrief truncates an oversized brief. */
 const CONTEXT_BRIEF_TRUNCATED = '[CONTEXT BRIEF TRUNCATED — exceeded 30k char cap (aligned to adapter delivery limit)]';
 
@@ -430,19 +438,33 @@ async function buildContextBrief(mgQuery, agentId, taskTitle, companyId, hboBrid
   } catch (e) { /* non-fatal */ }
 
   // 3. Agent's own pending tasks (what else is on their plate)
+  // Memgraph primary — SQLite fallback on unavailability
   try {
-    const pendingResult = await mgQuery(
-      `MATCH (t:Task {assigneeAgentId: $aid, companyId: $cid})
-       WHERE t.status IN ['todo', 'in_progress']
-       RETURN t.title AS title, t.status AS status, t.priority AS priority
-       ORDER BY CASE t.priority WHEN 'P0' THEN 0 WHEN 'P1' THEN 1 WHEN 'P2' THEN 2 ELSE 3 END
-       LIMIT 5`,
-      { aid: agentId, cid: companyId }
-    );
-    const pendingRecords = toRecords(pendingResult);
-    if (pendingRecords.length) {
+    let pendingRecords = null;
+    try {
+      const pendingResult = await mgQuery(
+        `MATCH (t:Task {assigneeAgentId: $aid, companyId: $cid})
+         WHERE t.status IN ['todo', 'in_progress']
+         RETURN t.title AS title, t.status AS status, t.priority AS priority
+         ORDER BY CASE t.priority WHEN 'P0' THEN 0 WHEN 'P1' THEN 1 WHEN 'P2' THEN 2 ELSE 3 END
+         LIMIT 5`,
+        { aid: agentId, cid: companyId }
+      );
+      pendingRecords = toRecords(pendingResult).map(r => ({ _title: r.get('title'), _status: r.get('status'), _priority: r.get('priority') }));
+    } catch (_mgErr) {
+      // Memgraph unavailable — fall back to SQLite pending tasks
+      try {
+        const _storeCE = _getHboStoreCE();
+        if (_storeCE && _storeCE.getTasksByCompanyStatus) {
+          const _rows = _storeCE.getTasksByCompanyStatus(companyId, ['todo', 'in_progress'])
+            .filter(t => t.assigneeAgentId === agentId);
+          pendingRecords = _rows.map(t => ({ _title: t.title, _status: t.status, _priority: t.priority }));
+        }
+      } catch (_sqliteErr) { /* SQLite also unavailable — skip pending tasks section */ }
+    }
+    if (pendingRecords && pendingRecords.length) {
       const lines = pendingRecords.map(r =>
-        `- [${r.get('priority') || 'P3'}] ${r.get('title')} (${r.get('status')})`
+        `- [${r._priority || 'P3'}] ${r._title} (${r._status})`
       );
       sections.push({ id: 'active-tasks', content: `## Your Other Tasks\n${lines.join('\n')}` });
     }
@@ -502,16 +524,32 @@ async function buildContextBrief(mgQuery, agentId, taskTitle, companyId, hboBrid
     }
   } catch (_) { /* non-fatal */ }
 
-  // § QUARTERLY OKR
+  // § QUARTERLY OKR — Memgraph primary, SQLite fallback on unavailability
   try {
-    const okrResult = await mgQuery(
-      `MATCH (o:QuarterlyOKR {companyId: $cid, status: 'active'})
-       WHERE o.agentId = $agentId OR o.assigneeAgentId = $agentId
-       RETURN o.objective AS objective, o.keyResults AS keyResults, o.quarter AS quarter
-       ORDER BY o.createdAt DESC LIMIT 1`,
-      { agentId, cid: companyId }
-    ).catch(() => null);
-    const okrRec = toRecords(okrResult)?.[0];
+    let okrRec = null;
+    try {
+      const okrResult = await mgQuery(
+        `MATCH (o:QuarterlyOKR {companyId: $cid, status: 'active'})
+         WHERE o.agentId = $agentId OR o.assigneeAgentId = $agentId
+         RETURN o.objective AS objective, o.keyResults AS keyResults, o.quarter AS quarter
+         ORDER BY o.createdAt DESC LIMIT 1`,
+        { agentId, cid: companyId }
+      );
+      okrRec = toRecords(okrResult)?.[0] ?? null;
+    } catch (_mgErr) {
+      // Memgraph unavailable — fall back to SQLite OKR data
+      try {
+        const _storeOKR = _getHboStoreCE();
+        if (_storeOKR && _storeOKR.getOKRsByCompanyType) {
+          const _okrs = _storeOKR.getOKRsByCompanyType(companyId, 'quarterly_okr')
+            .filter(o => (o.agentId === agentId || o.assigneeAgentId === agentId) && o.status === 'active');
+          if (_okrs.length) {
+            const o = _okrs[0];
+            okrRec = { get: (k) => o[k] ?? null };
+          }
+        }
+      } catch (_sqliteErr) { /* SQLite also unavailable — skip OKR section */ }
+    }
     if (okrRec) {
       const objective = okrRec.get('objective');
       const keyResults = okrRec.get('keyResults') || [];
@@ -1736,7 +1774,7 @@ ${lines.join('\n')}` });
     try {
       const excl = JSON.parse(pillarData.exclusions || '[]');
       if (excl.length > 0) exclusionNote = `\nDo NOT: ${excl.slice(0, 3).join(', ')}`;
-    } catch (_) {}
+  } catch (_) {}
     sections.push(
       `<north_star_reminder>\n` +
       `Before responding, confirm your answer serves this intent:\n` +

package/daemon/helios-api.js

@@ -42,6 +42,7 @@ const fs = require('fs');
 const path = require('path');
 const crypto = require('crypto');
 const { TranscriptStore } = require('./transcript-store');
+const hboStore = require('../lib/hbo-core-store');
 
 const TRANSCRIPTS_DIR = path.join(__dirname, 'transcripts');
 const _transcriptStore = new TranscriptStore(TRANSCRIPTS_DIR);
@@ -280,6 +281,26 @@ async function handleGetTasks(req, res, ctx) {
   let cypher;
   const params = { limit, cid: ctx.cid };
 
+  // SQLite-first task list read (P2-9)
+  try {
+    const _storeTasks = hboStore.getTasksByCompanyStatus
+      ? (q.status
+          ? hboStore.getTasksByCompanyStatus(ctx.cid, q.status)
+          : (() => { try { return require('../lib/hbo-core-store').getTasksByCompanyStatus(ctx.cid, ['todo','in_progress','done','andon_paused','help_pending','cancelled']); } catch(_) { return null; } })()
+        )
+      : null;
+    if (_storeTasks) {
+      let tasks = _storeTasks;
+      if (q.agentId) tasks = tasks.filter(t => t.assigneeAgentId === q.agentId);
+      tasks = tasks
+        .sort((a, b) => (b.createdAt ?? 0) - (a.createdAt ?? 0))
+        .slice(0, limit)
+        .map(t => ({ id: t.id, title: t.title, status: t.status, priority: t.priority, assignee: t.assigneeAgentId, body: t.body ?? null }));
+      jsonResponse(res, 200, { tasks, count: tasks.length });
+      return;
+    }
+  } catch (_) {}
+
   if (q.status && q.agentId) {
     cypher = `MATCH (t:Task {companyId: $cid, status: $status, assigneeAgentId: $agentId})
               RETURN t.id AS id, t.title AS title, t.status AS status,
@@ -503,6 +524,23 @@ async function handleGetApprovals(req, res, ctx) {
   let cypher;
   const params = { limit, cid: ctx.cid };
 
+  // SQLite-first approval list read (P2-9)
+  try {
+    const _storeApprovals = hboStore.getApprovalsByCompanyStatus
+      ? (q.status
+          ? hboStore.getApprovalsByCompanyStatus(ctx.cid, q.status)
+          : hboStore.getApprovalsByCompanyStatus(ctx.cid, ['pending','approved','rejected','expired'])
+        )
+      : null;
+    if (_storeApprovals) {
+      const approvals = _storeApprovals
+        .sort((a, b) => (b.createdAt ?? 0) - (a.createdAt ?? 0))
+        .slice(0, limit);
+      jsonResponse(res, 200, { approvals, count: approvals.length });
+      return;
+    }
+  } catch (_) {}
+
   if (q.status) {
     cypher = `MATCH (a:Approval {companyId: $cid, status: $status})${returnClause}`;
     params.status = q.status;
@@ -2848,6 +2886,10 @@ let interpretationRoute = () => false;
 let hedRoute = () => false;
 // D4: Department intelligence page handler — initialized lazily inside startApi()
 let handleGetDepartmentPage = null;
+// Domain-specific routes — initialized lazily inside startApi()
+let hitlRoute = null;
+let channelsRoute = null;
+let emailInfraRoute = null;
 const { handleOKRRoutes } = require('./routes/okrs');
 const { handleSenseiRoutes } = require('./routes/sensei');
 let suggestionsRoute, suggestionsCron;
@@ -2967,6 +3009,14 @@ async function route(req, res, ctx) {
         return;
       }
       const result = ctx.daemon.registerCompany(String(companyId));
+      // Also update ctx.companies so the tenant isolation check accepts this company
+      // on subsequent requests (the 403 guard checks ctx.companies, not _modulesByCompany).
+      // This is safe: ctx.companies is an array that only grows — no entries are removed.
+      if (!ctx.companies) ctx.companies = [];
+      const cidStr = String(companyId);
+      if (!ctx.companies.find(c => String(c.id) === cidStr)) {
+        try { ctx.companies.push({ id: cidStr, name: cidStr }); } catch (_) { /* frozen array — ignore */ }
+      }
       jsonResponse(res, 200, result);
     } catch (err) {
       jsonResponse(res, 500, { error: `registerCompany failed: ${err.message}` });
@@ -2974,6 +3024,51 @@ async function route(req, res, ctx) {
     return;
   }
 
+  // GET /api/v1/context-brief — build and return the agent context brief.
+  //
+  // Used by harbor tests (test_context_propagation.py) and desktop context panel.
+  // Calls buildContextBrief() from context-enrichment.js — the same function the
+  // daemon calls before every agent dispatch.
+  //
+  // Query params:
+  //   agentId    (required) — the BusinessAgent id
+  //   companyId  (optional) — defaults to primary company; checked against allowedCompanyIds
+  //   nocache    (optional) — if set, bypasses the 5-minute context cache
+  //
+  // Primary path: GET /api/v1/context-brief?agentId=X&companyId=Y
+  //   → buildContextBrief(mgQuery, agentId, '', companyId)
+  //   → { brief: string, agentId, companyId }
+  if (method === 'GET' && pathname === '/api/v1/context-brief') {
+    try {
+      const agentId = parsedUrl.searchParams.get('agentId');
+      const explicitCid = parsedUrl.searchParams.get('companyId');
+      if (!agentId) {
+        jsonResponse(res, 400, { error: 'agentId required' });
+        return;
+      }
+      if (!explicitCid) {
+        jsonResponse(res, 400, { error: 'companyId required' });
+        return;
+      }
+      const cid = resolvedCid;
+      const { mgQuery } = ctx;
+      if (!mgQuery) {
+        jsonResponse(res, 503, { error: 'Memgraph not available' });
+        return;
+      }
+      const { buildContextBrief, invalidateContextCache } = require('./context-enrichment');
+      // Bust 5-minute context cache when nocache=1 is set (used by tests after seeding)
+      if (parsedUrl.searchParams.get('nocache')) {
+        try { invalidateContextCache(cid); } catch (_) {}
+      }
+      const brief = await buildContextBrief(mgQuery, agentId, '', cid, null);
+      jsonResponse(res, 200, { brief: brief || '', agentId, companyId: cid });
+    } catch (err) {
+      jsonResponse(res, 500, { error: `context-brief failed: ${err.message}` });
+    }
+    return;
+  }
+
   // GET /api/headroom/health — proxy to Headroom compression proxy health endpoint
   // Used by HeliosInfraService (helios-desktop) to show proxy status in Ground Control.
   if (method === 'GET' && pathname === '/api/headroom/health') {
@@ -3163,18 +3258,49 @@ async function route(req, res, ctx) {
           const baseUrl = HeadroomProxyManager.getInstance().getBaseUrl();
           if (!baseUrl) return _origEnd(chunk, encoding, callback);
 
-          const headroomAi = require('headroom-ai');
-          const result = await headroomAi.compress(
-            [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'hbo_get', content: body }] }],
-            { model: 'claude', baseUrl }
-          );
+          // Direct HTTP POST /headroom/compress — no npm package required.
+          // Same pattern as context-compaction.ts and headroom-middleware.js.
+          const _payload = JSON.stringify({
+            messages: [{
+              role: 'user',
+              content: [{
+                type: 'tool_result',
+                tool_use_id: 'hbo_get',
+                content: body,
+              }],
+            }],
+          });
+          const _url = new URL(baseUrl);
+          const result = await new Promise((resolve, reject) => {
+            const http = require('http');
+            const req = http.request(
+              {
+                hostname: _url.hostname,
+                port: parseInt(_url.port || '8787', 10),
+                path: '/headroom/compress',
+                method: 'POST',
+                headers: {
+                  'Content-Type': 'application/json',
+                  'Content-Length': Buffer.byteLength(_payload),
+                },
+              },
+              (res) => {
+                let buf = '';
+                res.on('data', (c) => { buf += c; });
+                res.on('end', () => { try { resolve(JSON.parse(buf)); } catch { reject(new Error('bad json')); } });
+                res.on('error', reject);
+              }
+            );
+            req.setTimeout(8000, () => { req.destroy(); reject(new Error('timeout')); });
+            req.on('error', reject);
+            req.write(_payload);
+            req.end();
+          });
 
           let compressed = body;
           try {
-            const c = result?.messages?.[0]?.content;
-            const text = typeof c === 'string' ? c
-              : Array.isArray(c) ? (c[0]?.text ?? c[0]?.content ?? body) : body;
-            compressed = text;
+            const c = result?.messages?.[0]?.content?.[0]?.content;
+            if (c) compressed = c;
           } catch (_) {}
 
           if (result?.ccrHashes?.length) {
@@ -3495,41 +3621,27 @@ async function handleGetDepartmentPageRoute(req, res, ctx, department) {
       }
     }
 
-    // Cache miss or stale — regenerate
+    // Cache miss or stale — return 202 immediately and generate async
     if (!handleGetDepartmentPage) {
       jsonResponse(res, 503, { error: 'Department page generator not initialized' });
       return;
     }
 
-    const narrative = await handleGetDepartmentPage.generatePage(cid, department);
+    // D5: Return 202 immediately so the frontend can show a loading state.
+    // Generation runs in setImmediate (next event-loop tick) and broadcasts
+    // department:page:ready when complete so the client can refetch.
+    jsonResponse(res, 202, { ok: true, generating: true, department, companyId: cid });
 
-    // generatePage returns null on failure (LLM not configured, empty data, etc.).
-    // When null, query the DepartmentPage node for its status/error so the desktop
-    // can show a specific message (e.g. "AWS credentials required") instead of the
-    // generic "no intelligence page available" empty state.
-    if (!narrative) {
-      let failureError = null;
+    setImmediate(async () => {
       try {
-        const failedNode = await mg(
-          'MATCH (dp:DepartmentPage {companyId: $cid, department: $dept}) ' +
-          'WHERE dp.status = \'failed\' RETURN dp.error AS error ORDER BY dp.generatedAt DESC LIMIT 1',
-          { cid, dept: department }
-        ).catch(() => null);
-        const failRow = failedNode?.rows?.[0];
-        if (failRow) {
-          failureError = Array.isArray(failRow) ? failRow[0] : failRow.error;
+        const narrative = await handleGetDepartmentPage.generatePage(cid, department);
+        if (narrative) {
+          broadcast({ type: 'department:page:ready', companyId: cid, department });
         }
-      } catch (_) {}
-      jsonResponse(res, 200, {
-        department,
-        narrative: null,
-        error: failureError || 'Generation failed — check daemon logs',
-        cached: false,
-      });
-      return;
-    }
-
-    jsonResponse(res, 200, { department, narrative, generatedAt: new Date().toISOString(), cached: false });
+      } catch (_genErr) {
+        console.warn('[dept-page] async generation error:', _genErr && _genErr.message);
+      }
+    });
 
   } catch (e) {
     jsonResponse(res, 500, { error: 'Department page generation failed: ' + (e.message || 'unknown error') });
@@ -3753,7 +3865,7 @@ function startApi(mgQuery, config = {}, state = {}) {
   }
 
   // Domain-specific routes: HITL, channel setup, email infrastructure
-  let hitlRoute = null, channelsRoute = null, emailInfraRoute = null;
+  hitlRoute = null; channelsRoute = null; emailInfraRoute = null;
   try { hitlRoute       = require('./routes/hitl')({ mgQuery }); }              catch(e) { log('warn', `hitl route unavailable: ${e.message}`); }
   try { channelsRoute   = require('./routes/channels')({ mgQuery }); }          catch(e) { log('warn', `channels route unavailable: ${e.message}`); }
   try { emailInfraRoute = require('./routes/email-infrastructure')({ mgQuery }); } catch(e) { log('warn', `emailInfra route unavailable: ${e.message}`); }