@cgh567/agent 2.4.1 → 2.4.3

package/lib/__tests__/hbo-core-store.test.js

@@ -0,0 +1,238 @@
+'use strict';
+/**
+ * lib/__tests__/hbo-core-store.test.js
+ *
+ * Unit tests for hbo-core-store.ts (SQLite fallback store).
+ * Uses node:sqlite (Node.js v22+ built-in) — no native compilation required.
+ * Runs natively on Windows and macOS.
+ *
+ * Run: node --experimental-sqlite lib/__tests__/hbo-core-store.test.js
+ * Or:  pnpm run test:hbo-store
+ *
+ * Invariants verified:
+ *   - Memgraph is the primary data model; SQLite is the fallback
+ *   - No hardcoded company IDs (process.pid suffix)
+ *   - INSERT OR REPLACE is idempotent
+ *   - company_id isolation prevents cross-tenant data bleed
+ *   - company_id=undefined throws (never silently stores null)
+ *   - updateTask/updateApproval upsert when record is missing
+ */
+
+const os = require('os');
+const path = require('path');
+const fs = require('fs');
+
+// Unique per test run — no cross-run state pollution
+const TEST_ROOT = path.join(os.tmpdir(), 'hbo-core-store-test-' + process.pid);
+process.env.HELIOS_ROOT = TEST_ROOT;
+fs.mkdirSync(path.join(TEST_ROOT, 'data'), { recursive: true });
+
+// Must require AFTER setting HELIOS_ROOT
+const store = require('../hbo-core-store');
+
+// Unique company IDs — no hardcoded tenant names
+const CID_A = 'test:hbo-store:co-a:' + process.pid;
+const CID_B = 'test:hbo-store:co-b:' + process.pid;
+
+let passed = 0;
+let failed = 0;
+const failures = [];
+
+function assert(condition, message) {
+  if (!condition) {
+    const err = new Error('FAIL: ' + message);
+    failures.push(err);
+    failed++;
+    console.error('  ✗', message);
+  } else {
+    passed++;
+    console.log('  ✓', message);
+  }
+}
+
+function assertEqual(a, b, message) {
+  assert(a === b, `${message} (expected ${JSON.stringify(b)}, got ${JSON.stringify(a)})`);
+}
+
+function assertNull(a, message) {
+  assert(a === null, `${message} (expected null, got ${JSON.stringify(a)})`);
+}
+
+function assertThrows(fn, messagePart, label) {
+  try {
+    fn();
+    assert(false, `${label}: expected throw but did not throw`);
+  } catch (e) {
+    if (messagePart && !e.message.includes(messagePart)) {
+      assert(false, `${label}: threw but message "${e.message}" did not contain "${messagePart}"`);
+    } else {
+      assert(true, label);
+    }
+  }
+}
+
+// ── Task tests ────────────────────────────────────────────────────────────────
+
+console.log('\n[Tasks]');
+
+store.createTask({ id: 't-1', companyId: CID_A, title: 'Test Task', status: 'todo', priority: 1 });
+const t1 = store.getTask('t-1', CID_A);
+assert(t1 !== null, 'getTask returns record after createTask');
+assertEqual(t1.title, 'Test Task', 'getTask.title correct');
+assertEqual(t1.status, 'todo', 'getTask.status correct');
+
+assertNull(store.getTask('t-1', CID_B), 'getTask returns null for wrong company (isolation)');
+
+store.updateTask('t-1', CID_A, { status: 'in_progress' });
+assertEqual(store.getTask('t-1', CID_A).status, 'in_progress', 'updateTask changes status');
+
+// updateTask upsert: record not yet in SQLite → creates it
+store.updateTask('t-upsert', CID_A, { title: 'Upserted', status: 'todo' });
+const tu = store.getTask('t-upsert', CID_A);
+assert(tu !== null, 'updateTask upserts when record is missing');
+assertEqual(tu.status, 'todo', 'upserted task has correct status');
+
+store.createTask({ id: 't-2', companyId: CID_A, title: 'Task 2', status: 'done' });
+const todos = store.getTasksByCompanyStatus(CID_A, 'todo');
+assert(todos.some(t => t.id === 't-upsert'), 'getTasksByCompanyStatus(todo) includes todo tasks');
+assert(!todos.some(t => t.id === 't-2'), 'getTasksByCompanyStatus(todo) excludes done tasks');
+
+store.updateTask('t-1', CID_A, { status: 'in_progress' });
+const active = store.getTasksByCompanyStatus(CID_A, ['todo', 'in_progress']);
+assert(active.some(t => t.id === 't-1'), 'multi-status filter includes in_progress');
+assert(active.some(t => t.id === 't-upsert'), 'multi-status filter includes todo');
+assert(!active.some(t => t.id === 't-2'), 'multi-status filter excludes done');
+
+assertThrows(
+  () => store.createTask({ id: 't-nocompany', title: 'X' }),
+  'company_id is required',
+  'createTask throws when companyId is missing'
+);
+
+// Idempotency
+store.createTask({ id: 't-1', companyId: CID_A, title: 'Updated Title', status: 'in_progress' });
+const allTasks = store.getTasksByCompanyStatus(CID_A, ['todo', 'in_progress', 'done']);
+assertEqual(allTasks.filter(t => t.id === 't-1').length, 1, 'createTask is idempotent (no duplicate rows)');
+assertEqual(store.getTask('t-1', CID_A).title, 'Updated Title', 'second createTask updates the record');
+
+store.deleteTask('t-2', CID_A);
+assertNull(store.getTask('t-2', CID_A), 'deleteTask removes the record');
+
+// ── Approval tests ────────────────────────────────────────────────────────────
+
+console.log('\n[Approvals]');
+
+store.createApproval({ id: 'ap-1', companyId: CID_A, type: 'budget_exceeded', status: 'pending', requestedBy: 'agent:ceo' });
+const ap1 = store.getApproval('ap-1', CID_A);
+assert(ap1 !== null, 'getApproval returns record after createApproval');
+assertEqual(ap1.type, 'budget_exceeded', 'approval type correct');
+assertNull(store.getApproval('ap-1', CID_B), 'approval isolated from other company');
+
+store.updateApproval('ap-1', CID_A, { status: 'approved', followUpTaskCreated: false });
+assertEqual(store.getApproval('ap-1', CID_A).status, 'approved', 'updateApproval changes status');
+
+store.updateApproval('ap-upsert', CID_A, { status: 'pending', type: 'belief_confirmation' });
+assert(store.getApproval('ap-upsert', CID_A) !== null, 'updateApproval upserts when missing');
+
+const pendingApprovals = store.getApprovalsByCompanyStatus(CID_A, 'pending');
+assert(pendingApprovals.some(a => a.id === 'ap-upsert'), 'getApprovalsByCompanyStatus returns pending');
+assert(!pendingApprovals.some(a => a.id === 'ap-1'), 'getApprovalsByCompanyStatus excludes approved');
+
+assertThrows(
+  () => store.createApproval({ id: 'ap-nocompany', type: 'x' }),
+  'company_id is required',
+  'createApproval throws when companyId is missing'
+);
+
+// ── BudgetPolicy tests ────────────────────────────────────────────────────────
+
+console.log('\n[Goals]');
+
+store.createGoal({
+  id: 'goal-1',
+  companyId: CID_A,
+  title: 'Increase retention',
+  description: 'Company-level goal',
+  level: 'company',
+  status: 'active',
+});
+const goal1 = store.getGoal('goal-1', CID_A);
+assert(goal1 !== null, 'getGoal returns record after createGoal');
+assertEqual(goal1.title, 'Increase retention', 'getGoal.title correct');
+assertEqual(goal1.level, 'company', 'getGoal.level correct');
+assertNull(store.getGoal('goal-1', CID_B), 'getGoal returns null for wrong company (isolation)');
+
+store.updateGoal('goal-1', CID_A, { title: 'Increase net retention', status: 'paused' });
+const updatedGoal = store.getGoal('goal-1', CID_A);
+assertEqual(updatedGoal.title, 'Increase net retention', 'updateGoal changes title');
+assertEqual(updatedGoal.status, 'paused', 'updateGoal changes status');
+
+store.createGoal({
+  id: 'goal-child',
+  companyId: CID_A,
+  title: 'Improve onboarding',
+  level: 'department',
+  status: 'active',
+  parentId: 'goal-1',
+});
+const companyGoals = store.getGoalsByCompany(CID_A);
+assert(companyGoals.some(g => g.id === 'goal-1'), 'getGoalsByCompany includes parent goal');
+assert(companyGoals.some(g => g.id === 'goal-child' && g.parentId === 'goal-1'), 'getGoalsByCompany includes child goal with parentId');
+assert(!store.getGoalsByCompany(CID_B).some(g => g.id === 'goal-1'), 'goals isolated from other company');
+
+assertThrows(
+  () => store.createGoal({ id: 'goal-nocompany', title: 'X' }),
+  'company_id is required',
+  'createGoal throws when companyId is missing'
+);
+
+store.deleteGoal('goal-child', CID_A);
+assertNull(store.getGoal('goal-child', CID_A), 'deleteGoal removes the record');
+
+// ── BudgetPolicy tests ───────────────────────────────────────────────────────
+
+console.log('\n[BudgetPolicy]');
+
+store.upsertBudgetPolicy({ id: 'bp-1', companyId: CID_A, agentId: 'agent:sales', limitCents: 50000, warnPercent: 80, hardStopEnabled: true });
+assert(store.getBudgetPolicy('bp-1', CID_A) !== null, 'getBudgetPolicy returns record after upsert');
+assert(store.getBudgetPoliciesByCompany(CID_A).some(p => p.id === 'bp-1'), 'getBudgetPoliciesByCompany returns the policy');
+assert(!store.getBudgetPoliciesByCompany(CID_B).some(p => p.id === 'bp-1'), 'policy isolated from other company');
+
+// ── OKR tests ─────────────────────────────────────────────────────────────────
+
+console.log('\n[OKRs]');
+
+store.upsertOKR({ id: 'okr-1', companyId: CID_A, type: 'quarterly_okr', status: 'active', title: 'Q3 OKR' });
+assert(store.getOKRsByCompanyType(CID_A, 'quarterly_okr', 'active').some(o => o.id === 'okr-1'), 'getOKRsByCompanyType returns active quarterly OKR');
+assert(!store.getOKRsByCompanyType(CID_A, 'quarterly', 'active').some(o => o.id === 'okr-1'), "type='quarterly' does NOT match 'quarterly_okr'");
+assert(!store.getOKRsByCompanyType(CID_B, 'quarterly_okr', 'active').some(o => o.id === 'okr-1'), 'OKR isolated from other company');
+
+// ── Lead tests ────────────────────────────────────────────────────────────────
+
+console.log('\n[Leads]');
+
+store.upsertLead({ id: 'lead-1', companyId: CID_A, status: 'new', name: 'Alice', email: 'alice@example.com' });
+assert(store.getLeadsByCompanyStatus(CID_A, ['new', 'stale']).some(l => l.id === 'lead-1'), 'getLeadsByCompanyStatus returns new lead');
+assert(!store.getLeadsByCompanyStatus(CID_B, ['new']).some(l => l.id === 'lead-1'), 'lead isolated from other company');
+
+// ── CostEvent tests ───────────────────────────────────────────────────────────
+
+console.log('\n[CostEvents]');
+
+const now = Date.now();
+store.createCostEvent({ id: 'ce-1', companyId: CID_A, feature: 'llm', model: 'claude-sonnet', amountUsd: 0.05, createdAt: now });
+assert(store.getCostEventsByCompanyRange(CID_A, now - 1000, now + 1000).some(e => e.id === 'ce-1'), 'getCostEventsByCompanyRange returns event in range');
+assert(!store.getCostEventsByCompanyRange(CID_B, now - 1000, now + 1000).some(e => e.id === 'ce-1'), 'cost event isolated from other company');
+
+// ── Summary ───────────────────────────────────────────────────────────────────
+
+console.log('\n─────────────────────────────────────────');
+console.log(`Results: ${passed} passed, ${failed} failed`);
+if (failed > 0) {
+  console.error('\nFailures:');
+  failures.forEach(f => console.error(' -', f.message));
+  process.exit(1);
+} else {
+  console.log('All tests passed.');
+  process.exit(0);
+}

package/lib/compression/dist/ccr-store.js

@@ -0,0 +1,74 @@
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CcrStore = void 0;
+exports.getCcrStore = getCcrStore;
+class CcrStore {
+    store = new Map();
+    ttlMs;
+    cleanupInterval = null;
+    constructor(ttlSeconds = 7200) {
+        this.ttlMs = ttlSeconds * 1000;
+        // Cleanup every 10 minutes
+        this.cleanupInterval = setInterval(() => this._cleanup(), 10 * 60 * 1000);
+        // Don't hold the event loop open
+        if (this.cleanupInterval.unref)
+            this.cleanupInterval.unref();
+    }
+    set(hash, data) {
+        this.store.set(hash, {
+            data,
+            expiresAt: Date.now() + this.ttlMs,
+            originalBytes: data.length,
+        });
+    }
+    get(hash) {
+        const entry = this.store.get(hash);
+        if (!entry)
+            return null;
+        if (Date.now() > entry.expiresAt) {
+            this.store.delete(hash);
+            return null;
+        }
+        return entry.data;
+    }
+    delete(hash) {
+        this.store.delete(hash);
+    }
+    has(hash) {
+        return this.get(hash) !== null;
+    }
+    stats() {
+        let totalOriginalBytes = 0;
+        const now = Date.now();
+        let entries = 0;
+        for (const [, entry] of this.store) {
+            if (now <= entry.expiresAt) {
+                totalOriginalBytes += entry.originalBytes;
+                entries++;
+            }
+        }
+        return { entries, totalOriginalBytes };
+    }
+    _cleanup() {
+        const now = Date.now();
+        for (const [hash, entry] of this.store) {
+            if (now > entry.expiresAt)
+                this.store.delete(hash);
+        }
+    }
+    destroy() {
+        if (this.cleanupInterval) {
+            clearInterval(this.cleanupInterval);
+            this.cleanupInterval = null;
+        }
+        this.store.clear();
+    }
+}
+exports.CcrStore = CcrStore;
+// Singleton for the daemon process
+let _singleton = null;
+function getCcrStore(ttlSeconds = 7200) {
+    if (!_singleton)
+        _singleton = new CcrStore(ttlSeconds);
+    return _singleton;
+}

package/lib/compression/dist/content-router.js

@@ -0,0 +1,115 @@
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ContentRouter = void 0;
+exports.detectContentType = detectContentType;
+/**
+ * lib/compression/content-router.ts
+ *
+ * ContentRouter: detects content type in a message block and routes to the
+ * appropriate compressor.
+ *
+ * Current compressors:
+ *   JSON_ARRAY    → SmartCrusher (70–92% reduction)
+ *   PLAIN_TEXT    → Passthrough (Kompress-base ML model not available in TS)
+ *   SOURCE_CODE   → Passthrough (CodeCompressor not yet implemented)
+ *   other         → Passthrough
+ *
+ * The passthrough path is correct behavior — we improve compression as we
+ * add more compressors. SmartCrusher alone covers the dominant HBO payload
+ * type (arrays of lead/signal/task/goal objects).
+ *
+ * Protection rules (content never compressed):
+ *   - User messages (content might be misinterpreted if rewritten)
+ *   - Error outputs ≤ 8000 chars (must be preserved exactly)
+ *   - Tool outputs named 'headroom_retrieve' (infinite loop prevention)
+ *   - Content shorter than 500 chars (overhead > savings)
+ */
+const smart_crusher_js_1 = require("./smart-crusher.js");
+const CHARS_PER_TOKEN = 3.5; // Claude/Bedrock heuristic
+function estimateTokens(s) {
+    return Math.ceil(s.length / CHARS_PER_TOKEN);
+}
+/**
+ * Detect content type from a string block.
+ */
+function detectContentType(text) {
+    const trimmed = text.trim();
+    // JSON array: starts with [
+    if (trimmed.startsWith('[')) {
+        try {
+            const parsed = JSON.parse(trimmed);
+            if (Array.isArray(parsed))
+                return 'JSON_ARRAY';
+        }
+        catch { /* not valid JSON */ }
+    }
+    // JSON object: starts with {
+    if (trimmed.startsWith('{')) {
+        try {
+            JSON.parse(trimmed);
+            return 'JSON_OBJECT';
+        }
+        catch { /* not valid JSON */ }
+    }
+    // Source code heuristics (backtick code blocks, common keywords)
+    if (trimmed.startsWith('```') ||
+        trimmed.includes('\ndef ') ||
+        trimmed.includes('\nfunction ') ||
+        trimmed.includes('\nconst ') ||
+        trimmed.includes('\nclass ') ||
+        trimmed.includes('\nimport ')) {
+        return 'SOURCE_CODE';
+    }
+    return 'PLAIN_TEXT';
+}
+class ContentRouter {
+    crusher;
+    constructor(crusherConfig = {}) {
+        this.crusher = new smart_crusher_js_1.SmartCrusher(crusherConfig);
+    }
+    /**
+     * Route a single text block to the appropriate compressor.
+     * Returns the (possibly compressed) text plus metadata.
+     */
+    route(text, toolId, ccrStore) {
+        const passthrough = {
+            output: text,
+            contentType: 'UNKNOWN',
+            strategy: 'passthrough',
+            compressed: false,
+            ccrHash: null,
+            tokensSaved: 0,
+            rowsDropped: 0,
+        };
+        // Protection: too short
+        if (text.length < 500)
+            return passthrough;
+        // Protection: never re-compress CCR retrieve results
+        if (toolId === 'headroom_retrieve')
+            return passthrough;
+        // Protection: small error outputs
+        if (text.length <= 8000 &&
+            (text.toLowerCase().includes('"error"') || text.toLowerCase().includes('"traceback"'))) {
+            return passthrough;
+        }
+        const contentType = detectContentType(text);
+        switch (contentType) {
+            case 'JSON_ARRAY': {
+                const result = this.crusher.compress(text, ccrStore);
+                const tokensSaved = estimateTokens(text) - estimateTokens(result.output);
+                return {
+                    output: result.output,
+                    contentType,
+                    strategy: result.strategy,
+                    compressed: result.compressed,
+                    ccrHash: result.ccrHash,
+                    tokensSaved: Math.max(0, tokensSaved),
+                    rowsDropped: result.rowsDropped,
+                };
+            }
+            default:
+                return { ...passthrough, contentType };
+        }
+    }
+}
+exports.ContentRouter = ContentRouter;

package/lib/compression/dist/pipeline.js

@@ -0,0 +1,113 @@
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CompressionPipeline = void 0;
+/**
+ * lib/compression/pipeline.ts
+ *
+ * Message compression pipeline.
+ *
+ * Processes a messages array (Anthropic or OpenAI format) through the
+ * ContentRouter for each tool_result block.
+ *
+ * The pipeline mirrors what the headroom Python proxy does:
+ *   1. Skip system messages (cache prefix must stay stable)
+ *   2. Skip user messages (content must be preserved)
+ *   3. For each assistant + tool_result message: run ContentRouter
+ *   4. Collect total tokens saved and CCR hashes
+ *
+ * This runs inside the helios-agent daemon process — no HTTP round-trip.
+ * Every company's HBO tool results (signals, leads, pipeline, tasks) are
+ * compressed before they reach the LLM.
+ */
+const content_router_js_1 = require("./content-router.js");
+/**
+ * Deep-copy a messages array (shallow copy of each message, deep copy of
+ * content arrays so we don't mutate the caller's data).
+ */
+function deepCopyMessages(messages) {
+    return messages.map((msg) => ({
+        ...msg,
+        content: Array.isArray(msg.content)
+            ? msg.content.map((b) => ({ ...b }))
+            : msg.content,
+    }));
+}
+function extractText(content) {
+    if (!content)
+        return '';
+    if (typeof content === 'string')
+        return content;
+    for (const block of content) {
+        if (typeof block.content === 'string')
+            return block.content;
+        if (typeof block.text === 'string')
+            return block.text;
+    }
+    return '';
+}
+function originalLength(messages) {
+    return JSON.stringify(messages).length;
+}
+class CompressionPipeline {
+    router;
+    constructor() {
+        this.router = new content_router_js_1.ContentRouter();
+    }
+    /**
+     * Compress a messages array.
+     *
+     * Compresses tool_result blocks in ALL messages (user and assistant).
+     * In Anthropic format, tool_result blocks appear in user messages (they
+     * are the response to the assistant's tool_use calls). These are the
+     * dominant source of large payloads — HBO signals, leads, pipeline data.
+     *
+     * What we do NOT compress:
+     *   - Plain text content in user messages (user intent must be preserved)
+     *   - System messages (prompt cache stability)
+     *   - Anything shorter than 500 chars
+     */
+    compress(messages, ccrStore) {
+        const origLen = originalLength(messages);
+        const copied = deepCopyMessages(messages);
+        let totalTokensSaved = 0;
+        const ccrHashes = [];
+        const transformsApplied = new Set();
+        for (const msg of copied) {
+            // Skip system messages (preserve prompt cache prefix exactly)
+            if (msg.role === 'system')
+                continue;
+            if (!Array.isArray(msg.content))
+                continue;
+            for (const block of msg.content) {
+                // Only process tool_result blocks — this is where HBO data lives
+                // (tool_result appears in user messages in Anthropic format)
+                if (block.type !== 'tool_result')
+                    continue;
+                const text = typeof block.content === 'string'
+                    ? block.content
+                    : extractText(block.content);
+                if (!text)
+                    continue;
+                const toolId = block.tool_use_id;
+                const result = this.router.route(text, toolId, ccrStore);
+                if (result.compressed) {
+                    block.content = result.output;
+                    totalTokensSaved += result.tokensSaved;
+                    transformsApplied.add(result.strategy);
+                    if (result.ccrHash)
+                        ccrHashes.push(result.ccrHash);
+                }
+            }
+        }
+        const newLen = originalLength(copied);
+        const compressionRatio = origLen > 0 ? newLen / origLen : 1;
+        return {
+            messages: copied,
+            tokensSaved: totalTokensSaved,
+            ccrHashes,
+            compressionRatio,
+            transformsApplied: [...transformsApplied],
+        };
+    }
+}
+exports.CompressionPipeline = CompressionPipeline;