@cgh567/agent 2.4.1 → 2.4.3

package/lib/compression/dist/server.js

@@ -0,0 +1,265 @@
+'use strict';
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createCompressionServer = createCompressionServer;
+/**
+ * lib/compression/server.ts
+ *
+ * Helios Compression Server — TypeScript implementation of the Headroom proxy
+ * HTTP interface.
+ *
+ * Replaces the Python headroom proxy entirely. Starts in milliseconds (no ML
+ * model loading), works natively on Windows and macOS.
+ *
+ * Endpoints (headroom proxy compatible):
+ *   GET  /health                         → { status, version, uptime }
+ *   GET  /stats                          → compression statistics
+ *   POST /v1/messages                    → compress Anthropic message array
+ *   POST /v1/chat/completions            → compress OpenAI message array
+ *   GET  /v1/retrieve/:hash              → retrieve original CCR content
+ *   POST /headroom/compress              → direct compress endpoint
+ *
+ * All endpoints are cross-company — there is no tenant scoping in the
+ * compression layer. Compression is a pure function of content shape,
+ * not company identity.
+ */
+const http_1 = __importDefault(require("http"));
+const pipeline_js_1 = require("./pipeline.js");
+const ccr_store_js_1 = require("./ccr-store.js");
+const VERSION = '1.0.0-helios';
+const startedAt = Date.now();
+// Singleton pipeline and CCR store
+const pipeline = new pipeline_js_1.CompressionPipeline();
+const ccrStore = (0, ccr_store_js_1.getCcrStore)(7200);
+// Compression telemetry (process-lifetime counters)
+const stats = {
+    requests: { total: 0, compressed: 0, passthrough: 0, failed: 0 },
+    tokens: { saved: 0 },
+    ccrStore: { puts: 0 },
+};
+// ── HTTP helpers ──────────────────────────────────────────────────────────────
+function readBody(req) {
+    return new Promise((resolve, reject) => {
+        const chunks = [];
+        req.on('data', (c) => chunks.push(c));
+        req.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
+        req.on('error', reject);
+    });
+}
+function json(res, status, body) {
+    const payload = JSON.stringify(body);
+    res.writeHead(status, {
+        'Content-Type': 'application/json',
+        'Access-Control-Allow-Origin': '*',
+        'X-Headroom-Version': VERSION,
+    });
+    res.end(payload);
+}
+function error(res, status, message) {
+    json(res, status, { error: message });
+}
+// ── Route handlers ────────────────────────────────────────────────────────────
+function handleHealth(_req, res) {
+    json(res, 200, {
+        status: 'healthy',
+        version: VERSION,
+        uptime: (Date.now() - startedAt) / 1000,
+        available: true,
+    });
+}
+function handleStats(_req, res) {
+    const ccrStats = ccrStore.stats();
+    json(res, 200, {
+        available: true,
+        requests: stats.requests,
+        tokens: {
+            saved: stats.tokens.saved,
+            savingsPercent: stats.requests.total > 0
+                ? Math.round((stats.tokens.saved / Math.max(stats.tokens.saved + (stats.requests.total * 100), 1)) * 100)
+                : 0,
+        },
+        ccr: {
+            items: ccrStats.entries,
+            totalOriginalBytes: ccrStats.totalOriginalBytes,
+        },
+        uptime: (Date.now() - startedAt) / 1000,
+    });
+}
+async function handleCompressMessages(req, res) {
+    stats.requests.total++;
+    let body;
+    try {
+        body = JSON.parse(await readBody(req));
+    }
+    catch {
+        stats.requests.failed++;
+        return error(res, 400, 'Invalid JSON body');
+    }
+    const messages = body.messages;
+    if (!Array.isArray(messages)) {
+        // Not a messages request — passthrough
+        stats.requests.passthrough++;
+        json(res, 200, body);
+        return;
+    }
+    try {
+        const result = pipeline.compress(messages, ccrStore);
+        stats.tokens.saved += result.tokensSaved;
+        if (result.ccrHashes.length > 0)
+            stats.ccrStore.puts += result.ccrHashes.length;
+        if (result.transformsApplied.length > 0) {
+            stats.requests.compressed++;
+        }
+        else {
+            stats.requests.passthrough++;
+        }
+        // Return in same shape as input, with compressed messages
+        const responseBody = {
+            ...body,
+            messages: result.messages,
+        };
+        // Attach headroom metadata in response headers
+        res.setHeader('X-Headroom-Saved', String(result.tokensSaved));
+        res.setHeader('X-Headroom-Ratio', result.compressionRatio.toFixed(3));
+        if (result.ccrHashes.length > 0) {
+            res.setHeader('X-Headroom-CCR', result.ccrHashes.join(','));
+        }
+        if (result.transformsApplied.length > 0) {
+            res.setHeader('X-Headroom-Transforms', result.transformsApplied.join(','));
+        }
+        json(res, 200, responseBody);
+    }
+    catch (e) {
+        stats.requests.failed++;
+        const msg = e instanceof Error ? e.message : String(e);
+        error(res, 500, `Compression failed: ${msg}`);
+    }
+}
+async function handleDirectCompress(req, res) {
+    stats.requests.total++;
+    let body;
+    try {
+        body = JSON.parse(await readBody(req));
+    }
+    catch {
+        stats.requests.failed++;
+        return error(res, 400, 'Invalid JSON body');
+    }
+    const messages = body.messages;
+    if (!Array.isArray(messages)) {
+        stats.requests.passthrough++;
+        json(res, 200, { messages: body, tokensSaved: 0, compressionRatio: 1, transformsApplied: [], ccrHashes: [] });
+        return;
+    }
+    try {
+        const result = pipeline.compress(messages, ccrStore);
+        stats.tokens.saved += result.tokensSaved;
+        if (result.ccrHashes.length > 0)
+            stats.ccrStore.puts += result.ccrHashes.length;
+        if (result.transformsApplied.length > 0)
+            stats.requests.compressed++;
+        else
+            stats.requests.passthrough++;
+        json(res, 200, {
+            messages: result.messages,
+            tokensSaved: result.tokensSaved,
+            compressionRatio: result.compressionRatio,
+            transformsApplied: result.transformsApplied,
+            ccrHashes: result.ccrHashes,
+        });
+    }
+    catch (e) {
+        stats.requests.failed++;
+        const msg = e instanceof Error ? e.message : String(e);
+        error(res, 500, `Compression failed: ${msg}`);
+    }
+}
+function handleRetrieve(_req, res, hash) {
+    if (!hash || !/^[a-f0-9]{8,64}$/.test(hash)) {
+        return error(res, 400, 'Invalid hash format');
+    }
+    const data = ccrStore.get(hash);
+    if (!data) {
+        return error(res, 404, `CCR hash not found or expired: ${hash}`);
+    }
+    json(res, 200, {
+        hash,
+        data: JSON.parse(data),
+        originalBytes: data.length,
+    });
+}
+// ── Server ────────────────────────────────────────────────────────────────────
+function createCompressionServer() {
+    return http_1.default.createServer(async (req, res) => {
+        const url = req.url ?? '/';
+        const method = req.method ?? 'GET';
+        // CORS preflight
+        if (method === 'OPTIONS') {
+            res.writeHead(204, {
+                'Access-Control-Allow-Origin': '*',
+                'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
+                'Access-Control-Allow-Headers': 'Content-Type, Authorization, x-api-key, anthropic-version',
+            });
+            res.end();
+            return;
+        }
+        try {
+            // Health
+            if (url === '/health' || url === '/livez' || url === '/readyz') {
+                return handleHealth(req, res);
+            }
+            // Stats
+            if (url === '/stats' || url === '/metrics') {
+                return handleStats(req, res);
+            }
+            // Direct compress (used by helios-agent internals and context-compaction.ts)
+            if (url === '/headroom/compress' && method === 'POST') {
+                return await handleDirectCompress(req, res);
+            }
+            // headroom-ai SDK compress endpoint — the SDK calls POST /v1/compress
+            // with { messages, model, config } and expects the same shape back.
+            // We map this to handleDirectCompress which is identical in behavior.
+            if (url === '/v1/compress' && method === 'POST') {
+                return await handleDirectCompress(req, res);
+            }
+            // Anthropic messages endpoint (intercept + compress)
+            if (url === '/v1/messages' && method === 'POST') {
+                return await handleCompressMessages(req, res);
+            }
+            // OpenAI chat completions endpoint (intercept + compress)
+            if (url === '/v1/chat/completions' && method === 'POST') {
+                return await handleCompressMessages(req, res);
+            }
+            // CCR retrieval
+            const ccrMatch = url.match(/^\/v1\/retrieve\/([a-f0-9]{8,64})$/);
+            if (ccrMatch && method === 'GET') {
+                return handleRetrieve(req, res, ccrMatch[1]);
+            }
+            // 404
+            res.writeHead(404, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: `Unknown endpoint: ${method} ${url}` }));
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            res.writeHead(500, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: `Server error: ${msg}` }));
+        }
+    });
+}
+// ── Standalone entry point ────────────────────────────────────────────────────
+if (require.main === module) {
+    const port = parseInt(process.env.HEADROOM_PORT ?? '8787', 10);
+    const server = createCompressionServer();
+    server.listen(port, '127.0.0.1', () => {
+        process.stdout.write(JSON.stringify({ event: 'ready', port, version: VERSION }) + '\n');
+    });
+    server.on('error', (err) => {
+        if (err.code === 'EADDRINUSE') {
+            process.stderr.write(`[compression-server] Port ${port} already in use\n`);
+            process.exit(1);
+        }
+        throw err;
+    });
+}

package/lib/compression/dist/smart-crusher.js

@@ -0,0 +1,251 @@
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SmartCrusher = exports.DEFAULT_CRUSHER_CONFIG = void 0;
+/**
+ * lib/compression/smart-crusher.ts
+ *
+ * TypeScript implementation of the Headroom SmartCrusher algorithm.
+ *
+ * SmartCrusher compresses JSON arrays by:
+ *   1. Lossless path: for homogeneous arrays, render as CSV+schema (removes
+ *      repeated key names).  Wins when savings ≥ 15%.
+ *   2. Lossy path: keep first 30% + last 15% + importance-scored middle 55%.
+ *      Dropped rows are stored in the CCR store and retrievable on demand.
+ *
+ * Ported from headroom/crates/headroom-core (Rust) algorithm.
+ * No external dependencies — pure TypeScript.
+ *
+ * Every company's HBO data (signals, leads, pipeline, tasks, goals) flows
+ * through this compressor before reaching the LLM, reducing token cost
+ * by 70–92% for typical array payloads.
+ */
+const crypto_1 = require("crypto");
+exports.DEFAULT_CRUSHER_CONFIG = {
+    minItemsToAnalyze: 5,
+    minCharsToCompress: 200,
+    maxItemsAfterCrush: 15,
+    firstFraction: 0.3,
+    lastFraction: 0.15,
+    losslessMinSavingsRatio: 0.15,
+    compactionCoreFieldFraction: 0.8,
+};
+// ── Helpers ───────────────────────────────────────────────────────────────────
+function shortHash(data) {
+    return (0, crypto_1.createHash)('sha256').update(data).digest('hex').slice(0, 12);
+}
+/**
+ * Check if an array is homogeneous enough for lossless CSV compaction.
+ *
+ * Conditions (all must be true):
+ *   1. ≥90% of items are plain objects
+ *   2. At least 3 fields appear in ≥ coreFieldFraction of items
+ *      (single shared field like 'id' is not enough structure for CSV)
+ *   3. Core fields cover ≥50% of the average object's field count
+ *      (prevents trivial matches where most fields are non-shared)
+ */
+function isHomogeneousArray(items, coreFieldFraction) {
+    if (items.length === 0)
+        return { homogeneous: false, coreFields: [] };
+    const objectItems = items.filter((x) => x !== null && typeof x === 'object' && !Array.isArray(x));
+    if (objectItems.length / items.length < 0.9) {
+        return { homogeneous: false, coreFields: [] };
+    }
+    // Count field frequencies
+    const fieldFreq = new Map();
+    let totalFields = 0;
+    for (const obj of objectItems) {
+        const keys = Object.keys(obj);
+        totalFields += keys.length;
+        for (const key of keys) {
+            fieldFreq.set(key, (fieldFreq.get(key) ?? 0) + 1);
+        }
+    }
+    // Core fields: appear in ≥ coreFieldFraction of objects
+    const threshold = objectItems.length * coreFieldFraction;
+    const coreFields = [...fieldFreq.entries()]
+        .filter(([, count]) => count >= threshold)
+        .map(([key]) => key)
+        .sort();
+    // Gate 1: must have at least 3 core fields
+    // A single shared key (like 'id') is not enough structure for meaningful CSV
+    if (coreFields.length < 3) {
+        return { homogeneous: false, coreFields: [] };
+    }
+    // Gate 2: core fields must cover ≥50% of the average object's field count
+    // Prevents trivial CSV on objects where most fields are non-shared
+    const avgFieldCount = totalFields / objectItems.length;
+    const coverage = coreFields.length / avgFieldCount;
+    if (coverage < 0.5) {
+        return { homogeneous: false, coreFields: [] };
+    }
+    return { homogeneous: true, coreFields };
+}
+/**
+ * Lossless CSV compaction for homogeneous arrays.
+ * Format: header row + value rows, tab-separated within each row,
+ * pipe-separated rows. Significantly reduces token count for repeated keys.
+ */
+function losslessCsvCompress(items, coreFields) {
+    const header = coreFields.join('\t');
+    const rows = items.map((item) => coreFields
+        .map((f) => {
+        const v = item[f];
+        if (v === null || v === undefined)
+            return '';
+        if (typeof v === 'object')
+            return JSON.stringify(v);
+        return String(v);
+    })
+        .join('\t'));
+    return `[csv:${coreFields.length}cols×${items.length}rows]\n${header}\n${rows.join('\n')}`;
+}
+/**
+ * Score items for importance. Higher = more important to keep.
+ * Keeps items that: have errors, have non-null/non-zero values,
+ * or are at structural boundaries (first/last).
+ * Mid-range items are scored by field diversity (more unique values = more important).
+ */
+function scoreItems(items) {
+    return items.map((item) => {
+        let score = 0;
+        const str = JSON.stringify(item);
+        // Error/anomaly signals — always preserve
+        if (str.includes('"error"') ||
+            str.includes('"failed"') ||
+            str.includes('"ERROR"') ||
+            str.includes('"FAILED"')) {
+            score += 100;
+        }
+        // Non-trivial values (not null, not 0, not empty string)
+        const values = Object.values(item);
+        const nonTrivial = values.filter((v) => v !== null && v !== undefined && v !== '' && v !== 0 && v !== false).length;
+        score += (nonTrivial / Math.max(values.length, 1)) * 10;
+        // Field diversity (unique values signal interesting data)
+        const uniqueVals = new Set(values.map((v) => JSON.stringify(v))).size;
+        score += (uniqueVals / Math.max(values.length, 1)) * 5;
+        return score;
+    });
+}
+// ── Main SmartCrusher ─────────────────────────────────────────────────────────
+class SmartCrusher {
+    cfg;
+    constructor(config = {}) {
+        this.cfg = { ...exports.DEFAULT_CRUSHER_CONFIG, ...config };
+    }
+    /**
+     * Compress a JSON array string.
+     * If input is not a JSON array, returns passthrough.
+     * If array is too small, returns passthrough.
+     * Otherwise attempts lossless CSV then lossy sampling.
+     */
+    compress(input, ccrStore) {
+        const passthrough = {
+            output: input,
+            compressed: false,
+            strategy: 'passthrough',
+            ccrHash: null,
+            rowsDropped: 0,
+            originalCount: 0,
+            keptCount: 0,
+        };
+        // Gate: too short
+        if (input.length < this.cfg.minCharsToCompress)
+            return passthrough;
+        // Parse JSON
+        let parsed;
+        try {
+            parsed = JSON.parse(input);
+        }
+        catch {
+            return passthrough;
+        }
+        // Must be a non-empty array
+        if (!Array.isArray(parsed) || parsed.length < this.cfg.minItemsToAnalyze) {
+            return passthrough;
+        }
+        const items = parsed;
+        // ── Lossless path ────────────────────────────────────────────────────────
+        const { homogeneous, coreFields } = isHomogeneousArray(items, this.cfg.compactionCoreFieldFraction);
+        if (homogeneous) {
+            const objectItems = items.filter((x) => x !== null && typeof x === 'object' && !Array.isArray(x));
+            const losslessOutput = losslessCsvCompress(objectItems, coreFields);
+            const savingsRatio = 1 - losslessOutput.length / input.length;
+            if (savingsRatio >= this.cfg.losslessMinSavingsRatio) {
+                return {
+                    output: losslessOutput,
+                    compressed: true,
+                    strategy: 'lossless-csv',
+                    ccrHash: null,
+                    rowsDropped: 0,
+                    originalCount: items.length,
+                    keptCount: items.length,
+                };
+            }
+        }
+        // ── Lossy path ───────────────────────────────────────────────────────────
+        const n = items.length;
+        const maxKeep = this.cfg.maxItemsAfterCrush;
+        if (n <= maxKeep) {
+            // Already small enough — check if lossless was worth it
+            return passthrough;
+        }
+        const firstCount = Math.max(1, Math.floor(n * this.cfg.firstFraction));
+        const lastCount = Math.max(1, Math.floor(n * this.cfg.lastFraction));
+        const midBudget = Math.max(0, maxKeep - firstCount - lastCount);
+        const firstItems = items.slice(0, firstCount);
+        const lastItems = items.slice(n - lastCount);
+        const midItems = items.slice(firstCount, n - lastCount);
+        // Score mid-range items and keep the most important
+        const objectMidItems = midItems.filter((x) => x !== null && typeof x === 'object' && !Array.isArray(x));
+        let keptMid;
+        let droppedMid;
+        if (objectMidItems.length > 0 && midBudget > 0) {
+            const scores = scoreItems(objectMidItems);
+            const indexed = scores.map((s, i) => ({ s, i }));
+            indexed.sort((a, b) => b.s - a.s);
+            const keepIndices = new Set(indexed.slice(0, midBudget).map((x) => x.i));
+            keptMid = objectMidItems.filter((_, i) => keepIndices.has(i));
+            droppedMid = objectMidItems.filter((_, i) => !keepIndices.has(i));
+            // Dedup: remove items from keptMid that are identical to firstItems or lastItems
+            const existingSet = new Set([...firstItems, ...lastItems].map((x) => JSON.stringify(x)));
+            keptMid = keptMid.filter((x) => !existingSet.has(JSON.stringify(x)));
+        }
+        else {
+            keptMid = midBudget > 0 ? midItems.slice(0, midBudget) : [];
+            droppedMid = midItems.slice(midBudget);
+        }
+        const keptItems = [...firstItems, ...keptMid, ...lastItems];
+        const droppedItems = droppedMid;
+        const rowsDropped = droppedItems.length;
+        // Store dropped rows in CCR
+        let ccrHash = null;
+        if (rowsDropped > 0 && ccrStore) {
+            const droppedJson = JSON.stringify(droppedItems);
+            ccrHash = shortHash(droppedJson);
+            ccrStore.set(ccrHash, droppedJson);
+        }
+        // Build output
+        const ccrMarker = ccrHash && rowsDropped > 0
+            ? `\n{"_ccr_dropped":"<<ccr:${ccrHash} ${rowsDropped}_rows_offloaded>>"}`
+            : '';
+        const output = JSON.stringify(keptItems) + ccrMarker;
+        const savingsRatio = 1 - output.length / input.length;
+        // Only return lossy result if it actually saved something meaningful
+        if (savingsRatio < this.cfg.losslessMinSavingsRatio) {
+            // Clean up CCR entry — not worth it
+            if (ccrHash && ccrStore)
+                ccrStore.delete(ccrHash);
+            return passthrough;
+        }
+        return {
+            output,
+            compressed: true,
+            strategy: 'lossy-sample',
+            ccrHash,
+            rowsDropped,
+            originalCount: items.length,
+            keptCount: keptItems.length,
+        };
+    }
+}
+exports.SmartCrusher = SmartCrusher;

package/lib/event-bus.mts

@@ -46,7 +46,7 @@ interface EventMap {
     senderHandle: string;
     subject: string;
     snippet: string;
-    body: string;
+    // body intentionally excluded — PII (SEC-6)
     threadId: string;
     receivedAt: string;
     suggestedAction: string;

package/lib/graph/lib/utils.js

@@ -468,7 +468,7 @@ async function mgQueryAsync(cypher, params = {}, opts = {}) {
         let _queryTimer;
         const result = await Promise.race([
           session.run(cleanCypher, params),
-          new Promise((_, rej) => { _queryTimer = setTimeout(() => rej(new Error('direct bolt query timeout (15s)')), 15000); })
+          new Promise((_, rej) => { _queryTimer = setTimeout(() => rej(new Error('direct bolt query timeout (15s)')), parseInt(process.env.HELIOS_BOLT_QUERY_TIMEOUT_MS || '25000', 10)); })
         ]);
         clearTimeout(_queryTimer);
         const keys = result.records.length > 0 ? result.records[0].keys : [];

package/lib/graph-availability.js

@@ -0,0 +1,62 @@
+'use strict';
+/**
+ * lib/graph-availability.js
+ *
+ * Single source of truth for Memgraph availability state within a SINGLE
+ * Node.js process. Listeners are notified on state change only (not on
+ * redundant sets of the same value).
+ *
+ * ⚠️  PROCESS SCOPE WARNING:
+ * This module uses Node.js module-level state. Each OS process has its own
+ * module registry and therefore its own independent _available + _listeners.
+ * It is ONLY valid within a single process:
+ *
+ *   ✅ Pi extension process — memgraph-autostart sets state, cortex/hema read it
+ *   ✅ Eval container — same process, same state
+ *   ❌ helios-company-daemon.js — runs in its own process; state here is NEVER
+ *      set by memgraph-autostart (which runs in Pi's process). Do NOT use
+ *      isMemgraphAvailable() in daemon code to gate Memgraph writes — use the
+ *      try/catch + setImmediate fire-and-forget pattern instead.
+ *   ❌ broker/server.js — forked process; has its own availability state
+ *
+ * Usage (within Pi's extension process only):
+ *   const { isMemgraphAvailable, setMemgraphAvailable, onAvailabilityChange } = require('./graph-availability');
+ */
+
+let _available = false;
+let _listeners = [];
+
+function setMemgraphAvailable(available) {
+  const v = !!available;
+  if (v === _available) return;
+  _available = v;
+  const snap = _listeners.slice();
+  for (const fn of snap) {
+    try { fn(_available); } catch (e) {
+      process.stderr.write('[graph-availability] listener error: ' + String(e) + '\n');
+    }
+  }
+}
+
+function isMemgraphAvailable() {
+  return _available;
+}
+
+function onAvailabilityChange(fn) {
+  _listeners.push(fn);
+  return function unsubscribe() {
+    _listeners = _listeners.filter((l) => l !== fn);
+  };
+}
+
+function _resetForTest() {
+  _available = false;
+  _listeners = [];
+}
+
+module.exports = {
+  setMemgraphAvailable,
+  isMemgraphAvailable,
+  onAvailabilityChange,
+  _resetForTest,
+};