@cgh567/agent 2.4.1 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/helios +0 -0
- package/bin/helios-rpc-node-wrapper.cjs +0 -0
- package/bin/helios-rpc-wrapper.sh +0 -0
- package/daemon/adapters/helios-rpc-adapter.js +47 -25
- package/daemon/config/com.familiar.helios-daemon.plist +5 -0
- package/daemon/config/helios-daemon.service +4 -0
- package/daemon/context-enrichment.js +59 -21
- package/daemon/helios-api.js +149 -37
- package/daemon/helios-company-daemon.js +516 -124
- package/daemon/lib/harada/cascade-judge.js +12 -50
- package/daemon/lib/harada/mandala.js +20 -0
- package/daemon/lib/harada/pillar-dispatcher.js +1 -1
- package/daemon/lib/harada/project-factory.js +7 -2
- package/daemon/lib/hbo-bridge.js +31 -12
- package/daemon/lib/helios-hitl-host.js +15 -2
- package/daemon/lib/hitl-interaction-service.js +0 -0
- package/daemon/lib/memgraph-verify.js +38 -33
- package/daemon/lib/project-drift-detector.js +7 -17
- package/daemon/lib/project-semantic-updater.js +1 -14
- package/daemon/routes/channels.js +10 -5
- package/daemon/routes/harada-map.js +11 -48
- package/daemon/routes/hbo.js +89 -28
- package/daemon/routes/hitl.js +0 -0
- package/daemon/routes/project.js +4 -3
- package/daemon/routes/wizard.js +11 -4
- package/daemon/schema-migrations-hitl.js +0 -0
- package/extensions/001-tool-output-cap.ts +0 -0
- package/extensions/context-compaction.ts +45 -26
- package/extensions/cortex/activation-bridge.ts +5 -0
- package/extensions/cortex/learn.ts +26 -0
- package/extensions/email/backfill.ts +0 -0
- package/extensions/helios-governance/analysis/ambiguity.ts +0 -0
- package/extensions/helios-governance/analysis/compliance.ts +0 -0
- package/extensions/helios-governance/analysis/long-task-detector.ts +0 -0
- package/extensions/helios-governance/analysis/output-contract.ts +0 -0
- package/extensions/helios-governance/analysis/patterns.ts +0 -0
- package/extensions/helios-governance/analysis/preflight.ts +0 -0
- package/extensions/helios-governance/analysis/recurring-violations.ts +0 -0
- package/extensions/helios-governance/analysis/task-classification.ts +0 -0
- package/extensions/helios-governance/analysis/task-intent.ts +0 -0
- package/extensions/helios-governance/gates/high-impact.ts +1 -1
- package/extensions/helios-governance/handlers/_jiti-require.ts +15 -8
- package/extensions/helios-governance/handlers/proxy-test-detector.ts +0 -0
- package/extensions/hema-dispatch-v3/graph-memory.ts +10 -0
- package/extensions/hema-dispatch-v3/index.ts +59 -40
- package/extensions/lib/elo-engine.js +0 -0
- package/extensions/lib/elo-engine.test.js +0 -0
- package/extensions/memgraph-autostart.ts +13 -0
- package/extensions/neuroplastic-eval.ts +0 -0
- package/extensions/shadow-loop/index.ts +0 -0
- package/lib/brain-v2-budget.js +0 -0
- package/lib/brain-v2-circuit-breaker.js +0 -0
- package/lib/brain-v2.js +0 -0
- package/lib/broker/adaptive-throttle.js +0 -0
- package/lib/broker/batch-coalescer.js +0 -0
- package/lib/broker/bulkhead.js +0 -0
- package/lib/broker/channel-registry.js +0 -0
- package/lib/broker/circuit-breaker.js +0 -0
- package/lib/broker/evidence-cache.js +0 -0
- package/lib/broker/health-monitor.js +0 -0
- package/lib/broker/mage-queue.js +0 -0
- package/lib/broker/priority-queue.js +0 -0
- package/lib/broker/server.js.bak-error2-fix +0 -0
- package/lib/broker/session-registry.js +0 -0
- package/lib/broker/singleton-timers.js +0 -0
- package/lib/broker/types.d.ts +0 -0
- package/lib/broker/vegas-limit.js +0 -0
- package/lib/compression/dist/ccr-store.js +74 -0
- package/lib/compression/dist/content-router.js +115 -0
- package/lib/compression/dist/pipeline.js +113 -0
- package/lib/compression/dist/server.js +265 -0
- package/lib/compression/dist/smart-crusher.js +251 -0
- package/lib/context-budget.ts +0 -0
- package/lib/context-firewall.js +0 -0
- package/lib/crm/integration/triage-bridge.js +0 -0
- package/lib/email-utils.ts +0 -0
- package/lib/eval/__tests__/preflight-checker.test.ts +0 -0
- package/lib/eval/__tests__/task-instruction-parser.test.ts +0 -0
- package/lib/eval/__tests__/verifier-runner.test.ts +0 -0
- package/lib/eval/index.ts +0 -0
- package/lib/eval/preflight-checker.ts +0 -0
- package/lib/eval/task-domain-classifier.ts +0 -0
- package/lib/eval/task-instruction-parser.ts +0 -0
- package/lib/eval/verifier-runner.ts +0 -0
- package/lib/event-bus.d.ts +0 -0
- package/lib/governance-context-selector.ts +0 -0
- package/lib/graph/generate-extension-embeddings.js +0 -0
- package/lib/graph/generate-static-embeddings.js +0 -0
- package/lib/graph/lib/utils.js +1 -1
- package/lib/graph-audit.d.ts +0 -0
- package/lib/mesh-circuit-breaker.js +0 -0
- package/lib/mission-loop/lesson-extractor.ts +0 -0
- package/lib/mission-loop/mental-model-scorer.ts +0 -0
- package/lib/mission-loop/occ-detector.ts +0 -0
- package/lib/mission-loop/query-variants.ts +0 -0
- package/lib/mission-loop/verifier-check.ts +0 -0
- package/lib/skill-reference-builder.ts +0 -0
- package/lib/telemetry/token-breakdown.ts +0 -0
- package/lib/tool-compressor.ts +0 -0
- package/lib/triage-core/legal-routing.ts +0 -0
- package/lib/triage-core/mental-model/dunbar-classifier.ts +0 -0
- package/lib/triage-core/mental-model/enrich-all.ts +0 -0
- package/lib/triage-core/mental-model/identity-resolver.ts +0 -0
- package/lib/triage-core/mental-model/key-facts.ts +0 -0
- package/lib/triage-core/mental-model/model-assembler.ts +0 -0
- package/lib/triage-core/orchestrator.ts +0 -0
- package/lib/triage-core/orchestrator.ts.bak-r005-r006-r008 +0 -0
- package/package.json +10 -4
- package/skills/helios-business-operator/services/signals/upwork-signals.js +0 -0
- package/skills/talisman-ceo/SKILL.md +23 -25
- package/skills/talisman-comms/SKILL.md +5 -5
- package/skills/talisman-engineering/SKILL.md +5 -5
- package/skills/talisman-finance/SKILL.md +10 -8
- package/skills/talisman-marketing/SKILL.md +10 -10
- package/skills/talisman-sales/SKILL.md +12 -15
- package/skills/talisman-support/SKILL.md +5 -5
- package/agents/business/talisman-ceo.md +0 -183
- package/agents/business/talisman-comms.md +0 -257
- package/agents/business/talisman-cto.md +0 -153
- package/agents/business/talisman-finance.md +0 -246
- package/agents/business/talisman-marketing.md +0 -240
- package/agents/business/talisman-sales.md +0 -242
- package/agents/business/talisman-support.md +0 -236
- package/daemon/lib/approval-expiry.js +0 -162
- package/daemon/lib/blast-radius-analyzer.js +0 -75
- package/daemon/lib/domain-bootstrap-orchestrator.js +0 -267
- package/daemon/lib/forensic-log.js +0 -113
- package/daemon/lib/goal-research-pipeline.js +0 -644
- package/daemon/lib/harada/cascade-research-dispatcher.js +0 -261
- package/daemon/lib/headroom-middleware.js +0 -167
- package/daemon/lib/headroom-proxy-manager.js +0 -623
- package/daemon/lib/hed-engine.js +0 -307
- package/daemon/lib/mental-model-cache.js +0 -96
- package/daemon/lib/project-factory.js +0 -47
- package/daemon/lib/session-log-reader.js +0 -93
- package/daemon/routes/hed.js +0 -133
- package/lib/graph/learning/headroom-learn-bridge.js +0 -215
- package/skills/helios-bookkeeping/SKILL.md +0 -321
- package/skills/helios-briefer/SKILL.md +0 -44
- package/skills/helios-client-relations/SKILL.md +0 -322
- package/skills/helios-personal-triager/SKILL.md +0 -45
- package/skills/helios-recruitment/SKILL.md +0 -317
- package/skills/helios-relationship-nudger/SKILL.md +0 -77
- package/skills/helios-researcher/SKILL.md +0 -44
- package/skills/helios-scheduler/SKILL.md +0 -58
- package/skills/helios-tax-analyst/SKILL.md +0 -280
|
@@ -1,261 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
/**
|
|
3
|
-
* cascade-research-dispatcher.js — Dispatches L2 and L3 research tasks
|
|
4
|
-
* for the Hoshin Kanri cascade with full parent context in the task body.
|
|
5
|
-
*
|
|
6
|
-
* L2: Dept head agent receives company goal + GoalResearchBrief + pillar domain
|
|
7
|
-
* L3: Sub-agent receives pillar L2 content + action cell description
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
class CascadeResearchDispatcher {
|
|
11
|
-
/**
|
|
12
|
-
* @param {Function} mgQuery - (cypher, params) => Promise<result>
|
|
13
|
-
* @param {string} companyId - Company identifier
|
|
14
|
-
* @param {Function} [log] - Optional logger function
|
|
15
|
-
*/
|
|
16
|
-
constructor(mgQuery, companyId, log) {
|
|
17
|
-
this._mg = mgQuery;
|
|
18
|
-
this._companyId = companyId;
|
|
19
|
-
this._log = log || (() => {});
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
// ── L2 Research Dispatch ──────────────────────────────────────────────────
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* Dispatch a harada_l2_research task to the dept head agent for a pillar.
|
|
26
|
-
*
|
|
27
|
-
* @param {string} pillarId - GoalPillar node id
|
|
28
|
-
* @param {string} goalId - CompanyGoal node id
|
|
29
|
-
* @param {string} companyId - Company identifier (overrides constructor value if provided)
|
|
30
|
-
* @param {string} agentId - Assignee agent id
|
|
31
|
-
* @param {string} [l2ReviewCritique] - Previous CEO critique (for revision cycles)
|
|
32
|
-
* @returns {{ taskId: string, agentId: string }}
|
|
33
|
-
*/
|
|
34
|
-
async dispatchL2Research(pillarId, goalId, companyId, agentId, l2ReviewCritique) {
|
|
35
|
-
const cid = companyId || this._companyId;
|
|
36
|
-
|
|
37
|
-
// 1. Fetch GoalPillar name, GoalResearchBrief, and CompanyGoal
|
|
38
|
-
const contextRows = await this._mg(
|
|
39
|
-
`MATCH (p:GoalPillar {id: $pillarId})
|
|
40
|
-
OPTIONAL MATCH (g:CompanyGoal {id: $goalId})
|
|
41
|
-
OPTIONAL MATCH (grb:GoalResearchBrief {companyId: $cid})
|
|
42
|
-
WHERE grb.goalId = $goalId OR grb.id STARTS WITH ('grb:' + $goalId)
|
|
43
|
-
RETURN
|
|
44
|
-
p.name AS pillarName,
|
|
45
|
-
g.title AS goalTitle,
|
|
46
|
-
g.description AS goalDescription,
|
|
47
|
-
grb.tournamentWinner AS tournamentWinner,
|
|
48
|
-
grb.marketContext AS marketContext,
|
|
49
|
-
grb.benchmarks AS benchmarks,
|
|
50
|
-
grb.crmContext AS crmContext
|
|
51
|
-
LIMIT 1`,
|
|
52
|
-
{ pillarId, goalId, cid }
|
|
53
|
-
);
|
|
54
|
-
|
|
55
|
-
const row = contextRows && contextRows.rows && contextRows.rows[0]
|
|
56
|
-
? (Array.isArray(contextRows.rows[0])
|
|
57
|
-
? {
|
|
58
|
-
pillarName: contextRows.rows[0][0],
|
|
59
|
-
goalTitle: contextRows.rows[0][1],
|
|
60
|
-
goalDescription: contextRows.rows[0][2],
|
|
61
|
-
tournamentWinner: contextRows.rows[0][3],
|
|
62
|
-
marketContext: contextRows.rows[0][4],
|
|
63
|
-
benchmarks: contextRows.rows[0][5],
|
|
64
|
-
crmContext: contextRows.rows[0][6],
|
|
65
|
-
}
|
|
66
|
-
: contextRows.rows[0])
|
|
67
|
-
: {};
|
|
68
|
-
|
|
69
|
-
const pillarName = row.pillarName || pillarId;
|
|
70
|
-
const goalTitle = row.goalTitle || 'Company Goal';
|
|
71
|
-
const goalDescription = row.goalDescription || '';
|
|
72
|
-
const tournamentWinner = row.tournamentWinner || 'TBD';
|
|
73
|
-
const marketContext = row.marketContext || 'Not available';
|
|
74
|
-
const benchmarks = row.benchmarks || 'Not available';
|
|
75
|
-
const crmContext = row.crmContext || 'Not available';
|
|
76
|
-
|
|
77
|
-
// 2. Build task body
|
|
78
|
-
const previousCritiqueSection = l2ReviewCritique
|
|
79
|
-
? `\nPrevious CEO critique: ${l2ReviewCritique}`
|
|
80
|
-
: '';
|
|
81
|
-
|
|
82
|
-
const taskBody = `Goal: ${goalTitle}
|
|
83
|
-
Description: ${goalDescription}
|
|
84
|
-
Research Brief Summary: ${marketContext} | Tournament winner: ${tournamentWinner}
|
|
85
|
-
Benchmarks: ${benchmarks}
|
|
86
|
-
CRM context: ${crmContext}
|
|
87
|
-
Your pillar: ${pillarName} (id: ${pillarId})
|
|
88
|
-
Your role: ${pillarName} execution lead
|
|
89
|
-
|
|
90
|
-
Research your pillar domain using web search and your available tools.
|
|
91
|
-
Produce:
|
|
92
|
-
1. A 50-word purpose statement for this pillar
|
|
93
|
-
2. A strategy label (≤20 chars) matching or improving on the tournament winner
|
|
94
|
-
3. 3 key actions this pillar should execute
|
|
95
|
-
|
|
96
|
-
When complete, update pillar content by calling:
|
|
97
|
-
POST /api/hbo/pillar/${pillarId}/l2content
|
|
98
|
-
Body: { content: string, strategy: string, keyActions: string[] }
|
|
99
|
-
${previousCritiqueSection}`;
|
|
100
|
-
|
|
101
|
-
const taskId = `harada-l2:${pillarId}`;
|
|
102
|
-
|
|
103
|
-
// 3. MERGE Task
|
|
104
|
-
await this._mg(
|
|
105
|
-
`MERGE (t:Task {id: $taskId, companyId: $cid})
|
|
106
|
-
ON CREATE SET
|
|
107
|
-
t.originKind = $originKind,
|
|
108
|
-
t.assigneeAgentId = $agentId,
|
|
109
|
-
t.status = 'todo',
|
|
110
|
-
t.priority = 1,
|
|
111
|
-
t.body = $body,
|
|
112
|
-
t.createdAt = datetime()
|
|
113
|
-
ON MATCH SET
|
|
114
|
-
t.body = $body,
|
|
115
|
-
t.status = 'todo',
|
|
116
|
-
t.updatedAt = datetime()`,
|
|
117
|
-
{
|
|
118
|
-
taskId,
|
|
119
|
-
cid,
|
|
120
|
-
originKind: 'harada_l2_research',
|
|
121
|
-
agentId,
|
|
122
|
-
body: taskBody,
|
|
123
|
-
}
|
|
124
|
-
);
|
|
125
|
-
|
|
126
|
-
// 4. Create AgentReadySignal
|
|
127
|
-
const arsId = `ars:harada-l2:${pillarId}`;
|
|
128
|
-
await this._mg(
|
|
129
|
-
`MERGE (s:AgentReadySignal {id: $arsId, companyId: $cid})
|
|
130
|
-
ON CREATE SET
|
|
131
|
-
s.agentId = $agentId,
|
|
132
|
-
s.status = 'pending',
|
|
133
|
-
s.createdAt = datetime()
|
|
134
|
-
ON MATCH SET
|
|
135
|
-
s.status = 'pending',
|
|
136
|
-
s.updatedAt = datetime()`,
|
|
137
|
-
{ arsId, cid, agentId }
|
|
138
|
-
);
|
|
139
|
-
|
|
140
|
-
this._log(`[CascadeResearchDispatcher] Dispatched L2 research task ${taskId} to agent ${agentId}`);
|
|
141
|
-
|
|
142
|
-
// 5. Return
|
|
143
|
-
return { taskId, agentId };
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
// ── L3 Research Dispatch ──────────────────────────────────────────────────
|
|
147
|
-
|
|
148
|
-
/**
|
|
149
|
-
* Dispatch a harada_l3_research task to a sub-agent for an action cell.
|
|
150
|
-
*
|
|
151
|
-
* @param {string} cellId - ActionCell node id
|
|
152
|
-
* @param {string} pillarId - Parent GoalPillar node id
|
|
153
|
-
* @param {string} companyId - Company identifier (overrides constructor value if provided)
|
|
154
|
-
* @param {string} agentId - Assignee agent id
|
|
155
|
-
* @param {string} [l3ReviewCritique] - Previous dept head critique (for revision cycles)
|
|
156
|
-
* @returns {{ taskId: string, agentId: string }}
|
|
157
|
-
*/
|
|
158
|
-
async dispatchL3Research(cellId, pillarId, companyId, agentId, l3ReviewCritique) {
|
|
159
|
-
const cid = companyId || this._companyId;
|
|
160
|
-
|
|
161
|
-
// 1. Fetch ActionCell description, GoalPillar l2Content, l2Strategy, pillar name
|
|
162
|
-
// and goal info via the pillar
|
|
163
|
-
const contextRows = await this._mg(
|
|
164
|
-
`MATCH (p:GoalPillar {id: $pillarId})
|
|
165
|
-
OPTIONAL MATCH (c:ActionCell {id: $cellId})
|
|
166
|
-
OPTIONAL MATCH (g:CompanyGoal)-[:HAS_PILLAR]->(p)
|
|
167
|
-
RETURN
|
|
168
|
-
c.description AS cellDescription,
|
|
169
|
-
p.name AS pillarName,
|
|
170
|
-
p.l2Content AS l2Content,
|
|
171
|
-
p.l2Strategy AS l2Strategy,
|
|
172
|
-
g.title AS goalTitle
|
|
173
|
-
LIMIT 1`,
|
|
174
|
-
{ cellId, pillarId }
|
|
175
|
-
);
|
|
176
|
-
|
|
177
|
-
const row = contextRows && contextRows.rows && contextRows.rows[0]
|
|
178
|
-
? (Array.isArray(contextRows.rows[0])
|
|
179
|
-
? {
|
|
180
|
-
cellDescription: contextRows.rows[0][0],
|
|
181
|
-
pillarName: contextRows.rows[0][1],
|
|
182
|
-
l2Content: contextRows.rows[0][2],
|
|
183
|
-
l2Strategy: contextRows.rows[0][3],
|
|
184
|
-
goalTitle: contextRows.rows[0][4],
|
|
185
|
-
}
|
|
186
|
-
: contextRows.rows[0])
|
|
187
|
-
: {};
|
|
188
|
-
|
|
189
|
-
const cellDescription = row.cellDescription || cellId;
|
|
190
|
-
const pillarName = row.pillarName || pillarId;
|
|
191
|
-
const l2Content = row.l2Content || '';
|
|
192
|
-
const l2Strategy = row.l2Strategy || '';
|
|
193
|
-
const goalTitle = row.goalTitle || 'Company Goal';
|
|
194
|
-
|
|
195
|
-
// 2. Build task body
|
|
196
|
-
const previousCritiqueSection = l3ReviewCritique
|
|
197
|
-
? `\nPrevious dept head critique: ${l3ReviewCritique}`
|
|
198
|
-
: '';
|
|
199
|
-
|
|
200
|
-
const taskBody = `Goal: ${goalTitle}
|
|
201
|
-
Pillar: ${pillarName}
|
|
202
|
-
Pillar strategy: ${l2Strategy}
|
|
203
|
-
Pillar purpose: ${l2Content}
|
|
204
|
-
|
|
205
|
-
Your action cell: ${cellDescription} (id: ${cellId})
|
|
206
|
-
|
|
207
|
-
Research this specific action cell using available tools.
|
|
208
|
-
Produce a specific execution plan (≤100 words) for this action.
|
|
209
|
-
|
|
210
|
-
When complete, update by calling:
|
|
211
|
-
POST /api/hbo/actioncell/${cellId}/l3content
|
|
212
|
-
Body: { content: string }
|
|
213
|
-
${previousCritiqueSection}`;
|
|
214
|
-
|
|
215
|
-
const taskId = `harada-l3:${cellId}`;
|
|
216
|
-
|
|
217
|
-
// 3. MERGE Task
|
|
218
|
-
await this._mg(
|
|
219
|
-
`MERGE (t:Task {id: $taskId, companyId: $cid})
|
|
220
|
-
ON CREATE SET
|
|
221
|
-
t.originKind = $originKind,
|
|
222
|
-
t.assigneeAgentId = $agentId,
|
|
223
|
-
t.status = 'todo',
|
|
224
|
-
t.priority = 1,
|
|
225
|
-
t.body = $body,
|
|
226
|
-
t.createdAt = datetime()
|
|
227
|
-
ON MATCH SET
|
|
228
|
-
t.body = $body,
|
|
229
|
-
t.status = 'todo',
|
|
230
|
-
t.updatedAt = datetime()`,
|
|
231
|
-
{
|
|
232
|
-
taskId,
|
|
233
|
-
cid,
|
|
234
|
-
originKind: 'harada_l3_research',
|
|
235
|
-
agentId,
|
|
236
|
-
body: taskBody,
|
|
237
|
-
}
|
|
238
|
-
);
|
|
239
|
-
|
|
240
|
-
// 4. Create AgentReadySignal
|
|
241
|
-
const arsId = `ars:harada-l3:${cellId}`;
|
|
242
|
-
await this._mg(
|
|
243
|
-
`MERGE (s:AgentReadySignal {id: $arsId, companyId: $cid})
|
|
244
|
-
ON CREATE SET
|
|
245
|
-
s.agentId = $agentId,
|
|
246
|
-
s.status = 'pending',
|
|
247
|
-
s.createdAt = datetime()
|
|
248
|
-
ON MATCH SET
|
|
249
|
-
s.status = 'pending',
|
|
250
|
-
s.updatedAt = datetime()`,
|
|
251
|
-
{ arsId, cid, agentId }
|
|
252
|
-
);
|
|
253
|
-
|
|
254
|
-
this._log(`[CascadeResearchDispatcher] Dispatched L3 research task ${taskId} to agent ${agentId}`);
|
|
255
|
-
|
|
256
|
-
// 5. Return
|
|
257
|
-
return { taskId, agentId };
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
module.exports = { CascadeResearchDispatcher };
|
|
@@ -1,167 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* headroom-middleware.js
|
|
5
|
-
*
|
|
6
|
-
* Express-compatible middleware that compresses large JSON array responses
|
|
7
|
-
* from HBO API routes through the Headroom SmartCrusher before they are sent
|
|
8
|
-
* to the requesting agent or desktop client.
|
|
9
|
-
*
|
|
10
|
-
* Applied to all HBO routes that return arrays:
|
|
11
|
-
* signals, leads, tasks, goals, pipeline, activity-feed, agents, etc.
|
|
12
|
-
*
|
|
13
|
-
* Why this matters for all companies:
|
|
14
|
-
* A company tracking 500 leads produces ~80K tokens of JSON per pipeline call.
|
|
15
|
-
* SmartCrusher reduces that to ~8K tokens (90% reduction) while preserving
|
|
16
|
-
* statistical distribution and keeping all error/anomaly rows unconditionally.
|
|
17
|
-
* The original data is stored in CCR (Compress-Cache-Retrieve) and retrievable
|
|
18
|
-
* via GET /api/hbo/retrieve/:hash when a worker needs full fidelity.
|
|
19
|
-
*
|
|
20
|
-
* CCR hash: returned in X-Headroom-CCR response header.
|
|
21
|
-
* Workers call GET /api/hbo/retrieve/:hash to get the full original.
|
|
22
|
-
*
|
|
23
|
-
* Threshold: only compress arrays of ≥5 items or objects with array values ≥5.
|
|
24
|
-
* Smaller payloads have overhead that exceeds savings.
|
|
25
|
-
*/
|
|
26
|
-
|
|
27
|
-
const http = require('http');
|
|
28
|
-
|
|
29
|
-
const { HeadroomProxyManager } = require('./headroom-proxy-manager');
|
|
30
|
-
|
|
31
|
-
// Minimum array size to bother compressing
|
|
32
|
-
const MIN_ARRAY_LENGTH = 5;
|
|
33
|
-
|
|
34
|
-
/**
|
|
35
|
-
* Test whether a JSON payload warrants compression.
|
|
36
|
-
* Returns true for arrays of 5+ items, or objects containing arrays of 5+ items.
|
|
37
|
-
*/
|
|
38
|
-
function isLargeArrayPayload(data) {
|
|
39
|
-
if (Array.isArray(data) && data.length >= MIN_ARRAY_LENGTH) return true;
|
|
40
|
-
if (data && typeof data === 'object' && !Array.isArray(data)) {
|
|
41
|
-
for (const val of Object.values(data)) {
|
|
42
|
-
if (Array.isArray(val) && val.length >= MIN_ARRAY_LENGTH) return true;
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
return false;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
/**
|
|
49
|
-
* Compress a JSON payload through the Headroom proxy.
|
|
50
|
-
* Uses the Anthropic /v1/messages endpoint format that the proxy intercepts.
|
|
51
|
-
* Returns { compressed, ccrHashes, tokensSaved } or throws.
|
|
52
|
-
*/
|
|
53
|
-
async function compressPayload(data) {
|
|
54
|
-
const baseUrl = HeadroomProxyManager.getInstance().getBaseUrl();
|
|
55
|
-
if (!baseUrl) throw new Error('Headroom proxy not running');
|
|
56
|
-
|
|
57
|
-
const { compress } = require('headroom-ai');
|
|
58
|
-
const result = await compress(
|
|
59
|
-
[{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'hbo_response', content: JSON.stringify(data) }] }],
|
|
60
|
-
{
|
|
61
|
-
model: 'claude',
|
|
62
|
-
baseUrl,
|
|
63
|
-
}
|
|
64
|
-
);
|
|
65
|
-
|
|
66
|
-
let compressed = data;
|
|
67
|
-
try {
|
|
68
|
-
const content = result?.messages?.[0]?.content;
|
|
69
|
-
if (content) {
|
|
70
|
-
const text = typeof content === 'string' ? content
|
|
71
|
-
: Array.isArray(content) ? (content[0]?.text ?? content[0]?.content ?? JSON.stringify(data))
|
|
72
|
-
: JSON.stringify(data);
|
|
73
|
-
const parsed = JSON.parse(text);
|
|
74
|
-
if (parsed !== null && parsed !== undefined) compressed = parsed;
|
|
75
|
-
}
|
|
76
|
-
} catch (_) {
|
|
77
|
-
// If we can't parse the compressed output, return original data
|
|
78
|
-
compressed = data;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
return {
|
|
82
|
-
compressed,
|
|
83
|
-
ccrHashes: result?.ccrHashes ?? [],
|
|
84
|
-
tokensSaved: result?.tokensSaved ?? 0,
|
|
85
|
-
};
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
/**
|
|
89
|
-
* Express middleware factory.
|
|
90
|
-
* Wraps res.json() to compress large array payloads before sending.
|
|
91
|
-
* Set-and-forget: if compression fails for any reason the original is sent.
|
|
92
|
-
*/
|
|
93
|
-
function headroomResponseMiddleware() {
|
|
94
|
-
return async function(req, res, next) {
|
|
95
|
-
// Skip if proxy not running
|
|
96
|
-
if (!HeadroomProxyManager.getInstance().getBaseUrl()) return next();
|
|
97
|
-
|
|
98
|
-
const originalJson = res.json.bind(res);
|
|
99
|
-
|
|
100
|
-
res.json = async function(data) {
|
|
101
|
-
// Only compress large array payloads
|
|
102
|
-
if (!isLargeArrayPayload(data)) return originalJson(data);
|
|
103
|
-
|
|
104
|
-
try {
|
|
105
|
-
const { compressed, ccrHashes, tokensSaved } = await compressPayload(data);
|
|
106
|
-
|
|
107
|
-
// Attach CCR hashes in response headers so callers can retrieve originals
|
|
108
|
-
if (ccrHashes.length > 0) {
|
|
109
|
-
res.setHeader('X-Headroom-CCR', ccrHashes.join(','));
|
|
110
|
-
}
|
|
111
|
-
if (tokensSaved > 0) {
|
|
112
|
-
res.setHeader('X-Headroom-Saved', String(tokensSaved));
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
return originalJson(compressed);
|
|
116
|
-
} catch (err) {
|
|
117
|
-
// Never block a response — if compression fails, return original
|
|
118
|
-
process.stderr.write(`[headroom-middleware] compression failed (sending original): ${err.message}\n`);
|
|
119
|
-
return originalJson(data);
|
|
120
|
-
}
|
|
121
|
-
};
|
|
122
|
-
|
|
123
|
-
next();
|
|
124
|
-
};
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
/**
|
|
128
|
-
* CCR retrieve handler for use as an Express route.
|
|
129
|
-
* GET /api/hbo/retrieve/:hash
|
|
130
|
-
* Proxies to the Headroom proxy's /v1/retrieve/:hash endpoint.
|
|
131
|
-
*/
|
|
132
|
-
async function handleCcrRetrieve(req, res) {
|
|
133
|
-
const hash = req.params.hash;
|
|
134
|
-
if (!hash || !/^[a-f0-9]{8,64}$/.test(hash)) {
|
|
135
|
-
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
136
|
-
res.end(JSON.stringify({ error: 'Invalid hash format' }));
|
|
137
|
-
return;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
const baseUrl = HeadroomProxyManager.getInstance().getBaseUrl();
|
|
141
|
-
if (!baseUrl) {
|
|
142
|
-
res.writeHead(503, { 'Content-Type': 'application/json' });
|
|
143
|
-
res.end(JSON.stringify({ error: 'Headroom proxy not running' }));
|
|
144
|
-
return;
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
const url = new URL(baseUrl);
|
|
148
|
-
const proxyReq = http.request(
|
|
149
|
-
{
|
|
150
|
-
hostname: url.hostname,
|
|
151
|
-
port: parseInt(url.port || '8787', 10),
|
|
152
|
-
path: `/v1/retrieve/${hash}`,
|
|
153
|
-
method: 'GET',
|
|
154
|
-
},
|
|
155
|
-
(proxyRes) => {
|
|
156
|
-
res.writeHead(proxyRes.statusCode, proxyRes.headers);
|
|
157
|
-
proxyRes.pipe(res);
|
|
158
|
-
}
|
|
159
|
-
);
|
|
160
|
-
proxyReq.on('error', (err) => {
|
|
161
|
-
res.writeHead(502, { 'Content-Type': 'application/json' });
|
|
162
|
-
res.end(JSON.stringify({ error: `Proxy error: ${err.message}` }));
|
|
163
|
-
});
|
|
164
|
-
proxyReq.end();
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
module.exports = { headroomResponseMiddleware, handleCcrRetrieve, isLargeArrayPayload };
|