@stackbilt/aegis-core 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/package.json +96 -0
  2. package/schema.sql +586 -0
  3. package/src/adapters/voice/cloudflare-agent.ts +34 -0
  4. package/src/auth.ts +124 -0
  5. package/src/bluesky.ts +464 -0
  6. package/src/claude-tools/content.ts +188 -0
  7. package/src/claude-tools/email.ts +69 -0
  8. package/src/claude-tools/github.ts +440 -0
  9. package/src/claude-tools/goals.ts +116 -0
  10. package/src/claude-tools/index.ts +353 -0
  11. package/src/claude-tools/web.ts +59 -0
  12. package/src/claude.ts +406 -0
  13. package/src/codebeast.ts +200 -0
  14. package/src/composite.ts +715 -0
  15. package/src/content/column.ts +80 -0
  16. package/src/content/hero-image.ts +47 -0
  17. package/src/content/index.ts +27 -0
  18. package/src/content/journal.ts +91 -0
  19. package/src/content/roundtable.ts +163 -0
  20. package/src/core.ts +309 -0
  21. package/src/dashboard.ts +620 -0
  22. package/src/decision-docs.ts +284 -0
  23. package/src/dispatch.ts +13 -0
  24. package/src/edge-env.ts +58 -0
  25. package/src/email.ts +850 -0
  26. package/src/exports.ts +156 -0
  27. package/src/github-projects.ts +312 -0
  28. package/src/github.ts +670 -0
  29. package/src/groq.ts +247 -0
  30. package/src/health-page.ts +578 -0
  31. package/src/index.ts +89 -0
  32. package/src/kernel/argus-actions.ts +397 -0
  33. package/src/kernel/argus-correlation.ts +639 -0
  34. package/src/kernel/board.ts +91 -0
  35. package/src/kernel/briefing.ts +177 -0
  36. package/src/kernel/classify-memory-topic.ts +166 -0
  37. package/src/kernel/cognition.ts +377 -0
  38. package/src/kernel/court-cards.ts +163 -0
  39. package/src/kernel/dispatch.ts +587 -0
  40. package/src/kernel/domain.ts +50 -0
  41. package/src/kernel/dynamic-tools.ts +322 -0
  42. package/src/kernel/executor-port.ts +45 -0
  43. package/src/kernel/executors/claude.ts +73 -0
  44. package/src/kernel/executors/direct.ts +237 -0
  45. package/src/kernel/executors/groq.ts +18 -0
  46. package/src/kernel/executors/index.ts +87 -0
  47. package/src/kernel/executors/tarotscript.ts +104 -0
  48. package/src/kernel/executors/workers-ai.ts +54 -0
  49. package/src/kernel/insight-cache.ts +76 -0
  50. package/src/kernel/memory/agenda.ts +200 -0
  51. package/src/kernel/memory/blocks.ts +188 -0
  52. package/src/kernel/memory/consolidation.ts +194 -0
  53. package/src/kernel/memory/episodic.ts +241 -0
  54. package/src/kernel/memory/goals.ts +156 -0
  55. package/src/kernel/memory/graph.ts +290 -0
  56. package/src/kernel/memory/index.ts +11 -0
  57. package/src/kernel/memory/insights.ts +316 -0
  58. package/src/kernel/memory/procedural.ts +467 -0
  59. package/src/kernel/memory/pruning.ts +67 -0
  60. package/src/kernel/memory/recall.ts +367 -0
  61. package/src/kernel/memory/semantic.ts +315 -0
  62. package/src/kernel/memory/synthesis.ts +161 -0
  63. package/src/kernel/memory-adapter.ts +369 -0
  64. package/src/kernel/memory-guardrails.ts +76 -0
  65. package/src/kernel/port.ts +23 -0
  66. package/src/kernel/resilience.ts +322 -0
  67. package/src/kernel/router.ts +471 -0
  68. package/src/kernel/scheduled/agent-dispatch.ts +252 -0
  69. package/src/kernel/scheduled/argus-analytics.ts +247 -0
  70. package/src/kernel/scheduled/argus-heartbeat.ts +320 -0
  71. package/src/kernel/scheduled/argus-notify.ts +348 -0
  72. package/src/kernel/scheduled/board-sync.ts +110 -0
  73. package/src/kernel/scheduled/ci-watcher.ts +125 -0
  74. package/src/kernel/scheduled/cognitive-metrics.ts +377 -0
  75. package/src/kernel/scheduled/consolidation.ts +229 -0
  76. package/src/kernel/scheduled/content-drip.ts +47 -0
  77. package/src/kernel/scheduled/content.ts +6 -0
  78. package/src/kernel/scheduled/conversation-facts.ts +204 -0
  79. package/src/kernel/scheduled/cost-report.ts +84 -0
  80. package/src/kernel/scheduled/curiosity.ts +219 -0
  81. package/src/kernel/scheduled/dev-activity.ts +44 -0
  82. package/src/kernel/scheduled/digest.ts +317 -0
  83. package/src/kernel/scheduled/dreaming/agenda-triage.ts +115 -0
  84. package/src/kernel/scheduled/dreaming/facts.ts +239 -0
  85. package/src/kernel/scheduled/dreaming/index.ts +8 -0
  86. package/src/kernel/scheduled/dreaming/llm.ts +33 -0
  87. package/src/kernel/scheduled/dreaming/pattern-synthesis.ts +124 -0
  88. package/src/kernel/scheduled/dreaming/persona.ts +75 -0
  89. package/src/kernel/scheduled/dreaming/symbolic.ts +31 -0
  90. package/src/kernel/scheduled/dreaming/task-proposals.ts +80 -0
  91. package/src/kernel/scheduled/dreaming.ts +66 -0
  92. package/src/kernel/scheduled/entropy.ts +149 -0
  93. package/src/kernel/scheduled/escalation.ts +192 -0
  94. package/src/kernel/scheduled/feed-watcher.ts +206 -0
  95. package/src/kernel/scheduled/goals.ts +214 -0
  96. package/src/kernel/scheduled/governance.ts +41 -0
  97. package/src/kernel/scheduled/heartbeat.ts +220 -0
  98. package/src/kernel/scheduled/inbox-processor.ts +174 -0
  99. package/src/kernel/scheduled/index.ts +245 -0
  100. package/src/kernel/scheduled/issue-proposer.ts +478 -0
  101. package/src/kernel/scheduled/issue-watcher.ts +128 -0
  102. package/src/kernel/scheduled/pr-automerge.ts +213 -0
  103. package/src/kernel/scheduled/product-health.ts +107 -0
  104. package/src/kernel/scheduled/reflection.ts +373 -0
  105. package/src/kernel/scheduled/self-improvement.ts +114 -0
  106. package/src/kernel/scheduled/social-engage.ts +175 -0
  107. package/src/kernel/scheduled/task-audit.ts +60 -0
  108. package/src/kernel/symbolic.ts +156 -0
  109. package/src/kernel/types.ts +145 -0
  110. package/src/landing.ts +1190 -0
  111. package/src/lib/audit-chain/chain.ts +28 -0
  112. package/src/lib/audit-chain/types.ts +12 -0
  113. package/src/lib/observability/errors.ts +55 -0
  114. package/src/markdown.ts +164 -0
  115. package/src/mcp/handlers.ts +647 -0
  116. package/src/mcp/server.ts +184 -0
  117. package/src/mcp/tools.ts +316 -0
  118. package/src/mcp-client.ts +275 -0
  119. package/src/mcp-server.ts +2 -0
  120. package/src/operator/config.example.ts +60 -0
  121. package/src/operator/config.ts +60 -0
  122. package/src/operator/index.ts +46 -0
  123. package/src/operator/persona.example.ts +34 -0
  124. package/src/operator/persona.ts +34 -0
  125. package/src/operator/prompt-builder.ts +190 -0
  126. package/src/operator/types.ts +43 -0
  127. package/src/pulse.ts +1179 -0
  128. package/src/routes/bluesky.ts +116 -0
  129. package/src/routes/cc-tasks.ts +328 -0
  130. package/src/routes/codebeast.ts +1 -0
  131. package/src/routes/content.ts +194 -0
  132. package/src/routes/conversations.ts +25 -0
  133. package/src/routes/dynamic-tools.ts +111 -0
  134. package/src/routes/feedback.ts +192 -0
  135. package/src/routes/health.ts +147 -0
  136. package/src/routes/messages.ts +228 -0
  137. package/src/routes/observability.ts +82 -0
  138. package/src/routes/operator-logs.ts +42 -0
  139. package/src/routes/pages.ts +96 -0
  140. package/src/routes/sessions.ts +54 -0
  141. package/src/sanitize.ts +73 -0
  142. package/src/schema-enums.ts +155 -0
  143. package/src/search.ts +112 -0
  144. package/src/task-intelligence.ts +497 -0
  145. package/src/types.ts +194 -0
  146. package/src/ui.ts +5 -0
  147. package/src/version.ts +3 -0
  148. package/src/workers-ai-chat.ts +333 -0
@@ -0,0 +1,367 @@
1
+ // ─── Recall Pipeline: Blocks → Graph → Memory Worker → RRF Fusion ────
2
+ //
3
+ // Phase 2 of the unified graph engine. Replaces parallel memory lookups
4
+ // with a coordinated pipeline where each layer's output enriches the next.
5
+ //
6
+ // Cost budget: ~130-260ms total, $0 LLM.
7
+ // Design: research/2026-03-13-unified-graph-spike.md
8
+
9
+ import { activateGraph } from './graph.js';
10
+ import { getAllBlocks } from './blocks.js';
11
+ import type { MemoryServiceBinding, MemoryFragmentResult } from '../../types.js';
12
+
13
+ // ─── Types ───────────────────────────────────────────────────
14
+
15
+ export interface RecallResult {
16
+ facts: RecalledFact[];
17
+ graphExpansions: string[]; // labels from graph activation
18
+ blockContext: string[]; // active project names from blocks
19
+ mindspringHits: number; // count of MindSpring conversation matches
20
+ timing: {
21
+ blocks_ms: number;
22
+ graph_ms: number;
23
+ memory_ms: number;
24
+ mindspring_ms: number;
25
+ parallel_ms: number; // wall-clock for Memory Worker + MindSpring combined
26
+ fusion_ms: number;
27
+ total_ms: number;
28
+ };
29
+ }
30
+
31
+ export interface RecalledFact {
32
+ id: string;
33
+ text: string;
34
+ score: number; // fused score
35
+ semantic_score: number; // from Memory Worker
36
+ graph_score: number; // from activation (0 if no graph match)
37
+ source: 'memory_worker' | 'graph' | 'mindspring' | 'both';
38
+ }
39
+
40
+ // ─── MindSpring search result ────────────────────────────────
41
+ interface MindSpringResult {
42
+ id: string;
43
+ title: string;
44
+ text: string;
45
+ score: number;
46
+ }
47
+
48
+ // ─── RRF (Reciprocal Rank Fusion) ────────────────────────────
49
+
50
+ export function rrf(ranks: Map<string, number[]>, k = 60): Map<string, number> {
51
+ const scores = new Map<string, number>();
52
+ for (const [id, rankList] of ranks) {
53
+ let score = 0;
54
+ for (const rank of rankList) {
55
+ score += 1 / (k + rank);
56
+ }
57
+ scores.set(id, score);
58
+ }
59
+ return scores;
60
+ }
61
+
62
+ // ─── Pipeline Orchestrator ───────────────────────────────────
63
+
64
+ const TENANT = 'aegis';
65
+ const DEFAULT_MAX_RESULTS = 15;
66
+
67
+ export async function recallForQuery(
68
+ query: string,
69
+ env: { db: D1Database; memoryBinding?: MemoryServiceBinding; mindspringFetcher?: Fetcher; mindspringToken?: string },
70
+ options?: { maxResults?: number; includeGraph?: boolean },
71
+ ): Promise<RecallResult> {
72
+ const totalStart = Date.now();
73
+ const maxResults = options?.maxResults ?? DEFAULT_MAX_RESULTS;
74
+ const includeGraph = options?.includeGraph ?? true;
75
+
76
+ // ── Stage 1: Blocks (frame) ──────────────────────────────
77
+ const blocksStart = Date.now();
78
+ const blockContext = await loadBlockContext(env.db);
79
+ const blocks_ms = Date.now() - blocksStart;
80
+
81
+ // ── Stage 2: Graph (structure) ───────────────────────────
82
+ let graphExpansions: string[] = [];
83
+ let activatedNodeMap = new Map<string, number>(); // label → activation score
84
+ let nodeMemoryIds = new Map<string, string[]>(); // label → memory_ids from kg_nodes
85
+ const graphStart = Date.now();
86
+
87
+ if (includeGraph) {
88
+ try {
89
+ // Combine query entities with active project names from blocks as seeds
90
+ const seedQuery = blockContext.length > 0
91
+ ? `${query} ${blockContext.join(' ')}`
92
+ : query;
93
+
94
+ const activated = await activateGraph(env.db, seedQuery, 2);
95
+
96
+ for (const node of activated) {
97
+ graphExpansions.push(node.label);
98
+ activatedNodeMap.set(node.label.toLowerCase(), node.activation);
99
+ }
100
+
101
+ // Fetch memory_ids for activated nodes to enable fusion matching
102
+ if (activated.length > 0) {
103
+ nodeMemoryIds = await fetchNodeMemoryIds(env.db, activated.map(n => n.label));
104
+ }
105
+ } catch (err) {
106
+ console.warn('[recall] Graph activation failed:', err instanceof Error ? err.message : String(err));
107
+ }
108
+ }
109
+ const graph_ms = Date.now() - graphStart;
110
+
111
+ // ── Stage 3 + 3.5: Memory Worker + MindSpring (parallel) ──
112
+ // These are independent — both depend on graphExpansions but not each other.
113
+ const parallelStart = Date.now();
114
+ let memoryFragments: MemoryFragmentResult[] = [];
115
+ let mindspringResults: MindSpringResult[] = [];
116
+ let memory_ms = 0;
117
+ let mindspring_ms = 0;
118
+
119
+ const expandedQuery = graphExpansions.length > 0
120
+ ? `${query} ${graphExpansions.join(' ')}`
121
+ : query;
122
+
123
+ const memoryPromise = (async () => {
124
+ if (!env.memoryBinding) return;
125
+ const memoryStart = Date.now();
126
+ try {
127
+ memoryFragments = await env.memoryBinding.recall(TENANT, {
128
+ keywords: expandedQuery,
129
+ limit: maxResults + 10, // fetch extra for fusion headroom
130
+ });
131
+ } catch (err) {
132
+ console.warn('[recall] Memory Worker recall failed:', err instanceof Error ? err.message : String(err));
133
+ }
134
+ memory_ms = Date.now() - memoryStart;
135
+ })();
136
+
137
+ const mindspringPromise = (async () => {
138
+ if (!env.mindspringFetcher || !env.mindspringToken) return;
139
+ const mindspringStart = Date.now();
140
+ try {
141
+ const msQuery = graphExpansions.length > 0
142
+ ? `${query} ${graphExpansions.slice(0, 5).join(' ')}`
143
+ : query;
144
+
145
+ const msResponse = await env.mindspringFetcher.fetch(
146
+ `https://mindspring/api/search?q=${encodeURIComponent(msQuery)}&limit=5&threshold=0.4`,
147
+ { headers: { 'Authorization': `Bearer ${env.mindspringToken}` } },
148
+ );
149
+
150
+ if (msResponse.ok) {
151
+ const msData = await msResponse.json<{ results: MindSpringResult[] }>();
152
+ mindspringResults = msData.results ?? [];
153
+ }
154
+ } catch (err) {
155
+ console.warn('[recall] MindSpring search failed:', err instanceof Error ? err.message : String(err));
156
+ }
157
+ mindspring_ms = Date.now() - mindspringStart;
158
+ })();
159
+
160
+ await Promise.all([memoryPromise, mindspringPromise]);
161
+ const parallel_ms = Date.now() - parallelStart;
162
+
163
+ // Log parallel savings: sequential would be memory_ms + mindspring_ms
164
+ const sequential_ms = memory_ms + mindspring_ms;
165
+ if (sequential_ms > 0) {
166
+ console.log(`[recall] parallel recall: wall=${parallel_ms}ms (memory=${memory_ms}ms + mindspring=${mindspring_ms}ms sequential=${sequential_ms}ms, saved ~${sequential_ms - parallel_ms}ms)`);
167
+ }
168
+
169
+ // ── Stage 4: Fusion (RRF merge) ──────────────────────────
170
+ const fusionStart = Date.now();
171
+ const facts = fuseResults(memoryFragments, mindspringResults, activatedNodeMap, nodeMemoryIds, blockContext, maxResults);
172
+ const fusion_ms = Date.now() - fusionStart;
173
+
174
+ return {
175
+ facts,
176
+ graphExpansions,
177
+ blockContext,
178
+ mindspringHits: mindspringResults.length,
179
+ timing: {
180
+ blocks_ms,
181
+ graph_ms,
182
+ memory_ms,
183
+ mindspring_ms,
184
+ parallel_ms,
185
+ fusion_ms,
186
+ total_ms: Date.now() - totalStart,
187
+ },
188
+ };
189
+ }
190
+
191
+ // ─── Stage 1: Load block context (active project names) ─────
192
+
193
+ async function loadBlockContext(db: D1Database): Promise<string[]> {
194
+ try {
195
+ const blocks = await getAllBlocks(db);
196
+ const projectNames: string[] = [];
197
+
198
+ for (const block of blocks) {
199
+ // Extract project names from active_context block
200
+ if (block.id === 'active_context') {
201
+ // Look for project references in the block content
202
+ const projectMatches = block.content.matchAll(/\*\*([a-z][\w-]*)\*\*/gi);
203
+ for (const match of projectMatches) {
204
+ const name = match[1].toLowerCase();
205
+ if (name.length >= 3 && !projectNames.includes(name)) {
206
+ projectNames.push(name);
207
+ }
208
+ }
209
+ }
210
+ }
211
+
212
+ return projectNames;
213
+ } catch (err) {
214
+ console.warn('[recall] Block context load failed:', err instanceof Error ? err.message : String(err));
215
+ return [];
216
+ }
217
+ }
218
+
219
+ // ─── Fetch memory_ids from kg_nodes for activated nodes ─────
220
+
221
+ async function fetchNodeMemoryIds(
222
+ db: D1Database,
223
+ labels: string[],
224
+ ): Promise<Map<string, string[]>> {
225
+ const result = new Map<string, string[]>();
226
+ if (labels.length === 0) return result;
227
+
228
+ // Batch fetch in chunks of 20
229
+ for (let i = 0; i < labels.length; i += 20) {
230
+ const chunk = labels.slice(i, i + 20);
231
+ const placeholders = chunk.map(() => 'LOWER(?) = LOWER(label)').join(' OR ');
232
+
233
+ try {
234
+ const rows = await db.prepare(
235
+ `SELECT label, memory_ids FROM kg_nodes WHERE ${placeholders}`
236
+ ).bind(...chunk).all<{ label: string; memory_ids: string }>();
237
+
238
+ for (const row of rows.results) {
239
+ try {
240
+ const ids = JSON.parse(row.memory_ids || '[]');
241
+ if (Array.isArray(ids) && ids.length > 0) {
242
+ result.set(row.label.toLowerCase(), ids.map(String));
243
+ }
244
+ } catch {
245
+ // Invalid JSON in memory_ids — skip
246
+ }
247
+ }
248
+ } catch (err) {
249
+ console.warn('[recall] fetchNodeMemoryIds failed:', err instanceof Error ? err.message : String(err));
250
+ }
251
+ }
252
+
253
+ return result;
254
+ }
255
+
256
+ // ─── Stage 4: RRF Fusion ────────────────────────────────────
257
+
258
+ function fuseResults(
259
+ memoryFragments: MemoryFragmentResult[],
260
+ mindspringResults: MindSpringResult[],
261
+ activatedNodeMap: Map<string, number>,
262
+ nodeMemoryIds: Map<string, string[]>,
263
+ blockContext: string[],
264
+ maxResults: number,
265
+ ): RecalledFact[] {
266
+ // Build a set of memory_ids that are linked to activated graph nodes
267
+ const graphLinkedIds = new Set<string>();
268
+ const graphScoreById = new Map<string, number>();
269
+
270
+ for (const [label, memIds] of nodeMemoryIds) {
271
+ const activation = activatedNodeMap.get(label.toLowerCase()) ?? 0;
272
+ for (const id of memIds) {
273
+ graphLinkedIds.add(id);
274
+ const existing = graphScoreById.get(id) ?? 0;
275
+ graphScoreById.set(id, Math.max(existing, activation));
276
+ }
277
+ }
278
+
279
+ // Build RRF rank lists
280
+ const ranks = new Map<string, number[]>();
281
+ const factById = new Map<string, { text: string; semanticScore: number }>();
282
+
283
+ // Rank list 1: Memory Worker semantic order
284
+ for (let i = 0; i < memoryFragments.length; i++) {
285
+ const f = memoryFragments[i];
286
+ const id = f.id;
287
+ factById.set(id, { text: f.content, semanticScore: f.confidence });
288
+
289
+ const existing = ranks.get(id) ?? [];
290
+ existing.push(i + 1); // 1-indexed rank
291
+ ranks.set(id, existing);
292
+ }
293
+
294
+ // Rank list 2: Graph activation order (for fragments that have graph links)
295
+ const graphRankedIds = [...graphScoreById.entries()]
296
+ .sort((a, b) => b[1] - a[1]);
297
+ for (let i = 0; i < graphRankedIds.length; i++) {
298
+ const [id] = graphRankedIds[i];
299
+ // Only include if we have the fact text (from memory worker or add it)
300
+ const existing = ranks.get(id) ?? [];
301
+ existing.push(i + 1);
302
+ ranks.set(id, existing);
303
+ }
304
+
305
+ // Rank list 3: Block relevance boost (facts mentioning active projects)
306
+ if (blockContext.length > 0) {
307
+ const blockBoosted: Array<{ id: string; matches: number }> = [];
308
+ for (const [id, info] of factById) {
309
+ const text = info.text.toLowerCase();
310
+ let matches = 0;
311
+ for (const project of blockContext) {
312
+ if (text.includes(project.toLowerCase())) matches++;
313
+ }
314
+ if (matches > 0) blockBoosted.push({ id, matches });
315
+ }
316
+ blockBoosted.sort((a, b) => b.matches - a.matches);
317
+ for (let i = 0; i < blockBoosted.length; i++) {
318
+ const existing = ranks.get(blockBoosted[i].id) ?? [];
319
+ existing.push(i + 1);
320
+ ranks.set(blockBoosted[i].id, existing);
321
+ }
322
+ }
323
+
324
+ // Rank list 4: MindSpring conversation matches (historical context)
325
+ for (let i = 0; i < mindspringResults.length; i++) {
326
+ const ms = mindspringResults[i];
327
+ const id = `ms:${ms.id}`;
328
+ // Synthesize a fact from the conversation match: title + truncated text
329
+ const text = `[Prior conversation: "${ms.title}"] ${ms.text.slice(0, 500)}`;
330
+ factById.set(id, { text, semanticScore: ms.score });
331
+
332
+ const existing = ranks.get(id) ?? [];
333
+ existing.push(i + 1);
334
+ ranks.set(id, existing);
335
+ }
336
+
337
+ // Apply RRF
338
+ const fusedScores = rrf(ranks);
339
+
340
+ // Build result facts
341
+ const results: RecalledFact[] = [];
342
+ const seen = new Set<string>();
343
+
344
+ for (const [id, score] of fusedScores) {
345
+ if (seen.has(id)) continue;
346
+ seen.add(id);
347
+
348
+ const info = factById.get(id);
349
+ const graphScore = graphScoreById.get(id) ?? 0;
350
+ const hasGraph = graphScore > 0;
351
+ const hasSemantic = !!info;
352
+ const isMindspring = id.startsWith('ms:');
353
+
354
+ results.push({
355
+ id,
356
+ text: info?.text ?? '', // empty if only graph-linked without memory worker match
357
+ score,
358
+ semantic_score: info?.semanticScore ?? 0,
359
+ graph_score: graphScore,
360
+ source: isMindspring ? 'mindspring' : hasGraph && hasSemantic ? 'both' : hasGraph ? 'graph' : 'memory_worker',
361
+ });
362
+ }
363
+
364
+ // Sort by fused score, filter out empty-text entries, take top-K
365
+ results.sort((a, b) => b.score - a.score);
366
+ return results.filter(f => f.text.length > 0).slice(0, maxResults);
367
+ }
@@ -0,0 +1,315 @@
1
+ import type { MemoryEntry } from '../types.js';
2
+
3
+ // ─── Temporal Relevance Decay (#53) ─────────────────────────
4
+ export const BASE_HALF_LIFE_DAYS = 14;
5
+ const RECENCY_WEIGHT = 0.6;
6
+ const STRENGTH_WEIGHT = 0.3;
7
+ const CONFIDENCE_WEIGHT = 0.1;
8
+ const MEMORY_CONTEXT_LIMIT = 50;
9
+ const MEMORY_FETCH_POOL = 80;
10
+
11
+ // ─── Topic Taxonomy ──────────────────────────────────────────
12
+ // Canonical topic map: normalized variant → canonical name.
13
+ // Two-pass normalizeTopic(): string cleanup → canonical lookup.
14
+ // Add new entries here when a new conceptual cluster emerges.
15
+ const CANONICAL_TOPICS: Record<string, string> = {
16
+ // AEGIS internals — all facts about the cognitive kernel itself
17
+ 'aegis_development': 'aegis',
18
+ 'aegis_architecture': 'aegis',
19
+ 'aegis_infrastructure': 'aegis',
20
+ 'aegis_governance': 'aegis',
21
+ 'aegis_status': 'aegis',
22
+ // img-forge — normalize the inconsistent prefix
23
+ 'imgforge': 'img_forge',
24
+ 'img_forge_economics': 'img_forge',
25
+ 'img_forge_integration': 'img_forge',
26
+ // BizOps — operational mutations and improvements are the same domain
27
+ 'bizops_improvements': 'bizops',
28
+ 'bizops_mutate': 'bizops',
29
+ // Content pipelines — roundtable, dispatch, column all live here
30
+ 'dispatch_generation': 'content',
31
+ 'roundtable_generation': 'content',
32
+ 'research_dispatch': 'content',
33
+ 'research_synthesis': 'content',
34
+ 'column_generation': 'content',
35
+ 'content_generation': 'content',
36
+ // Self-improvement — outcomes belong with the source topic
37
+ 'self_improvement_outcomes': 'self_improvement',
38
+ // Organization-level governance and priorities → root organization
39
+ 'organization_governance': 'organization',
40
+ 'organization_priorities': 'organization',
41
+ };
42
+
43
+ // Normalize topics to lowercase_underscore to prevent fragmentation
44
+ // ("Compliance Monitoring", "compliance_monitoring", "Compliance" → "compliance_monitoring")
45
+ export function normalizeTopic(topic: string): string {
46
+ // Pass 1: string cleanup — lowercase, underscores, strip non-alphanum
47
+ const normalized = topic.toLowerCase().replace(/\s+/g, '_').replace(/[^a-z0-9_]/g, '').replace(/_+/g, '_').replace(/^_|_$/g, '');
48
+ // Pass 2: canonical lookup — collapse semantic variants to the same bucket
49
+ return CANONICAL_TOPICS[normalized] ?? normalized;
50
+ }
51
+
52
+ // ─── Semantic Memory ────────────────────────────────────────
53
+
54
+ // Simple djb2-style hash for deduplication — no crypto needed, collision risk acceptable for this use
55
+ export function factHash(topic: string, fact: string): string {
56
+ const s = `${topic}::${fact.toLowerCase().replace(/\s+/g, ' ').trim()}`;
57
+ let h = 5381;
58
+ for (let i = 0; i < s.length; i++) h = ((h * 33) ^ s.charCodeAt(i)) >>> 0;
59
+ return h.toString(16).padStart(8, '0');
60
+ }
61
+
62
+ // ─── Semantic Dedup Helpers (#37) ────────────────────────────
63
+
64
+ const STOP_WORDS = new Set([
65
+ 'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
66
+ 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
67
+ 'should', 'may', 'might', 'shall', 'can', 'to', 'of', 'in', 'for',
68
+ 'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'during',
69
+ 'before', 'after', 'and', 'but', 'or', 'nor', 'not', 'so', 'yet',
70
+ 'both', 'either', 'neither', 'each', 'every', 'all', 'any', 'few',
71
+ 'more', 'most', 'other', 'some', 'such', 'no', 'only', 'own', 'same',
72
+ 'than', 'too', 'very', 'just', 'also', 'it', 'its', 'this', 'that',
73
+ 'these', 'those', 'he', 'she', 'they', 'we', 'you', 'i', 'me', 'my',
74
+ 'his', 'her', 'our', 'your', 'their', 'what', 'which', 'who', 'whom',
75
+ ]);
76
+
77
+ export function tokenize(text: string): Set<string> {
78
+ return new Set(
79
+ text.toLowerCase()
80
+ .replace(/[^a-z0-9\s]/g, ' ')
81
+ .split(/\s+/)
82
+ .filter(w => w.length > 1 && !STOP_WORDS.has(w))
83
+ );
84
+ }
85
+
86
+ export function jaccardSimilarity(a: Set<string>, b: Set<string>): number {
87
+ if (a.size === 0 && b.size === 0) return 1;
88
+ let intersection = 0;
89
+ for (const word of a) {
90
+ if (b.has(word)) intersection++;
91
+ }
92
+ const union = a.size + b.size - intersection;
93
+ return union === 0 ? 0 : intersection / union;
94
+ }
95
+
96
+ const SEMANTIC_DEDUP_THRESHOLD = 0.6;
97
+ const COSINE_DEDUP_THRESHOLD = 0.80;
98
+
99
+ // Cosine similarity for embedding-based dedup
100
+ export function cosineSimilarity(a: number[], b: number[]): number {
101
+ if (a.length !== b.length || a.length === 0) return 0;
102
+ let dot = 0, normA = 0, normB = 0;
103
+ for (let i = 0; i < a.length; i++) {
104
+ dot += a[i] * b[i];
105
+ normA += a[i] * a[i];
106
+ normB += b[i] * b[i];
107
+ }
108
+ const denom = Math.sqrt(normA) * Math.sqrt(normB);
109
+ return denom === 0 ? 0 : dot / denom;
110
+ }
111
+
112
+ // Three-phase dedup: exact hash → semantic similarity → insert new (#1, #37)
113
+ // When memoryBinding is provided, Phase 2 uses cosine similarity on BGE embeddings
114
+ // instead of Jaccard token overlap — catches paraphrased duplicates.
115
+ export async function recordMemory(
116
+ db: D1Database, topic: string, fact: string, confidence: number, source: string,
117
+ memoryBinding?: { embed(tenantId: string, texts: string[]): Promise<{ embeddings: number[][] }> },
118
+ ): Promise<void> {
119
+ topic = normalizeTopic(topic);
120
+ const hash = factHash(topic, fact);
121
+
122
+ // Phase 1: Exact hash dedup — fast path (active entries only)
123
+ const existing = await db.prepare(
124
+ 'SELECT id, confidence FROM memory_entries WHERE topic = ? AND fact_hash = ? AND valid_until IS NULL LIMIT 1'
125
+ ).bind(topic, hash).first<{ id: number; confidence: number }>();
126
+
127
+ if (existing) {
128
+ await db.prepare(
129
+ "UPDATE memory_entries SET confidence = MAX(confidence, ?), source = ?, updated_at = datetime('now') WHERE id = ?"
130
+ ).bind(confidence, source, existing.id).run();
131
+ return;
132
+ }
133
+
134
+ // Phase 2: Semantic dedup — cosine similarity (preferred) or Jaccard fallback
135
+ const topicEntries = await db.prepare(
136
+ 'SELECT id, fact, confidence FROM memory_entries WHERE topic = ? AND valid_until IS NULL ORDER BY confidence DESC LIMIT 20'
137
+ ).bind(topic).all<{ id: number; fact: string; confidence: number }>();
138
+
139
+ // Try cosine similarity via Memory Worker embeddings first
140
+ let matchedEntry: { id: number; fact: string; confidence: number } | null = null;
141
+
142
+ if (memoryBinding && topicEntries.results.length > 0) {
143
+ try {
144
+ const textsToEmbed = [fact, ...topicEntries.results.map(e => e.fact)];
145
+ const { embeddings } = await memoryBinding.embed('aegis', textsToEmbed);
146
+ const newEmb = embeddings[0];
147
+ for (let i = 0; i < topicEntries.results.length; i++) {
148
+ if (cosineSimilarity(newEmb, embeddings[i + 1]) >= COSINE_DEDUP_THRESHOLD) {
149
+ matchedEntry = topicEntries.results[i];
150
+ break;
151
+ }
152
+ }
153
+ } catch (err) {
154
+ console.error('[semantic] cosine dedup failed, falling back to Jaccard:', err instanceof Error ? err.message : String(err));
155
+ // Fall through to Jaccard below
156
+ }
157
+ }
158
+
159
+ // Jaccard fallback when memoryBinding unavailable or cosine dedup failed
160
+ if (!matchedEntry && (!memoryBinding || topicEntries.results.length > 0)) {
161
+ const newTokens = tokenize(fact);
162
+ for (const entry of topicEntries.results) {
163
+ const existingTokens = tokenize(entry.fact);
164
+ if (jaccardSimilarity(newTokens, existingTokens) >= SEMANTIC_DEDUP_THRESHOLD) {
165
+ matchedEntry = entry;
166
+ break;
167
+ }
168
+ }
169
+ }
170
+
171
+ if (matchedEntry) {
172
+ // Supersede: invalidate old entry, insert new with audit link
173
+ const mergedFact = fact.length > matchedEntry.fact.length ? fact : matchedEntry.fact;
174
+ const mergedHash = factHash(topic, mergedFact);
175
+ const mergedConfidence = Math.min(1.0, Math.max(confidence, matchedEntry.confidence) + 0.05);
176
+ // Invalidate old entry
177
+ await db.prepare(
178
+ "UPDATE memory_entries SET valid_until = datetime('now'), updated_at = datetime('now') WHERE id = ?"
179
+ ).bind(matchedEntry.id).run();
180
+ // Insert replacement with superseded_by link
181
+ const insertResult = await db.prepare(
182
+ 'INSERT INTO memory_entries (topic, fact, fact_hash, confidence, source, superseded_by) VALUES (?, ?, ?, ?, ?, ?)'
183
+ ).bind(topic, mergedFact, mergedHash, mergedConfidence, source, matchedEntry.id).run();
184
+ // Back-link: set superseded_by on the old entry to point to new entry
185
+ if (insertResult.meta.last_row_id) {
186
+ await db.prepare(
187
+ 'UPDATE memory_entries SET superseded_by = ? WHERE id = ?'
188
+ ).bind(insertResult.meta.last_row_id, matchedEntry.id).run();
189
+ }
190
+ return;
191
+ }
192
+
193
+ // Phase 3: No match — insert new entry
194
+ await db.prepare(
195
+ 'INSERT INTO memory_entries (topic, fact, fact_hash, confidence, source) VALUES (?, ?, ?, ?, ?)'
196
+ ).bind(topic, fact, hash, confidence, source).run();
197
+ }
198
+
199
+ // ─── Targeted Memory Search (#memory-recall-fix) ─────────────
200
+ // Keyword-based search across all memory facts — returns entries where
201
+ // the fact text contains any of the significant words from the query.
202
+ // Used by dispatch to augment memory_recall prompts with relevant context
203
+ // instead of relying on the model to find it in a 50-entry dump.
204
+
205
+ export async function searchMemoryByKeywords(
206
+ db: D1Database,
207
+ query: string,
208
+ limit = 10,
209
+ ): Promise<Array<{ id: number; topic: string; fact: string; confidence: number }>> {
210
+ // Extract significant keywords (>3 chars, not stop words)
211
+ const keywords = query.toLowerCase()
212
+ .replace(/[^a-z0-9\s]/g, ' ')
213
+ .split(/\s+/)
214
+ .filter(w => w.length > 3 && !STOP_WORDS.has(w));
215
+
216
+ if (keywords.length === 0) return [];
217
+
218
+ // Build OR-based LIKE search — each keyword gets a LIKE clause
219
+ // D1 doesn't support full-text search, so LIKE is our best option
220
+ const likeClauses = keywords.map(() => "LOWER(fact) LIKE ?").join(' OR ');
221
+ const likeParams = keywords.map(k => `%${k}%`);
222
+
223
+ const result = await db.prepare(
224
+ `SELECT id, topic, fact, confidence FROM memory_entries
225
+ WHERE valid_until IS NULL AND (${likeClauses})
226
+ ORDER BY confidence DESC, updated_at DESC
227
+ LIMIT ?`
228
+ ).bind(...likeParams, limit).all<{ id: number; topic: string; fact: string; confidence: number }>();
229
+
230
+ return result.results;
231
+ }
232
+
233
+ // ─── Memory Retrieval + Scoring ─────────────────────────────
234
+
235
+ export async function getMemoryEntries(db: D1Database, topic?: string): Promise<MemoryEntry[]> {
236
+ if (topic) {
237
+ const result = await db.prepare(
238
+ 'SELECT * FROM memory_entries WHERE topic = ? AND valid_until IS NULL ORDER BY created_at DESC'
239
+ ).bind(normalizeTopic(topic)).all();
240
+ return result.results as unknown as MemoryEntry[];
241
+ }
242
+ const result = await db.prepare(
243
+ 'SELECT * FROM memory_entries WHERE valid_until IS NULL ORDER BY created_at DESC LIMIT 100'
244
+ ).all();
245
+ return result.results as unknown as MemoryEntry[];
246
+ }
247
+
248
+ // ─── Memory Recall Tracking (#52) ────────────────────────────
249
+
250
+ // Increment strength and update last_recalled_at when facts are used in responses.
251
+ // Single batch UPDATE instead of N individual queries — 50x fewer D1 operations at recall.
252
+ export async function recallMemory(db: D1Database, ids: number[]): Promise<void> {
253
+ if (ids.length === 0) return;
254
+ const placeholders = ids.map(() => '?').join(',');
255
+ await db.prepare(
256
+ `UPDATE memory_entries SET strength = strength + 1, last_recalled_at = datetime('now') WHERE id IN (${placeholders})`
257
+ ).bind(...ids).run();
258
+ }
259
+
260
+ // EWA (Ebbinghaus-Weighted Attention) score for temporal relevance ranking
261
+ // Combines recency decay, reinforcement strength, and source confidence
262
+ export function computeEwaScore(
263
+ daysSinceActive: number,
264
+ strength: number,
265
+ confidence: number,
266
+ ): number {
267
+ // Recency: Ebbinghaus retention curve — 2^(-t / (s * halfLife))
268
+ const recency = Math.pow(2, -daysSinceActive / (strength * BASE_HALF_LIFE_DAYS));
269
+ // Strength: logarithmic diminishing returns, capped at 5
270
+ const strengthScore = Math.log2(strength + 1) / 5;
271
+ // Combined weighted score
272
+ return RECENCY_WEIGHT * recency + STRENGTH_WEIGHT * strengthScore + CONFIDENCE_WEIGHT * confidence;
273
+ }
274
+
275
+ export async function getAllMemoryForContext(db: D1Database): Promise<{ text: string; ids: number[] }> {
276
+ const result = await db.prepare(
277
+ 'SELECT id, topic, fact, confidence, strength, last_recalled_at, created_at FROM memory_entries WHERE valid_until IS NULL ORDER BY topic, created_at DESC LIMIT ?'
278
+ ).bind(MEMORY_FETCH_POOL).all();
279
+ const entries = result.results as unknown as {
280
+ id: number; topic: string; fact: string; confidence: number;
281
+ strength: number; last_recalled_at: string | null; created_at: string;
282
+ }[];
283
+
284
+ if (entries.length === 0) return { text: '', ids: [] };
285
+
286
+ const now = Date.now();
287
+
288
+ // Score each entry with EWA temporal relevance
289
+ const scored = entries.map(e => {
290
+ const activeDate = e.last_recalled_at ?? e.created_at;
291
+ const ts = activeDate.endsWith('Z') ? activeDate : activeDate + 'Z';
292
+ const daysSince = Math.max(0, (now - new Date(ts).getTime()) / 86_400_000);
293
+ const score = computeEwaScore(daysSince, e.strength ?? 1, e.confidence);
294
+ return { ...e, score };
295
+ });
296
+
297
+ // Sort by score descending, take top MEMORY_CONTEXT_LIMIT
298
+ scored.sort((a, b) => b.score - a.score);
299
+ const top = scored.slice(0, MEMORY_CONTEXT_LIMIT);
300
+
301
+ const ids = top.map(e => e.id);
302
+
303
+ const byTopic = new Map<string, string[]>();
304
+ for (const e of top) {
305
+ const list = byTopic.get(e.topic) || [];
306
+ list.push(`- ${e.fact} (confidence: ${e.confidence})`);
307
+ byTopic.set(e.topic, list);
308
+ }
309
+
310
+ let text = '\n## Agent Memory\n';
311
+ for (const [topic, facts] of byTopic) {
312
+ text += `\n### ${topic}\n${facts.join('\n')}\n`;
313
+ }
314
+ return { text, ids };
315
+ }