agent-working-memory 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -8
- package/dist/adapters/common.d.ts.map +1 -1
- package/dist/adapters/common.js +9 -1
- package/dist/adapters/common.js.map +1 -1
- package/dist/api/routes.d.ts.map +1 -1
- package/dist/api/routes.js +108 -10
- package/dist/api/routes.js.map +1 -1
- package/dist/cli.js +103 -103
- package/dist/core/auto-tagger.d.ts +29 -0
- package/dist/core/auto-tagger.d.ts.map +1 -0
- package/dist/core/auto-tagger.js +139 -0
- package/dist/core/auto-tagger.js.map +1 -0
- package/dist/core/hebbian.d.ts +25 -1
- package/dist/core/hebbian.d.ts.map +1 -1
- package/dist/core/hebbian.js +71 -3
- package/dist/core/hebbian.js.map +1 -1
- package/dist/core/query-expander.d.ts.map +1 -1
- package/dist/core/query-expander.js.map +1 -1
- package/dist/core/reranker.d.ts.map +1 -1
- package/dist/core/reranker.js.map +1 -1
- package/dist/engine/activation.d.ts +22 -4
- package/dist/engine/activation.d.ts.map +1 -1
- package/dist/engine/activation.js +136 -73
- package/dist/engine/activation.js.map +1 -1
- package/dist/engine/consolidation.d.ts +1 -0
- package/dist/engine/consolidation.d.ts.map +1 -1
- package/dist/engine/consolidation.js +149 -9
- package/dist/engine/consolidation.js.map +1 -1
- package/dist/index.js +1 -1
- package/dist/mcp.js +123 -84
- package/dist/mcp.js.map +1 -1
- package/dist/storage/sqlite.d.ts +17 -0
- package/dist/storage/sqlite.d.ts.map +1 -1
- package/dist/storage/sqlite.js +73 -0
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/types/engram.d.ts +2 -0
- package/dist/types/engram.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/adapters/common.ts +9 -1
- package/src/api/routes.ts +723 -600
- package/src/cli.ts +719 -719
- package/src/core/auto-tagger.ts +168 -0
- package/src/core/hebbian.ts +84 -3
- package/src/core/query-expander.ts +0 -1
- package/src/core/reranker.ts +0 -1
- package/src/engine/activation.ts +136 -70
- package/src/engine/consolidation.ts +165 -9
- package/src/index.ts +199 -199
- package/src/mcp.ts +1134 -1099
- package/src/storage/sqlite.ts +77 -0
- package/src/types/engram.ts +2 -0
|
@@ -99,8 +99,19 @@ export interface ConsolidationResult {
|
|
|
99
99
|
stagingPromoted: number;
|
|
100
100
|
stagingDiscarded: number;
|
|
101
101
|
engramsProcessed: number;
|
|
102
|
+
synthesesCreated: number;
|
|
102
103
|
}
|
|
103
104
|
|
|
105
|
+
const MAX_SYNTHESES_PER_CYCLE = 5;
|
|
106
|
+
const MIN_CLUSTER_SIZE_FOR_SYNTHESIS = 3;
|
|
107
|
+
|
|
108
|
+
/** Shared stopwords for synthesis keyword extraction */
|
|
109
|
+
const SYNTH_STOPWORDS = new Set(['the', 'is', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'for',
|
|
110
|
+
'on', 'with', 'that', 'this', 'it', 'was', 'are', 'be', 'has', 'had', 'but', 'not', 'from',
|
|
111
|
+
'by', 'as', 'at', 'i', 'you', 'we', 'my', 'your', 'can', 'will', 'do', 'did', 'if', 'user',
|
|
112
|
+
'assistant', 'would', 'like', 'just', 'also', 'about', 'really', 'think', 'know', 'want',
|
|
113
|
+
'here', 'there', 'some', 'more', 'very', 'been', 'have', 'what', 'when', 'how', 'they']);
|
|
114
|
+
|
|
104
115
|
export class ConsolidationEngine {
|
|
105
116
|
private store: EngramStore;
|
|
106
117
|
|
|
@@ -136,6 +147,7 @@ export class ConsolidationEngine {
|
|
|
136
147
|
stagingPromoted: 0,
|
|
137
148
|
stagingDiscarded: 0,
|
|
138
149
|
engramsProcessed: 0,
|
|
150
|
+
synthesesCreated: 0,
|
|
139
151
|
};
|
|
140
152
|
|
|
141
153
|
// --- Phase 1: Replay ---
|
|
@@ -208,6 +220,153 @@ export class ConsolidationEngine {
|
|
|
208
220
|
}
|
|
209
221
|
}
|
|
210
222
|
|
|
223
|
+
// --- Phase 2.5: Two types of synthesis ---
|
|
224
|
+
//
|
|
225
|
+
// Type A: SESSION SYNTHESIS (perfect recall)
|
|
226
|
+
// Groups by shared metadata tags (sid=, proj=, topic=).
|
|
227
|
+
// Summarizes what happened in a conversation/project session.
|
|
228
|
+
// Helps find specific facts by providing a topical anchor.
|
|
229
|
+
//
|
|
230
|
+
// Type B: PATTERN SYNTHESIS (novel recall)
|
|
231
|
+
// Uses the existing vector-similarity clusters.
|
|
232
|
+
// Finds structural patterns across disparate topics.
|
|
233
|
+
// "Debugging X by Y" + "Resolving A by B" → pattern: "conflict → ordering"
|
|
234
|
+
// Lower confidence — these are speculative connections, not facts.
|
|
235
|
+
|
|
236
|
+
let synthCount = 0;
|
|
237
|
+
|
|
238
|
+
// --- Type A: Session synthesis (tag-based grouping) ---
|
|
239
|
+
// Group engrams by shared session/project tags, NOT vector similarity
|
|
240
|
+
const tagGroups = new Map<string, Engram[]>();
|
|
241
|
+
for (const e of engrams) {
|
|
242
|
+
if (e.tags.includes('synth=true')) continue; // Skip existing syntheses
|
|
243
|
+
for (const tag of e.tags) {
|
|
244
|
+
if (tag.startsWith('sid=') || tag.startsWith('proj=') || tag.startsWith('topic=')) {
|
|
245
|
+
const group = tagGroups.get(tag) ?? [];
|
|
246
|
+
group.push(e);
|
|
247
|
+
tagGroups.set(tag, group);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
for (const [tag, group] of tagGroups) {
|
|
253
|
+
if (group.length < MIN_CLUSTER_SIZE_FOR_SYNTHESIS) continue;
|
|
254
|
+
if (synthCount >= MAX_SYNTHESES_PER_CYCLE) break;
|
|
255
|
+
|
|
256
|
+
// Check if a synthesis for this tag already exists
|
|
257
|
+
const existing = engrams.find(e =>
|
|
258
|
+
e.tags.includes('synth=true') && e.tags.includes(tag)
|
|
259
|
+
);
|
|
260
|
+
if (existing) continue;
|
|
261
|
+
|
|
262
|
+
// Extract key terms from this group
|
|
263
|
+
const wordCounts = new Map<string, number>();
|
|
264
|
+
for (const e of group) {
|
|
265
|
+
const words = e.content.toLowerCase().replace(/[^\w\s]/g, '').split(/\s+/);
|
|
266
|
+
for (const w of words) {
|
|
267
|
+
if (w.length > 3 && !SYNTH_STOPWORDS.has(w)) {
|
|
268
|
+
wordCounts.set(w, (wordCounts.get(w) ?? 0) + 1);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
const keyTerms = [...wordCounts.entries()]
|
|
273
|
+
.sort((a, b) => b[1] - a[1])
|
|
274
|
+
.slice(0, 15)
|
|
275
|
+
.map(([word]) => word);
|
|
276
|
+
|
|
277
|
+
// Extract unique concepts (deduplicated)
|
|
278
|
+
const concepts = [...new Set(group.map(e => e.concept))];
|
|
279
|
+
|
|
280
|
+
const synthContent = [
|
|
281
|
+
`Session summary (${tag}, ${group.length} turns).`,
|
|
282
|
+
`Key topics: ${keyTerms.slice(0, 8).join(', ')}.`,
|
|
283
|
+
`Discussed: ${keyTerms.slice(8).join(', ')}.`,
|
|
284
|
+
].join(' ');
|
|
285
|
+
|
|
286
|
+
const synthEngram = this.store.createEngram({
|
|
287
|
+
agentId,
|
|
288
|
+
concept: `session: ${tag} (${keyTerms.slice(0, 3).join(', ')})`,
|
|
289
|
+
content: synthContent,
|
|
290
|
+
tags: [tag, 'synth=true', 'synth-type=session'],
|
|
291
|
+
salience: 0.6,
|
|
292
|
+
confidence: 0.55,
|
|
293
|
+
memoryType: 'semantic',
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
// Set embedding to group centroid
|
|
297
|
+
const centroid = this.computeCentroid(group);
|
|
298
|
+
if (centroid.length > 0) {
|
|
299
|
+
this.store.updateEmbedding(synthEngram.id, centroid);
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// Link to sources
|
|
303
|
+
for (const source of group.slice(0, 10)) { // Cap links to prevent explosion
|
|
304
|
+
this.store.upsertAssociation(synthEngram.id, source.id, 0.4, 'causal');
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
synthCount++;
|
|
308
|
+
result.synthesesCreated++;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// --- Type B: Pattern synthesis (vector-similarity clusters, speculative) ---
|
|
312
|
+
// Only create these for clusters where members come from DIFFERENT sessions/projects.
|
|
313
|
+
// This finds cross-domain patterns: "debugging technique A" + "architecture pattern B"
|
|
314
|
+
for (const cluster of clusters) {
|
|
315
|
+
if (cluster.length < MIN_CLUSTER_SIZE_FOR_SYNTHESIS) continue;
|
|
316
|
+
if (synthCount >= MAX_SYNTHESES_PER_CYCLE) break;
|
|
317
|
+
if (cluster.some(e => e.tags.includes('synth=true'))) continue;
|
|
318
|
+
|
|
319
|
+
// Only create pattern synthesis if cluster spans multiple sessions
|
|
320
|
+
const sessionTags = new Set<string>();
|
|
321
|
+
for (const e of cluster) {
|
|
322
|
+
for (const tag of e.tags) {
|
|
323
|
+
if (tag.startsWith('sid=') || tag.startsWith('proj=')) sessionTags.add(tag);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
if (sessionTags.size < 2) continue; // Same session → skip (Type A handles it)
|
|
327
|
+
|
|
328
|
+
const wordCounts = new Map<string, number>();
|
|
329
|
+
for (const e of cluster) {
|
|
330
|
+
const words = e.content.toLowerCase().replace(/[^\w\s]/g, '').split(/\s+/);
|
|
331
|
+
for (const w of words) {
|
|
332
|
+
if (w.length > 3 && !SYNTH_STOPWORDS.has(w)) {
|
|
333
|
+
wordCounts.set(w, (wordCounts.get(w) ?? 0) + 1);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
const keyTerms = [...wordCounts.entries()]
|
|
338
|
+
.sort((a, b) => b[1] - a[1])
|
|
339
|
+
.slice(0, 10)
|
|
340
|
+
.map(([word]) => word);
|
|
341
|
+
|
|
342
|
+
const synthContent = [
|
|
343
|
+
`Pattern across ${sessionTags.size} sessions (${cluster.length} memories).`,
|
|
344
|
+
`Common themes: ${keyTerms.join(', ')}.`,
|
|
345
|
+
].join(' ');
|
|
346
|
+
|
|
347
|
+
const synthEngram = this.store.createEngram({
|
|
348
|
+
agentId,
|
|
349
|
+
concept: `pattern: ${keyTerms.slice(0, 3).join(', ')}`,
|
|
350
|
+
content: synthContent,
|
|
351
|
+
tags: [...sessionTags, 'synth=true', 'synth-type=pattern'],
|
|
352
|
+
salience: 0.5, // Lower — speculative
|
|
353
|
+
confidence: 0.4, // Lower — these are hypotheses not facts
|
|
354
|
+
memoryType: 'semantic',
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
const centroid = this.computeCentroid(cluster);
|
|
358
|
+
if (centroid.length > 0) {
|
|
359
|
+
this.store.updateEmbedding(synthEngram.id, centroid);
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
for (const source of cluster.slice(0, 8)) {
|
|
363
|
+
this.store.upsertAssociation(synthEngram.id, source.id, 0.3, 'bridge');
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
synthCount++;
|
|
367
|
+
result.synthesesCreated++;
|
|
368
|
+
}
|
|
369
|
+
|
|
211
370
|
// --- Phase 3: Direct cross-cluster bridging ---
|
|
212
371
|
// Find the closest pair of memories between each cluster pair and bridge them.
|
|
213
372
|
if (clusters.length >= 2) {
|
|
@@ -244,21 +403,18 @@ export class ConsolidationEngine {
|
|
|
244
403
|
// that received positive feedback are more durable — just like how
|
|
245
404
|
// practiced memories are more resistant to forgetting in the brain.
|
|
246
405
|
// Base half-life: 7 days. High-confidence (0.8+) gets up to 30 days.
|
|
247
|
-
const
|
|
406
|
+
const engramMap = new Map(engrams.map(e => [e.id, e]));
|
|
248
407
|
const associations = this.store.getAllAssociations(agentId);
|
|
249
408
|
for (const assoc of associations) {
|
|
250
409
|
const daysSince =
|
|
251
410
|
(Date.now() - assoc.lastActivated.getTime()) / (1000 * 60 * 60 * 24);
|
|
252
|
-
if (daysSince < 0.5) continue;
|
|
411
|
+
if (daysSince < 0.5) continue;
|
|
253
412
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
const
|
|
258
|
-
const toConf = engramConfMap.get(assoc.toEngramId) ?? 0.5;
|
|
413
|
+
const fromEngram = engramMap.get(assoc.fromEngramId);
|
|
414
|
+
const toEngram = engramMap.get(assoc.toEngramId);
|
|
415
|
+
const fromConf = fromEngram?.confidence ?? 0.5;
|
|
416
|
+
const toConf = toEngram?.confidence ?? 0.5;
|
|
259
417
|
const maxConf = Math.max(fromConf, toConf);
|
|
260
|
-
const fromEngram = engrams.find(e => e.id === assoc.fromEngramId);
|
|
261
|
-
const toEngram = engrams.find(e => e.id === assoc.toEngramId);
|
|
262
418
|
const maxAccess = Math.max(fromEngram?.accessCount ?? 0, toEngram?.accessCount ?? 0);
|
|
263
419
|
const accessBoost = Math.min(2.0, 1.0 + 0.5 * Math.log1p(maxAccess));
|
|
264
420
|
const halfLifeDays = Math.min(
|
package/src/index.ts
CHANGED
|
@@ -1,199 +1,199 @@
|
|
|
1
|
-
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
import { readFileSync, copyFileSync, existsSync, mkdirSync, readdirSync, unlinkSync } from 'node:fs';
|
|
4
|
-
import { resolve, dirname, basename } from 'node:path';
|
|
5
|
-
import Fastify from 'fastify';
|
|
6
|
-
|
|
7
|
-
// Load .env file if present (no external dependency)
|
|
8
|
-
try {
|
|
9
|
-
const envPath = resolve(process.cwd(), '.env');
|
|
10
|
-
const envContent = readFileSync(envPath, 'utf-8');
|
|
11
|
-
for (const line of envContent.split('\n')) {
|
|
12
|
-
const trimmed = line.trim();
|
|
13
|
-
if (!trimmed || trimmed.startsWith('#')) continue;
|
|
14
|
-
const eqIdx = trimmed.indexOf('=');
|
|
15
|
-
if (eqIdx === -1) continue;
|
|
16
|
-
const key = trimmed.slice(0, eqIdx).trim();
|
|
17
|
-
const val = trimmed.slice(eqIdx + 1).trim().replace(/^["']|["']$/g, '');
|
|
18
|
-
if (!process.env[key]) process.env[key] = val; // Don't override existing env
|
|
19
|
-
}
|
|
20
|
-
} catch { /* No .env file — that's fine */ }
|
|
21
|
-
import { EngramStore } from './storage/sqlite.js';
|
|
22
|
-
import { ActivationEngine } from './engine/activation.js';
|
|
23
|
-
import { ConnectionEngine } from './engine/connections.js';
|
|
24
|
-
import { StagingBuffer } from './engine/staging.js';
|
|
25
|
-
import { EvictionEngine } from './engine/eviction.js';
|
|
26
|
-
import { RetractionEngine } from './engine/retraction.js';
|
|
27
|
-
import { EvalEngine } from './engine/eval.js';
|
|
28
|
-
import { ConsolidationEngine } from './engine/consolidation.js';
|
|
29
|
-
import { ConsolidationScheduler } from './engine/consolidation-scheduler.js';
|
|
30
|
-
import { registerRoutes } from './api/routes.js';
|
|
31
|
-
import { DEFAULT_AGENT_CONFIG } from './types/agent.js';
|
|
32
|
-
import { getEmbedder } from './core/embeddings.js';
|
|
33
|
-
import { getReranker } from './core/reranker.js';
|
|
34
|
-
import { getExpander } from './core/query-expander.js';
|
|
35
|
-
import { initLogger } from './core/logger.js';
|
|
36
|
-
|
|
37
|
-
const PORT = parseInt(process.env.AWM_PORT ?? '8400', 10);
|
|
38
|
-
const DB_PATH = process.env.AWM_DB_PATH ?? 'memory.db';
|
|
39
|
-
const API_KEY = process.env.AWM_API_KEY ?? null;
|
|
40
|
-
|
|
41
|
-
async function main() {
|
|
42
|
-
// Auto-backup: copy DB to backups/ on startup (cheap insurance)
|
|
43
|
-
if (existsSync(DB_PATH)) {
|
|
44
|
-
const dbDir = dirname(resolve(DB_PATH));
|
|
45
|
-
const backupDir = resolve(dbDir, 'backups');
|
|
46
|
-
mkdirSync(backupDir, { recursive: true });
|
|
47
|
-
const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
48
|
-
const backupPath = resolve(backupDir, `${basename(DB_PATH, '.db')}-${ts}.db`);
|
|
49
|
-
try {
|
|
50
|
-
copyFileSync(resolve(DB_PATH), backupPath);
|
|
51
|
-
console.log(`Backup: ${backupPath}`);
|
|
52
|
-
} catch (err) {
|
|
53
|
-
console.log(`Backup skipped: ${(err as Error).message}`);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
// Logger — write activity to awm.log alongside the DB
|
|
58
|
-
initLogger(DB_PATH);
|
|
59
|
-
|
|
60
|
-
// Storage
|
|
61
|
-
const store = new EngramStore(DB_PATH);
|
|
62
|
-
|
|
63
|
-
// Integrity check
|
|
64
|
-
const integrity = store.integrityCheck();
|
|
65
|
-
if (!integrity.ok) {
|
|
66
|
-
console.error(`DB integrity check FAILED: ${integrity.result}`);
|
|
67
|
-
// Close corrupt DB, restore from backup, and exit for process manager to restart
|
|
68
|
-
store.close();
|
|
69
|
-
const dbDir = dirname(resolve(DB_PATH));
|
|
70
|
-
const backupDir = resolve(dbDir, 'backups');
|
|
71
|
-
if (existsSync(backupDir)) {
|
|
72
|
-
const backups = readdirSync(backupDir)
|
|
73
|
-
.filter(f => f.endsWith('.db'))
|
|
74
|
-
.sort()
|
|
75
|
-
.reverse();
|
|
76
|
-
if (backups.length > 0) {
|
|
77
|
-
const restorePath = resolve(backupDir, backups[0]);
|
|
78
|
-
console.error(`Attempting restore from: ${restorePath}`);
|
|
79
|
-
try {
|
|
80
|
-
copyFileSync(restorePath, resolve(DB_PATH));
|
|
81
|
-
console.error('Restore complete — exiting for restart with restored DB');
|
|
82
|
-
process.exit(1);
|
|
83
|
-
} catch (restoreErr) {
|
|
84
|
-
console.error(`Restore failed: ${(restoreErr as Error).message}`);
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
console.error('No backup available — continuing with potentially corrupt DB');
|
|
89
|
-
} else {
|
|
90
|
-
console.log(' DB integrity check: ok');
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
// Engines
|
|
94
|
-
const activationEngine = new ActivationEngine(store);
|
|
95
|
-
const connectionEngine = new ConnectionEngine(store, activationEngine);
|
|
96
|
-
const stagingBuffer = new StagingBuffer(store, activationEngine);
|
|
97
|
-
const evictionEngine = new EvictionEngine(store);
|
|
98
|
-
const retractionEngine = new RetractionEngine(store);
|
|
99
|
-
const evalEngine = new EvalEngine(store);
|
|
100
|
-
const consolidationEngine = new ConsolidationEngine(store);
|
|
101
|
-
const consolidationScheduler = new ConsolidationScheduler(store, consolidationEngine);
|
|
102
|
-
|
|
103
|
-
// API — disable Fastify's default request logging (too noisy for hive polling)
|
|
104
|
-
// bodyLimit: 512KB to prevent Content-Length mismatch errors with large task payloads
|
|
105
|
-
const app = Fastify({ logger: false, bodyLimit: 512_000 });
|
|
106
|
-
|
|
107
|
-
// Bearer token auth — only enforced when AWM_API_KEY is explicitly set and non-empty
|
|
108
|
-
if (API_KEY && API_KEY !== 'NONE' && API_KEY.length > 1) {
|
|
109
|
-
app.addHook('onRequest', async (req, reply) => {
|
|
110
|
-
if (req.url === '/health') return; // Health check is always public
|
|
111
|
-
const bearer = req.headers.authorization;
|
|
112
|
-
const xApiKey = req.headers['x-api-key'] as string | undefined;
|
|
113
|
-
if (bearer === `Bearer ${API_KEY}` || xApiKey === API_KEY) return;
|
|
114
|
-
reply.code(401).send({ error: 'Unauthorized' });
|
|
115
|
-
});
|
|
116
|
-
console.log('API key auth enabled (AWM_API_KEY set)');
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
registerRoutes(app, {
|
|
120
|
-
store, activationEngine, connectionEngine,
|
|
121
|
-
evictionEngine, retractionEngine, evalEngine,
|
|
122
|
-
consolidationEngine, consolidationScheduler,
|
|
123
|
-
});
|
|
124
|
-
|
|
125
|
-
// Coordination module (opt-in via AWM_COORDINATION=true)
|
|
126
|
-
const { isCoordinationEnabled, initCoordination, stopCoordinationCleanup } = await import('./coordination/index.js');
|
|
127
|
-
if (isCoordinationEnabled()) {
|
|
128
|
-
initCoordination(app, store.getDb(), store);
|
|
129
|
-
} else {
|
|
130
|
-
console.log(' Coordination module disabled (set AWM_COORDINATION=true to enable)');
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
// Background tasks
|
|
134
|
-
stagingBuffer.start(DEFAULT_AGENT_CONFIG.stagingTtlMs);
|
|
135
|
-
consolidationScheduler.start();
|
|
136
|
-
|
|
137
|
-
// Periodic hot backup every 10 minutes (keep last 6 = 1hr coverage)
|
|
138
|
-
const dbDir = dirname(resolve(DB_PATH));
|
|
139
|
-
const backupDir = resolve(dbDir, 'backups');
|
|
140
|
-
mkdirSync(backupDir, { recursive: true });
|
|
141
|
-
|
|
142
|
-
// Cleanup old backups on startup (older than 2 hours)
|
|
143
|
-
try {
|
|
144
|
-
const TWO_HOURS_MS = 2 * 60 * 60 * 1000;
|
|
145
|
-
const now = Date.now();
|
|
146
|
-
for (const f of readdirSync(backupDir).filter(f => f.endsWith('.db'))) {
|
|
147
|
-
const match = f.match(/(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})/);
|
|
148
|
-
if (match) {
|
|
149
|
-
const fileDate = new Date(`${match[1]}-${match[2]}-${match[3]}T${match[4]}:${match[5]}:${match[6]}Z`);
|
|
150
|
-
if (now - fileDate.getTime() > TWO_HOURS_MS) {
|
|
151
|
-
unlinkSync(resolve(backupDir, f));
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
} catch { /* cleanup is non-fatal */ }
|
|
156
|
-
|
|
157
|
-
const backupTimer = setInterval(() => {
|
|
158
|
-
try {
|
|
159
|
-
const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
160
|
-
const backupPath = resolve(backupDir, `${basename(DB_PATH, '.db')}-${ts}.db`);
|
|
161
|
-
store.backup(backupPath);
|
|
162
|
-
// Prune: keep only last 6 backups
|
|
163
|
-
const backups = readdirSync(backupDir).filter(f => f.endsWith('.db')).sort();
|
|
164
|
-
while (backups.length > 6) {
|
|
165
|
-
const old = backups.shift()!;
|
|
166
|
-
try { unlinkSync(resolve(backupDir, old)); } catch { /* non-fatal */ }
|
|
167
|
-
}
|
|
168
|
-
} catch (err) {
|
|
169
|
-
console.warn(`[backup] failed: ${(err as Error).message}`);
|
|
170
|
-
}
|
|
171
|
-
}, 10 * 60_000); // 10 minutes
|
|
172
|
-
|
|
173
|
-
// Pre-load ML models (downloads on first run: embeddings ~22MB, reranker ~22MB, expander ~80MB)
|
|
174
|
-
getEmbedder().catch(err => console.warn('Embedding model unavailable:', err.message));
|
|
175
|
-
getReranker().catch(err => console.warn('Reranker model unavailable:', err.message));
|
|
176
|
-
getExpander().catch(err => console.warn('Query expander model unavailable:', err.message));
|
|
177
|
-
|
|
178
|
-
// Start server
|
|
179
|
-
await app.listen({ port: PORT, host: '0.0.0.0' });
|
|
180
|
-
console.log(`AgentWorkingMemory v0.
|
|
181
|
-
|
|
182
|
-
// Graceful shutdown
|
|
183
|
-
const shutdown = async () => {
|
|
184
|
-
clearInterval(backupTimer);
|
|
185
|
-
await stopCoordinationCleanup();
|
|
186
|
-
consolidationScheduler.stop();
|
|
187
|
-
stagingBuffer.stop();
|
|
188
|
-
try { store.walCheckpoint(); } catch { /* non-fatal */ }
|
|
189
|
-
store.close();
|
|
190
|
-
process.exit(0);
|
|
191
|
-
};
|
|
192
|
-
process.on('SIGINT', shutdown);
|
|
193
|
-
process.on('SIGTERM', shutdown);
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
main().catch(err => {
|
|
197
|
-
console.error('Failed to start:', err);
|
|
198
|
-
process.exit(1);
|
|
199
|
-
});
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
import { readFileSync, copyFileSync, existsSync, mkdirSync, readdirSync, unlinkSync } from 'node:fs';
|
|
4
|
+
import { resolve, dirname, basename } from 'node:path';
|
|
5
|
+
import Fastify from 'fastify';
|
|
6
|
+
|
|
7
|
+
// Load .env file if present (no external dependency)
|
|
8
|
+
try {
|
|
9
|
+
const envPath = resolve(process.cwd(), '.env');
|
|
10
|
+
const envContent = readFileSync(envPath, 'utf-8');
|
|
11
|
+
for (const line of envContent.split('\n')) {
|
|
12
|
+
const trimmed = line.trim();
|
|
13
|
+
if (!trimmed || trimmed.startsWith('#')) continue;
|
|
14
|
+
const eqIdx = trimmed.indexOf('=');
|
|
15
|
+
if (eqIdx === -1) continue;
|
|
16
|
+
const key = trimmed.slice(0, eqIdx).trim();
|
|
17
|
+
const val = trimmed.slice(eqIdx + 1).trim().replace(/^["']|["']$/g, '');
|
|
18
|
+
if (!process.env[key]) process.env[key] = val; // Don't override existing env
|
|
19
|
+
}
|
|
20
|
+
} catch { /* No .env file — that's fine */ }
|
|
21
|
+
import { EngramStore } from './storage/sqlite.js';
|
|
22
|
+
import { ActivationEngine } from './engine/activation.js';
|
|
23
|
+
import { ConnectionEngine } from './engine/connections.js';
|
|
24
|
+
import { StagingBuffer } from './engine/staging.js';
|
|
25
|
+
import { EvictionEngine } from './engine/eviction.js';
|
|
26
|
+
import { RetractionEngine } from './engine/retraction.js';
|
|
27
|
+
import { EvalEngine } from './engine/eval.js';
|
|
28
|
+
import { ConsolidationEngine } from './engine/consolidation.js';
|
|
29
|
+
import { ConsolidationScheduler } from './engine/consolidation-scheduler.js';
|
|
30
|
+
import { registerRoutes } from './api/routes.js';
|
|
31
|
+
import { DEFAULT_AGENT_CONFIG } from './types/agent.js';
|
|
32
|
+
import { getEmbedder } from './core/embeddings.js';
|
|
33
|
+
import { getReranker } from './core/reranker.js';
|
|
34
|
+
import { getExpander } from './core/query-expander.js';
|
|
35
|
+
import { initLogger } from './core/logger.js';
|
|
36
|
+
|
|
37
|
+
const PORT = parseInt(process.env.AWM_PORT ?? '8400', 10);
|
|
38
|
+
const DB_PATH = process.env.AWM_DB_PATH ?? 'memory.db';
|
|
39
|
+
const API_KEY = process.env.AWM_API_KEY ?? null;
|
|
40
|
+
|
|
41
|
+
async function main() {
|
|
42
|
+
// Auto-backup: copy DB to backups/ on startup (cheap insurance)
|
|
43
|
+
if (existsSync(DB_PATH)) {
|
|
44
|
+
const dbDir = dirname(resolve(DB_PATH));
|
|
45
|
+
const backupDir = resolve(dbDir, 'backups');
|
|
46
|
+
mkdirSync(backupDir, { recursive: true });
|
|
47
|
+
const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
48
|
+
const backupPath = resolve(backupDir, `${basename(DB_PATH, '.db')}-${ts}.db`);
|
|
49
|
+
try {
|
|
50
|
+
copyFileSync(resolve(DB_PATH), backupPath);
|
|
51
|
+
console.log(`Backup: ${backupPath}`);
|
|
52
|
+
} catch (err) {
|
|
53
|
+
console.log(`Backup skipped: ${(err as Error).message}`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Logger — write activity to awm.log alongside the DB
|
|
58
|
+
initLogger(DB_PATH);
|
|
59
|
+
|
|
60
|
+
// Storage
|
|
61
|
+
const store = new EngramStore(DB_PATH);
|
|
62
|
+
|
|
63
|
+
// Integrity check
|
|
64
|
+
const integrity = store.integrityCheck();
|
|
65
|
+
if (!integrity.ok) {
|
|
66
|
+
console.error(`DB integrity check FAILED: ${integrity.result}`);
|
|
67
|
+
// Close corrupt DB, restore from backup, and exit for process manager to restart
|
|
68
|
+
store.close();
|
|
69
|
+
const dbDir = dirname(resolve(DB_PATH));
|
|
70
|
+
const backupDir = resolve(dbDir, 'backups');
|
|
71
|
+
if (existsSync(backupDir)) {
|
|
72
|
+
const backups = readdirSync(backupDir)
|
|
73
|
+
.filter(f => f.endsWith('.db'))
|
|
74
|
+
.sort()
|
|
75
|
+
.reverse();
|
|
76
|
+
if (backups.length > 0) {
|
|
77
|
+
const restorePath = resolve(backupDir, backups[0]);
|
|
78
|
+
console.error(`Attempting restore from: ${restorePath}`);
|
|
79
|
+
try {
|
|
80
|
+
copyFileSync(restorePath, resolve(DB_PATH));
|
|
81
|
+
console.error('Restore complete — exiting for restart with restored DB');
|
|
82
|
+
process.exit(1);
|
|
83
|
+
} catch (restoreErr) {
|
|
84
|
+
console.error(`Restore failed: ${(restoreErr as Error).message}`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
console.error('No backup available — continuing with potentially corrupt DB');
|
|
89
|
+
} else {
|
|
90
|
+
console.log(' DB integrity check: ok');
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Engines
|
|
94
|
+
const activationEngine = new ActivationEngine(store);
|
|
95
|
+
const connectionEngine = new ConnectionEngine(store, activationEngine);
|
|
96
|
+
const stagingBuffer = new StagingBuffer(store, activationEngine);
|
|
97
|
+
const evictionEngine = new EvictionEngine(store);
|
|
98
|
+
const retractionEngine = new RetractionEngine(store);
|
|
99
|
+
const evalEngine = new EvalEngine(store);
|
|
100
|
+
const consolidationEngine = new ConsolidationEngine(store);
|
|
101
|
+
const consolidationScheduler = new ConsolidationScheduler(store, consolidationEngine);
|
|
102
|
+
|
|
103
|
+
// API — disable Fastify's default request logging (too noisy for hive polling)
|
|
104
|
+
// bodyLimit: 512KB to prevent Content-Length mismatch errors with large task payloads
|
|
105
|
+
const app = Fastify({ logger: false, bodyLimit: 512_000 });
|
|
106
|
+
|
|
107
|
+
// Bearer token auth — only enforced when AWM_API_KEY is explicitly set and non-empty
|
|
108
|
+
if (API_KEY && API_KEY !== 'NONE' && API_KEY.length > 1) {
|
|
109
|
+
app.addHook('onRequest', async (req, reply) => {
|
|
110
|
+
if (req.url === '/health') return; // Health check is always public
|
|
111
|
+
const bearer = req.headers.authorization;
|
|
112
|
+
const xApiKey = req.headers['x-api-key'] as string | undefined;
|
|
113
|
+
if (bearer === `Bearer ${API_KEY}` || xApiKey === API_KEY) return;
|
|
114
|
+
reply.code(401).send({ error: 'Unauthorized' });
|
|
115
|
+
});
|
|
116
|
+
console.log('API key auth enabled (AWM_API_KEY set)');
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
registerRoutes(app, {
|
|
120
|
+
store, activationEngine, connectionEngine,
|
|
121
|
+
evictionEngine, retractionEngine, evalEngine,
|
|
122
|
+
consolidationEngine, consolidationScheduler,
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
// Coordination module (opt-in via AWM_COORDINATION=true)
|
|
126
|
+
const { isCoordinationEnabled, initCoordination, stopCoordinationCleanup } = await import('./coordination/index.js');
|
|
127
|
+
if (isCoordinationEnabled()) {
|
|
128
|
+
initCoordination(app, store.getDb(), store);
|
|
129
|
+
} else {
|
|
130
|
+
console.log(' Coordination module disabled (set AWM_COORDINATION=true to enable)');
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Background tasks
|
|
134
|
+
stagingBuffer.start(DEFAULT_AGENT_CONFIG.stagingTtlMs);
|
|
135
|
+
consolidationScheduler.start();
|
|
136
|
+
|
|
137
|
+
// Periodic hot backup every 10 minutes (keep last 6 = 1hr coverage)
|
|
138
|
+
const dbDir = dirname(resolve(DB_PATH));
|
|
139
|
+
const backupDir = resolve(dbDir, 'backups');
|
|
140
|
+
mkdirSync(backupDir, { recursive: true });
|
|
141
|
+
|
|
142
|
+
// Cleanup old backups on startup (older than 2 hours)
|
|
143
|
+
try {
|
|
144
|
+
const TWO_HOURS_MS = 2 * 60 * 60 * 1000;
|
|
145
|
+
const now = Date.now();
|
|
146
|
+
for (const f of readdirSync(backupDir).filter(f => f.endsWith('.db'))) {
|
|
147
|
+
const match = f.match(/(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})/);
|
|
148
|
+
if (match) {
|
|
149
|
+
const fileDate = new Date(`${match[1]}-${match[2]}-${match[3]}T${match[4]}:${match[5]}:${match[6]}Z`);
|
|
150
|
+
if (now - fileDate.getTime() > TWO_HOURS_MS) {
|
|
151
|
+
unlinkSync(resolve(backupDir, f));
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
} catch { /* cleanup is non-fatal */ }
|
|
156
|
+
|
|
157
|
+
const backupTimer = setInterval(() => {
|
|
158
|
+
try {
|
|
159
|
+
const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
160
|
+
const backupPath = resolve(backupDir, `${basename(DB_PATH, '.db')}-${ts}.db`);
|
|
161
|
+
store.backup(backupPath);
|
|
162
|
+
// Prune: keep only last 6 backups
|
|
163
|
+
const backups = readdirSync(backupDir).filter(f => f.endsWith('.db')).sort();
|
|
164
|
+
while (backups.length > 6) {
|
|
165
|
+
const old = backups.shift()!;
|
|
166
|
+
try { unlinkSync(resolve(backupDir, old)); } catch { /* non-fatal */ }
|
|
167
|
+
}
|
|
168
|
+
} catch (err) {
|
|
169
|
+
console.warn(`[backup] failed: ${(err as Error).message}`);
|
|
170
|
+
}
|
|
171
|
+
}, 10 * 60_000); // 10 minutes
|
|
172
|
+
|
|
173
|
+
// Pre-load ML models (downloads on first run: embeddings ~22MB, reranker ~22MB, expander ~80MB)
|
|
174
|
+
getEmbedder().catch(err => console.warn('Embedding model unavailable:', err.message));
|
|
175
|
+
getReranker().catch(err => console.warn('Reranker model unavailable:', err.message));
|
|
176
|
+
getExpander().catch(err => console.warn('Query expander model unavailable:', err.message));
|
|
177
|
+
|
|
178
|
+
// Start server
|
|
179
|
+
await app.listen({ port: PORT, host: '0.0.0.0' });
|
|
180
|
+
console.log(`AgentWorkingMemory v0.7.1 listening on port ${PORT}`);
|
|
181
|
+
|
|
182
|
+
// Graceful shutdown
|
|
183
|
+
const shutdown = async () => {
|
|
184
|
+
clearInterval(backupTimer);
|
|
185
|
+
await stopCoordinationCleanup();
|
|
186
|
+
consolidationScheduler.stop();
|
|
187
|
+
stagingBuffer.stop();
|
|
188
|
+
try { store.walCheckpoint(); } catch { /* non-fatal */ }
|
|
189
|
+
store.close();
|
|
190
|
+
process.exit(0);
|
|
191
|
+
};
|
|
192
|
+
process.on('SIGINT', shutdown);
|
|
193
|
+
process.on('SIGTERM', shutdown);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
main().catch(err => {
|
|
197
|
+
console.error('Failed to start:', err);
|
|
198
|
+
process.exit(1);
|
|
199
|
+
});
|