niahere 0.2.43 → 0.2.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/channels/slack.ts +19 -14
- package/src/channels/telegram.ts +12 -7
- package/src/chat/engine.ts +73 -8
- package/src/cli/index.ts +15 -0
- package/src/commands/backup.ts +146 -0
- package/src/core/consolidator.ts +138 -0
- package/src/core/runner.ts +7 -0
- package/src/db/migrations/008_message_delivery_status.ts +10 -0
- package/src/db/models/message.ts +37 -4
- package/src/types/engine.ts +2 -0
package/package.json
CHANGED
package/src/channels/slack.ts
CHANGED
|
@@ -7,7 +7,7 @@ import type { Channel, ChatState, Attachment, AttachmentType } from "../types";
|
|
|
7
7
|
import { getConfig, updateRawConfig, resetConfig } from "../utils/config";
|
|
8
8
|
import { relativeTime } from "../utils/format";
|
|
9
9
|
import { runMigrations } from "../db/migrate";
|
|
10
|
-
import { Session } from "../db/models";
|
|
10
|
+
import { Session, Message } from "../db/models";
|
|
11
11
|
import { log } from "../utils/log";
|
|
12
12
|
import { getMcpServers } from "../mcp";
|
|
13
13
|
import { getNiaHome, getPaths } from "../utils/paths";
|
|
@@ -493,7 +493,7 @@ class SlackChannel implements Channel {
|
|
|
493
493
|
.catch((err) => log.debug({ err, channel: msg.channel }, "slack: failed to add thinking reaction"));
|
|
494
494
|
|
|
495
495
|
try {
|
|
496
|
-
const { result } = await state.engine.send(text, {
|
|
496
|
+
const { result, messageId } = await state.engine.send(text, {
|
|
497
497
|
onActivity(status) {
|
|
498
498
|
log.debug({ status }, "slack engine activity");
|
|
499
499
|
},
|
|
@@ -504,22 +504,27 @@ class SlackChannel implements Channel {
|
|
|
504
504
|
// [NO_REPLY] or empty = agent chose not to respond (thread judgement)
|
|
505
505
|
if (!reply || cleanSentinel(reply) === "[NO_REPLY]") {
|
|
506
506
|
log.info({ channel: msg.channel, key }, "slack: agent chose not to reply");
|
|
507
|
+
if (messageId) await Message.updateDeliveryStatus(messageId, "sent").catch(() => {});
|
|
507
508
|
return;
|
|
508
509
|
}
|
|
509
510
|
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
511
|
+
try {
|
|
512
|
+
if (replyThreadTs) {
|
|
513
|
+
await client.chat.postMessage({
|
|
514
|
+
channel: msg.channel,
|
|
515
|
+
text: reply,
|
|
516
|
+
thread_ts: replyThreadTs,
|
|
517
|
+
});
|
|
518
|
+
} else {
|
|
519
|
+
await say(reply);
|
|
520
|
+
}
|
|
521
|
+
if (messageId) await Message.updateDeliveryStatus(messageId, "sent").catch(() => {});
|
|
522
|
+
log.info({ channel: msg.channel, key, chars: reply.length }, "slack reply sent");
|
|
523
|
+
} catch (sendErr) {
|
|
524
|
+
if (messageId) await Message.updateDeliveryStatus(messageId, "failed").catch(() => {});
|
|
525
|
+
throw sendErr;
|
|
518
526
|
}
|
|
519
|
-
|
|
520
|
-
log.info({ channel: msg.channel, key, chars: reply.length }, "slack reply sent");
|
|
521
527
|
} catch (err) {
|
|
522
|
-
|
|
523
528
|
const errText = err instanceof Error ? err.message : String(err);
|
|
524
529
|
log.error({ err, channel: msg.channel }, "slack message processing failed");
|
|
525
530
|
|
|
@@ -530,7 +535,7 @@ class SlackChannel implements Channel {
|
|
|
530
535
|
thread_ts: replyThreadTs,
|
|
531
536
|
}).catch(() => {});
|
|
532
537
|
} else {
|
|
533
|
-
await say(`[error] ${errText}`);
|
|
538
|
+
await say(`[error] ${errText}`).catch(() => {});
|
|
534
539
|
}
|
|
535
540
|
} finally {
|
|
536
541
|
await client.reactions.remove({ channel: msg.channel, timestamp: msg.ts, name: "thinking_face" })
|
package/src/channels/telegram.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { createChatEngine } from "../chat/engine";
|
|
|
3
3
|
import type { Channel, ChatState, Attachment } from "../types";
|
|
4
4
|
import { getConfig, updateRawConfig } from "../utils/config";
|
|
5
5
|
import { runMigrations } from "../db/migrate";
|
|
6
|
-
import { Session } from "../db/models";
|
|
6
|
+
import { Session, Message } from "../db/models";
|
|
7
7
|
import { log } from "../utils/log";
|
|
8
8
|
import { getMcpServers } from "../mcp";
|
|
9
9
|
import { classifyMime, validateAttachment, prepareImage } from "../utils/attachment";
|
|
@@ -136,16 +136,21 @@ class TelegramChannel implements Channel {
|
|
|
136
136
|
bot.api.sendChatAction(chatId, "typing").catch(() => {});
|
|
137
137
|
|
|
138
138
|
try {
|
|
139
|
-
const { result } = await state.engine.send(text, {}, attachments);
|
|
139
|
+
const { result, messageId } = await state.engine.send(text, {}, attachments);
|
|
140
140
|
|
|
141
141
|
const reply = result.trim() || "(no response)";
|
|
142
142
|
try {
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
143
|
+
try {
|
|
144
|
+
await bot.api.sendMessage(chatId, reply, { parse_mode: "MarkdownV2" });
|
|
145
|
+
} catch {
|
|
146
|
+
await bot.api.sendMessage(chatId, reply);
|
|
147
|
+
}
|
|
148
|
+
if (messageId) await Message.updateDeliveryStatus(messageId, "sent").catch(() => {});
|
|
149
|
+
log.info({ chatId, chars: result.length }, "telegram reply sent");
|
|
150
|
+
} catch (sendErr) {
|
|
151
|
+
if (messageId) await Message.updateDeliveryStatus(messageId, "failed").catch(() => {});
|
|
152
|
+
throw sendErr;
|
|
146
153
|
}
|
|
147
|
-
|
|
148
|
-
log.info({ chatId, chars: result.length }, "telegram reply sent");
|
|
149
154
|
} catch (err) {
|
|
150
155
|
const errText = err instanceof Error ? err.message : String(err);
|
|
151
156
|
log.error({ err, chatId }, "telegram message processing failed");
|
package/src/chat/engine.ts
CHANGED
|
@@ -10,8 +10,11 @@ import { getAgentDefinitions } from "../core/agents";
|
|
|
10
10
|
import { Session, Message, ActiveEngine } from "../db/models";
|
|
11
11
|
import type { Attachment, SendResult, StreamCallback, ActivityCallback, SendCallbacks, ChatEngine, EngineOptions } from "../types";
|
|
12
12
|
import { truncate, formatToolUse } from "../utils/format-activity";
|
|
13
|
+
import { consolidateSession } from "../core/consolidator";
|
|
14
|
+
import { log } from "../utils/log";
|
|
13
15
|
|
|
14
16
|
const IDLE_TIMEOUT = 10 * 60 * 1000; // 10 minutes
|
|
17
|
+
const LONG_RUNNING_WARN = 30 * 60 * 1000; // 30 minutes
|
|
15
18
|
|
|
16
19
|
interface SDKUserMessage {
|
|
17
20
|
type: "user";
|
|
@@ -128,20 +131,57 @@ export async function createChatEngine(opts: EngineOptions): Promise<ChatEngine>
|
|
|
128
131
|
let queryHandle: Query | null = null;
|
|
129
132
|
let pending: PendingResult | null = null;
|
|
130
133
|
let idleTimer: ReturnType<typeof setTimeout> | null = null;
|
|
134
|
+
let longRunningTimer: ReturnType<typeof setTimeout> | null = null;
|
|
135
|
+
let longRunningWarned = false;
|
|
131
136
|
let alive = false;
|
|
137
|
+
let messageCount = 0;
|
|
138
|
+
|
|
139
|
+
function clearIdleTimer() {
|
|
140
|
+
if (idleTimer) {
|
|
141
|
+
clearTimeout(idleTimer);
|
|
142
|
+
idleTimer = null;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
132
145
|
|
|
133
146
|
function resetIdleTimer() {
|
|
134
|
-
|
|
135
|
-
idleTimer = setTimeout(() => {
|
|
147
|
+
clearIdleTimer();
|
|
148
|
+
idleTimer = setTimeout(async () => {
|
|
149
|
+
if (pending) {
|
|
150
|
+
// Don't tear down while a request is in flight
|
|
151
|
+
log.warn({ room }, "idle timer fired while request pending, skipping teardown");
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
// Memory consolidation — "hippocampal replay" before sleep
|
|
155
|
+
if (sessionId && messageCount > 0) {
|
|
156
|
+
consolidateSession(sessionId, room).catch((err) => {
|
|
157
|
+
log.error({ err, room }, "consolidation failed during idle teardown");
|
|
158
|
+
});
|
|
159
|
+
}
|
|
136
160
|
teardown();
|
|
137
161
|
}, IDLE_TIMEOUT);
|
|
138
162
|
}
|
|
139
163
|
|
|
140
|
-
function
|
|
141
|
-
if (
|
|
142
|
-
clearTimeout(
|
|
143
|
-
|
|
164
|
+
function clearLongRunningTimer() {
|
|
165
|
+
if (longRunningTimer) {
|
|
166
|
+
clearTimeout(longRunningTimer);
|
|
167
|
+
longRunningTimer = null;
|
|
144
168
|
}
|
|
169
|
+
longRunningWarned = false;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function startLongRunningTimer() {
|
|
173
|
+
clearLongRunningTimer();
|
|
174
|
+
longRunningTimer = setTimeout(() => {
|
|
175
|
+
if (pending) {
|
|
176
|
+
longRunningWarned = true;
|
|
177
|
+
log.warn({ room, elapsed: LONG_RUNNING_WARN / 1000 }, "engine request running for 30+ minutes");
|
|
178
|
+
}
|
|
179
|
+
}, LONG_RUNNING_WARN);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function teardown() {
|
|
183
|
+
clearIdleTimer();
|
|
184
|
+
clearLongRunningTimer();
|
|
145
185
|
if (stream) {
|
|
146
186
|
stream.end();
|
|
147
187
|
stream = null;
|
|
@@ -207,6 +247,7 @@ export async function createChatEngine(opts: EngineOptions): Promise<ChatEngine>
|
|
|
207
247
|
content: pending.userMessage,
|
|
208
248
|
isFromAgent: false,
|
|
209
249
|
});
|
|
250
|
+
messageCount++;
|
|
210
251
|
}
|
|
211
252
|
}
|
|
212
253
|
|
|
@@ -285,31 +326,44 @@ export async function createChatEngine(opts: EngineOptions): Promise<ChatEngine>
|
|
|
285
326
|
const costUsd = (message as any).total_cost_usd as number;
|
|
286
327
|
const turns = (message as any).num_turns as number;
|
|
287
328
|
|
|
329
|
+
let messageId: number | undefined;
|
|
288
330
|
if (sessionId && resultText) {
|
|
289
|
-
await Message.save({
|
|
331
|
+
messageId = await Message.save({
|
|
290
332
|
sessionId,
|
|
291
333
|
room,
|
|
292
334
|
sender: "nia",
|
|
293
335
|
content: resultText,
|
|
294
336
|
isFromAgent: true,
|
|
337
|
+
deliveryStatus: "pending",
|
|
295
338
|
});
|
|
296
339
|
await Session.touch(sessionId);
|
|
297
340
|
}
|
|
298
341
|
|
|
299
342
|
await ActiveEngine.unregister(room);
|
|
300
|
-
|
|
343
|
+
clearLongRunningTimer();
|
|
344
|
+
pending.resolve({ result: resultText, costUsd, turns, messageId });
|
|
301
345
|
pending = null;
|
|
302
346
|
resetIdleTimer();
|
|
303
347
|
} else {
|
|
304
348
|
const errors = (message as any).errors;
|
|
305
349
|
const errorText = `[error] ${errors?.join(", ") || "unknown error"}`;
|
|
306
350
|
await ActiveEngine.unregister(room);
|
|
351
|
+
clearLongRunningTimer();
|
|
307
352
|
pending.resolve({ result: errorText, costUsd: 0, turns: 0 });
|
|
308
353
|
pending = null;
|
|
309
354
|
resetIdleTimer();
|
|
310
355
|
}
|
|
311
356
|
}
|
|
312
357
|
}
|
|
358
|
+
|
|
359
|
+
// Stream ended without a result — subprocess exited or was killed
|
|
360
|
+
if (pending) {
|
|
361
|
+
const partial = pending.accumulatedText;
|
|
362
|
+
log.error({ room, partialChars: partial.length }, "query stream ended without result, rejecting pending request");
|
|
363
|
+
await ActiveEngine.unregister(room).catch(() => {});
|
|
364
|
+
pending.reject(new Error(`stream ended without result (${partial.length} chars accumulated)`));
|
|
365
|
+
pending = null;
|
|
366
|
+
}
|
|
313
367
|
} catch (err) {
|
|
314
368
|
if (pending) {
|
|
315
369
|
await ActiveEngine.unregister(room).catch(() => {});
|
|
@@ -317,6 +371,7 @@ export async function createChatEngine(opts: EngineOptions): Promise<ChatEngine>
|
|
|
317
371
|
pending = null;
|
|
318
372
|
}
|
|
319
373
|
} finally {
|
|
374
|
+
clearLongRunningTimer();
|
|
320
375
|
alive = false;
|
|
321
376
|
stream = null;
|
|
322
377
|
queryHandle = null;
|
|
@@ -334,6 +389,10 @@ export async function createChatEngine(opts: EngineOptions): Promise<ChatEngine>
|
|
|
334
389
|
},
|
|
335
390
|
|
|
336
391
|
async send(userMessage: string, callbacks?: SendCallbacks, attachments?: Attachment[]) {
|
|
392
|
+
// Clear idle timer — engine is not idle while processing a request
|
|
393
|
+
clearIdleTimer();
|
|
394
|
+
startLongRunningTimer();
|
|
395
|
+
|
|
337
396
|
await ActiveEngine.register(room, channel);
|
|
338
397
|
|
|
339
398
|
if (!alive || !stream) {
|
|
@@ -356,6 +415,12 @@ export async function createChatEngine(opts: EngineOptions): Promise<ChatEngine>
|
|
|
356
415
|
},
|
|
357
416
|
|
|
358
417
|
close() {
|
|
418
|
+
// Memory consolidation on explicit close (skip if mid-turn — transcript is incomplete)
|
|
419
|
+
if (sessionId && messageCount > 0 && !pending) {
|
|
420
|
+
consolidateSession(sessionId, room).catch((err) => {
|
|
421
|
+
log.error({ err, room }, "consolidation failed during close");
|
|
422
|
+
});
|
|
423
|
+
}
|
|
359
424
|
teardown();
|
|
360
425
|
ActiveEngine.unregister(room).catch(() => {});
|
|
361
426
|
},
|
package/src/cli/index.ts
CHANGED
|
@@ -433,6 +433,12 @@ switch (command) {
|
|
|
433
433
|
process.exit(exitCode);
|
|
434
434
|
}
|
|
435
435
|
|
|
436
|
+
case "backup": {
|
|
437
|
+
const { backupCommand } = await import("../commands/backup");
|
|
438
|
+
await backupCommand();
|
|
439
|
+
break;
|
|
440
|
+
}
|
|
441
|
+
|
|
436
442
|
case "validate": {
|
|
437
443
|
const { validateConfig } = await import("../commands/validate");
|
|
438
444
|
const result = validateConfig();
|
|
@@ -444,6 +450,15 @@ switch (command) {
|
|
|
444
450
|
case "update": {
|
|
445
451
|
const { version: currentVersion } = await import("../../package.json");
|
|
446
452
|
console.log(`Current: v${currentVersion}`);
|
|
453
|
+
// Auto-backup before update
|
|
454
|
+
try {
|
|
455
|
+
const { createBackup } = await import("../commands/backup");
|
|
456
|
+
console.log("Backing up...");
|
|
457
|
+
await createBackup(true);
|
|
458
|
+
console.log("✓ pre-update backup created");
|
|
459
|
+
} catch (err) {
|
|
460
|
+
console.log(`⚠ backup skipped: ${errMsg(err)}`);
|
|
461
|
+
}
|
|
447
462
|
console.log("Updating...");
|
|
448
463
|
const install = Bun.spawn(["npm", "i", "-g", "niahere@latest"], { stdio: ["ignore", "inherit", "inherit"] });
|
|
449
464
|
const installExit = await install.exited;
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readdirSync, statSync, unlinkSync } from "fs";
|
|
2
|
+
import { join, basename } from "path";
|
|
3
|
+
import { getNiaHome } from "../utils/paths";
|
|
4
|
+
import { getConfig } from "../utils/config";
|
|
5
|
+
|
|
6
|
+
const MAX_BACKUPS = 10;
|
|
7
|
+
|
|
8
|
+
function getBackupDir(): string {
|
|
9
|
+
const dir = join(getNiaHome(), "backups");
|
|
10
|
+
mkdirSync(dir, { recursive: true });
|
|
11
|
+
return dir;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function humanDate(): string {
|
|
15
|
+
const now = new Date();
|
|
16
|
+
const y = now.getFullYear();
|
|
17
|
+
const m = String(now.getMonth() + 1).padStart(2, "0");
|
|
18
|
+
const d = String(now.getDate()).padStart(2, "0");
|
|
19
|
+
return `${y}-${m}-${d}`;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function formatSize(bytes: number): string {
|
|
23
|
+
if (bytes < 1024) return `${bytes}B`;
|
|
24
|
+
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
|
|
25
|
+
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function pruneOldBackups(dir: string): void {
|
|
29
|
+
const files = readdirSync(dir)
|
|
30
|
+
.filter((f) => f.startsWith("niahere-") && f.endsWith(".tar.gz"))
|
|
31
|
+
.map((f) => ({ name: f, mtime: statSync(join(dir, f)).mtimeMs }))
|
|
32
|
+
.sort((a, b) => b.mtime - a.mtime);
|
|
33
|
+
|
|
34
|
+
for (const file of files.slice(MAX_BACKUPS)) {
|
|
35
|
+
unlinkSync(join(dir, file.name));
|
|
36
|
+
console.log(` pruned old backup: ${file.name}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export async function createBackup(silent = false): Promise<string> {
|
|
41
|
+
const home = getNiaHome();
|
|
42
|
+
const backupDir = getBackupDir();
|
|
43
|
+
const filename = `niahere-${humanDate()}-${Math.floor(Date.now() / 1000)}.tar.gz`;
|
|
44
|
+
const outPath = join(backupDir, filename);
|
|
45
|
+
|
|
46
|
+
// Directories/files to include (relative to home)
|
|
47
|
+
const includes: string[] = [];
|
|
48
|
+
if (existsSync(join(home, "config.yaml"))) includes.push("config.yaml");
|
|
49
|
+
if (existsSync(join(home, "self"))) includes.push("self");
|
|
50
|
+
if (existsSync(join(home, "agents"))) includes.push("agents");
|
|
51
|
+
if (existsSync(join(home, "skills"))) includes.push("skills");
|
|
52
|
+
|
|
53
|
+
// Database dump
|
|
54
|
+
const config = getConfig();
|
|
55
|
+
const dbUrl = config.database_url;
|
|
56
|
+
let dbDumped = false;
|
|
57
|
+
if (dbUrl) {
|
|
58
|
+
const dumpPath = join(home, "tmp", "db-backup.sql");
|
|
59
|
+
mkdirSync(join(home, "tmp"), { recursive: true });
|
|
60
|
+
const pg = Bun.spawn(["pg_dump", dbUrl, "-f", dumpPath], {
|
|
61
|
+
stdout: "pipe",
|
|
62
|
+
stderr: "pipe",
|
|
63
|
+
});
|
|
64
|
+
const exitCode = await pg.exited;
|
|
65
|
+
if (exitCode === 0 && existsSync(dumpPath)) {
|
|
66
|
+
// Copy to a relative path for tar
|
|
67
|
+
const relDump = "db-backup.sql";
|
|
68
|
+
const { copyFileSync } = await import("fs");
|
|
69
|
+
copyFileSync(dumpPath, join(home, relDump));
|
|
70
|
+
includes.push(relDump);
|
|
71
|
+
dbDumped = true;
|
|
72
|
+
} else if (!silent) {
|
|
73
|
+
const stderr = await new Response(pg.stderr).text();
|
|
74
|
+
console.log(` ⚠ db dump skipped: ${stderr.trim() || `exit ${exitCode}`}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
if (includes.length === 0) {
|
|
79
|
+
console.log("Nothing to back up.");
|
|
80
|
+
return "";
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Create tar.gz
|
|
84
|
+
const tar = Bun.spawn(["tar", "czf", outPath, ...includes], {
|
|
85
|
+
cwd: home,
|
|
86
|
+
stdout: "pipe",
|
|
87
|
+
stderr: "pipe",
|
|
88
|
+
});
|
|
89
|
+
const tarExit = await tar.exited;
|
|
90
|
+
if (tarExit !== 0) {
|
|
91
|
+
const stderr = await new Response(tar.stderr).text();
|
|
92
|
+
throw new Error(`tar failed: ${stderr.trim()}`);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Clean up temp db dump
|
|
96
|
+
if (dbDumped) {
|
|
97
|
+
try { unlinkSync(join(home, "db-backup.sql")); } catch {}
|
|
98
|
+
try { unlinkSync(join(home, "tmp", "db-backup.sql")); } catch {}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const size = statSync(outPath).size;
|
|
102
|
+
if (!silent) {
|
|
103
|
+
console.log(`✓ backup created: ${filename} (${formatSize(size)})`);
|
|
104
|
+
if (dbDumped) console.log(" includes: files + database");
|
|
105
|
+
else console.log(" includes: files only (no database)");
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
pruneOldBackups(backupDir);
|
|
109
|
+
|
|
110
|
+
return outPath;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function listBackups(): void {
|
|
114
|
+
const dir = getBackupDir();
|
|
115
|
+
const files = readdirSync(dir)
|
|
116
|
+
.filter((f) => f.startsWith("niahere-") && f.endsWith(".tar.gz"))
|
|
117
|
+
.map((f) => {
|
|
118
|
+
const stat = statSync(join(dir, f));
|
|
119
|
+
return { name: f, size: stat.size, mtime: stat.mtimeMs };
|
|
120
|
+
})
|
|
121
|
+
.sort((a, b) => b.mtime - a.mtime);
|
|
122
|
+
|
|
123
|
+
if (files.length === 0) {
|
|
124
|
+
console.log("No backups found.");
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
console.log(`${files.length} backup(s) in ${dir}:\n`);
|
|
129
|
+
for (const f of files) {
|
|
130
|
+
const date = new Date(f.mtime).toLocaleString();
|
|
131
|
+
console.log(` ${f.name} ${formatSize(f.size)} ${date}`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export async function backupCommand(): Promise<void> {
|
|
136
|
+
const sub = process.argv[3];
|
|
137
|
+
if (sub === "list") {
|
|
138
|
+
listBackups();
|
|
139
|
+
} else if (!sub) {
|
|
140
|
+
await createBackup();
|
|
141
|
+
} else {
|
|
142
|
+
console.log("Usage:");
|
|
143
|
+
console.log(" nia backup — create a backup");
|
|
144
|
+
console.log(" nia backup list — list existing backups");
|
|
145
|
+
}
|
|
146
|
+
}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory consolidator — "hippocampal replay" for Nia.
|
|
3
|
+
*
|
|
4
|
+
* After a chat session goes idle or a job completes, this module reviews
|
|
5
|
+
* what happened and saves memories worth keeping.
|
|
6
|
+
*
|
|
7
|
+
* This decouples memory formation from task execution: during a conversation,
|
|
8
|
+
* the agent focuses on the task. Afterward, a background pass extracts what's
|
|
9
|
+
* worth remembering — just like the brain consolidates memories during sleep.
|
|
10
|
+
*
|
|
11
|
+
* The consolidator uses the same agent loop as cron jobs — full Nia system
|
|
12
|
+
* prompt, full tool access, same runner. It's just a specialized job.
|
|
13
|
+
*
|
|
14
|
+
* Research basis:
|
|
15
|
+
* - LangChain: "background" memory formation avoids latency + competing optimization pressures
|
|
16
|
+
* - Mem0: LLM-driven extraction with ADD/UPDATE/NOOP decisions against existing memories
|
|
17
|
+
* - Cognitive science: hippocampal replay consolidates experiences after the fact, not during
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { Message } from "../db/models";
|
|
21
|
+
import { buildSystemPrompt } from "../chat/identity";
|
|
22
|
+
import { runJobWithClaude } from "./runner";
|
|
23
|
+
import { log } from "../utils/log";
|
|
24
|
+
import { homedir } from "os";
|
|
25
|
+
import type { SessionMessage } from "../types";
|
|
26
|
+
|
|
27
|
+
/** Track sessions already consolidated to prevent double runs. */
|
|
28
|
+
const consolidated = new Set<string>();
|
|
29
|
+
|
|
30
|
+
/** Max messages to include in transcript (most recent). Keeps prompt size bounded. */
|
|
31
|
+
const MAX_TRANSCRIPT_MESSAGES = 50;
|
|
32
|
+
|
|
33
|
+
/** Rooms to skip (placeholder sessions). */
|
|
34
|
+
function shouldSkip(room: string): boolean {
|
|
35
|
+
return room.includes("placeholder");
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Format conversation transcript for the extraction prompt. Cap to recent messages. */
|
|
39
|
+
function formatTranscript(messages: SessionMessage[]): string {
|
|
40
|
+
const recent = messages.slice(-MAX_TRANSCRIPT_MESSAGES);
|
|
41
|
+
const skipped = messages.length - recent.length;
|
|
42
|
+
const prefix = skipped > 0 ? `[...${skipped} earlier messages omitted]\n\n` : "";
|
|
43
|
+
|
|
44
|
+
return prefix + recent
|
|
45
|
+
.map((m) => `[${m.sender}] (${m.createdAt}): ${m.content.slice(0, 2000)}`)
|
|
46
|
+
.join("\n\n");
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Build the extraction prompt from a conversation transcript. */
|
|
50
|
+
function buildConsolidationPrompt(transcript: string, source: string): string {
|
|
51
|
+
return `Job: memory-consolidation (triggered by ${source})
|
|
52
|
+
|
|
53
|
+
You just finished a session. It has gone idle.
|
|
54
|
+
Your task: review the transcript below and save anything worth keeping for future sessions.
|
|
55
|
+
|
|
56
|
+
## Transcript
|
|
57
|
+
${transcript}
|
|
58
|
+
|
|
59
|
+
## Instructions
|
|
60
|
+
1. First, read your existing memories (read_memory tool) and rules (read rules.md) to avoid duplicates
|
|
61
|
+
2. Review the transcript for things worth persisting. Use the RIGHT tool for each:
|
|
62
|
+
|
|
63
|
+
**Use add_memory for FACTS** (nouns — things that are true):
|
|
64
|
+
- People: names, roles, orgs, relationships
|
|
65
|
+
- Decisions: what was decided or agreed on
|
|
66
|
+
- Technical facts: system details, API quirks, config gotchas
|
|
67
|
+
- Patterns: recurring issues, user behaviors, workflow tendencies
|
|
68
|
+
- Events: travel, deadlines, incidents, milestones with dates
|
|
69
|
+
|
|
70
|
+
**Use add_rule for INSTRUCTIONS** (verbs — how to behave):
|
|
71
|
+
- User corrected your tone, format, or approach
|
|
72
|
+
- User said "from now on" / "always" / "never" / "stop doing X"
|
|
73
|
+
- User expressed a preference about how you communicate or work
|
|
74
|
+
|
|
75
|
+
3. Skip anything already in existing memories or rules (no duplicates)
|
|
76
|
+
4. Skip small talk, greetings, conversational filler
|
|
77
|
+
5. Skip transient state ("currently working on X")
|
|
78
|
+
6. Quality over quantity — saving nothing is fine if the conversation was trivial
|
|
79
|
+
7. If existing memories are outdated based on new info, note what should be updated
|
|
80
|
+
|
|
81
|
+
Do NOT message the user about this. Save silently and report a brief summary of what you saved.`;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Run the consolidation agent loop. */
|
|
85
|
+
async function runConsolidation(transcript: string, source: string): Promise<void> {
|
|
86
|
+
const systemPrompt = buildSystemPrompt("job");
|
|
87
|
+
const jobPrompt = buildConsolidationPrompt(transcript, source);
|
|
88
|
+
const output = await runJobWithClaude(systemPrompt, jobPrompt, homedir());
|
|
89
|
+
|
|
90
|
+
if (output.error) {
|
|
91
|
+
log.error({ source, error: output.error }, "consolidator: extraction failed");
|
|
92
|
+
} else {
|
|
93
|
+
log.info({ source, resultChars: output.agentText.length }, "consolidator: done");
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Consolidate a chat session's conversation into memories.
|
|
99
|
+
* Called when a chat engine goes idle or is explicitly closed.
|
|
100
|
+
*/
|
|
101
|
+
export async function consolidateSession(sessionId: string, room: string): Promise<void> {
|
|
102
|
+
if (shouldSkip(room)) return;
|
|
103
|
+
if (consolidated.has(sessionId)) return;
|
|
104
|
+
consolidated.add(sessionId);
|
|
105
|
+
|
|
106
|
+
try {
|
|
107
|
+
const messages = await Message.getBySession(sessionId);
|
|
108
|
+
if (messages.length < 2) return;
|
|
109
|
+
|
|
110
|
+
log.info({ sessionId, room, messageCount: messages.length }, "consolidator: extracting memories from chat");
|
|
111
|
+
|
|
112
|
+
const transcript = formatTranscript(messages);
|
|
113
|
+
await runConsolidation(transcript, `chat session idle — ${room}`);
|
|
114
|
+
} catch (err) {
|
|
115
|
+
log.error({ err, sessionId, room }, "consolidator: chat extraction failed");
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Consolidate a job run's output into memories.
|
|
121
|
+
* Called after a job completes in the runner.
|
|
122
|
+
*/
|
|
123
|
+
export async function consolidateJobRun(jobName: string, jobPrompt: string, result: string): Promise<void> {
|
|
124
|
+
// Skip if the job itself is the consolidator (prevent infinite loop)
|
|
125
|
+
if (jobName === "memory-consolidation") return;
|
|
126
|
+
|
|
127
|
+
const transcript = `[job-prompt]: ${jobPrompt}\n\n[job-result]: ${result}`;
|
|
128
|
+
|
|
129
|
+
// Skip trivial results
|
|
130
|
+
if (result.length < 50) return;
|
|
131
|
+
|
|
132
|
+
try {
|
|
133
|
+
log.info({ jobName, resultChars: result.length }, "consolidator: extracting memories from job");
|
|
134
|
+
await runConsolidation(transcript, `job run — ${jobName}`);
|
|
135
|
+
} catch (err) {
|
|
136
|
+
log.error({ err, jobName }, "consolidator: job extraction failed");
|
|
137
|
+
}
|
|
138
|
+
}
|
package/src/core/runner.ts
CHANGED
|
@@ -257,6 +257,13 @@ export async function runJob(job: JobInput, onActivity?: ActivityCallback): Prom
|
|
|
257
257
|
};
|
|
258
258
|
writeState(freshState);
|
|
259
259
|
|
|
260
|
+
// Memory consolidation — review what the job learned (fire-and-forget)
|
|
261
|
+
if (ok && result.result) {
|
|
262
|
+
import("./consolidator").then(({ consolidateJobRun }) => {
|
|
263
|
+
consolidateJobRun(job.name, jobPrompt, result.result).catch(() => {});
|
|
264
|
+
}).catch(() => {});
|
|
265
|
+
}
|
|
266
|
+
|
|
260
267
|
return result;
|
|
261
268
|
} catch (err) {
|
|
262
269
|
const duration_ms = Math.round(performance.now() - startMs);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type postgres from "postgres";
|
|
2
|
+
|
|
3
|
+
export const name = "008_message_delivery_status";
|
|
4
|
+
|
|
5
|
+
export async function up(sql: postgres.Sql): Promise<void> {
|
|
6
|
+
await sql`
|
|
7
|
+
ALTER TABLE messages
|
|
8
|
+
ADD COLUMN IF NOT EXISTS delivery_status TEXT DEFAULT 'sent'
|
|
9
|
+
`;
|
|
10
|
+
}
|
package/src/db/models/message.ts
CHANGED
|
@@ -1,12 +1,45 @@
|
|
|
1
1
|
import { getSql } from "../connection";
|
|
2
2
|
import type { SaveMessageParams, RoomStats, RecentMessage, SearchResult, SessionMessage } from "../../types";
|
|
3
3
|
|
|
4
|
-
export
|
|
4
|
+
export type DeliveryStatus = "pending" | "sent" | "failed";
|
|
5
|
+
|
|
6
|
+
export async function save(params: SaveMessageParams & { deliveryStatus?: DeliveryStatus }): Promise<number> {
|
|
5
7
|
const sql = getSql();
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
8
|
+
const status = params.deliveryStatus || "sent";
|
|
9
|
+
const rows = await sql`
|
|
10
|
+
INSERT INTO messages (session_id, room, sender, content, is_from_agent, delivery_status)
|
|
11
|
+
VALUES (${params.sessionId}, ${params.room}, ${params.sender}, ${params.content}, ${params.isFromAgent}, ${status})
|
|
12
|
+
RETURNING id
|
|
9
13
|
`;
|
|
14
|
+
return rows[0].id;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export async function updateDeliveryStatus(id: number, status: DeliveryStatus): Promise<void> {
|
|
18
|
+
const sql = getSql();
|
|
19
|
+
await sql`UPDATE messages SET delivery_status = ${status} WHERE id = ${id}`;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export async function getUndelivered(room?: string): Promise<Array<{ id: number; room: string; content: string; createdAt: string }>> {
|
|
23
|
+
const sql = getSql();
|
|
24
|
+
const rows = room
|
|
25
|
+
? await sql`
|
|
26
|
+
SELECT id, room, content, created_at
|
|
27
|
+
FROM messages
|
|
28
|
+
WHERE delivery_status = 'failed' AND is_from_agent = true AND room = ${room}
|
|
29
|
+
ORDER BY created_at ASC
|
|
30
|
+
`
|
|
31
|
+
: await sql`
|
|
32
|
+
SELECT id, room, content, created_at
|
|
33
|
+
FROM messages
|
|
34
|
+
WHERE delivery_status = 'failed' AND is_from_agent = true
|
|
35
|
+
ORDER BY created_at ASC
|
|
36
|
+
`;
|
|
37
|
+
return rows.map((r) => ({
|
|
38
|
+
id: r.id,
|
|
39
|
+
room: r.room,
|
|
40
|
+
content: r.content,
|
|
41
|
+
createdAt: String(r.created_at),
|
|
42
|
+
}));
|
|
10
43
|
}
|
|
11
44
|
|
|
12
45
|
export async function getRecent(limit = 20, room?: string): Promise<RecentMessage[]> {
|
package/src/types/engine.ts
CHANGED