kongbrain 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/package.json +1 -1
- package/src/context-engine.ts +16 -8
- package/src/daemon-manager.ts +183 -99
- package/src/daemon-types.ts +1 -48
- package/src/deferred-cleanup.ts +141 -0
- package/src/handoff-file.ts +55 -0
- package/src/hooks/llm-output.ts +12 -91
- package/src/index.ts +90 -18
- package/src/memory-daemon.ts +181 -376
- package/src/reflection.ts +1 -1
- package/src/schema.surql +2 -0
- package/src/state.ts +6 -1
- package/src/surreal.ts +24 -0
- package/src/wakeup.ts +13 -1
package/README.md
CHANGED
|
@@ -4,8 +4,9 @@
|
|
|
4
4
|
|
|
5
5
|

|
|
6
6
|
|
|
7
|
+
[](https://www.npmjs.com/package/kongbrain)
|
|
7
8
|
[](https://github.com/42U/kongbrain)
|
|
8
|
-
[](https://opensource.org/licenses/MIT)
|
|
9
10
|
[](https://nodejs.org)
|
|
10
11
|
[](https://surrealdb.com)
|
|
11
12
|
[](https://github.com/openclaw/openclaw)
|
|
@@ -270,7 +271,7 @@ Triggers at session end when metrics indicate problems:
|
|
|
270
271
|
| Steering candidates | any detected |
|
|
271
272
|
| Context waste | > 0.5% of context window |
|
|
272
273
|
|
|
273
|
-
|
|
274
|
+
The LLM generates a 2-4 sentence reflection: root cause, error pattern, what to do differently. Stored with importance 7.0, deduped at 0.85 cosine similarity.
|
|
274
275
|
|
|
275
276
|
</details>
|
|
276
277
|
|
|
@@ -297,7 +298,7 @@ Context Injection ─ Vector search -> graph expand -> 6-signal scoring -> budge
|
|
|
297
298
|
| Scores: similarity, recency, importance, access, neighbor, utility
|
|
298
299
|
| Budget: 21% of context window reserved for retrieval
|
|
299
300
|
v
|
|
300
|
-
Agent Loop ────────
|
|
301
|
+
Agent Loop ──────── LLM + tool execution
|
|
301
302
|
| Planning gate: announces plan before touching tools
|
|
302
303
|
| Smart truncation: preserves tail of large tool outputs
|
|
303
304
|
v
|
|
@@ -307,7 +308,7 @@ Turn Storage ────── Every message embedded + stored + linked via gra
|
|
|
307
308
|
Quality Eval ────── Measures retrieval utilization (text overlap, trigrams, unigrams)
|
|
308
309
|
| Tracks tool success, context waste, feeds ACAN training
|
|
309
310
|
v
|
|
310
|
-
Memory Daemon ───── Worker thread extracts 9 knowledge types via
|
|
311
|
+
Memory Daemon ───── Worker thread extracts 9 knowledge types via LLM:
|
|
311
312
|
| causal chains, monologues, concepts, corrections,
|
|
312
313
|
| preferences, artifacts, decisions, skills, resolved memories
|
|
313
314
|
v
|
|
@@ -323,7 +324,7 @@ At session start, a wake-up briefing is synthesized from the handoff, recent mon
|
|
|
323
324
|
<details>
|
|
324
325
|
<summary><strong>Memory Daemon</strong>: background knowledge extraction</summary>
|
|
325
326
|
|
|
326
|
-
A worker thread running throughout the session. Batches turns every ~12K tokens, calls
|
|
327
|
+
A worker thread running throughout the session. Batches turns every ~12K tokens, calls the configured LLM to extract:
|
|
327
328
|
|
|
328
329
|
- **Causal chains**: trigger/outcome sequences with success/confidence
|
|
329
330
|
- **Monologue traces**: thinking blocks that reveal problem-solving approach
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "kongbrain",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.3",
|
|
4
4
|
"description": "Graph-backed persistent memory engine for OpenClaw. Replaces the default context window with SurrealDB + vector embeddings that learn across sessions.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/src/context-engine.ts
CHANGED
|
@@ -53,7 +53,7 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
53
53
|
readonly info: ContextEngineInfo = {
|
|
54
54
|
id: "kongbrain",
|
|
55
55
|
name: "KongBrain",
|
|
56
|
-
version: "0.1.
|
|
56
|
+
version: "0.1.2",
|
|
57
57
|
ownsCompaction: true,
|
|
58
58
|
};
|
|
59
59
|
|
|
@@ -103,10 +103,13 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
103
103
|
.catch(e => swallow.warn("bootstrap:linkTaskToProject", e));
|
|
104
104
|
|
|
105
105
|
const surrealSessionId = await store.createSession(session.agentId);
|
|
106
|
+
await store.markSessionActive(surrealSessionId)
|
|
107
|
+
.catch(e => swallow.warn("bootstrap:markActive", e));
|
|
106
108
|
await store.linkSessionToTask(surrealSessionId, session.taskId)
|
|
107
109
|
.catch(e => swallow.warn("bootstrap:linkSessionToTask", e));
|
|
108
110
|
|
|
109
|
-
//
|
|
111
|
+
// Store the DB session ID for cleanup tracking
|
|
112
|
+
session.surrealSessionId = surrealSessionId;
|
|
110
113
|
session.lastUserTurnId = "";
|
|
111
114
|
} catch (e) {
|
|
112
115
|
swallow.error("bootstrap:5pillar", e);
|
|
@@ -248,8 +251,10 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
248
251
|
});
|
|
249
252
|
|
|
250
253
|
if (turnId) {
|
|
251
|
-
|
|
252
|
-
.
|
|
254
|
+
if (session.surrealSessionId) {
|
|
255
|
+
await store.relate(turnId, "part_of", session.surrealSessionId)
|
|
256
|
+
.catch(e => swallow.warn("ingest:relate", e));
|
|
257
|
+
}
|
|
253
258
|
|
|
254
259
|
// Link to previous user turn for responds_to edge
|
|
255
260
|
if (role === "assistant" && session.lastUserTurnId) {
|
|
@@ -257,8 +262,8 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
257
262
|
.catch(e => swallow.warn("ingest:responds_to", e));
|
|
258
263
|
}
|
|
259
264
|
|
|
260
|
-
// Extract and link concepts for user turns
|
|
261
|
-
if (
|
|
265
|
+
// Extract and link concepts for both user and assistant turns
|
|
266
|
+
if (worthEmbedding) {
|
|
262
267
|
extractAndLinkConcepts(turnId, text, this.state)
|
|
263
268
|
.catch(e => swallow.warn("ingest:concepts", e));
|
|
264
269
|
}
|
|
@@ -381,8 +386,10 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
381
386
|
session.newContentTokens += Math.ceil(session.lastAssistantText.length / 4);
|
|
382
387
|
}
|
|
383
388
|
|
|
384
|
-
// Flush to daemon when token threshold is reached
|
|
385
|
-
|
|
389
|
+
// Flush to daemon when token threshold OR turn count threshold is reached
|
|
390
|
+
const tokenReady = session.newContentTokens >= session.DAEMON_TOKEN_THRESHOLD;
|
|
391
|
+
const turnReady = session.userTurnCount >= session.lastDaemonFlushTurnCount + 3;
|
|
392
|
+
if (session.daemon && (tokenReady || turnReady)) {
|
|
386
393
|
try {
|
|
387
394
|
const recentTurns = await store.getSessionTurns(session.sessionId, 20);
|
|
388
395
|
const turnData = recentTurns.map(t => ({
|
|
@@ -404,6 +411,7 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
404
411
|
);
|
|
405
412
|
|
|
406
413
|
session.newContentTokens = 0;
|
|
414
|
+
session.lastDaemonFlushTurnCount = session.userTurnCount;
|
|
407
415
|
session.pendingThinking.length = 0;
|
|
408
416
|
} catch (e) {
|
|
409
417
|
swallow.warn("afterTurn:daemonBatch", e);
|
package/src/daemon-manager.ts
CHANGED
|
@@ -1,20 +1,18 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Daemon Manager —
|
|
2
|
+
* Daemon Manager — runs memory extraction in-process.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* Originally used a Worker thread, but OpenClaw loads plugins via jiti
|
|
5
|
+
* (TypeScript only, no compiled JS), and Node's Worker constructor requires
|
|
6
|
+
* .js files. Refactored to run extraction async in the main thread.
|
|
7
|
+
* The extraction is I/O-bound (LLM calls + DB writes), not CPU-bound,
|
|
8
|
+
* so in-process execution is fine.
|
|
8
9
|
*/
|
|
9
|
-
import { Worker } from "node:worker_threads";
|
|
10
|
-
import { join, dirname } from "node:path";
|
|
11
|
-
import { fileURLToPath } from "node:url";
|
|
12
10
|
import type { SurrealConfig, EmbeddingConfig } from "./config.js";
|
|
13
|
-
import type {
|
|
11
|
+
import type { TurnData, PriorExtractions } from "./daemon-types.js";
|
|
12
|
+
import { SurrealStore } from "./surreal.js";
|
|
13
|
+
import { EmbeddingService } from "./embeddings.js";
|
|
14
14
|
import { swallow } from "./errors.js";
|
|
15
15
|
|
|
16
|
-
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
17
|
-
|
|
18
16
|
export type { TurnData } from "./daemon-types.js";
|
|
19
17
|
|
|
20
18
|
export interface MemoryDaemon {
|
|
@@ -25,11 +23,11 @@ export interface MemoryDaemon {
|
|
|
25
23
|
retrievedMemories: { id: string; text: string }[],
|
|
26
24
|
priorExtractions?: PriorExtractions,
|
|
27
25
|
): void;
|
|
28
|
-
/** Request current daemon status
|
|
29
|
-
getStatus(): Promise<
|
|
30
|
-
/** Graceful shutdown: waits for current extraction, then
|
|
26
|
+
/** Request current daemon status. */
|
|
27
|
+
getStatus(): Promise<{ type: "status"; extractedTurns: number; pendingBatches: number; errors: number }>;
|
|
28
|
+
/** Graceful shutdown: waits for current extraction, then cleans up. */
|
|
31
29
|
shutdown(timeoutMs?: number): Promise<void>;
|
|
32
|
-
/**
|
|
30
|
+
/** How many turns has the daemon already extracted? */
|
|
33
31
|
getExtractedTurnCount(): number;
|
|
34
32
|
}
|
|
35
33
|
|
|
@@ -39,106 +37,192 @@ export function startMemoryDaemon(
|
|
|
39
37
|
sessionId: string,
|
|
40
38
|
llmConfig?: { provider?: string; model?: string },
|
|
41
39
|
): MemoryDaemon {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
40
|
+
// Daemon-local DB and embedding instances (separate connections)
|
|
41
|
+
let store: SurrealStore | null = null;
|
|
42
|
+
let embeddings: EmbeddingService | null = null;
|
|
43
|
+
let initialized = false;
|
|
44
|
+
let initFailed = false;
|
|
45
|
+
let processing = false;
|
|
46
|
+
let shuttingDown = false;
|
|
47
|
+
let extractedTurnCount = 0;
|
|
48
|
+
let errorCount = 0;
|
|
49
|
+
|
|
50
|
+
const priorState: PriorExtractions = {
|
|
51
|
+
conceptNames: [], artifactPaths: [], skillNames: [],
|
|
48
52
|
};
|
|
49
53
|
|
|
50
|
-
|
|
54
|
+
// Lazy init — connect on first batch, not at startup
|
|
55
|
+
async function ensureInit(): Promise<boolean> {
|
|
56
|
+
if (initialized) return true;
|
|
57
|
+
if (initFailed) return false;
|
|
58
|
+
try {
|
|
59
|
+
store = new SurrealStore(surrealConfig);
|
|
60
|
+
await store.initialize();
|
|
61
|
+
embeddings = new EmbeddingService(embeddingConfig);
|
|
62
|
+
await embeddings.initialize();
|
|
63
|
+
initialized = true;
|
|
64
|
+
return true;
|
|
65
|
+
} catch (e) {
|
|
66
|
+
swallow.warn("daemon:init", e);
|
|
67
|
+
initFailed = true;
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
51
71
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
72
|
+
// Import extraction logic lazily to avoid circular deps
|
|
73
|
+
async function runExtraction(
|
|
74
|
+
turns: TurnData[],
|
|
75
|
+
thinking: string[],
|
|
76
|
+
retrievedMemories: { id: string; text: string }[],
|
|
77
|
+
incomingPrior?: PriorExtractions,
|
|
78
|
+
): Promise<void> {
|
|
79
|
+
if (!store || !embeddings) return;
|
|
80
|
+
if (turns.length < 2) return;
|
|
81
|
+
|
|
82
|
+
const provider = llmConfig?.provider;
|
|
83
|
+
const modelId = llmConfig?.model;
|
|
84
|
+
if (!provider || !modelId) {
|
|
85
|
+
swallow.warn("daemon:extraction", new Error("Missing llmProvider/llmModel"));
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Merge incoming prior state
|
|
90
|
+
if (incomingPrior) {
|
|
91
|
+
for (const name of incomingPrior.conceptNames) {
|
|
92
|
+
if (!priorState.conceptNames.includes(name)) priorState.conceptNames.push(name);
|
|
93
|
+
}
|
|
94
|
+
for (const path of incomingPrior.artifactPaths) {
|
|
95
|
+
if (!priorState.artifactPaths.includes(path)) priorState.artifactPaths.push(path);
|
|
96
|
+
}
|
|
97
|
+
for (const name of incomingPrior.skillNames) {
|
|
98
|
+
if (!priorState.skillNames.includes(name)) priorState.skillNames.push(name);
|
|
99
|
+
}
|
|
71
100
|
}
|
|
72
|
-
});
|
|
73
101
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
});
|
|
102
|
+
// Dynamically import the extraction helpers from memory-daemon
|
|
103
|
+
const { buildSystemPrompt, buildTranscript, writeExtractionResults } = await import("./memory-daemon.js");
|
|
77
104
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
105
|
+
const transcript = buildTranscript(turns);
|
|
106
|
+
const sections: string[] = [`[TRANSCRIPT]\n${transcript.slice(0, 60000)}`];
|
|
107
|
+
|
|
108
|
+
if (thinking.length > 0) {
|
|
109
|
+
sections.push(`[THINKING]\n${thinking.slice(-8).join("\n---\n").slice(0, 4000)}`);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
if (retrievedMemories.length > 0) {
|
|
113
|
+
const memList = retrievedMemories.map(m => `${m.id}: ${String(m.text).slice(0, 200)}`).join("\n");
|
|
114
|
+
sections.push(`[RETRIEVED MEMORIES]\nMark any that have been fully addressed/fixed/completed.\n${memList}`);
|
|
82
115
|
}
|
|
83
|
-
|
|
116
|
+
|
|
117
|
+
const systemPrompt = buildSystemPrompt(thinking.length > 0, retrievedMemories.length > 0, priorState);
|
|
118
|
+
|
|
119
|
+
const { completeSimple, getModel } = await import("@mariozechner/pi-ai");
|
|
120
|
+
const model = (getModel as any)(provider, modelId);
|
|
121
|
+
|
|
122
|
+
const response = await completeSimple(model, {
|
|
123
|
+
systemPrompt,
|
|
124
|
+
messages: [{
|
|
125
|
+
role: "user",
|
|
126
|
+
timestamp: Date.now(),
|
|
127
|
+
content: sections.join("\n\n"),
|
|
128
|
+
}],
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
const responseText = response.content
|
|
132
|
+
.filter((c: any) => c.type === "text")
|
|
133
|
+
.map((c: any) => c.text)
|
|
134
|
+
.join("");
|
|
135
|
+
|
|
136
|
+
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
|
137
|
+
if (!jsonMatch) return;
|
|
138
|
+
|
|
139
|
+
let result: Record<string, any>;
|
|
140
|
+
try {
|
|
141
|
+
result = JSON.parse(jsonMatch[0]);
|
|
142
|
+
} catch {
|
|
143
|
+
try {
|
|
144
|
+
result = JSON.parse(jsonMatch[0].replace(/,\s*([}\]])/g, "$1"));
|
|
145
|
+
} catch {
|
|
146
|
+
result = {};
|
|
147
|
+
const fields = ["causal", "monologue", "resolved", "concepts", "corrections", "preferences", "artifacts", "decisions", "skills"];
|
|
148
|
+
for (const field of fields) {
|
|
149
|
+
const fieldMatch = jsonMatch[0].match(new RegExp(`"${field}"\\s*:\\s*(\\[[\\s\\S]*?\\])(?=\\s*[,}]\\s*"[a-z]|\\s*\\}$)`, "m"));
|
|
150
|
+
if (fieldMatch) {
|
|
151
|
+
try { result[field] = JSON.parse(fieldMatch[1]); } catch { /* skip */ }
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
if (Object.keys(result).length === 0) return;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const counts = await writeExtractionResults(result, sessionId, store, embeddings, priorState);
|
|
159
|
+
extractedTurnCount = turns.length;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// Pending batch (only keep latest — newer batch supersedes older)
|
|
163
|
+
let pendingBatch: {
|
|
164
|
+
turns: TurnData[];
|
|
165
|
+
thinking: string[];
|
|
166
|
+
retrievedMemories: { id: string; text: string }[];
|
|
167
|
+
priorExtractions?: PriorExtractions;
|
|
168
|
+
} | null = null;
|
|
169
|
+
|
|
170
|
+
async function processPending(): Promise<void> {
|
|
171
|
+
if (processing || shuttingDown) return;
|
|
172
|
+
while (pendingBatch) {
|
|
173
|
+
processing = true;
|
|
174
|
+
const batch = pendingBatch;
|
|
175
|
+
pendingBatch = null;
|
|
176
|
+
try {
|
|
177
|
+
await runExtraction(batch.turns, batch.thinking, batch.retrievedMemories, batch.priorExtractions);
|
|
178
|
+
} catch (e) {
|
|
179
|
+
errorCount++;
|
|
180
|
+
swallow.warn("daemon:extraction", e);
|
|
181
|
+
} finally {
|
|
182
|
+
processing = false;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
84
186
|
|
|
85
187
|
return {
|
|
86
188
|
sendTurnBatch(turns, thinking, retrievedMemories, priorExtractions) {
|
|
87
|
-
if (
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
retrievedMemories,
|
|
94
|
-
sessionId,
|
|
95
|
-
priorExtractions,
|
|
96
|
-
} satisfies DaemonMessage);
|
|
97
|
-
} catch (e) { swallow.warn("daemon-manager:sendBatch", e); }
|
|
189
|
+
if (shuttingDown) return;
|
|
190
|
+
pendingBatch = { turns, thinking, retrievedMemories, priorExtractions };
|
|
191
|
+
// Fire-and-forget: init if needed, then process
|
|
192
|
+
ensureInit()
|
|
193
|
+
.then(ok => { if (ok) return processPending(); })
|
|
194
|
+
.catch(e => swallow.warn("daemon:sendBatch", e));
|
|
98
195
|
},
|
|
99
196
|
|
|
100
197
|
async getStatus() {
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
pendingStatusResolve = (resp) => {
|
|
108
|
-
clearTimeout(timer);
|
|
109
|
-
resolve(resp);
|
|
110
|
-
};
|
|
111
|
-
worker.postMessage({ type: "status_request" } satisfies DaemonMessage);
|
|
112
|
-
});
|
|
198
|
+
return {
|
|
199
|
+
type: "status" as const,
|
|
200
|
+
extractedTurns: extractedTurnCount,
|
|
201
|
+
pendingBatches: pendingBatch ? 1 : 0,
|
|
202
|
+
errors: errorCount,
|
|
203
|
+
};
|
|
113
204
|
},
|
|
114
205
|
|
|
115
206
|
async shutdown(timeoutMs = 45_000) {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
worker.postMessage({ type: "shutdown" } satisfies DaemonMessage);
|
|
136
|
-
} catch {
|
|
137
|
-
clearTimeout(timer);
|
|
138
|
-
terminated = true;
|
|
139
|
-
resolve();
|
|
140
|
-
}
|
|
141
|
-
});
|
|
207
|
+
shuttingDown = true;
|
|
208
|
+
// Wait for current extraction to finish
|
|
209
|
+
if (processing) {
|
|
210
|
+
await Promise.race([
|
|
211
|
+
new Promise<void>(resolve => {
|
|
212
|
+
const check = setInterval(() => {
|
|
213
|
+
if (!processing) { clearInterval(check); resolve(); }
|
|
214
|
+
}, 100);
|
|
215
|
+
}),
|
|
216
|
+
new Promise<void>(resolve => setTimeout(resolve, timeoutMs)),
|
|
217
|
+
]);
|
|
218
|
+
}
|
|
219
|
+
// Clean up daemon-local connections
|
|
220
|
+
await Promise.allSettled([
|
|
221
|
+
store?.dispose(),
|
|
222
|
+
embeddings?.dispose(),
|
|
223
|
+
]).catch(() => {});
|
|
224
|
+
store = null;
|
|
225
|
+
embeddings = null;
|
|
142
226
|
},
|
|
143
227
|
|
|
144
228
|
getExtractedTurnCount() {
|
package/src/daemon-types.ts
CHANGED
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared types for the memory daemon system.
|
|
3
|
-
* Imported by both the worker thread (memory-daemon.ts) and the
|
|
4
|
-
* main thread manager (daemon-manager.ts).
|
|
5
3
|
*/
|
|
6
|
-
import type { SurrealConfig, EmbeddingConfig } from "./config.js";
|
|
7
4
|
|
|
8
5
|
export interface TurnData {
|
|
9
6
|
role: string;
|
|
@@ -13,53 +10,9 @@ export interface TurnData {
|
|
|
13
10
|
file_paths?: string[];
|
|
14
11
|
}
|
|
15
12
|
|
|
16
|
-
/**
|
|
17
|
-
export interface DaemonWorkerData {
|
|
18
|
-
surrealConfig: SurrealConfig;
|
|
19
|
-
embeddingConfig: EmbeddingConfig;
|
|
20
|
-
sessionId: string;
|
|
21
|
-
/** LLM provider name (resolved from OpenClaw config at daemon start). */
|
|
22
|
-
llmProvider?: string;
|
|
23
|
-
/** LLM model ID (resolved from OpenClaw config at daemon start). */
|
|
24
|
-
llmModel?: string;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
/** Previously extracted item names — for dedup across daemon runs. */
|
|
13
|
+
/** Previously extracted item names for dedup across daemon runs. */
|
|
28
14
|
export interface PriorExtractions {
|
|
29
15
|
conceptNames: string[];
|
|
30
16
|
artifactPaths: string[];
|
|
31
17
|
skillNames: string[];
|
|
32
18
|
}
|
|
33
|
-
|
|
34
|
-
/** Messages from main thread -> daemon worker. */
|
|
35
|
-
export type DaemonMessage =
|
|
36
|
-
| {
|
|
37
|
-
type: "turn_batch";
|
|
38
|
-
turns: TurnData[];
|
|
39
|
-
thinking: string[];
|
|
40
|
-
retrievedMemories: { id: string; text: string }[];
|
|
41
|
-
sessionId: string;
|
|
42
|
-
priorExtractions?: PriorExtractions;
|
|
43
|
-
}
|
|
44
|
-
| { type: "shutdown" }
|
|
45
|
-
| { type: "status_request" };
|
|
46
|
-
|
|
47
|
-
/** Messages from daemon worker -> main thread. */
|
|
48
|
-
export type DaemonResponse =
|
|
49
|
-
| {
|
|
50
|
-
type: "extraction_complete";
|
|
51
|
-
extractedTurnCount: number;
|
|
52
|
-
causalCount: number;
|
|
53
|
-
monologueCount: number;
|
|
54
|
-
resolvedCount: number;
|
|
55
|
-
conceptCount: number;
|
|
56
|
-
correctionCount: number;
|
|
57
|
-
preferenceCount: number;
|
|
58
|
-
artifactCount: number;
|
|
59
|
-
decisionCount: number;
|
|
60
|
-
skillCount: number;
|
|
61
|
-
extractedNames?: PriorExtractions;
|
|
62
|
-
}
|
|
63
|
-
| { type: "status"; extractedTurns: number; pendingBatches: number; errors: number }
|
|
64
|
-
| { type: "shutdown_complete" }
|
|
65
|
-
| { type: "error"; message: string };
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deferred Cleanup — extract knowledge from orphaned sessions.
|
|
3
|
+
*
|
|
4
|
+
* When the process dies abruptly (Ctrl+C×2), session cleanup never runs.
|
|
5
|
+
* On next session start, this module finds orphaned sessions (started but
|
|
6
|
+
* never marked cleanup_completed), loads their turns, runs daemon extraction,
|
|
7
|
+
* generates a handoff note, and marks them complete.
|
|
8
|
+
*
|
|
9
|
+
* Turns are already persisted via afterTurn → ingest. This just processes them.
|
|
10
|
+
*/
|
|
11
|
+
import type { SurrealStore } from "./surreal.js";
|
|
12
|
+
import type { EmbeddingService } from "./embeddings.js";
|
|
13
|
+
import type { CompleteFn } from "./state.js";
|
|
14
|
+
import { buildSystemPrompt, buildTranscript, writeExtractionResults } from "./memory-daemon.js";
|
|
15
|
+
import type { PriorExtractions } from "./daemon-types.js";
|
|
16
|
+
import { swallow } from "./errors.js";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Find and process orphaned sessions. Runs with a 30s total timeout.
|
|
20
|
+
* Fire-and-forget from session_start — does not block the new session.
|
|
21
|
+
*/
|
|
22
|
+
export async function runDeferredCleanup(
|
|
23
|
+
store: SurrealStore,
|
|
24
|
+
embeddings: EmbeddingService,
|
|
25
|
+
complete: CompleteFn,
|
|
26
|
+
): Promise<number> {
|
|
27
|
+
if (!store.isAvailable()) return 0;
|
|
28
|
+
|
|
29
|
+
const orphaned = await store.getOrphanedSessions(3).catch(() => []);
|
|
30
|
+
if (orphaned.length === 0) return 0;
|
|
31
|
+
|
|
32
|
+
let processed = 0;
|
|
33
|
+
|
|
34
|
+
const cleanup = async () => {
|
|
35
|
+
for (const session of orphaned) {
|
|
36
|
+
try {
|
|
37
|
+
await processOrphanedSession(session.id, store, embeddings, complete);
|
|
38
|
+
processed++;
|
|
39
|
+
} catch (e) {
|
|
40
|
+
swallow.warn("deferredCleanup:session", e);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
// 30s timeout — don't hold up the new session forever
|
|
46
|
+
await Promise.race([
|
|
47
|
+
cleanup(),
|
|
48
|
+
new Promise<void>(resolve => setTimeout(resolve, 30_000)),
|
|
49
|
+
]);
|
|
50
|
+
|
|
51
|
+
return processed;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async function processOrphanedSession(
|
|
55
|
+
surrealSessionId: string,
|
|
56
|
+
store: SurrealStore,
|
|
57
|
+
embeddings: EmbeddingService,
|
|
58
|
+
complete: CompleteFn,
|
|
59
|
+
): Promise<void> {
|
|
60
|
+
// Find the OpenClaw session ID from turns stored in this session
|
|
61
|
+
// (turns use the OpenClaw session_id, not the surreal record ID)
|
|
62
|
+
const sessionTurns = await store.queryFirst<{ session_id: string }>(
|
|
63
|
+
`SELECT session_id FROM turn WHERE session_id != NONE ORDER BY created_at DESC LIMIT 1`,
|
|
64
|
+
).catch(() => []);
|
|
65
|
+
|
|
66
|
+
// Load turns for extraction
|
|
67
|
+
// We need to find turns associated with this DB session via the part_of edge
|
|
68
|
+
const turns = await store.queryFirst<{ role: string; text: string; tool_name?: string }>(
|
|
69
|
+
`SELECT role, text, tool_name FROM turn
|
|
70
|
+
WHERE session_id IN (SELECT VALUE out FROM part_of WHERE in = $sid)
|
|
71
|
+
OR session_id = $sid
|
|
72
|
+
ORDER BY created_at ASC LIMIT 50`,
|
|
73
|
+
{ sid: surrealSessionId },
|
|
74
|
+
).catch(() => []);
|
|
75
|
+
|
|
76
|
+
if (turns.length < 2) {
|
|
77
|
+
// Nothing to extract, just mark complete
|
|
78
|
+
await store.markSessionEnded(surrealSessionId).catch(e => swallow("deferred:markEmpty", e));
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Run daemon extraction
|
|
83
|
+
const priorState: PriorExtractions = { conceptNames: [], artifactPaths: [], skillNames: [] };
|
|
84
|
+
const turnData = turns.map(t => ({ role: t.role, text: t.text, tool_name: t.tool_name }));
|
|
85
|
+
const transcript = buildTranscript(turnData);
|
|
86
|
+
const systemPrompt = buildSystemPrompt(false, false, priorState);
|
|
87
|
+
|
|
88
|
+
try {
|
|
89
|
+
const response = await complete({
|
|
90
|
+
system: systemPrompt,
|
|
91
|
+
messages: [{ role: "user", content: `[TRANSCRIPT]\n${transcript.slice(0, 60000)}` }],
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
const responseText = response.text;
|
|
95
|
+
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
|
96
|
+
if (jsonMatch) {
|
|
97
|
+
let result: Record<string, any>;
|
|
98
|
+
try {
|
|
99
|
+
result = JSON.parse(jsonMatch[0]);
|
|
100
|
+
} catch {
|
|
101
|
+
try {
|
|
102
|
+
result = JSON.parse(jsonMatch[0].replace(/,\s*([}\]])/g, "$1"));
|
|
103
|
+
} catch { result = {}; }
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (Object.keys(result).length > 0) {
|
|
107
|
+
const sessionId = surrealSessionId; // Use DB ID as session reference
|
|
108
|
+
await writeExtractionResults(result, sessionId, store, embeddings, priorState);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
} catch (e) {
|
|
112
|
+
swallow.warn("deferredCleanup:extraction", e);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Generate handoff note
|
|
116
|
+
try {
|
|
117
|
+
const lastTurns = turns.slice(-15);
|
|
118
|
+
const turnSummary = lastTurns
|
|
119
|
+
.map(t => `[${t.role}] ${t.text.slice(0, 200)}`)
|
|
120
|
+
.join("\n");
|
|
121
|
+
|
|
122
|
+
const handoffResponse = await complete({
|
|
123
|
+
system: "Summarize this session for handoff to your next self. What was worked on, what's unfinished, what to remember. 2-3 sentences. Write in first person.",
|
|
124
|
+
messages: [{ role: "user", content: turnSummary }],
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
const handoffText = handoffResponse.text.trim();
|
|
128
|
+
if (handoffText.length > 20) {
|
|
129
|
+
let emb: number[] | null = null;
|
|
130
|
+
if (embeddings.isAvailable()) {
|
|
131
|
+
try { emb = await embeddings.embed(handoffText); } catch { /* ok */ }
|
|
132
|
+
}
|
|
133
|
+
await store.createMemory(handoffText, emb, 8, "handoff", surrealSessionId);
|
|
134
|
+
}
|
|
135
|
+
} catch (e) {
|
|
136
|
+
swallow.warn("deferredCleanup:handoff", e);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Mark session as cleaned up
|
|
140
|
+
await store.markSessionEnded(surrealSessionId).catch(e => swallow("deferred:markDone", e));
|
|
141
|
+
}
|