kongbrain 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +45 -0
- package/.github/workflows/pr-check.yml +16 -0
- package/CHANGELOG.md +64 -0
- package/README.github.md +40 -1
- package/SKILL.md +1 -1
- package/TOKEN_FLOW.md +184 -0
- package/package.json +1 -1
- package/src/acan.ts +28 -5
- package/src/causal.ts +18 -25
- package/src/cognitive-bootstrap.ts +6 -6
- package/src/cognitive-check.ts +17 -19
- package/src/config.ts +1 -1
- package/src/context-engine.ts +81 -48
- package/src/daemon-manager.ts +51 -17
- package/src/deferred-cleanup.ts +11 -9
- package/src/embeddings.ts +6 -7
- package/src/errors.ts +5 -3
- package/src/graph-context.ts +269 -173
- package/src/hooks/after-tool-call.ts +2 -1
- package/src/hooks/before-tool-call.ts +15 -11
- package/src/hooks/llm-output.ts +18 -10
- package/src/index.ts +17 -12
- package/src/intent.ts +9 -8
- package/src/log.ts +11 -0
- package/src/orchestrator.ts +11 -4
- package/src/prefetch.ts +2 -2
- package/src/reflection.ts +9 -2
- package/src/schema.surql +4 -0
- package/src/skills.ts +32 -10
- package/src/soul.ts +17 -1
- package/src/state.ts +31 -0
- package/src/surreal.ts +134 -110
- package/src/tools/introspect.ts +1 -1
- package/src/wakeup.ts +0 -142
package/src/context-engine.ts
CHANGED
|
@@ -51,11 +51,12 @@ import { graduateCausalToSkills } from "./skills.js";
|
|
|
51
51
|
import { attemptGraduation, evolveSoul, checkStageTransition } from "./soul.js";
|
|
52
52
|
import { swallow } from "./errors.js";
|
|
53
53
|
|
|
54
|
+
/** OpenClaw ContextEngine backed by SurrealDB graph retrieval and BGE-M3 embeddings. */
|
|
54
55
|
export class KongBrainContextEngine implements ContextEngine {
|
|
55
56
|
readonly info: ContextEngineInfo = {
|
|
56
57
|
id: "kongbrain",
|
|
57
58
|
name: "KongBrain",
|
|
58
|
-
version: "0.
|
|
59
|
+
version: "0.4.2",
|
|
59
60
|
ownsCompaction: true,
|
|
60
61
|
};
|
|
61
62
|
|
|
@@ -63,6 +64,7 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
63
64
|
|
|
64
65
|
// ── Bootstrap ──────────────────────────────────────────────────────────
|
|
65
66
|
|
|
67
|
+
/** Initialize schema, create 5-pillar graph nodes, and start the memory daemon. */
|
|
66
68
|
async bootstrap(params: {
|
|
67
69
|
sessionId: string;
|
|
68
70
|
sessionKey?: string;
|
|
@@ -139,6 +141,7 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
139
141
|
|
|
140
142
|
// ── Assemble ───────────────────────────────────────────────────────────
|
|
141
143
|
|
|
144
|
+
/** Build the context window: graph retrieval + system prompt additions + budget trimming. */
|
|
142
145
|
async assemble(params: {
|
|
143
146
|
sessionId: string;
|
|
144
147
|
sessionKey?: string;
|
|
@@ -173,26 +176,22 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
173
176
|
if (systemPromptSection) additions.push(systemPromptSection);
|
|
174
177
|
|
|
175
178
|
// Compaction summary (claw-code: compact.rs structured signals — inject once after compaction)
|
|
176
|
-
const compactionSummary =
|
|
179
|
+
const compactionSummary = session._compactionSummary;
|
|
177
180
|
if (compactionSummary) {
|
|
178
181
|
additions.push("[POST-COMPACTION CONTEXT]\n" + compactionSummary);
|
|
179
|
-
|
|
182
|
+
session._compactionSummary = undefined;
|
|
180
183
|
}
|
|
181
184
|
|
|
182
185
|
// Wakeup briefing (synthesized at session start, may still be in-flight)
|
|
183
|
-
const wakeupPromise =
|
|
186
|
+
const wakeupPromise = session._wakeupPromise;
|
|
184
187
|
if (wakeupPromise) {
|
|
185
188
|
const wakeupBriefing = await wakeupPromise;
|
|
186
|
-
|
|
189
|
+
session._wakeupPromise = undefined; // Only inject once
|
|
187
190
|
if (wakeupBriefing) additions.push(wakeupBriefing);
|
|
188
191
|
}
|
|
189
192
|
|
|
190
193
|
// Graduation celebration — tell the agent it just graduated so it can share with the user
|
|
191
|
-
const graduation =
|
|
192
|
-
qualityScore: number;
|
|
193
|
-
volumeScore: number;
|
|
194
|
-
soulSummary: string;
|
|
195
|
-
} | undefined;
|
|
194
|
+
const graduation = session._graduationCelebration;
|
|
196
195
|
if (graduation) {
|
|
197
196
|
let graduationBlock =
|
|
198
197
|
"[SOUL GRADUATION — CELEBRATE WITH THE USER]\n" +
|
|
@@ -211,11 +210,11 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
211
210
|
"identity emerging from YOUR experience. Don't be robotic about it. This only happens once.";
|
|
212
211
|
|
|
213
212
|
additions.push(graduationBlock);
|
|
214
|
-
|
|
213
|
+
session._graduationCelebration = undefined; // Only inject once
|
|
215
214
|
}
|
|
216
215
|
|
|
217
216
|
// Migration nudge — tell the agent there are workspace files to offer migrating
|
|
218
|
-
if (
|
|
217
|
+
if (session._hasMigratableFiles) {
|
|
219
218
|
additions.push(
|
|
220
219
|
"[MIGRATION AVAILABLE] This workspace has files from the default context engine " +
|
|
221
220
|
"(IDENTITY.md, MEMORY.md, skills/, etc.). You can offer to migrate them into the graph " +
|
|
@@ -226,15 +225,31 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
226
225
|
);
|
|
227
226
|
}
|
|
228
227
|
|
|
228
|
+
// Apply SPA priority budget — drop lowest-priority sections if over budget
|
|
229
|
+
// (dropped sections aren't lost — they're in the graph, retrievable on demand)
|
|
230
|
+
const BYTES_PER_TOKEN = 4; // claw-code: roughTokenCountEstimation default
|
|
231
|
+
const SPA_BUDGET_CHARS = Math.round(contextWindow * 0.08 * BYTES_PER_TOKEN);
|
|
232
|
+
let spaTotalChars = 0;
|
|
233
|
+
const keptAdditions: string[] = [];
|
|
234
|
+
for (const section of additions) { // additions are already in priority order
|
|
235
|
+
if (spaTotalChars + section.length > SPA_BUDGET_CHARS && keptAdditions.length > 0) break;
|
|
236
|
+
keptAdditions.push(section);
|
|
237
|
+
spaTotalChars += section.length;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const spaText = keptAdditions.length > 0 ? keptAdditions.join("\n\n") : undefined;
|
|
241
|
+
const spaTokens = spaText ? Math.ceil(spaText.length / BYTES_PER_TOKEN) : 0;
|
|
242
|
+
|
|
229
243
|
return {
|
|
230
244
|
messages,
|
|
231
|
-
estimatedTokens: stats.sentTokens,
|
|
232
|
-
systemPromptAddition:
|
|
245
|
+
estimatedTokens: stats.sentTokens + spaTokens,
|
|
246
|
+
systemPromptAddition: spaText,
|
|
233
247
|
};
|
|
234
248
|
}
|
|
235
249
|
|
|
236
250
|
// ── Ingest ─────────────────────────────────────────────────────────────
|
|
237
251
|
|
|
252
|
+
/** Embed and store a single user or assistant message as a turn node. */
|
|
238
253
|
async ingest(params: {
|
|
239
254
|
sessionId: string;
|
|
240
255
|
sessionKey?: string;
|
|
@@ -247,7 +262,7 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
247
262
|
const msg = params.message;
|
|
248
263
|
|
|
249
264
|
try {
|
|
250
|
-
const role = (msg as
|
|
265
|
+
const role = "role" in msg ? (msg as { role: string }).role : "";
|
|
251
266
|
if (role === "user" || role === "assistant") {
|
|
252
267
|
const text = extractMessageText(msg);
|
|
253
268
|
if (!text) return { ingested: false };
|
|
@@ -256,11 +271,16 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
256
271
|
let embedding: number[] | null = null;
|
|
257
272
|
if (worthEmbedding && embeddings.isAvailable()) {
|
|
258
273
|
try {
|
|
259
|
-
const
|
|
260
|
-
embedding = await embeddings.embed(text.slice(0,
|
|
274
|
+
const INGEST_EMBED_CHAR_LIMIT = 22_282; // ~6,554 tokens at 3.4 chars/token (BGE-M3 8192-token window * 0.8 safety margin)
|
|
275
|
+
embedding = await embeddings.embed(text.slice(0, INGEST_EMBED_CHAR_LIMIT));
|
|
261
276
|
} catch (e) { swallow("ingest:embed", e); }
|
|
262
277
|
}
|
|
263
278
|
|
|
279
|
+
// Stash user embedding for reuse in buildContextualQueryVec (avoids re-embedding)
|
|
280
|
+
if (role === "user" && embedding) {
|
|
281
|
+
session.lastUserEmbedding = embedding;
|
|
282
|
+
}
|
|
283
|
+
|
|
264
284
|
const turnId = await store.upsertTurn({
|
|
265
285
|
session_id: session.sessionId,
|
|
266
286
|
role,
|
|
@@ -327,6 +347,7 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
327
347
|
|
|
328
348
|
// ── Compact ────────────────────────────────────────────────────────────
|
|
329
349
|
|
|
350
|
+
/** Extract structured signals (pending work, key files, errors) for post-compaction injection. */
|
|
330
351
|
async compact(params: {
|
|
331
352
|
sessionId: string;
|
|
332
353
|
sessionKey?: string;
|
|
@@ -346,8 +367,9 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
346
367
|
|
|
347
368
|
// Extract structured compaction signals from stored turns
|
|
348
369
|
let summary: string | undefined;
|
|
370
|
+
const { store } = this.state;
|
|
371
|
+
const contextWindow = params.tokenBudget ?? 200_000;
|
|
349
372
|
try {
|
|
350
|
-
const { store } = this.state;
|
|
351
373
|
if (store.isAvailable()) {
|
|
352
374
|
const turns = await store.getSessionTurnsRich(params.sessionId, 30);
|
|
353
375
|
if (turns.length > 0) {
|
|
@@ -370,6 +392,12 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
370
392
|
turns.filter(t => t.tool_name).map(t => t.tool_name!)
|
|
371
393
|
)];
|
|
372
394
|
|
|
395
|
+
// Recent errors — preserve tool failure context across compaction
|
|
396
|
+
const errorRe = /\b(error|failed|exception|crash|panic|TypeError|ReferenceError)\b[^.\n]{0,120}/gi;
|
|
397
|
+
const recentErrors = [...fullText.matchAll(errorRe)]
|
|
398
|
+
.map(m => m[0].trim().slice(0, 160))
|
|
399
|
+
.slice(-3); // last 3 errors only
|
|
400
|
+
|
|
373
401
|
// Current work inference (claw-code: compact.rs:272-279)
|
|
374
402
|
const lastText = turns.filter(t => t.text.length > 10).at(-1)?.text.slice(0, 200) ?? "";
|
|
375
403
|
|
|
@@ -377,6 +405,7 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
377
405
|
if (pendingMatches.length > 0) parts.push(`PENDING: ${pendingMatches.join("; ")}`);
|
|
378
406
|
if (filePaths.length > 0) parts.push(`FILES: ${filePaths.join(", ")}`);
|
|
379
407
|
if (toolNames.length > 0) parts.push(`TOOLS USED: ${toolNames.join(", ")}`);
|
|
408
|
+
if (recentErrors.length > 0) parts.push(`RECENT ERRORS: ${recentErrors.join("; ")}`);
|
|
380
409
|
if (lastText) parts.push(`LAST: ${lastText}`);
|
|
381
410
|
parts.push("Resume directly — do not recap what was happening.");
|
|
382
411
|
|
|
@@ -384,25 +413,34 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
384
413
|
summary = parts.join("\n");
|
|
385
414
|
// Stash for next assemble() to inject
|
|
386
415
|
if (session) {
|
|
387
|
-
|
|
416
|
+
session._compactionSummary = summary;
|
|
388
417
|
}
|
|
389
418
|
}
|
|
390
419
|
}
|
|
391
420
|
}
|
|
392
421
|
} catch { /* non-critical */ }
|
|
393
422
|
|
|
423
|
+
// Compaction checkpoint — diagnostic trail for debugging
|
|
424
|
+
if (store.isAvailable() && session) {
|
|
425
|
+
store.createCompactionCheckpoint(params.sessionId, 0, session.userTurnCount)
|
|
426
|
+
.catch(e => swallow.warn("compact:checkpoint", e));
|
|
427
|
+
}
|
|
428
|
+
|
|
394
429
|
return {
|
|
395
430
|
ok: true,
|
|
396
|
-
compacted:
|
|
397
|
-
reason:
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
431
|
+
compacted: true,
|
|
432
|
+
reason: "Graph-curated context window: assemble() selects relevant context each turn.",
|
|
433
|
+
result: summary ? {
|
|
434
|
+
summary,
|
|
435
|
+
tokensBefore: Math.round(summary.length / 4), // 4 bytes/token (claw-code ratio)
|
|
436
|
+
tokensAfter: Math.round(contextWindow * 0.325),
|
|
437
|
+
} : undefined,
|
|
401
438
|
};
|
|
402
439
|
}
|
|
403
440
|
|
|
404
441
|
// ── After turn ─────────────────────────────────────────────────────────
|
|
405
442
|
|
|
443
|
+
/** Post-turn: ingest messages, evaluate retrieval quality, flush daemon, and run periodic maintenance. */
|
|
406
444
|
async afterTurn?(params: {
|
|
407
445
|
sessionId: string;
|
|
408
446
|
sessionKey?: string;
|
|
@@ -442,11 +480,12 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
442
480
|
.catch(e => swallow.warn("afterTurn:evaluateRetrieval", e));
|
|
443
481
|
}
|
|
444
482
|
|
|
483
|
+
// Single fetch for all downstream consumers (cognitive check, daemon flush, handoff)
|
|
484
|
+
const allSessionTurns = await store.getSessionTurns(session.sessionId, 50)
|
|
485
|
+
.catch(() => [] as { role: string; text: string }[]);
|
|
486
|
+
|
|
445
487
|
// Cognitive check: periodic reasoning over retrieved context
|
|
446
488
|
if (shouldRunCheck(session.userTurnCount, session) && stagedSnapshot.length > 0) {
|
|
447
|
-
const recentTurns = await store.getSessionTurns(session.sessionId, 6)
|
|
448
|
-
.catch(() => [] as { role: string; text: string }[]);
|
|
449
|
-
|
|
450
489
|
runCognitiveCheck({
|
|
451
490
|
sessionId: session.sessionId,
|
|
452
491
|
userQuery: session.lastUserText,
|
|
@@ -457,7 +496,7 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
457
496
|
score: n.finalScore ?? 0,
|
|
458
497
|
table: n.table,
|
|
459
498
|
})),
|
|
460
|
-
recentTurns,
|
|
499
|
+
recentTurns: allSessionTurns.slice(-6),
|
|
461
500
|
}, session, store, this.state.complete).catch(e => swallow.warn("afterTurn:cognitiveCheck", e));
|
|
462
501
|
}
|
|
463
502
|
|
|
@@ -466,11 +505,11 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
466
505
|
const turnReady = session.userTurnCount >= session.lastDaemonFlushTurnCount + 3;
|
|
467
506
|
if (session.daemon && (tokenReady || turnReady)) {
|
|
468
507
|
try {
|
|
469
|
-
const recentTurns =
|
|
508
|
+
const recentTurns = allSessionTurns.slice(-20);
|
|
470
509
|
const turnData = recentTurns.map(t => ({
|
|
471
510
|
role: t.role as "user" | "assistant",
|
|
472
511
|
text: t.text,
|
|
473
|
-
turnId: String((t as
|
|
512
|
+
turnId: String((t as { id?: string }).id ?? ""),
|
|
474
513
|
}));
|
|
475
514
|
|
|
476
515
|
// Gather retrieved memory IDs for dedup
|
|
@@ -503,20 +542,14 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
503
542
|
// Fire-and-forget: these are non-critical background operations
|
|
504
543
|
const cleanupOps: Promise<unknown>[] = [];
|
|
505
544
|
|
|
506
|
-
// Final daemon flush with full transcript before cleanup
|
|
545
|
+
// Final daemon flush with full transcript before cleanup (reuse allSessionTurns)
|
|
507
546
|
if (session.daemon) {
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
turnId: String((t as any).id ?? ""),
|
|
515
|
-
}));
|
|
516
|
-
session.daemon!.sendTurnBatch(turnData, [...session.pendingThinking], []);
|
|
517
|
-
})
|
|
518
|
-
.catch(e => swallow.warn("midCleanup:daemonFlush", e)),
|
|
519
|
-
);
|
|
547
|
+
const turnData = allSessionTurns.map(t => ({
|
|
548
|
+
role: t.role as "user" | "assistant",
|
|
549
|
+
text: t.text,
|
|
550
|
+
turnId: String((t as { id?: string }).id ?? ""),
|
|
551
|
+
}));
|
|
552
|
+
session.daemon.sendTurnBatch(turnData, [...session.pendingThinking], []);
|
|
520
553
|
}
|
|
521
554
|
|
|
522
555
|
if (session.taskId) {
|
|
@@ -542,10 +575,10 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
542
575
|
.catch(e => swallow("midCleanup:acan", e)),
|
|
543
576
|
);
|
|
544
577
|
|
|
545
|
-
// Handoff note — snapshot for wakeup even if session continues
|
|
578
|
+
// Handoff note — snapshot for wakeup even if session continues (reuse allSessionTurns)
|
|
546
579
|
cleanupOps.push(
|
|
547
580
|
(async () => {
|
|
548
|
-
const recentTurns =
|
|
581
|
+
const recentTurns = allSessionTurns.slice(-15);
|
|
549
582
|
if (recentTurns.length < 2) return;
|
|
550
583
|
const turnSummary = recentTurns
|
|
551
584
|
.map(t => `[${t.role}] ${t.text.slice(0, 200)}`)
|
|
@@ -635,12 +668,12 @@ export class KongBrainContextEngine implements ContextEngine {
|
|
|
635
668
|
// ── Helpers ────────────────────────────────────────────────────────────────────
|
|
636
669
|
|
|
637
670
|
function extractMessageText(msg: AgentMessage): string {
|
|
638
|
-
const m = msg as
|
|
671
|
+
const m = msg as { content?: string | { type: string; text?: string }[] };
|
|
639
672
|
if (typeof m.content === "string") return m.content;
|
|
640
673
|
if (Array.isArray(m.content)) {
|
|
641
674
|
return m.content
|
|
642
|
-
.filter((c
|
|
643
|
-
.map((c
|
|
675
|
+
.filter((c) => c.type === "text")
|
|
676
|
+
.map((c) => c.text ?? "")
|
|
644
677
|
.join("\n");
|
|
645
678
|
}
|
|
646
679
|
return "";
|
package/src/daemon-manager.ts
CHANGED
|
@@ -79,10 +79,10 @@ export function startMemoryDaemon(
|
|
|
79
79
|
const { buildSystemPrompt, buildTranscript, writeExtractionResults } = await import("./memory-daemon.js");
|
|
80
80
|
|
|
81
81
|
const transcript = buildTranscript(turns);
|
|
82
|
-
const sections: string[] = [`[TRANSCRIPT]\n${transcript.slice(0,
|
|
82
|
+
const sections: string[] = [`[TRANSCRIPT]\n${transcript.slice(0, 30000)}`];
|
|
83
83
|
|
|
84
84
|
if (thinking.length > 0) {
|
|
85
|
-
sections.push(`[THINKING]\n${thinking.slice(-
|
|
85
|
+
sections.push(`[THINKING]\n${thinking.slice(-3).join("\n---\n").slice(0, 2000)}`);
|
|
86
86
|
}
|
|
87
87
|
|
|
88
88
|
if (retrievedMemories.length > 0) {
|
|
@@ -92,33 +92,62 @@ export function startMemoryDaemon(
|
|
|
92
92
|
|
|
93
93
|
const systemPrompt = buildSystemPrompt(thinking.length > 0, retrievedMemories.length > 0, priorState);
|
|
94
94
|
|
|
95
|
+
// Structured output schema — forces API to return valid JSON (no markdown, no preamble)
|
|
96
|
+
const extractionSchema = {
|
|
97
|
+
type: "object" as const,
|
|
98
|
+
properties: {
|
|
99
|
+
causal: { type: "array", items: { type: "object" } },
|
|
100
|
+
monologue: { type: "array", items: { type: "object" } },
|
|
101
|
+
resolved: { type: "array", items: { type: "string" } },
|
|
102
|
+
concepts: { type: "array", items: { type: "object" } },
|
|
103
|
+
corrections: { type: "array", items: { type: "object" } },
|
|
104
|
+
preferences: { type: "array", items: { type: "object" } },
|
|
105
|
+
artifacts: { type: "array", items: { type: "object" } },
|
|
106
|
+
decisions: { type: "array", items: { type: "object" } },
|
|
107
|
+
skills: { type: "array", items: { type: "object" } },
|
|
108
|
+
},
|
|
109
|
+
required: ["causal", "monologue", "resolved", "concepts", "corrections", "preferences", "artifacts", "decisions", "skills"],
|
|
110
|
+
};
|
|
111
|
+
|
|
95
112
|
const response = await complete({
|
|
96
113
|
system: systemPrompt,
|
|
97
114
|
messages: [{ role: "user", content: sections.join("\n\n") }],
|
|
115
|
+
outputFormat: { type: "json_schema", schema: extractionSchema },
|
|
98
116
|
});
|
|
99
117
|
|
|
100
118
|
const responseText = response.text;
|
|
101
119
|
|
|
102
|
-
|
|
103
|
-
if
|
|
104
|
-
|
|
120
|
+
// With structured output the response should be valid JSON directly.
|
|
121
|
+
// Fall back to regex extraction if the provider doesn't support outputFormat.
|
|
105
122
|
let result: Record<string, any>;
|
|
106
123
|
try {
|
|
107
|
-
result = JSON.parse(
|
|
124
|
+
result = JSON.parse(responseText);
|
|
108
125
|
} catch {
|
|
126
|
+
const jsonMatch = responseText.match(/\{[\s\S]*?\}/);
|
|
127
|
+
if (!jsonMatch) {
|
|
128
|
+
swallow.warn("daemon:noJson", new Error(`LLM response contained no JSON (${responseText.length} chars)`));
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
109
131
|
try {
|
|
110
|
-
result = JSON.parse(jsonMatch[0]
|
|
132
|
+
result = JSON.parse(jsonMatch[0]);
|
|
111
133
|
} catch {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
134
|
+
try {
|
|
135
|
+
result = JSON.parse(jsonMatch[0].replace(/,\s*([}\]])/g, "$1"));
|
|
136
|
+
} catch {
|
|
137
|
+
result = {};
|
|
138
|
+
const fields = ["causal", "monologue", "resolved", "concepts", "corrections", "preferences", "artifacts", "decisions", "skills"];
|
|
139
|
+
for (const field of fields) {
|
|
140
|
+
const fieldMatch = jsonMatch[0].match(new RegExp(`"${field}"\\s*:\\s*(\\[[\\s\\S]*?\\])(?=\\s*[,}]\\s*"[a-z]|\\s*\\}$)`, "m"));
|
|
141
|
+
if (fieldMatch) {
|
|
142
|
+
try { result[field] = JSON.parse(fieldMatch[1]); } catch { /* skip */ }
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const PRIMARY_FIELDS = ["causal", "monologue", "artifacts"];
|
|
146
|
+
if (!PRIMARY_FIELDS.some(f => f in result)) {
|
|
147
|
+
swallow.warn("daemon:fallbackFailed", new Error(`Regex fallback extracted no primary fields from: ${jsonMatch[0].slice(0, 100)}`));
|
|
148
|
+
return;
|
|
118
149
|
}
|
|
119
150
|
}
|
|
120
|
-
const PRIMARY_FIELDS = ["causal", "monologue", "artifacts"];
|
|
121
|
-
if (!PRIMARY_FIELDS.some(f => f in result)) return;
|
|
122
151
|
}
|
|
123
152
|
}
|
|
124
153
|
|
|
@@ -164,9 +193,14 @@ export function startMemoryDaemon(
|
|
|
164
193
|
sendTurnBatch(turns, thinking, retrievedMemories, priorExtractions) {
|
|
165
194
|
if (shuttingDown) return;
|
|
166
195
|
if (pendingBatch) {
|
|
167
|
-
|
|
196
|
+
// Merge into pending batch instead of discarding — prevents turn data loss
|
|
197
|
+
pendingBatch.turns = [...pendingBatch.turns, ...turns];
|
|
198
|
+
pendingBatch.thinking = [...pendingBatch.thinking, ...thinking];
|
|
199
|
+
pendingBatch.retrievedMemories = [...pendingBatch.retrievedMemories, ...retrievedMemories];
|
|
200
|
+
pendingBatch.priorExtractions = priorExtractions ?? pendingBatch.priorExtractions;
|
|
201
|
+
} else {
|
|
202
|
+
pendingBatch = { turns, thinking, retrievedMemories, priorExtractions };
|
|
168
203
|
}
|
|
169
|
-
pendingBatch = { turns, thinking, retrievedMemories, priorExtractions };
|
|
170
204
|
// Fire-and-forget
|
|
171
205
|
processPending().catch(e => swallow.warn("daemon:sendBatch", e));
|
|
172
206
|
},
|
package/src/deferred-cleanup.ts
CHANGED
|
@@ -14,10 +14,12 @@ import type { CompleteFn } from "./state.js";
|
|
|
14
14
|
import { buildSystemPrompt, buildTranscript, writeExtractionResults } from "./memory-daemon.js";
|
|
15
15
|
import type { PriorExtractions } from "./daemon-types.js";
|
|
16
16
|
import { swallow } from "./errors.js";
|
|
17
|
+
import { log } from "./log.js";
|
|
17
18
|
|
|
18
19
|
// Process-global flag — deferred cleanup runs AT MOST ONCE per process.
|
|
19
20
|
// Using Symbol.for so it survives Jiti re-importing this module.
|
|
20
21
|
const RAN_KEY = Symbol.for("kongbrain.deferredCleanup.ran");
|
|
22
|
+
const _g = globalThis as Record<symbol, unknown>;
|
|
21
23
|
|
|
22
24
|
/**
|
|
23
25
|
* Find and process orphaned sessions. Runs with a 30s total timeout.
|
|
@@ -30,8 +32,8 @@ export async function runDeferredCleanup(
|
|
|
30
32
|
complete: CompleteFn,
|
|
31
33
|
): Promise<number> {
|
|
32
34
|
// Once per process — never re-run even if first run times out
|
|
33
|
-
if (
|
|
34
|
-
|
|
35
|
+
if (_g[RAN_KEY]) return 0;
|
|
36
|
+
_g[RAN_KEY] = true;
|
|
35
37
|
|
|
36
38
|
try {
|
|
37
39
|
return await runDeferredCleanupInner(store, embeddings, complete);
|
|
@@ -101,7 +103,7 @@ async function processOrphanedSession(
|
|
|
101
103
|
const systemPrompt = buildSystemPrompt(false, false, priorState);
|
|
102
104
|
|
|
103
105
|
try {
|
|
104
|
-
|
|
106
|
+
log.info(`[deferred] extracting session ${surrealSessionId} (${turns.length} turns, transcript ${transcript.length} chars)`);
|
|
105
107
|
const LLM_CALL_TIMEOUT_MS = 30_000;
|
|
106
108
|
const response = await Promise.race([
|
|
107
109
|
complete({
|
|
@@ -114,7 +116,7 @@ async function processOrphanedSession(
|
|
|
114
116
|
]);
|
|
115
117
|
|
|
116
118
|
const responseText = response.text;
|
|
117
|
-
|
|
119
|
+
log.info(`[deferred] extraction response: ${responseText.length} chars`);
|
|
118
120
|
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
|
119
121
|
if (jsonMatch) {
|
|
120
122
|
let result: Record<string, any>;
|
|
@@ -128,17 +130,17 @@ async function processOrphanedSession(
|
|
|
128
130
|
// Strip prototype pollution keys from LLM-generated JSON
|
|
129
131
|
const BANNED_KEYS = new Set(["__proto__", "constructor", "prototype"]);
|
|
130
132
|
for (const key of Object.keys(result)) {
|
|
131
|
-
if (BANNED_KEYS.has(key)) delete
|
|
133
|
+
if (BANNED_KEYS.has(key)) delete result[key];
|
|
132
134
|
}
|
|
133
135
|
|
|
134
136
|
const keys = Object.keys(result);
|
|
135
|
-
|
|
137
|
+
log.info(`[deferred] parsed ${keys.length} keys: ${keys.join(", ")}`);
|
|
136
138
|
if (keys.length > 0) {
|
|
137
139
|
await writeExtractionResults(result, surrealSessionId, store, embeddings, priorState, undefined, undefined, turnData);
|
|
138
|
-
|
|
140
|
+
log.info(`[deferred] wrote extraction results for ${surrealSessionId}`);
|
|
139
141
|
}
|
|
140
142
|
} else {
|
|
141
|
-
|
|
143
|
+
log.warn(`[deferred] no JSON found in response`);
|
|
142
144
|
}
|
|
143
145
|
} catch (e) {
|
|
144
146
|
swallow.warn("deferredCleanup:extraction", e);
|
|
@@ -162,7 +164,7 @@ async function processOrphanedSession(
|
|
|
162
164
|
]);
|
|
163
165
|
|
|
164
166
|
const handoffText = handoffResponse.text.trim();
|
|
165
|
-
|
|
167
|
+
log.info(`[deferred] handoff response: ${handoffText.length} chars`);
|
|
166
168
|
if (handoffText.length > 20) {
|
|
167
169
|
let emb: number[] | null = null;
|
|
168
170
|
if (embeddings.isAvailable()) {
|
package/src/embeddings.ts
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { existsSync } from "node:fs";
|
|
2
2
|
import type { EmbeddingConfig } from "./config.js";
|
|
3
3
|
import { swallow } from "./errors.js";
|
|
4
|
+
import { log } from "./log.js";
|
|
4
5
|
|
|
5
6
|
// Lazy-import node-llama-cpp to avoid top-level await issues with jiti.
|
|
6
7
|
// The actual import happens inside initialize() at runtime.
|
|
7
8
|
type LlamaEmbeddingContext = import("node-llama-cpp").LlamaEmbeddingContext;
|
|
8
9
|
type LlamaModel = import("node-llama-cpp").LlamaModel;
|
|
9
10
|
|
|
11
|
+
/** BGE-M3 embedding service (1024-dim via GGUF) with an LRU cache of up to 512 entries. */
|
|
10
12
|
export class EmbeddingService {
|
|
11
13
|
private model: LlamaModel | null = null;
|
|
12
14
|
private ctx: LlamaEmbeddingContext | null = null;
|
|
@@ -30,8 +32,8 @@ export class EmbeddingService {
|
|
|
30
32
|
logLevel: LlamaLogLevel.error,
|
|
31
33
|
logger: (level, message) => {
|
|
32
34
|
if (message.includes("missing newline token")) return;
|
|
33
|
-
if (level === LlamaLogLevel.error)
|
|
34
|
-
else if (level === LlamaLogLevel.warn)
|
|
35
|
+
if (level === LlamaLogLevel.error) log.error(`[llama] ${message}`);
|
|
36
|
+
else if (level === LlamaLogLevel.warn) log.warn(`[llama] ${message}`);
|
|
35
37
|
},
|
|
36
38
|
});
|
|
37
39
|
this.model = await llama.loadModel({ modelPath: this.config.modelPath });
|
|
@@ -40,6 +42,7 @@ export class EmbeddingService {
|
|
|
40
42
|
return true;
|
|
41
43
|
}
|
|
42
44
|
|
|
45
|
+
/** Return the embedding vector for text, serving from LRU cache on repeat calls. */
|
|
43
46
|
async embed(text: string): Promise<number[]> {
|
|
44
47
|
if (!this.ready || !this.ctx) throw new Error("Embeddings not initialized");
|
|
45
48
|
const cached = this.cache.get(text);
|
|
@@ -61,11 +64,7 @@ export class EmbeddingService {
|
|
|
61
64
|
|
|
62
65
|
async embedBatch(texts: string[]): Promise<number[][]> {
|
|
63
66
|
if (texts.length === 0) return [];
|
|
64
|
-
|
|
65
|
-
for (const text of texts) {
|
|
66
|
-
results.push(await this.embed(text));
|
|
67
|
-
}
|
|
68
|
-
return results;
|
|
67
|
+
return Promise.all(texts.map(text => this.embed(text)));
|
|
69
68
|
}
|
|
70
69
|
|
|
71
70
|
isAvailable(): boolean {
|
package/src/errors.ts
CHANGED
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
* Always logged to stderr with stack trace.
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
+
import { log } from "./log.js";
|
|
13
|
+
|
|
12
14
|
const DEBUG = process.env.KONGBRAIN_DEBUG === "1";
|
|
13
15
|
|
|
14
16
|
/**
|
|
@@ -18,7 +20,7 @@ const DEBUG = process.env.KONGBRAIN_DEBUG === "1";
|
|
|
18
20
|
function swallow(context: string, err?: unknown): void {
|
|
19
21
|
if (!DEBUG) return;
|
|
20
22
|
const msg = err instanceof Error ? err.message : String(err ?? "unknown");
|
|
21
|
-
|
|
23
|
+
log.debug(`[swallow] ${context}: ${msg}`);
|
|
22
24
|
}
|
|
23
25
|
|
|
24
26
|
/**
|
|
@@ -27,7 +29,7 @@ function swallow(context: string, err?: unknown): void {
|
|
|
27
29
|
*/
|
|
28
30
|
swallow.warn = function swallowWarn(context: string, err?: unknown): void {
|
|
29
31
|
const msg = err instanceof Error ? err.message : String(err ?? "unknown");
|
|
30
|
-
|
|
32
|
+
log.warn(`${context}: ${msg}`);
|
|
31
33
|
};
|
|
32
34
|
|
|
33
35
|
/**
|
|
@@ -37,7 +39,7 @@ swallow.warn = function swallowWarn(context: string, err?: unknown): void {
|
|
|
37
39
|
swallow.error = function swallowError(context: string, err?: unknown): void {
|
|
38
40
|
const msg = err instanceof Error ? err.message : String(err ?? "unknown");
|
|
39
41
|
const stack = err instanceof Error ? `\n${err.stack}` : "";
|
|
40
|
-
|
|
42
|
+
log.error(`${context}: ${msg}${stack}`);
|
|
41
43
|
};
|
|
42
44
|
|
|
43
45
|
export { swallow };
|