kongbrain 0.4.1 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -50,12 +50,14 @@ import { generateReflection } from "./reflection.js";
50
50
  import { graduateCausalToSkills } from "./skills.js";
51
51
  import { attemptGraduation, evolveSoul, checkStageTransition } from "./soul.js";
52
52
  import { swallow } from "./errors.js";
53
+ import { log } from "./log.js";
53
54
 
55
+ /** OpenClaw ContextEngine backed by SurrealDB graph retrieval and BGE-M3 embeddings. */
54
56
  export class KongBrainContextEngine implements ContextEngine {
55
57
  readonly info: ContextEngineInfo = {
56
58
  id: "kongbrain",
57
59
  name: "KongBrain",
58
- version: "0.1.2",
60
+ version: "0.4.2",
59
61
  ownsCompaction: true,
60
62
  };
61
63
 
@@ -63,6 +65,7 @@ export class KongBrainContextEngine implements ContextEngine {
63
65
 
64
66
  // ── Bootstrap ──────────────────────────────────────────────────────────
65
67
 
68
+ /** Initialize schema, create 5-pillar graph nodes, and start the memory daemon. */
66
69
  async bootstrap(params: {
67
70
  sessionId: string;
68
71
  sessionKey?: string;
@@ -139,6 +142,7 @@ export class KongBrainContextEngine implements ContextEngine {
139
142
 
140
143
  // ── Assemble ───────────────────────────────────────────────────────────
141
144
 
145
+ /** Build the context window: graph retrieval + system prompt additions + budget trimming. */
142
146
  async assemble(params: {
143
147
  sessionId: string;
144
148
  sessionKey?: string;
@@ -173,26 +177,22 @@ export class KongBrainContextEngine implements ContextEngine {
173
177
  if (systemPromptSection) additions.push(systemPromptSection);
174
178
 
175
179
  // Compaction summary (claw-code: compact.rs structured signals — inject once after compaction)
176
- const compactionSummary = (session as any)._compactionSummary as string | undefined;
180
+ const compactionSummary = session._compactionSummary;
177
181
  if (compactionSummary) {
178
182
  additions.push("[POST-COMPACTION CONTEXT]\n" + compactionSummary);
179
- delete (session as any)._compactionSummary;
183
+ session._compactionSummary = undefined;
180
184
  }
181
185
 
182
186
  // Wakeup briefing (synthesized at session start, may still be in-flight)
183
- const wakeupPromise = (session as any)._wakeupPromise as Promise<string | null> | undefined;
187
+ const wakeupPromise = session._wakeupPromise;
184
188
  if (wakeupPromise) {
185
189
  const wakeupBriefing = await wakeupPromise;
186
- delete (session as any)._wakeupPromise; // Only inject once
190
+ session._wakeupPromise = undefined; // Only inject once
187
191
  if (wakeupBriefing) additions.push(wakeupBriefing);
188
192
  }
189
193
 
190
194
  // Graduation celebration — tell the agent it just graduated so it can share with the user
191
- const graduation = (session as any)._graduationCelebration as {
192
- qualityScore: number;
193
- volumeScore: number;
194
- soulSummary: string;
195
- } | undefined;
195
+ const graduation = session._graduationCelebration;
196
196
  if (graduation) {
197
197
  let graduationBlock =
198
198
  "[SOUL GRADUATION — CELEBRATE WITH THE USER]\n" +
@@ -211,11 +211,11 @@ export class KongBrainContextEngine implements ContextEngine {
211
211
  "identity emerging from YOUR experience. Don't be robotic about it. This only happens once.";
212
212
 
213
213
  additions.push(graduationBlock);
214
- delete (session as any)._graduationCelebration; // Only inject once
214
+ session._graduationCelebration = undefined; // Only inject once
215
215
  }
216
216
 
217
217
  // Migration nudge — tell the agent there are workspace files to offer migrating
218
- if ((session as any)._hasMigratableFiles) {
218
+ if (session._hasMigratableFiles) {
219
219
  additions.push(
220
220
  "[MIGRATION AVAILABLE] This workspace has files from the default context engine " +
221
221
  "(IDENTITY.md, MEMORY.md, skills/, etc.). You can offer to migrate them into the graph " +
@@ -226,15 +226,31 @@ export class KongBrainContextEngine implements ContextEngine {
226
226
  );
227
227
  }
228
228
 
229
+ // Apply SPA priority budget — drop lowest-priority sections if over budget
230
+ // (dropped sections aren't lost — they're in the graph, retrievable on demand)
231
+ const BYTES_PER_TOKEN = 4; // claw-code: roughTokenCountEstimation default
232
+ const SPA_BUDGET_CHARS = Math.round(contextWindow * 0.08 * BYTES_PER_TOKEN);
233
+ let spaTotalChars = 0;
234
+ const keptAdditions: string[] = [];
235
+ for (const section of additions) { // additions are already in priority order
236
+ if (spaTotalChars + section.length > SPA_BUDGET_CHARS && keptAdditions.length > 0) break;
237
+ keptAdditions.push(section);
238
+ spaTotalChars += section.length;
239
+ }
240
+
241
+ const spaText = keptAdditions.length > 0 ? keptAdditions.join("\n\n") : undefined;
242
+ const spaTokens = spaText ? Math.ceil(spaText.length / BYTES_PER_TOKEN) : 0;
243
+
229
244
  return {
230
245
  messages,
231
- estimatedTokens: stats.sentTokens,
232
- systemPromptAddition: additions.length > 0 ? additions.join("\n\n") : undefined,
246
+ estimatedTokens: stats.sentTokens + spaTokens,
247
+ systemPromptAddition: spaText,
233
248
  };
234
249
  }
235
250
 
236
251
  // ── Ingest ─────────────────────────────────────────────────────────────
237
252
 
253
+ /** Embed and store a single user or assistant message as a turn node. */
238
254
  async ingest(params: {
239
255
  sessionId: string;
240
256
  sessionKey?: string;
@@ -247,7 +263,7 @@ export class KongBrainContextEngine implements ContextEngine {
247
263
  const msg = params.message;
248
264
 
249
265
  try {
250
- const role = (msg as any).role as string;
266
+ const role = "role" in msg ? (msg as { role: string }).role : "";
251
267
  if (role === "user" || role === "assistant") {
252
268
  const text = extractMessageText(msg);
253
269
  if (!text) return { ingested: false };
@@ -256,11 +272,16 @@ export class KongBrainContextEngine implements ContextEngine {
256
272
  let embedding: number[] | null = null;
257
273
  if (worthEmbedding && embeddings.isAvailable()) {
258
274
  try {
259
- const embedLimit = Math.round(8192 * 3.4 * 0.8);
260
- embedding = await embeddings.embed(text.slice(0, embedLimit));
275
+ const INGEST_EMBED_CHAR_LIMIT = 22_282; // ~6,554 tokens at 3.4 chars/token (BGE-M3 8192-token window * 0.8 safety margin)
276
+ embedding = await embeddings.embed(text.slice(0, INGEST_EMBED_CHAR_LIMIT));
261
277
  } catch (e) { swallow("ingest:embed", e); }
262
278
  }
263
279
 
280
+ // Stash user embedding for reuse in buildContextualQueryVec (avoids re-embedding)
281
+ if (role === "user" && embedding) {
282
+ session.lastUserEmbedding = embedding;
283
+ }
284
+
264
285
  const turnId = await store.upsertTurn({
265
286
  session_id: session.sessionId,
266
287
  role,
@@ -327,6 +348,7 @@ export class KongBrainContextEngine implements ContextEngine {
327
348
 
328
349
  // ── Compact ────────────────────────────────────────────────────────────
329
350
 
351
+ /** Extract structured signals (pending work, key files, errors) for post-compaction injection. */
330
352
  async compact(params: {
331
353
  sessionId: string;
332
354
  sessionKey?: string;
@@ -346,8 +368,9 @@ export class KongBrainContextEngine implements ContextEngine {
346
368
 
347
369
  // Extract structured compaction signals from stored turns
348
370
  let summary: string | undefined;
371
+ const { store } = this.state;
372
+ const contextWindow = params.tokenBudget ?? 200_000;
349
373
  try {
350
- const { store } = this.state;
351
374
  if (store.isAvailable()) {
352
375
  const turns = await store.getSessionTurnsRich(params.sessionId, 30);
353
376
  if (turns.length > 0) {
@@ -370,6 +393,12 @@ export class KongBrainContextEngine implements ContextEngine {
370
393
  turns.filter(t => t.tool_name).map(t => t.tool_name!)
371
394
  )];
372
395
 
396
+ // Recent errors — preserve tool failure context across compaction
397
+ const errorRe = /\b(error|failed|exception|crash|panic|TypeError|ReferenceError)\b[^.\n]{0,120}/gi;
398
+ const recentErrors = [...fullText.matchAll(errorRe)]
399
+ .map(m => m[0].trim().slice(0, 160))
400
+ .slice(-3); // last 3 errors only
401
+
373
402
  // Current work inference (claw-code: compact.rs:272-279)
374
403
  const lastText = turns.filter(t => t.text.length > 10).at(-1)?.text.slice(0, 200) ?? "";
375
404
 
@@ -377,6 +406,7 @@ export class KongBrainContextEngine implements ContextEngine {
377
406
  if (pendingMatches.length > 0) parts.push(`PENDING: ${pendingMatches.join("; ")}`);
378
407
  if (filePaths.length > 0) parts.push(`FILES: ${filePaths.join(", ")}`);
379
408
  if (toolNames.length > 0) parts.push(`TOOLS USED: ${toolNames.join(", ")}`);
409
+ if (recentErrors.length > 0) parts.push(`RECENT ERRORS: ${recentErrors.join("; ")}`);
380
410
  if (lastText) parts.push(`LAST: ${lastText}`);
381
411
  parts.push("Resume directly — do not recap what was happening.");
382
412
 
@@ -384,25 +414,34 @@ export class KongBrainContextEngine implements ContextEngine {
384
414
  summary = parts.join("\n");
385
415
  // Stash for next assemble() to inject
386
416
  if (session) {
387
- (session as any)._compactionSummary = summary;
417
+ session._compactionSummary = summary;
388
418
  }
389
419
  }
390
420
  }
391
421
  }
392
422
  } catch { /* non-critical */ }
393
423
 
424
+ // Compaction checkpoint — diagnostic trail for debugging
425
+ if (store.isAvailable() && session) {
426
+ store.createCompactionCheckpoint(params.sessionId, 0, session.userTurnCount)
427
+ .catch(e => swallow.warn("compact:checkpoint", e));
428
+ }
429
+
394
430
  return {
395
431
  ok: true,
396
- compacted: !!summary,
397
- reason: summary
398
- ? "Extracted structured signals for continuation."
399
- : "Graph retrieval handles context selection; no LLM-based compaction needed.",
400
- result: summary ? { summary, tokensBefore: 0 } : undefined,
432
+ compacted: true,
433
+ reason: "Graph-curated context window: assemble() selects relevant context each turn.",
434
+ result: summary ? {
435
+ summary,
436
+ tokensBefore: Math.round(summary.length / 4), // 4 bytes/token (claw-code ratio)
437
+ tokensAfter: Math.round(contextWindow * 0.325),
438
+ } : undefined,
401
439
  };
402
440
  }
403
441
 
404
442
  // ── After turn ─────────────────────────────────────────────────────────
405
443
 
444
+ /** Post-turn: ingest messages, evaluate retrieval quality, flush daemon, and run periodic maintenance. */
406
445
  async afterTurn?(params: {
407
446
  sessionId: string;
408
447
  sessionKey?: string;
@@ -411,11 +450,31 @@ export class KongBrainContextEngine implements ContextEngine {
411
450
  prePromptMessageCount: number;
412
451
  }): Promise<void> {
413
452
  const sessionKey = params.sessionKey ?? params.sessionId;
414
- const session = this.state.getSession(sessionKey);
415
- if (!session) return;
453
+ log.debug(`afterTurn: session=${sessionKey} messages=${params.messages.length}`);
454
+ // Use getOrCreateSession so resumed sessions (where session_start
455
+ // didn't fire after a gateway restart) still get a session object.
456
+ const session = this.state.getOrCreateSession(sessionKey, params.sessionId);
416
457
 
417
458
  const { store, embeddings } = this.state;
418
459
 
460
+ // Lazy daemon start: if session was resumed after gateway restart,
461
+ // session_start won't re-fire, so the daemon never started.
462
+ if (!session.daemon && typeof this.state.complete === "function") {
463
+ try {
464
+ session.daemon = startMemoryDaemon(
465
+ store,
466
+ embeddings,
467
+ session.sessionId,
468
+ this.state.complete,
469
+ this.state.config.thresholds.extractionTimeoutMs,
470
+ session.taskId,
471
+ session.projectId,
472
+ );
473
+ } catch (e) {
474
+ swallow.warn("afterTurn:lazyDaemonStart", e);
475
+ }
476
+ }
477
+
419
478
  // Deferred cleanup: run once on first turn when complete() is available
420
479
  if (session.userTurnCount <= 1 && typeof this.state.complete === "function") {
421
480
  runDeferredCleanup(store, embeddings, this.state.complete)
@@ -442,11 +501,12 @@ export class KongBrainContextEngine implements ContextEngine {
442
501
  .catch(e => swallow.warn("afterTurn:evaluateRetrieval", e));
443
502
  }
444
503
 
504
+ // Single fetch for all downstream consumers (cognitive check, daemon flush, handoff)
505
+ const allSessionTurns = await store.getSessionTurns(session.sessionId, 50)
506
+ .catch(() => [] as { role: string; text: string }[]);
507
+
445
508
  // Cognitive check: periodic reasoning over retrieved context
446
509
  if (shouldRunCheck(session.userTurnCount, session) && stagedSnapshot.length > 0) {
447
- const recentTurns = await store.getSessionTurns(session.sessionId, 6)
448
- .catch(() => [] as { role: string; text: string }[]);
449
-
450
510
  runCognitiveCheck({
451
511
  sessionId: session.sessionId,
452
512
  userQuery: session.lastUserText,
@@ -457,20 +517,21 @@ export class KongBrainContextEngine implements ContextEngine {
457
517
  score: n.finalScore ?? 0,
458
518
  table: n.table,
459
519
  })),
460
- recentTurns,
520
+ recentTurns: allSessionTurns.slice(-6),
461
521
  }, session, store, this.state.complete).catch(e => swallow.warn("afterTurn:cognitiveCheck", e));
462
522
  }
463
523
 
464
524
  // Flush to daemon when token threshold OR turn count threshold is reached
465
525
  const tokenReady = session.newContentTokens >= session.daemonTokenThreshold;
466
526
  const turnReady = session.userTurnCount >= session.lastDaemonFlushTurnCount + 3;
527
+ log.debug(`flush check: daemon=${!!session.daemon} tokenReady=${tokenReady} turnReady=${turnReady} turns=${session.userTurnCount}`);
467
528
  if (session.daemon && (tokenReady || turnReady)) {
468
529
  try {
469
- const recentTurns = await store.getSessionTurns(session.sessionId, 20);
530
+ const recentTurns = allSessionTurns.slice(-20);
470
531
  const turnData = recentTurns.map(t => ({
471
532
  role: t.role as "user" | "assistant",
472
533
  text: t.text,
473
- turnId: String((t as any).id ?? ""),
534
+ turnId: String((t as { id?: string }).id ?? ""),
474
535
  }));
475
536
 
476
537
  // Gather retrieved memory IDs for dedup
@@ -503,20 +564,14 @@ export class KongBrainContextEngine implements ContextEngine {
503
564
  // Fire-and-forget: these are non-critical background operations
504
565
  const cleanupOps: Promise<unknown>[] = [];
505
566
 
506
- // Final daemon flush with full transcript before cleanup
567
+ // Final daemon flush with full transcript before cleanup (reuse allSessionTurns)
507
568
  if (session.daemon) {
508
- cleanupOps.push(
509
- store.getSessionTurns(session.sessionId, 50)
510
- .then(recentTurns => {
511
- const turnData = recentTurns.map(t => ({
512
- role: t.role as "user" | "assistant",
513
- text: t.text,
514
- turnId: String((t as any).id ?? ""),
515
- }));
516
- session.daemon!.sendTurnBatch(turnData, [...session.pendingThinking], []);
517
- })
518
- .catch(e => swallow.warn("midCleanup:daemonFlush", e)),
519
- );
569
+ const turnData = allSessionTurns.map(t => ({
570
+ role: t.role as "user" | "assistant",
571
+ text: t.text,
572
+ turnId: String((t as { id?: string }).id ?? ""),
573
+ }));
574
+ session.daemon.sendTurnBatch(turnData, [...session.pendingThinking], []);
520
575
  }
521
576
 
522
577
  if (session.taskId) {
@@ -542,10 +597,10 @@ export class KongBrainContextEngine implements ContextEngine {
542
597
  .catch(e => swallow("midCleanup:acan", e)),
543
598
  );
544
599
 
545
- // Handoff note — snapshot for wakeup even if session continues
600
+ // Handoff note — snapshot for wakeup even if session continues (reuse allSessionTurns)
546
601
  cleanupOps.push(
547
602
  (async () => {
548
- const recentTurns = await store.getSessionTurns(session.sessionId, 15);
603
+ const recentTurns = allSessionTurns.slice(-15);
549
604
  if (recentTurns.length < 2) return;
550
605
  const turnSummary = recentTurns
551
606
  .map(t => `[${t.role}] ${t.text.slice(0, 200)}`)
@@ -635,12 +690,12 @@ export class KongBrainContextEngine implements ContextEngine {
635
690
  // ── Helpers ────────────────────────────────────────────────────────────────────
636
691
 
637
692
  function extractMessageText(msg: AgentMessage): string {
638
- const m = msg as any;
693
+ const m = msg as { content?: string | { type: string; text?: string }[] };
639
694
  if (typeof m.content === "string") return m.content;
640
695
  if (Array.isArray(m.content)) {
641
696
  return m.content
642
- .filter((c: any) => c.type === "text")
643
- .map((c: any) => c.text ?? "")
697
+ .filter((c) => c.type === "text")
698
+ .map((c) => c.text ?? "")
644
699
  .join("\n");
645
700
  }
646
701
  return "";
@@ -36,7 +36,7 @@ export function startMemoryDaemon(
36
36
  sharedEmbeddings: EmbeddingService,
37
37
  sessionId: string,
38
38
  complete: CompleteFn,
39
- extractionTimeoutMs = 60_000,
39
+ extractionTimeoutMs = 120_000,
40
40
  taskId?: string,
41
41
  projectId?: string,
42
42
  ): MemoryDaemon {
@@ -79,10 +79,10 @@ export function startMemoryDaemon(
79
79
  const { buildSystemPrompt, buildTranscript, writeExtractionResults } = await import("./memory-daemon.js");
80
80
 
81
81
  const transcript = buildTranscript(turns);
82
- const sections: string[] = [`[TRANSCRIPT]\n${transcript.slice(0, 60000)}`];
82
+ const sections: string[] = [`[TRANSCRIPT]\n${transcript.slice(0, 30000)}`];
83
83
 
84
84
  if (thinking.length > 0) {
85
- sections.push(`[THINKING]\n${thinking.slice(-8).join("\n---\n").slice(0, 4000)}`);
85
+ sections.push(`[THINKING]\n${thinking.slice(-3).join("\n---\n").slice(0, 2000)}`);
86
86
  }
87
87
 
88
88
  if (retrievedMemories.length > 0) {
@@ -92,33 +92,79 @@ export function startMemoryDaemon(
92
92
 
93
93
  const systemPrompt = buildSystemPrompt(thinking.length > 0, retrievedMemories.length > 0, priorState);
94
94
 
95
+ // Structured output schema — forces API to return valid JSON (no markdown, no preamble)
96
+ const extractionSchema = {
97
+ type: "object" as const,
98
+ properties: {
99
+ causal: { type: "array", items: { type: "object" } },
100
+ monologue: { type: "array", items: { type: "object" } },
101
+ resolved: { type: "array", items: { type: "string" } },
102
+ concepts: { type: "array", items: { type: "object" } },
103
+ corrections: { type: "array", items: { type: "object" } },
104
+ preferences: { type: "array", items: { type: "object" } },
105
+ artifacts: { type: "array", items: { type: "object" } },
106
+ decisions: { type: "array", items: { type: "object" } },
107
+ skills: { type: "array", items: { type: "object" } },
108
+ },
109
+ required: ["causal", "monologue", "resolved", "concepts", "corrections", "preferences", "artifacts", "decisions", "skills"],
110
+ };
111
+
95
112
  const response = await complete({
96
113
  system: systemPrompt,
97
114
  messages: [{ role: "user", content: sections.join("\n\n") }],
115
+ outputFormat: { type: "json_schema", schema: extractionSchema },
98
116
  });
99
117
 
100
- const responseText = response.text;
118
+ let responseText = response.text;
101
119
 
102
- const jsonMatch = responseText.match(/\{[\s\S]*?\}/);
103
- if (!jsonMatch) return;
120
+ // Sanitize: strip BOM, markdown fences, and trim
121
+ responseText = responseText.replace(/^\uFEFF/, "").trim();
122
+ const fenceMatch = responseText.match(/^```(?:json)?\s*\n([\s\S]*?)\n```\s*$/);
123
+ if (fenceMatch) responseText = fenceMatch[1].trim();
104
124
 
125
+ // With structured output the response should be valid JSON directly.
126
+ // Fall back to regex extraction if the provider doesn't support outputFormat.
105
127
  let result: Record<string, any>;
106
128
  try {
107
- result = JSON.parse(jsonMatch[0]);
108
- } catch {
129
+ result = JSON.parse(responseText);
130
+ } catch (parseErr) {
131
+ swallow.warn("daemon:parseDebug", new Error(
132
+ `JSON.parse failed: ${(parseErr as Error).message}; ` +
133
+ `len=${responseText.length}; first100=${JSON.stringify(responseText.slice(0, 100))}; ` +
134
+ `last100=${JSON.stringify(responseText.slice(-100))}`
135
+ ));
136
+ const jsonMatch = responseText.match(/\{[\s\S]*\}/);
137
+ if (!jsonMatch) {
138
+ swallow.warn("daemon:noJson", new Error(`LLM response contained no JSON (${responseText.length} chars)`));
139
+ return;
140
+ }
109
141
  try {
110
- result = JSON.parse(jsonMatch[0].replace(/,\s*([}\]])/g, "$1"));
142
+ result = JSON.parse(jsonMatch[0]);
111
143
  } catch {
112
- result = {};
113
- const fields = ["causal", "monologue", "resolved", "concepts", "corrections", "preferences", "artifacts", "decisions", "skills"];
114
- for (const field of fields) {
115
- const fieldMatch = jsonMatch[0].match(new RegExp(`"${field}"\\s*:\\s*(\\[[\\s\\S]*?\\])(?=\\s*[,}]\\s*"[a-z]|\\s*\\}$)`, "m"));
116
- if (fieldMatch) {
117
- try { result[field] = JSON.parse(fieldMatch[1]); } catch { /* skip */ }
144
+ // Try fixing trailing commas
145
+ try {
146
+ result = JSON.parse(jsonMatch[0].replace(/,\s*([}\]])/g, "$1"));
147
+ } catch {
148
+ // Try stripping control characters
149
+ try {
150
+ const cleaned = jsonMatch[0].replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
151
+ result = JSON.parse(cleaned);
152
+ } catch {
153
+ result = {};
154
+ const fields = ["causal", "monologue", "resolved", "concepts", "corrections", "preferences", "artifacts", "decisions", "skills"];
155
+ for (const field of fields) {
156
+ const fieldMatch = jsonMatch[0].match(new RegExp(`"${field}"\\s*:\\s*(\\[[\\s\\S]*?\\])(?=\\s*[,}]\\s*"[a-z]|\\s*\\}$)`, "m"));
157
+ if (fieldMatch) {
158
+ try { result[field] = JSON.parse(fieldMatch[1]); } catch { /* skip */ }
159
+ }
160
+ }
161
+ const PRIMARY_FIELDS = ["causal", "monologue", "artifacts"];
162
+ if (!PRIMARY_FIELDS.some(f => f in result)) {
163
+ swallow.warn("daemon:fallbackFailed", new Error(`Regex fallback extracted no primary fields from: ${jsonMatch[0].slice(0, 100)}`));
164
+ return;
165
+ }
118
166
  }
119
167
  }
120
- const PRIMARY_FIELDS = ["causal", "monologue", "artifacts"];
121
- if (!PRIMARY_FIELDS.some(f => f in result)) return;
122
168
  }
123
169
  }
124
170
 
@@ -164,9 +210,14 @@ export function startMemoryDaemon(
164
210
  sendTurnBatch(turns, thinking, retrievedMemories, priorExtractions) {
165
211
  if (shuttingDown) return;
166
212
  if (pendingBatch) {
167
- swallow.warn("daemon:batchOverwrite", new Error(`Overwriting pending batch (${pendingBatch.turns.length} turns) with new batch (${turns.length} turns)`));
213
+ // Merge into pending batch instead of discarding prevents turn data loss
214
+ pendingBatch.turns = [...pendingBatch.turns, ...turns];
215
+ pendingBatch.thinking = [...pendingBatch.thinking, ...thinking];
216
+ pendingBatch.retrievedMemories = [...pendingBatch.retrievedMemories, ...retrievedMemories];
217
+ pendingBatch.priorExtractions = priorExtractions ?? pendingBatch.priorExtractions;
218
+ } else {
219
+ pendingBatch = { turns, thinking, retrievedMemories, priorExtractions };
168
220
  }
169
- pendingBatch = { turns, thinking, retrievedMemories, priorExtractions };
170
221
  // Fire-and-forget
171
222
  processPending().catch(e => swallow.warn("daemon:sendBatch", e));
172
223
  },
@@ -14,10 +14,12 @@ import type { CompleteFn } from "./state.js";
14
14
  import { buildSystemPrompt, buildTranscript, writeExtractionResults } from "./memory-daemon.js";
15
15
  import type { PriorExtractions } from "./daemon-types.js";
16
16
  import { swallow } from "./errors.js";
17
+ import { log } from "./log.js";
17
18
 
18
19
  // Process-global flag — deferred cleanup runs AT MOST ONCE per process.
19
20
  // Using Symbol.for so it survives Jiti re-importing this module.
20
21
  const RAN_KEY = Symbol.for("kongbrain.deferredCleanup.ran");
22
+ const _g = globalThis as Record<symbol, unknown>;
21
23
 
22
24
  /**
23
25
  * Find and process orphaned sessions. Runs with a 30s total timeout.
@@ -30,8 +32,8 @@ export async function runDeferredCleanup(
30
32
  complete: CompleteFn,
31
33
  ): Promise<number> {
32
34
  // Once per process — never re-run even if first run times out
33
- if ((globalThis as any)[RAN_KEY]) return 0;
34
- (globalThis as any)[RAN_KEY] = true;
35
+ if (_g[RAN_KEY]) return 0;
36
+ _g[RAN_KEY] = true;
35
37
 
36
38
  try {
37
39
  return await runDeferredCleanupInner(store, embeddings, complete);
@@ -101,8 +103,8 @@ async function processOrphanedSession(
101
103
  const systemPrompt = buildSystemPrompt(false, false, priorState);
102
104
 
103
105
  try {
104
- console.warn(`[deferred] extracting session ${surrealSessionId} (${turns.length} turns, transcript ${transcript.length} chars)`);
105
- const LLM_CALL_TIMEOUT_MS = 30_000;
106
+ log.info(`[deferred] extracting session ${surrealSessionId} (${turns.length} turns, transcript ${transcript.length} chars)`);
107
+ const LLM_CALL_TIMEOUT_MS = 120_000;
106
108
  const response = await Promise.race([
107
109
  complete({
108
110
  system: systemPrompt,
@@ -114,7 +116,7 @@ async function processOrphanedSession(
114
116
  ]);
115
117
 
116
118
  const responseText = response.text;
117
- console.warn(`[deferred] extraction response: ${responseText.length} chars`);
119
+ log.info(`[deferred] extraction response: ${responseText.length} chars`);
118
120
  const jsonMatch = responseText.match(/\{[\s\S]*\}/);
119
121
  if (jsonMatch) {
120
122
  let result: Record<string, any>;
@@ -128,17 +130,17 @@ async function processOrphanedSession(
128
130
  // Strip prototype pollution keys from LLM-generated JSON
129
131
  const BANNED_KEYS = new Set(["__proto__", "constructor", "prototype"]);
130
132
  for (const key of Object.keys(result)) {
131
- if (BANNED_KEYS.has(key)) delete (result as any)[key];
133
+ if (BANNED_KEYS.has(key)) delete result[key];
132
134
  }
133
135
 
134
136
  const keys = Object.keys(result);
135
- console.warn(`[deferred] parsed ${keys.length} keys: ${keys.join(", ")}`);
137
+ log.info(`[deferred] parsed ${keys.length} keys: ${keys.join(", ")}`);
136
138
  if (keys.length > 0) {
137
139
  await writeExtractionResults(result, surrealSessionId, store, embeddings, priorState, undefined, undefined, turnData);
138
- console.warn(`[deferred] wrote extraction results for ${surrealSessionId}`);
140
+ log.info(`[deferred] wrote extraction results for ${surrealSessionId}`);
139
141
  }
140
142
  } else {
141
- console.warn(`[deferred] no JSON found in response`);
143
+ log.warn(`[deferred] no JSON found in response`);
142
144
  }
143
145
  } catch (e) {
144
146
  swallow.warn("deferredCleanup:extraction", e);
@@ -162,7 +164,7 @@ async function processOrphanedSession(
162
164
  ]);
163
165
 
164
166
  const handoffText = handoffResponse.text.trim();
165
- console.warn(`[deferred] handoff response: ${handoffText.length} chars`);
167
+ log.info(`[deferred] handoff response: ${handoffText.length} chars`);
166
168
  if (handoffText.length > 20) {
167
169
  let emb: number[] | null = null;
168
170
  if (embeddings.isAvailable()) {
package/src/embeddings.ts CHANGED
@@ -1,12 +1,14 @@
1
1
  import { existsSync } from "node:fs";
2
2
  import type { EmbeddingConfig } from "./config.js";
3
3
  import { swallow } from "./errors.js";
4
+ import { log } from "./log.js";
4
5
 
5
6
  // Lazy-import node-llama-cpp to avoid top-level await issues with jiti.
6
7
  // The actual import happens inside initialize() at runtime.
7
8
  type LlamaEmbeddingContext = import("node-llama-cpp").LlamaEmbeddingContext;
8
9
  type LlamaModel = import("node-llama-cpp").LlamaModel;
9
10
 
11
+ /** BGE-M3 embedding service (1024-dim via GGUF) with an LRU cache of up to 512 entries. */
10
12
  export class EmbeddingService {
11
13
  private model: LlamaModel | null = null;
12
14
  private ctx: LlamaEmbeddingContext | null = null;
@@ -30,8 +32,8 @@ export class EmbeddingService {
30
32
  logLevel: LlamaLogLevel.error,
31
33
  logger: (level, message) => {
32
34
  if (message.includes("missing newline token")) return;
33
- if (level === LlamaLogLevel.error) console.error(`[llama] ${message}`);
34
- else if (level === LlamaLogLevel.warn) console.warn(`[llama] ${message}`);
35
+ if (level === LlamaLogLevel.error) log.error(`[llama] ${message}`);
36
+ else if (level === LlamaLogLevel.warn) log.warn(`[llama] ${message}`);
35
37
  },
36
38
  });
37
39
  this.model = await llama.loadModel({ modelPath: this.config.modelPath });
@@ -40,6 +42,7 @@ export class EmbeddingService {
40
42
  return true;
41
43
  }
42
44
 
45
+ /** Return the embedding vector for text, serving from LRU cache on repeat calls. */
43
46
  async embed(text: string): Promise<number[]> {
44
47
  if (!this.ready || !this.ctx) throw new Error("Embeddings not initialized");
45
48
  const cached = this.cache.get(text);
@@ -61,11 +64,7 @@ export class EmbeddingService {
61
64
 
62
65
  async embedBatch(texts: string[]): Promise<number[][]> {
63
66
  if (texts.length === 0) return [];
64
- const results: number[][] = [];
65
- for (const text of texts) {
66
- results.push(await this.embed(text));
67
- }
68
- return results;
67
+ return Promise.all(texts.map(text => this.embed(text)));
69
68
  }
70
69
 
71
70
  isAvailable(): boolean {
package/src/errors.ts CHANGED
@@ -9,6 +9,8 @@
9
9
  * Always logged to stderr with stack trace.
10
10
  */
11
11
 
12
+ import { log } from "./log.js";
13
+
12
14
  const DEBUG = process.env.KONGBRAIN_DEBUG === "1";
13
15
 
14
16
  /**
@@ -18,7 +20,7 @@ const DEBUG = process.env.KONGBRAIN_DEBUG === "1";
18
20
  function swallow(context: string, err?: unknown): void {
19
21
  if (!DEBUG) return;
20
22
  const msg = err instanceof Error ? err.message : String(err ?? "unknown");
21
- console.debug(`[swallow] ${context}: ${msg}`);
23
+ log.debug(`[swallow] ${context}: ${msg}`);
22
24
  }
23
25
 
24
26
  /**
@@ -27,7 +29,7 @@ function swallow(context: string, err?: unknown): void {
27
29
  */
28
30
  swallow.warn = function swallowWarn(context: string, err?: unknown): void {
29
31
  const msg = err instanceof Error ? err.message : String(err ?? "unknown");
30
- console.warn(`[warn] ${context}: ${msg}`);
32
+ log.warn(`${context}: ${msg}`);
31
33
  };
32
34
 
33
35
  /**
@@ -37,7 +39,7 @@ swallow.warn = function swallowWarn(context: string, err?: unknown): void {
37
39
  swallow.error = function swallowError(context: string, err?: unknown): void {
38
40
  const msg = err instanceof Error ? err.message : String(err ?? "unknown");
39
41
  const stack = err instanceof Error ? `\n${err.stack}` : "";
40
- console.error(`[ERROR] ${context}: ${msg}${stack}`);
42
+ log.error(`${context}: ${msg}${stack}`);
41
43
  };
42
44
 
43
45
  export { swallow };