kongbrain 0.2.1 → 0.3.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kongbrain",
-  "version": "0.2.1",
+  "version": "0.3.1",
   "description": "Graph-backed persistent memory engine for OpenClaw. Replaces the default context window with SurrealDB + vector embeddings that learn across sessions.",
   "type": "module",
   "license": "MIT",

package/src/acan.ts

@@ -282,8 +282,9 @@ function trainInBackground(
 const STALENESS_GROWTH_FACTOR = 0.5;
 const STALENESS_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
 
-export async function checkACANReadiness(store?: SurrealStore): Promise<void> {
+export async function checkACANReadiness(store?: SurrealStore, trainingThreshold?: number): Promise<void> {
   if (!store) return;
+  const threshold = trainingThreshold ?? TRAINING_THRESHOLD;
   const weightsPath = join(getKongDir(), WEIGHTS_FILENAME);
   const hasWeights = initACAN();
   const count = await getTrainingDataCount(store);
@@ -295,13 +296,13 @@ export async function checkACANReadiness(store?: SurrealStore): Promise<void> {
     const ageMs = Date.now() - trainedAt;
     const isStale = growthRatio >= STALENESS_GROWTH_FACTOR || ageMs >= STALENESS_MAX_AGE_MS;
     if (!isStale) return;
-  } else if (count < TRAINING_THRESHOLD) {
+  } else if (count < threshold) {
     return;
   }
 
   try {
     const samples = await fetchTrainingData(store);
-    if (samples.length < TRAINING_THRESHOLD) return;
+    if (samples.length < threshold) return;
     trainInBackground(samples, weightsPath, hasWeights ? _weights ?? undefined : undefined);
   } catch {
     // training is best-effort

package/src/config.ts

@@ -15,9 +15,23 @@ export interface EmbeddingConfig {
   dimensions: number;
 }
 
+export interface ThresholdConfig {
+  /** Tokens accumulated before daemon flushes extraction (default: 4000) */
+  daemonTokenThreshold: number;
+  /** Cumulative tokens before mid-session cleanup fires (default: 100000) */
+  midSessionCleanupThreshold: number;
+  /** Per-extraction timeout in ms (default: 60000) */
+  extractionTimeoutMs: number;
+  /** Max pending thinking blocks kept in memory (default: 20) */
+  maxPendingThinking: number;
+  /** Retrieval outcome samples needed before ACAN training (default: 5000) */
+  acanTrainingThreshold: number;
+}
+
 export interface KongBrainConfig {
   surreal: SurrealConfig;
   embedding: EmbeddingConfig;
+  thresholds: ThresholdConfig;
 }
 
 /**
@@ -27,6 +41,7 @@ export interface KongBrainConfig {
 export function parsePluginConfig(raw?: Record<string, unknown>): KongBrainConfig {
   const surreal = (raw?.surreal ?? {}) as Record<string, unknown>;
   const embedding = (raw?.embedding ?? {}) as Record<string, unknown>;
+  const thresholds = (raw?.thresholds ?? {}) as Record<string, unknown>;
 
   // Priority: plugin config > env vars > defaults
   const url =
@@ -60,5 +75,17 @@ export function parsePluginConfig(raw?: Record<string, unknown>): KongBrainConfi
       dimensions:
         typeof embedding.dimensions === "number" ? embedding.dimensions : 1024,
     },
+    thresholds: {
+      daemonTokenThreshold:
+        typeof thresholds.daemonTokenThreshold === "number" ? thresholds.daemonTokenThreshold : 4000,
+      midSessionCleanupThreshold:
+        typeof thresholds.midSessionCleanupThreshold === "number" ? thresholds.midSessionCleanupThreshold : 100_000,
+      extractionTimeoutMs:
+        typeof thresholds.extractionTimeoutMs === "number" ? thresholds.extractionTimeoutMs : 60_000,
+      maxPendingThinking:
+        typeof thresholds.maxPendingThinking === "number" ? thresholds.maxPendingThinking : 20,
+      acanTrainingThreshold:
+        typeof thresholds.acanTrainingThreshold === "number" ? thresholds.acanTrainingThreshold : 5000,
+    },
   };
 }

package/src/context-engine.ts

@@ -73,19 +73,22 @@ export class KongBrainContextEngine implements ContextEngine {
   }): Promise<BootstrapResult> {
     const { store, embeddings } = this.state;
 
-    // Run schema if first bootstrap
-    try {
-      const schemaPath = join(__dirname, "..", "src", "schema.surql");
-      let schemaSql: string;
+    // Run schema once per process (idempotent but expensive on every bootstrap)
+    if (!this.state.schemaApplied) {
       try {
-        schemaSql = readFileSync(schemaPath, "utf-8");
-      } catch {
-        // Fallback: try relative to compiled output
-        schemaSql = readFileSync(join(__dirname, "schema.surql"), "utf-8");
+        const schemaPath = join(__dirname, "..", "src", "schema.surql");
+        let schemaSql: string;
+        try {
+          schemaSql = readFileSync(schemaPath, "utf-8");
+        } catch {
+          // Fallback: try relative to compiled output
+          schemaSql = readFileSync(join(__dirname, "schema.surql"), "utf-8");
+        }
+        await store.queryExec(schemaSql);
+        this.state.schemaApplied = true;
+      } catch (e) {
+        swallow.warn("context-engine:schema", e);
       }
-      await store.queryExec(schemaSql);
-    } catch (e) {
-      swallow.warn("context-engine:schema", e);
     }
 
     // 5-pillar graph init
@@ -122,6 +125,7 @@ export class KongBrainContextEngine implements ContextEngine {
         if (!session.daemon) {
           session.daemon = startMemoryDaemon(
             store, embeddings, session.sessionId, this.state.complete,
+            this.state.config.thresholds.extractionTimeoutMs,
           );
         }
       } catch (e) {
@@ -135,7 +139,7 @@ export class KongBrainContextEngine implements ContextEngine {
       store.archiveOldTurns(),
       store.consolidateMemories((text) => embeddings.embed(text)),
       store.garbageCollectMemories(),
-      checkACANReadiness(store),
+      checkACANReadiness(store, this.state.config.thresholds.acanTrainingThreshold),
       // Deferred cleanup is triggered on first afterTurn() when complete() is available
     ]).catch(e => swallow.warn("bootstrap:maintenance", e));
 
@@ -404,7 +408,7 @@ export class KongBrainContextEngine implements ContextEngine {
     }
 
     // Flush to daemon when token threshold OR turn count threshold is reached
-    const tokenReady = session.newContentTokens >= session.DAEMON_TOKEN_THRESHOLD;
+    const tokenReady = session.newContentTokens >= session.daemonTokenThreshold;
     const turnReady = session.userTurnCount >= session.lastDaemonFlushTurnCount + 3;
     if (session.daemon && (tokenReady || turnReady)) {
       try {
@@ -439,7 +443,7 @@ export class KongBrainContextEngine implements ContextEngine {
     // OpenClaw exits via Ctrl+C×2 (no async window), so session_end never fires.
     // Run reflection, skill extraction, and causal graduation periodically.
     const tokensSinceCleanup = session.cumulativeTokens - session.lastCleanupTokens;
-    if (tokensSinceCleanup >= session.MID_SESSION_CLEANUP_THRESHOLD && typeof this.state.complete === "function") {
+    if (tokensSinceCleanup >= session.midSessionCleanupThreshold && typeof this.state.complete === "function") {
       session.lastCleanupTokens = session.cumulativeTokens;
 
       // Fire-and-forget: these are non-critical background operations
@@ -478,6 +482,12 @@ export class KongBrainContextEngine implements ContextEngine {
           .catch(e => swallow.warn("midCleanup:graduateCausal", e)),
       );
 
+      // ACAN: check if new retrieval outcomes warrant retraining
+      cleanupOps.push(
+        checkACANReadiness(store, this.state.config.thresholds.acanTrainingThreshold)
+          .catch(e => swallow("midCleanup:acan", e)),
+      );
+
       // Handoff note — snapshot for wakeup even if session continues
       cleanupOps.push(
         (async () => {

package/src/daemon-manager.ts

@@ -36,6 +36,7 @@ export function startMemoryDaemon(
   sharedEmbeddings: EmbeddingService,
   sessionId: string,
   complete: CompleteFn,
+  extractionTimeoutMs = 60_000,
 ): MemoryDaemon {
   // Use shared store/embeddings from global state (no duplicate connections)
   const store = sharedStore;
@@ -137,7 +138,12 @@ export function startMemoryDaemon(
       const batch = pendingBatch;
       pendingBatch = null;
       try {
-        await runExtraction(batch.turns, batch.thinking, batch.retrievedMemories, batch.priorExtractions);
+        await Promise.race([
+          runExtraction(batch.turns, batch.thinking, batch.retrievedMemories, batch.priorExtractions),
+          new Promise<void>((_, reject) =>
+            setTimeout(() => reject(new Error(`Extraction timed out after ${extractionTimeoutMs}ms`)), extractionTimeoutMs),
+          ),
+        ]);
       } catch (e) {
         errorCount++;
         swallow.warn("daemon:extraction", e);

package/src/embeddings.ts

@@ -11,7 +11,6 @@ export class EmbeddingService {
   private model: LlamaModel | null = null;
   private ctx: LlamaEmbeddingContext | null = null;
   private ready = false;
-  private embedCallCount = 0;
 
   constructor(private readonly config: EmbeddingConfig) {}
 
@@ -40,7 +39,6 @@ export class EmbeddingService {
 
   async embed(text: string): Promise<number[]> {
     if (!this.ready || !this.ctx) throw new Error("Embeddings not initialized");
-    this.embedCallCount++;
     const result = await this.ctx.getEmbeddingFor(text);
     return Array.from(result.vector);
   }
@@ -58,16 +56,6 @@ export class EmbeddingService {
     return this.ready;
   }
 
-  drainEmbedCallCount(): number {
-    const count = this.embedCallCount;
-    this.embedCallCount = 0;
-    return count;
-  }
-
-  getEmbedCallCount(): number {
-    return this.embedCallCount;
-  }
-
   async dispose(): Promise<void> {
     try {
       await this.ctx?.dispose();

package/src/graph-context.ts

@@ -634,9 +634,9 @@ async function formatContextMessage(
 
 function truncateToolResult(msg: AgentMessage, maxChars: number): AgentMessage {
   if (!isToolResult(msg)) return msg;
-  const totalLen = msg.content.reduce((s, c) => s + ((c as TextContent).text?.length ?? 0), 0);
+  const totalLen = msg.content.reduce((s: number, c: any) => s + ((c as TextContent).text?.length ?? 0), 0);
   if (totalLen <= maxChars) return msg;
-  const content = msg.content.map((c) => {
+  const content = msg.content.map((c: any) => {
     if (c.type !== "text") return c;
     const tc = c as TextContent;
     const allowed = Math.max(200, Math.floor((tc.text.length / totalLen) * maxChars));
@@ -654,8 +654,8 @@ function getRecentTurns(messages: AgentMessage[], maxTokens: number, contextWind
   const clean = messages.map((m) => {
     if (isAssistant(m) && m.stopReason === "error") {
       const errorText = m.content
-        .filter((c): c is TextContent => c.type === "text")
-        .map((c) => c.text)
+        .filter((c: any): c is TextContent => c.type === "text")
+        .map((c: any) => c.text)
         .join("")
         .slice(0, 150);
       return {
@@ -672,7 +672,7 @@ function getRecentTurns(messages: AgentMessage[], maxTokens: number, contextWind
   let i = 0;
   while (i < clean.length) {
     const msg = clean[i];
-    if (isAssistant(msg) && msg.content.some((c) => c.type === "toolCall")) {
+    if (isAssistant(msg) && msg.content.some((c: any) => c.type === "toolCall")) {
       const group: AgentMessage[] = [clean[i]];
       let j = i + 1;
       while (j < clean.length && isToolResult(clean[j])) {
@@ -837,9 +837,19 @@ async function graphTransformInner(
   const config = session.currentConfig;
   const skipRetrieval = config?.skipRetrieval ?? false;
   const currentIntent = config?.intent ?? "unknown";
-  const vectorSearchLimits = config?.vectorSearchLimits ?? {
+  const baseLimits = config?.vectorSearchLimits ?? {
     turn: 25, identity: 10, concept: 20, memory: 20, artifact: 10,
   };
+  // Scale search limits with context window — larger windows can use more results
+  const cwScale = Math.max(0.5, Math.min(2.0, contextWindow / 200_000));
+  const vectorSearchLimits = {
+    turn: Math.round((baseLimits.turn ?? 25) * cwScale),
+    identity: baseLimits.identity,  // always load full identity
+    concept: Math.round((baseLimits.concept ?? 20) * cwScale),
+    memory: Math.round((baseLimits.memory ?? 20) * cwScale),
+    artifact: Math.round((baseLimits.artifact ?? 10) * cwScale),
+    monologue: Math.round(8 * cwScale),
+  };
   let tokenBudget = Math.min(config?.tokenBudget ?? 6000, budgets.retrieval);
 
   // Pressure-based adaptive scaling

package/src/hooks/llm-output.ts

@@ -30,9 +30,16 @@ export function createLlmOutputHandler(state: GlobalPluginState) {
     const session = state.getSession(sessionKey);
     if (!session) return;
 
-    // Extract token counts (0 if provider didn't report usage)
-    const inputTokens = event.usage?.input ?? 0;
-    const outputTokens = event.usage?.output ?? 0;
+    // Measure assistant text output (used for token estimation and planning gate)
+    const textLen = event.assistantTexts.reduce((s, t) => s + t.length, 0);
+
+    // Extract token counts — fall back to text-length estimate when provider
+    // doesn't report usage (OpenClaw often passes 0 or undefined)
+    let inputTokens = event.usage?.input ?? 0;
+    let outputTokens = event.usage?.output ?? 0;
+    if (inputTokens + outputTokens === 0 && textLen > 0) {
+      outputTokens = Math.ceil(textLen / 4); // ~4 chars per token
+    }
 
     // Always update session stats — turn_count must increment even without usage data
     if (session.surrealSessionId) {
@@ -47,14 +54,11 @@ export function createLlmOutputHandler(state: GlobalPluginState) {
       }
     }
 
-    // Accumulate for daemon batching (only when real tokens present)
-    if (inputTokens + outputTokens > 0) {
-      session.newContentTokens += inputTokens + outputTokens;
-      session.cumulativeTokens += inputTokens + outputTokens;
-    }
+    // Accumulate for daemon batching and mid-session cleanup
+    session.newContentTokens += inputTokens + outputTokens;
+    session.cumulativeTokens += inputTokens + outputTokens;
 
     // Track accumulated text output for planning gate
-    const textLen = event.assistantTexts.reduce((s, t) => s + t.length, 0);
     session.turnTextLength += textLen;
 
     if (textLen > 50) {
@@ -78,6 +82,11 @@ export function createLlmOutputHandler(state: GlobalPluginState) {
           const thinking = block.thinking ?? block.text ?? "";
           if (thinking.length > 50) {
             session.pendingThinking.push(thinking);
+            // Cap to prevent unbounded growth in long sessions
+            const max = state.config.thresholds.maxPendingThinking;
+            if (session.pendingThinking.length > max) {
+              session.pendingThinking.splice(0, session.pendingThinking.length - max);
+            }
           }
         }
       }

package/src/index.ts

@@ -463,6 +463,7 @@ export default definePluginEntry({
           globalState!.embeddings,
           session.sessionId,
           globalState!.complete,
+          globalState!.config.thresholds.extractionTimeoutMs,
         );
       } catch (e) {
         swallow.warn("index:startDaemon", e);
@@ -476,7 +477,7 @@ export default definePluginEntry({
               (session as any)._hasMigratableFiles = true;
             }
           })
-          .catch(e => swallow("index:migrationCheck", e));
+          .catch(e => swallow.warn("index:migrationCheck", e));
       }
 
       // Set reflection context window from config

package/src/schema.surql

@@ -76,8 +76,6 @@ DEFINE FIELD IF NOT EXISTS model ON turn TYPE option<string>;
 DEFINE FIELD IF NOT EXISTS usage ON turn TYPE option<object>;
 DEFINE INDEX IF NOT EXISTS turn_vec_idx ON turn FIELDS embedding HNSW DIMENSION 1024 DIST COSINE;
 DEFINE INDEX IF NOT EXISTS turn_session_idx ON turn FIELDS session_id;
--- Migration: backfill created_at from timestamp for existing turns
-UPDATE turn SET created_at = timestamp WHERE created_at IS NONE AND timestamp IS NOT NONE;
 
 -- Identity chunks (agent persona / identity)
 DEFINE TABLE IF NOT EXISTS identity_chunk SCHEMALESS;
@@ -112,7 +110,6 @@ DEFINE FIELD IF NOT EXISTS source ON memory TYPE option<string>;
 DEFINE FIELD IF NOT EXISTS created_at ON memory TYPE datetime DEFAULT time::now();
 DEFINE FIELD IF NOT EXISTS last_accessed ON memory TYPE option<datetime>;
 DEFINE FIELD IF NOT EXISTS status ON memory TYPE option<string> DEFAULT "active";
-UPDATE memory SET status = "active" WHERE status IS NONE;
 DEFINE FIELD IF NOT EXISTS resolved_at ON memory TYPE option<datetime>;
 DEFINE FIELD IF NOT EXISTS resolved_by ON memory TYPE option<string>;
 DEFINE INDEX IF NOT EXISTS memory_vec_idx ON memory FIELDS embedding HNSW DIMENSION 1024 DIST COSINE;

package/src/skills.ts

@@ -154,7 +154,7 @@ export async function supersedeOldSkills(
         );
       }
     }
-  } catch (e) { swallow("skills:supersedeOld", e); }
+  } catch (e) { swallow.warn("skills:supersedeOld", e); }
 }
 
 // --- Skill Retrieval ---

package/src/state.ts

@@ -54,13 +54,13 @@ export class SessionState {
   // Memory daemon
   daemon: MemoryDaemon | null = null;
   newContentTokens = 0;
-  readonly DAEMON_TOKEN_THRESHOLD = 4000;
+  daemonTokenThreshold = 4000;
   lastDaemonFlushTurnCount = 0;
 
   // Cumulative session token tracking (for mid-session cleanup trigger)
   cumulativeTokens = 0;
   lastCleanupTokens = 0;
-  readonly MID_SESSION_CLEANUP_THRESHOLD = 100_000;
+  midSessionCleanupThreshold = 100_000;
 
   // Cleanup tracking
   cleanedUp = false;
@@ -106,6 +106,7 @@ export class GlobalPluginState {
   complete: CompleteFn;
   workspaceDir?: string;
   enqueueSystemEvent?: EnqueueSystemEventFn;
+  schemaApplied = false;
   private sessions = new Map<string, SessionState>();
 
   constructor(
@@ -125,6 +126,8 @@ export class GlobalPluginState {
     let session = this.sessions.get(sessionKey);
     if (!session) {
       session = new SessionState(sessionId, sessionKey);
+      session.daemonTokenThreshold = this.config.thresholds.daemonTokenThreshold;
+      session.midSessionCleanupThreshold = this.config.thresholds.midSessionCleanupThreshold;
       this.sessions.set(sessionKey, session);
     }
     return session;

package/src/surreal.ts

@@ -1062,16 +1062,16 @@ export class SurrealStore {
 
   async runMemoryMaintenance(): Promise<void> {
     try {
-      await this.queryExec(
-        `UPDATE memory SET importance = math::max([importance * 0.95, 2.0]) WHERE importance > 2.0`,
-      );
-      await this.queryExec(
-        `UPDATE memory SET importance = math::max([importance, 3 + ((
+      // Single round-trip to reduce transaction conflict window
+      await this.queryExec(`
+        UPDATE memory SET importance = math::max([importance * 0.95, 2.0]) WHERE importance > 2.0;
+        UPDATE memory SET importance = math::max([importance, 3 + ((
           SELECT VALUE avg_utilization FROM memory_utility_cache WHERE memory_id = string::concat(meta::tb(id), ":", meta::id(id)) LIMIT 1
-        )[0] ?? 0) * 4]) WHERE importance < 7`,
-      );
+        )[0] ?? 0) * 4]) WHERE importance < 7;
+      `);
     } catch (e) {
-      swallow.warn("surreal:runMemoryMaintenance", e);
+      // Transaction conflicts expected when daemon writes concurrently — silent
+      swallow("surreal:runMemoryMaintenance", e);
     }
   }