kongbrain 0.1.0

package/src/acan.ts

@@ -0,0 +1,309 @@
+/**
+ * ACAN — Attentive Cross-Attention Network for learned memory scoring.
+ *
+ * Replaces the fixed 6-signal WMR weights in scoreResults() with a learned
+ * cross-attention model. Ships dormant — auto-trains and activates when
+ * enough retrieval outcome data accumulates (5000+ labeled pairs).
+ *
+ * Ported from kongbrain — uses SurrealStore instead of module-level DB.
+ */
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { Worker } from "node:worker_threads";
+import type { SurrealStore } from "./surreal.js";
+import { swallow } from "./errors.js";
+
+// ── Types ──
+
+export interface ACANWeights {
+  W_q: number[][];
+  W_k: number[][];
+  W_final: number[];
+  bias: number;
+  version: number;
+  trainedAt?: number;
+  trainedOnSamples?: number;
+}
+
+export interface ACANCandidate {
+  embedding: number[];
+  recency: number;
+  importance: number;
+  access: number;
+  neighborBonus: number;
+  provenUtility: number;
+  reflectionBoost?: number;
+}
+
+interface TrainingSample {
+  query_embedding: number[];
+  memory_embedding: number[];
+  retrieval_score: number;
+  was_neighbor: boolean;
+  utilization: number;
+  importance: number;
+  access_count: number;
+  recency: number;
+}
+
+interface TrainingConfig {
+  epochs: number;
+  lr: number;
+  earlyStopPatience: number;
+  lrDecayPatience: number;
+  lrFloor: number;
+  valSplit: number;
+}
+
+// ── Module state ──
+
+let _weights: ACANWeights | null = null;
+let _active = false;
+
+const ATTN_DIM = 64;
+const EMBED_DIM = 1024;
+const FEATURE_COUNT = 7;
+const WEIGHTS_FILENAME = "acan_weights.json";
+const TRAINING_THRESHOLD = 5000;
+
+function getKongDir(): string {
+  const dir = join(homedir(), ".kongbrain");
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+const DEFAULT_TRAINING_CONFIG: TrainingConfig = {
+  epochs: 80,
+  lr: 0.001,
+  earlyStopPatience: 8,
+  lrDecayPatience: 4,
+  lrFloor: 0.00005,
+  valSplit: 0.2,
+};
+
+// ── Weight loading / saving ──
+
+function loadWeights(path: string): ACANWeights | null {
+  try {
+    if (!existsSync(path)) return null;
+    const raw = JSON.parse(readFileSync(path, "utf-8"));
+    if (raw.version !== 1) return null;
+    if (!Array.isArray(raw.W_q) || raw.W_q.length !== EMBED_DIM) return null;
+    if (!Array.isArray(raw.W_k) || raw.W_k.length !== EMBED_DIM) return null;
+    if (!Array.isArray(raw.W_final) || raw.W_final.length !== FEATURE_COUNT) return null;
+    if (typeof raw.bias !== "number") return null;
+    if (raw.W_q[0].length !== ATTN_DIM || raw.W_k[0].length !== ATTN_DIM) return null;
+    return raw as ACANWeights;
+  } catch (e) {
+    swallow("acan:loadWeights", e);
+    return null;
+  }
+}
+
+function saveWeights(weights: ACANWeights, path: string): void {
+  const dir = join(path, "..");
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  writeFileSync(path, JSON.stringify(weights), "utf-8");
+}
+
+export function initACAN(weightsDir?: string): boolean {
+  const dir = weightsDir ?? getKongDir();
+  _weights = loadWeights(join(dir, WEIGHTS_FILENAME));
+  _active = _weights !== null;
+  return _active;
+}
+
+export function isACANActive(): boolean {
+  return _active;
+}
+
+// ── Linear algebra ──
+
+function dot(a: number[], b: number[]): number {
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) sum += a[i] * b[i];
+  return sum;
+}
+
+function projectVec(vec: number[], matrix: number[][]): number[] {
+  const out = new Array(matrix[0].length).fill(0);
+  for (let i = 0; i < vec.length; i++) {
+    if (vec[i] === 0) continue;
+    const row = matrix[i];
+    for (let j = 0; j < out.length; j++) out[j] += vec[i] * row[j];
+  }
+  return out;
+}
+
+// ── ACAN inference ──
+
+export function scoreWithACAN(queryEmbedding: number[], candidates: ACANCandidate[]): number[] {
+  if (!_weights || candidates.length === 0) return [];
+
+  const q = projectVec(queryEmbedding, _weights.W_q);
+  const scale = Math.sqrt(ATTN_DIM);
+  const scores: number[] = [];
+
+  for (const cand of candidates) {
+    const k = projectVec(cand.embedding, _weights.W_k);
+    const attnLogit = dot(q, k) / scale;
+    const features = [
+      attnLogit, cand.recency, cand.importance, cand.access,
+      cand.neighborBonus, cand.provenUtility, cand.reflectionBoost ?? 0,
+    ];
+    scores.push(dot(features, _weights.W_final) + _weights.bias);
+  }
+
+  return scores;
+}
+
+// ── Training data fetching ──
+
+async function getTrainingDataCount(store: SurrealStore): Promise<number> {
+  if (!store.isAvailable()) return 0;
+  try {
+    const flat = await store.queryFirst<{ count: number }>(
+      `SELECT count() AS count FROM retrieval_outcome WHERE query_embedding != NONE GROUP ALL`,
+    );
+    return flat[0]?.count ?? 0;
+  } catch (e) {
+    swallow("acan:count", e);
+    return 0;
+  }
+}
+
+async function fetchTrainingData(store: SurrealStore): Promise<TrainingSample[]> {
+  if (!store.isAvailable()) return [];
+
+  const outcomes = await store.queryFirst<any>(
+    `SELECT query_embedding, memory_id, memory_table,
+            IF llm_relevance != NONE THEN llm_relevance ELSE utilization END AS utilization,
+            retrieval_score, was_neighbor,
+            importance, access_count, recency, created_at
+     FROM retrieval_outcome
+     WHERE query_embedding != NONE
+     ORDER BY created_at ASC`,
+  );
+  if (outcomes.length === 0) return [];
+
+  const uniqueMemIds = [...new Set(outcomes.map((r: any) => String(r.memory_id)))];
+  const embeddingMap = new Map<string, number[]>();
+  for (const mid of uniqueMemIds) {
+    try {
+      const flat = await store.queryFirst<{ id: string; embedding: number[] }>(
+        `SELECT id, embedding FROM type::record($mid) WHERE embedding != NONE`,
+        { mid },
+      );
+      if (flat[0]?.embedding) embeddingMap.set(mid, flat[0].embedding);
+    } catch (e) { swallow("acan:fetchEmb", e); }
+  }
+
+  const samples: TrainingSample[] = [];
+  for (const row of outcomes) {
+    const memEmb = embeddingMap.get(String(row.memory_id));
+    if (!memEmb || !row.query_embedding) continue;
+    samples.push({
+      query_embedding: row.query_embedding,
+      memory_embedding: memEmb,
+      retrieval_score: row.retrieval_score ?? 0,
+      was_neighbor: row.was_neighbor ?? false,
+      utilization: row.utilization ?? 0,
+      importance: row.importance ?? 0.5,
+      access_count: row.access_count ?? 0,
+      recency: row.recency ?? 0.5,
+    });
+  }
+  return samples;
+}
+
+// ── Background training ──
+
+function trainInBackground(
+  samples: TrainingSample[],
+  weightsPath: string,
+  warmStart?: ACANWeights,
+  config?: Partial<TrainingConfig>,
+): void {
+  const cfg = { ...DEFAULT_TRAINING_CONFIG, ...config };
+
+  const workerCode = `
+    import { parentPort, workerData } from "node:worker_threads";
+    const { samples, cfg, warmStart, EMBED_DIM, ATTN_DIM, FEATURE_COUNT } = workerData;
+    function dot(a, b) { let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; return s; }
+    function projectVec(vec, matrix) {
+      const out = new Array(matrix[0].length).fill(0);
+      for (let i = 0; i < vec.length; i++) { if (vec[i] === 0) continue; const row = matrix[i]; for (let j = 0; j < out.length; j++) out[j] += vec[i] * row[j]; }
+      return out;
+    }
+    function shuffle(arr) { for (let i = arr.length - 1; i > 0; i--) { const j = Math.floor(Math.random() * (i + 1)); [arr[i], arr[j]] = [arr[j], arr[i]]; } return arr; }
+    const n = samples.length;
+    const auxFeatures = samples.map(s => [s.recency, s.importance, s.access_count, s.was_neighbor ? 1.0 : 0.0, 0.0, 0.0]);
+    const indices = shuffle(Array.from({ length: n }, (_, i) => i));
+    const valSize = Math.max(1, Math.floor(n * cfg.valSplit));
+    const valIdx = indices.slice(0, valSize);
+    const trainIdx = indices.slice(valSize);
+    const nTrain = trainIdx.length;
+    let W_q, W_k, W_final, bias;
+    if (warmStart) { W_q = JSON.parse(JSON.stringify(warmStart.W_q)); W_k = JSON.parse(JSON.stringify(warmStart.W_k)); W_final = [...warmStart.W_final]; bias = warmStart.bias; }
+    else { const xQK = Math.sqrt(2/(EMBED_DIM+ATTN_DIM)), xF = Math.sqrt(2/(FEATURE_COUNT+1)); W_q = []; W_k = []; for (let i = 0; i < EMBED_DIM; i++) { W_q.push(Array.from({length:ATTN_DIM}, () => (Math.random()*2-1)*xQK)); W_k.push(Array.from({length:ATTN_DIM}, () => (Math.random()*2-1)*xQK)); } W_final = Array.from({length:FEATURE_COUNT}, () => (Math.random()*2-1)*xF); W_final[0] = 0.3; bias = 0.0; }
+    const scale = Math.sqrt(ATTN_DIM);
+    function evalLoss(idxList) { let total = 0; for (const si of idxList) { const s = samples[si]; const q = projectVec(s.query_embedding, W_q); const k = projectVec(s.memory_embedding, W_k); const attn = dot(q,k)/scale; const features = [attn,...auxFeatures[si]]; const score = dot(features, W_final)+bias; const err = score - s.utilization; total += err*err; } return total/idxList.length; }
+    let lr = cfg.lr, bestValLoss = Infinity, epochsSinceImprovement = 0, epochsSinceLrDecay = 0, lastTrainLoss = Infinity, actualEpochs = 0;
+    for (let epoch = 0; epoch < cfg.epochs; epoch++) {
+      actualEpochs = epoch+1; shuffle(trainIdx); let totalLoss = 0;
+      for (const si of trainIdx) { const s = samples[si]; const q = projectVec(s.query_embedding, W_q); const k = projectVec(s.memory_embedding, W_k); const attn = dot(q,k)/scale; const features = [attn,...auxFeatures[si]]; const score = dot(features, W_final)+bias; const err = score - s.utilization; totalLoss += err*err; const dScore = (2/nTrain)*err; for (let j = 0; j < FEATURE_COUNT; j++) W_final[j] -= lr*dScore*features[j]; bias -= lr*dScore; const dAttn = dScore*W_final[0]; const dQ = new Array(ATTN_DIM), dK = new Array(ATTN_DIM); for (let j = 0; j < ATTN_DIM; j++) { dQ[j] = dAttn*k[j]/scale; dK[j] = dAttn*q[j]/scale; } for (let i = 0; i < EMBED_DIM; i++) { if (s.query_embedding[i]!==0) { const qi=s.query_embedding[i], row=W_q[i]; for (let j=0;j<ATTN_DIM;j++) row[j]-=lr*dQ[j]*qi; } if (s.memory_embedding[i]!==0) { const mi=s.memory_embedding[i], row=W_k[i]; for (let j=0;j<ATTN_DIM;j++) row[j]-=lr*dK[j]*mi; } } }
+      lastTrainLoss = totalLoss/nTrain; const valLoss = evalLoss(valIdx);
+      if (valLoss < bestValLoss) { bestValLoss = valLoss; epochsSinceImprovement = 0; epochsSinceLrDecay = 0; } else { epochsSinceImprovement++; epochsSinceLrDecay++; }
+      if (epochsSinceLrDecay >= cfg.lrDecayPatience && lr > cfg.lrFloor) { lr = Math.max(lr*0.5, cfg.lrFloor); epochsSinceLrDecay = 0; }
+      if (epochsSinceImprovement >= cfg.earlyStopPatience) break;
+    }
+    parentPort.postMessage({ weights: { W_q, W_k, W_final, bias, version: 1, trainedAt: Date.now(), trainedOnSamples: n }, trainLoss: lastTrainLoss, valLoss: bestValLoss, actualEpochs, finalLr: lr, config: cfg });
+  `;
+
+  const worker = new Worker(new URL(`data:text/javascript,${encodeURIComponent(workerCode)}`), {
+    workerData: { samples, cfg, warmStart: warmStart ?? null, EMBED_DIM, ATTN_DIM, FEATURE_COUNT },
+  });
+
+  worker.on("message", (msg: any) => {
+    try {
+      saveWeights(msg.weights, weightsPath);
+      _weights = msg.weights;
+      _active = true;
+    } catch { /* non-fatal */ }
+  });
+
+  worker.on("error", () => { /* training failure is non-fatal */ });
+}
+
+// ── Startup: auto-train and activate ──
+
+const STALENESS_GROWTH_FACTOR = 0.5;
+const STALENESS_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+
+export async function checkACANReadiness(store?: SurrealStore): Promise<void> {
+  if (!store) return;
+  const weightsPath = join(getKongDir(), WEIGHTS_FILENAME);
+  const hasWeights = initACAN();
+  const count = await getTrainingDataCount(store);
+
+  if (hasWeights && _weights) {
+    const trainedOn = _weights.trainedOnSamples ?? 0;
+    const trainedAt = _weights.trainedAt ?? 0;
+    const growthRatio = trainedOn > 0 ? (count - trainedOn) / trainedOn : Infinity;
+    const ageMs = Date.now() - trainedAt;
+    const isStale = growthRatio >= STALENESS_GROWTH_FACTOR || ageMs >= STALENESS_MAX_AGE_MS;
+    if (!isStale) return;
+  } else if (count < TRAINING_THRESHOLD) {
+    return;
+  }
+
+  try {
+    const samples = await fetchTrainingData(store);
+    if (samples.length < TRAINING_THRESHOLD) return;
+    trainInBackground(samples, weightsPath, hasWeights ? _weights ?? undefined : undefined);
+  } catch {
+    // training is best-effort
+  }
+}

package/src/causal.ts

@@ -0,0 +1,237 @@
+/**
+ * Causal Memory Graph
+ *
+ * Activates the dormant caused_by/supports/contradicts edges in the graph.
+ * At session end, analyzes the conversation for cause-effect sequences
+ * (bug->investigation->fix->outcome) and creates causal chains linking memories.
+ * During retrieval, traverses causal edges to pull full chains as context.
+ *
+ * Ported from kongbrain — takes SurrealStore/EmbeddingService as params.
+ */
+
+import type { EmbeddingService } from "./embeddings.js";
+import type { SurrealStore, VectorSearchResult } from "./surreal.js";
+import { swallow } from "./errors.js";
+
+// --- Types ---
+
+export interface CausalChain {
+  triggerText: string;
+  outcomeText: string;
+  chainType: "debug" | "refactor" | "feature" | "fix";
+  success: boolean;
+  confidence: number;
+  description: string;
+}
+
+/**
+ * Create memory nodes for each end of the chain and link them with
+ * caused_by/supports/contradicts edges.
+ */
+export async function linkCausalEdges(
+  chains: CausalChain[],
+  sessionId: string,
+  store: SurrealStore,
+  embeddings: EmbeddingService,
+): Promise<void> {
+  if (chains.length === 0 || !store.isAvailable()) return;
+
+  for (const chain of chains) {
+    try {
+      // Create trigger memory
+      let triggerEmb: number[] | null = null;
+      if (embeddings.isAvailable()) {
+        try { triggerEmb = await embeddings.embed(chain.triggerText); } catch (e) { swallow("causal:ok", e); }
+      }
+      const triggerId = await store.createMemory(
+        chain.triggerText, triggerEmb, 5, `causal_trigger_${chain.chainType}`, sessionId,
+      );
+
+      // Create outcome memory
+      let outcomeEmb: number[] | null = null;
+      if (embeddings.isAvailable()) {
+        try { outcomeEmb = await embeddings.embed(chain.outcomeText); } catch (e) { swallow("causal:ok", e); }
+      }
+      const outcomeId = await store.createMemory(
+        chain.outcomeText, outcomeEmb, 6, `causal_outcome_${chain.chainType}`, sessionId,
+      );
+
+      if (!triggerId || !outcomeId) continue;
+
+      // Create causal edges
+      await store.relate(outcomeId, "caused_by", triggerId).catch(e => swallow.warn("causal:relateCausedBy", e));
+      if (chain.success) {
+        await store.relate(outcomeId, "supports", triggerId).catch(e => swallow.warn("causal:relateSupports", e));
+      } else {
+        await store.relate(outcomeId, "contradicts", triggerId).catch(e => swallow.warn("causal:relateContradicts", e));
+      }
+
+      // Embed the description as a searchable memory node
+      let descriptionId: string | null = null;
+      if (chain.description && chain.description.length > 10) {
+        const descText = `[${chain.chainType}${chain.success ? "" : " FAILED"}] ${chain.description}`;
+        let descEmb: number[] | null = null;
+        if (embeddings.isAvailable()) {
+          try { descEmb = await embeddings.embed(descText); } catch (e) { swallow("causal:ok", e); }
+        }
+        descriptionId = await store.createMemory(
+          descText, descEmb, 5, `causal_description_${chain.chainType}`, sessionId,
+        );
+        if (descriptionId) {
+          await store.relate(descriptionId, "describes", triggerId).catch(e => swallow.warn("causal:relateDescTrigger", e));
+          await store.relate(descriptionId, "describes", outcomeId).catch(e => swallow.warn("causal:relateDescOutcome", e));
+        }
+      }
+
+      // Store chain metadata
+      await store.queryExec(`CREATE causal_chain CONTENT $data`, {
+        data: {
+          session_id: sessionId,
+          trigger_memory: triggerId,
+          outcome_memory: outcomeId,
+          description_memory: descriptionId,
+          chain_type: chain.chainType,
+          success: chain.success,
+          confidence: chain.confidence,
+          description: chain.description,
+        },
+      }).catch(e => swallow.warn("causal:storeChain", e));
+    } catch (e) {
+      swallow("causal:silent", e);
+    }
+  }
+}
+
+// --- Causal Context Retrieval ---
+
+/**
+ * Given seed memory IDs from vector search, traverse causal edges
+ * (caused_by, supports, contradicts) up to `hops` deep.
+ * Computes cosine similarity server-side so results compete fairly in scoring.
+ */
+export async function queryCausalContext(
+  seedIds: string[],
+  queryVec: number[],
+  hops = 2,
+  minConfidence = 0.4,
+  store?: SurrealStore,
+): Promise<VectorSearchResult[]> {
+  if (seedIds.length === 0 || !store?.isAvailable()) return [];
+
+  const RECORD_ID_RE = /^[a-zA-Z_][a-zA-Z0-9_]*:[a-zA-Z0-9_]+$/;
+  const validIds = seedIds.filter((id) => RECORD_ID_RE.test(id)).slice(0, 10);
+  if (validIds.length === 0) return [];
+
+  const causalEdges = ["caused_by", "supports", "contradicts", "describes"];
+  const seen = new Set<string>(validIds);
+  let frontier = validIds;
+  const results: VectorSearchResult[] = [];
+  const bindings = { vec: queryVec };
+
+  const scoreExpr = `, IF embedding != NONE AND array::len(embedding) > 0
+         THEN vector::similarity::cosine(embedding, $vec)
+         ELSE 0 END AS score`;
+
+  for (let hop = 0; hop < hops && frontier.length > 0; hop++) {
+    const queries = frontier.flatMap((id) =>
+      causalEdges.map((edge) =>
+        store.queryFirst<any>(
+          `SELECT id, text, importance, access_count AS accessCount,
+                  created_at AS timestamp, category, meta::tb(id) AS table${scoreExpr}
+           FROM ${id}->${edge}->? LIMIT 3`,
+          bindings,
+        ).catch(e => { swallow.warn("causal:edge-query", e); return [] as any[]; }),
+      ),
+    );
+
+    const reverseQueries = frontier.flatMap((id) =>
+      causalEdges.map((edge) =>
+        store.queryFirst<any>(
+          `SELECT id, text, importance, access_count AS accessCount,
+                  created_at AS timestamp, category, meta::tb(id) AS table${scoreExpr}
+           FROM ${id}<-${edge}<-? LIMIT 3`,
+          bindings,
+        ).catch(e => { swallow.warn("causal:edge-query", e); return [] as any[]; }),
+      ),
+    );
+
+    const allQueryResults = await Promise.all([...queries, ...reverseQueries]);
+    const nextFrontier: string[] = [];
+
+    for (const rows of allQueryResults) {
+      for (const row of rows) {
+        const nodeId = String(row.id);
+        if (seen.has(nodeId)) continue;
+        seen.add(nodeId);
+
+        const text = row.text ?? "";
+        if (text) {
+          results.push({
+            id: nodeId,
+            text,
+            score: row.score ?? 0,
+            importance: row.importance,
+            accessCount: row.accessCount,
+            timestamp: row.timestamp,
+            table: String(row.table ?? "memory"),
+            source: row.category,
+          });
+          if (RECORD_ID_RE.test(nodeId)) {
+            nextFrontier.push(nodeId);
+          }
+        }
+      }
+    }
+
+    frontier = nextFrontier.slice(0, 5);
+  }
+
+  // Filter by causal_chain confidence
+  if (results.length > 0 && minConfidence > 0) {
+    const resultIds = results.map(r => r.id);
+    try {
+      const chains = await store.queryFirst<{ trigger_memory: string; outcome_memory: string; confidence: number }>(
+        `SELECT trigger_memory, outcome_memory, confidence FROM causal_chain
+         WHERE confidence >= $minConf AND (trigger_memory IN $ids OR outcome_memory IN $ids)`,
+        { minConf: minConfidence, ids: resultIds },
+      );
+      const allowedIds = new Set<string>();
+      for (const c of chains) {
+        allowedIds.add(String(c.trigger_memory));
+        allowedIds.add(String(c.outcome_memory));
+      }
+      return results.filter(r => allowedIds.has(r.id));
+    } catch (e) {
+      swallow.warn("causal:confidence-filter", e);
+      return results;
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Get causal chain metadata for a session (for metrics/display).
+ */
+export async function getSessionCausalChains(
+  sessionId: string,
+  store: SurrealStore,
+): Promise<{ count: number; successRate: number }> {
+  try {
+    if (!store.isAvailable()) return { count: 0, successRate: 0 };
+    const rows = await store.queryFirst<{ total: number; successes: number }>(
+      `SELECT count() AS total, math::sum(IF success THEN 1 ELSE 0 END) AS successes
+       FROM causal_chain WHERE session_id = $sid GROUP ALL`,
+      { sid: sessionId },
+    );
+    const row = rows[0];
+    if (!row || !row.total) return { count: 0, successRate: 0 };
+    return {
+      count: Number(row.total),
+      successRate: Number(row.successes) / Number(row.total),
+    };
+  } catch (e) {
+    swallow("causal:metrics", e);
+    return { count: 0, successRate: 0 };
+  }
+}