kongbrain 0.1.0

package/src/reflection.ts

@@ -0,0 +1,280 @@
+/**
+ * Metacognitive Reflection
+ *
+ * At session end, reviews own performance: tool failures, runaway detections,
+ * low retrieval utilization, wasted tokens. If problems exceeded thresholds,
+ * generates a structured reflection via Opus, stored as high-importance memory.
+ * Retrieved when similar situations arise in future sessions.
+ *
+ * Ported from kongbrain — takes SurrealStore/EmbeddingService as params.
+ */
+
+import type { CompleteFn } from "./state.js";
+import type { EmbeddingService } from "./embeddings.js";
+import type { SurrealStore } from "./surreal.js";
+import { swallow } from "./errors.js";
+
+// --- Types ---
+
+export interface ReflectionMetrics {
+  avgUtilization: number;
+  toolFailureRate: number;
+  steeringCandidates: number;
+  wastedTokens: number;
+  totalToolCalls: number;
+  totalTurns: number;
+}
+
+export interface Reflection {
+  id: string;
+  text: string;
+  category: string;
+  severity: string;
+  importance: number;
+  score?: number;
+}
+
+// --- Thresholds ---
+
+const UTIL_THRESHOLD = 0.2;
+const TOOL_FAILURE_THRESHOLD = 0.2;
+const STEERING_THRESHOLD = 1;
+
+let _reflectionContextWindow = 200000;
+
+export function setReflectionContextWindow(cw: number): void {
+  _reflectionContextWindow = cw;
+}
+
+function getWasteThreshold(): number {
+  return Math.round(_reflectionContextWindow * 0.005);
+}
+
+// --- Reflection Generation ---
+
+/**
+ * Gather session metrics and determine if reflection is warranted.
+ */
+export async function gatherSessionMetrics(
+  sessionId: string,
+  store: SurrealStore,
+): Promise<ReflectionMetrics | null> {
+  if (!store.isAvailable()) return null;
+
+  try {
+    const metricsRows = await store.queryFirst<any>(
+      `SELECT
+         count() AS totalTurns,
+         math::sum(actual_tool_calls) AS totalTools,
+         math::sum(steering_candidates) AS totalSteering
+       FROM orchestrator_metrics WHERE session_id = $sid GROUP ALL`,
+      { sid: sessionId },
+    );
+    const metrics = metricsRows[0];
+
+    const qualityRows = await store.queryFirst<any>(
+      `SELECT
+         count() AS totalRetrievals,
+         math::mean(utilization) AS avgUtil,
+         math::sum(context_tokens) AS totalContextTokens,
+         math::sum(IF tool_success = false THEN 1 ELSE 0 END) AS toolFailures,
+         math::sum(IF utilization < 0.1 THEN context_tokens ELSE 0 END) AS wastedTokens
+       FROM retrieval_outcome WHERE session_id = $sid GROUP ALL`,
+      { sid: sessionId },
+    );
+    const quality = qualityRows[0];
+
+    const totalTurns = Number(metrics?.totalTurns ?? 0);
+    const totalTools = Number(metrics?.totalTools ?? 0);
+    const totalSteering = Number(metrics?.totalSteering ?? 0);
+    const totalRetrievals = Number(quality?.totalRetrievals ?? 0);
+    const avgUtilization = Number(quality?.avgUtil ?? 1);
+    const toolFailures = Number(quality?.toolFailures ?? 0);
+    const wastedTokens = Number(quality?.wastedTokens ?? 0);
+
+    const toolFailureRate = totalRetrievals > 0 ? toolFailures / totalRetrievals : 0;
+
+    return {
+      avgUtilization,
+      toolFailureRate,
+      steeringCandidates: totalSteering,
+      wastedTokens,
+      totalToolCalls: totalTools,
+      totalTurns,
+    };
+  } catch (e) {
+    swallow.warn("reflection:gatherMetrics", e);
+    return null;
+  }
+}
+
+/**
+ * Determine if session performance warrants a reflection.
+ */
+export function shouldReflect(metrics: ReflectionMetrics): { reflect: boolean; reasons: string[] } {
+  const reasons: string[] = [];
+
+  if (metrics.avgUtilization < UTIL_THRESHOLD && metrics.totalTurns > 1) {
+    reasons.push(`Low retrieval utilization: ${(metrics.avgUtilization * 100).toFixed(0)}% (threshold: ${UTIL_THRESHOLD * 100}%)`);
+  }
+  if (metrics.toolFailureRate > TOOL_FAILURE_THRESHOLD) {
+    reasons.push(`High tool failure rate: ${(metrics.toolFailureRate * 100).toFixed(0)}% (threshold: ${TOOL_FAILURE_THRESHOLD * 100}%)`);
+  }
+  if (metrics.steeringCandidates >= STEERING_THRESHOLD) {
+    reasons.push(`${metrics.steeringCandidates} steering candidate(s) detected`);
+  }
+  if (metrics.wastedTokens > getWasteThreshold()) {
+    reasons.push(`~${metrics.wastedTokens} wasted context tokens`);
+  }
+
+  return { reflect: reasons.length > 0, reasons };
+}
+
+/**
+ * Generate a structured reflection from session performance data.
+ * Only called when shouldReflect() returns true.
+ */
+export async function generateReflection(
+  sessionId: string,
+  store: SurrealStore,
+  embeddings: EmbeddingService,
+  complete: CompleteFn,
+): Promise<void> {
+  const metrics = await gatherSessionMetrics(sessionId, store);
+  if (!metrics) return;
+
+  const { reflect, reasons } = shouldReflect(metrics);
+  if (!reflect) return;
+
+  const severity = reasons.length >= 3 ? "critical" : reasons.length >= 2 ? "moderate" : "minor";
+
+  let category = "efficiency";
+  if (metrics.toolFailureRate > TOOL_FAILURE_THRESHOLD) category = "failure_pattern";
+  if (metrics.steeringCandidates >= STEERING_THRESHOLD) category = "approach_strategy";
+
+  try {
+    const response = await complete({
+      system: `Write 2-4 sentences: root cause, error pattern, what to do differently. Be specific. Example: "Spent 8 tool calls reading source before checking error log. For timeout bugs, check logs first."`,
+      messages: [{
+        role: "user",
+        content: `${metrics.totalTurns} turns, ${metrics.totalToolCalls} tools, ${(metrics.avgUtilization * 100).toFixed(0)}% util, ${(metrics.toolFailureRate * 100).toFixed(0)}% fail, ~${metrics.wastedTokens} wasted tokens\nIssues: ${reasons.join("; ")}`,
+      }],
+    });
+
+    const reflectionText = response.text.trim();
+
+    if (reflectionText.length < 20) return;
+
+    let reflEmb: number[] | null = null;
+    if (embeddings.isAvailable()) {
+      try { reflEmb = await embeddings.embed(reflectionText); } catch (e) { swallow("reflection:ok", e); }
+    }
+
+    // Dedup: skip if a very similar reflection already exists
+    if (reflEmb?.length) {
+      const existing = await store.queryFirst<{ id: string; importance: number; score: number }>(
+        `SELECT id, importance,
+                vector::similarity::cosine(embedding, $vec) AS score
+         FROM reflection
+         WHERE embedding != NONE AND array::len(embedding) > 0
+         ORDER BY score DESC LIMIT 1`,
+        { vec: reflEmb },
+      );
+      const top = existing[0];
+      if (top && (top.score ?? 0) > 0.85) {
+        const newImportance = Math.min(10, (top.importance ?? 7) + 0.5);
+        await store.queryFirst<any>(
+          `UPDATE $id SET importance = $imp, updated_at = time::now()`,
+          { id: top.id, imp: newImportance },
+        );
+        return;
+      }
+    }
+
+    const record: Record<string, unknown> = {
+      session_id: sessionId,
+      text: reflectionText,
+      category,
+      severity,
+      importance: 7.0,
+    };
+    if (reflEmb?.length) record.embedding = reflEmb;
+
+    const rows = await store.queryFirst<{ id: string }>(
+      `CREATE reflection CONTENT $record RETURN id`,
+      { record },
+    );
+    const reflectionId = String(rows[0]?.id ?? "");
+
+    if (reflectionId) {
+      await store.relate(reflectionId, "reflects_on", sessionId).catch(e => swallow.warn("reflection:relate", e));
+    }
+  } catch (e) {
+    swallow("reflection:silent", e);
+  }
+}
+
+// --- Reflection Retrieval ---
+
+/**
+ * Vector search on the reflection table.
+ */
+export async function retrieveReflections(
+  queryVec: number[],
+  limit = 3,
+  store?: SurrealStore,
+): Promise<Reflection[]> {
+  if (!store?.isAvailable()) return [];
+
+  try {
+    const rows = await store.queryFirst<any>(
+      `SELECT id, text, category, severity, importance,
+              vector::similarity::cosine(embedding, $vec) AS score
+       FROM reflection
+       WHERE embedding != NONE AND array::len(embedding) > 0
+       ORDER BY score DESC LIMIT $lim`,
+      { vec: queryVec, lim: limit },
+    );
+
+    return rows
+      .filter((r: any) => (r.score ?? 0) > 0.35)
+      .map((r: any) => ({
+        id: String(r.id),
+        text: r.text ?? "",
+        category: r.category ?? "efficiency",
+        severity: r.severity ?? "minor",
+        importance: Number(r.importance ?? 7.0),
+        score: r.score,
+      }));
+  } catch (e) {
+    swallow.warn("reflection:retrieve", e);
+    return [];
+  }
+}
+
+/**
+ * Format reflections as a context block for the LLM.
+ */
+export function formatReflectionContext(reflections: Reflection[]): string {
+  if (reflections.length === 0) return "";
+
+  const lines = reflections.map((r) => {
+    return `[reflection/${r.category}] ${r.text}`;
+  });
+
+  return `\n<reflection_context>\n[Lessons from past sessions — avoid repeating these mistakes]\n${lines.join("\n\n")}\n</reflection_context>`;
+}
+
+/**
+ * Get reflection count (for /stats display).
+ */
+export async function getReflectionCount(store: SurrealStore): Promise<number> {
+  try {
+    if (!store.isAvailable()) return 0;
+    const rows = await store.queryFirst<{ count: number }>(`SELECT count() AS count FROM reflection GROUP ALL`);
+    return Number(rows[0]?.count ?? 0);
+  } catch (e) {
+    swallow.warn("reflection:count", e);
+    return 0;
+  }
+}

package/src/retrieval-quality.ts

@@ -0,0 +1,266 @@
+/**
+ * Retrieval Quality Tracker
+ *
+ * Measures whether retrieved context was actually useful, not just relevant.
+ * Tracks 6 signals from research:
+ * 1. Referenced in response (text overlap)
+ * 2. Task success (tool executions)
+ * 3. Retrieval stability
+ * 4. Access patterns
+ * 5. Context waste
+ * 6. Contradiction detection
+ *
+ * Ported from kongbrain — uses SurrealStore instead of module-level DB.
+ */
+
+import type { SurrealStore, VectorSearchResult } from "./surreal.js";
+import { swallow } from "./errors.js";
+
+export type RetrievedItem = VectorSearchResult & {
+  finalScore?: number;
+  fromNeighbor?: boolean;
+};
+
+interface QualitySignals {
+  utilization: number;
+  toolSuccess: boolean | null;
+  contextTokens: number;
+  wasNeighbor: boolean;
+  recency: number;
+}
+
+// Per-turn state — module-level since only one turn is active at a time
+let _pendingRetrieval: {
+  sessionId: string;
+  items: RetrievedItem[];
+  toolResults: { success: boolean }[];
+  queryEmbedding?: number[];
+} | null = null;
+
+export function getStagedItems(): RetrievedItem[] {
+  return _pendingRetrieval?.items ? [..._pendingRetrieval.items] : [];
+}
+
+export function stageRetrieval(
+  sessionId: string,
+  items: RetrievedItem[],
+  queryEmbedding?: number[],
+): void {
+  _pendingRetrieval = {
+    sessionId,
+    items,
+    toolResults: [],
+    queryEmbedding,
+  };
+}
+
+export function recordToolOutcome(success: boolean): void {
+  if (_pendingRetrieval) {
+    _pendingRetrieval.toolResults.push({ success });
+  }
+}
+
+/**
+ * Evaluate retrieval quality after assistant response.
+ */
+export async function evaluateRetrieval(
+  responseTurnId: string,
+  responseText: string,
+  store: SurrealStore,
+): Promise<void> {
+  if (!_pendingRetrieval || _pendingRetrieval.items.length === 0) {
+    _pendingRetrieval = null;
+    return;
+  }
+
+  const { sessionId, items, toolResults, queryEmbedding } = _pendingRetrieval;
+  _pendingRetrieval = null;
+
+  const toolSuccess = toolResults.length > 0
+    ? toolResults.every((r) => r.success)
+    : null;
+
+  const responseLower = responseText.toLowerCase();
+
+  for (const item of items) {
+    const signals = computeSignals(item, responseLower, toolSuccess);
+
+    try {
+      const record: Record<string, unknown> = {
+        session_id: sessionId,
+        turn_id: responseTurnId,
+        memory_id: String(item.id),
+        memory_table: item.table,
+        retrieval_score: item.finalScore ?? 0,
+        utilization: signals.utilization,
+        context_tokens: signals.contextTokens,
+        was_neighbor: signals.wasNeighbor,
+        importance: ((item.importance ?? 5) / 10),
+        access_count: Math.min((item.accessCount ?? 0) / 50, 1),
+        recency: signals.recency,
+      };
+      if (signals.toolSuccess != null) {
+        record.tool_success = signals.toolSuccess;
+      }
+      if (queryEmbedding) {
+        record.query_embedding = queryEmbedding;
+      }
+      await store.queryExec(`CREATE retrieval_outcome CONTENT $data`, { data: record });
+      store.updateUtilityCache(String(item.id), signals.utilization)
+        .catch(e => swallow.warn("retrieval-quality:utilityCache", e));
+    } catch {
+      // non-critical telemetry
+    }
+  }
+}
+
+// --- Signal computation ---
+
+function computeSignals(
+  item: RetrievedItem,
+  responseLower: string,
+  toolSuccess: boolean | null,
+): QualitySignals {
+  const rawText = item.text ?? "";
+  const memText = rawText.toLowerCase();
+  const contextTokens = Math.ceil(rawText.length / 4);
+
+  const keyTermScore = keyTermOverlap(rawText, responseLower);
+  const trigramScore = trigramOverlap(memText, responseLower);
+  const unigramScore = unigramOverlap(memText, responseLower);
+  const utilization = Math.max(keyTermScore, trigramScore, unigramScore * 0.5);
+
+  let recency = 0.5;
+  if (item.timestamp) {
+    const ageHours = (Date.now() - new Date(item.timestamp).getTime()) / 3_600_000;
+    recency = Math.exp(-ageHours / 168);
+  }
+
+  return { utilization, toolSuccess, contextTokens, wasNeighbor: item.fromNeighbor ?? false, recency };
+}
+
+function stripPunctuation(text: string): string {
+  return text.replace(/[.,;:!?()"'\[\]{}<>—–…]/g, " ");
+}
+
+const KEY_TERM_PATTERNS = [
+  /`([^`]{2,60})`/g,
+  /\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b/g,
+  /\b([A-Z]{2,}(?:[-_][A-Z0-9]+)*)\b/g,
+  /\b([A-Z][a-z]*[A-Z]\w*)\b/g,
+  /\b([A-Z][a-z]{2,})\b/g,
+  /\b(\w+(?:[-_]\w+){1,3})\b/g,
+];
+
+const STOP_WORDS = new Set([
+  "the", "a", "an", "but", "and", "or", "if", "when", "this", "that",
+  "for", "with", "from", "into", "not", "are", "was", "were", "has",
+  "have", "been", "its", "can", "will", "may", "also", "just", "then",
+  "than", "too", "very", "such", "each", "all", "any", "most", "more",
+  "some", "other", "about", "over", "only", "new", "used", "how", "where",
+  "what", "which", "who", "whom", "does", "did", "had", "could", "would",
+  "should", "shall", "let", "get", "got", "set", "put", "run", "see",
+  "try", "use", "one", "two", "now", "way", "own", "same", "here",
+  "there", "still", "yet", "both", "few", "many", "much", "well",
+]);
+
+function extractKeyTerms(text: string): Set<string> {
+  const terms = new Set<string>();
+  for (const pattern of KEY_TERM_PATTERNS) {
+    for (const match of text.matchAll(pattern)) {
+      const term = match[1].trim().toLowerCase();
+      if (term.length >= 3 && !STOP_WORDS.has(term)) terms.add(term);
+    }
+  }
+  return terms;
+}
+
+function keyTermOverlap(source: string, targetLower: string): number {
+  const terms = extractKeyTerms(source);
+  if (terms.size === 0) return 0;
+  const cleanTarget = stripPunctuation(targetLower);
+  let found = 0;
+  for (const term of terms) { if (cleanTarget.includes(term)) found++; }
+  return found / terms.size;
+}
+
+function trigramOverlap(source: string, target: string): number {
+  const srcGrams = extractNgrams(stripPunctuation(source));
+  if (srcGrams.size === 0) return 0;
+  const tgtGrams = extractNgrams(stripPunctuation(target));
+  let matches = 0;
+  for (const gram of srcGrams) { if (tgtGrams.has(gram)) matches++; }
+  return matches / srcGrams.size;
+}
+
+function extractNgrams(text: string): Set<string> {
+  const words = text.split(/\s+/).filter((w) => w.length > 2);
+  const grams = new Set<string>();
+  if (words.length >= 3) {
+    for (let i = 0; i <= words.length - 3; i++) {
+      grams.add(`${words[i]} ${words[i + 1]} ${words[i + 2]}`);
+    }
+  } else if (words.length === 2) {
+    grams.add(`${words[0]} ${words[1]}`);
+  } else if (words.length === 1) {
+    grams.add(words[0]);
+  }
+  return grams;
+}
+
+function unigramOverlap(source: string, target: string): number {
+  const srcWords = new Set(
+    stripPunctuation(source).split(/\s+/)
+      .filter((w) => w.length >= 5 && !STOP_WORDS.has(w)),
+  );
+  if (srcWords.size === 0) return 0;
+  const cleanTarget = " " + stripPunctuation(target) + " ";
+  let found = 0;
+  for (const word of srcWords) {
+    if (cleanTarget.includes(` ${word} `) || cleanTarget.includes(` ${word}s `)) found++;
+  }
+  return found / srcWords.size;
+}
+
+// --- Historical utility queries ---
+
+export async function getHistoricalUtilityBatch(
+  ids: string[],
+  store?: SurrealStore,
+): Promise<Map<string, number>> {
+  const result = new Map<string, number>();
+  if (ids.length === 0 || !store) return result;
+  try {
+    const flat = await store.queryFirst<{ memory_id: string; avg: number }>(
+      `SELECT memory_id,
+        math::mean(IF llm_relevance != NONE THEN llm_relevance ELSE utilization END) AS avg
+       FROM retrieval_outcome
+       WHERE memory_id IN $ids AND (utilization > 0 OR llm_relevance != NONE)
+       GROUP BY memory_id`,
+      { ids },
+    );
+    for (const row of flat) {
+      if (row.avg != null) result.set(String(row.memory_id), row.avg);
+    }
+  } catch (e) {
+    swallow("retrieval-quality:batch", e);
+  }
+  return result;
+}
+
+export async function getRecentUtilizationAvg(
+  sessionId: string,
+  windowSize = 10,
+  store?: SurrealStore,
+): Promise<number | null> {
+  if (!store) return null;
+  try {
+    const rows = await store.queryFirst<{ avg: number }>(
+      `SELECT math::mean(utilization) AS avg FROM (SELECT utilization, created_at FROM retrieval_outcome WHERE session_id = $sid ORDER BY created_at DESC LIMIT $lim)`,
+      { sid: sessionId, lim: windowSize },
+    );
+    return rows[0]?.avg ?? null;
+  } catch {
+    return null;
+  }
+}