@jonathangu/openclawbrain 0.3.0

package/src/brain-core/pack.ts

@@ -0,0 +1,117 @@
+/**
+ * Immutable pack management.
+ *
+ * Gateway reads only promoted pack. Daemon writes only mutable state.
+ * Promotion requires passing replay gate and health bounds.
+ */
+
+import type { Pack, HealthMetrics, Episode } from "./types.js";
+import type { BrainGraph } from "./graph.js";
+import { computeHealth } from "./health.js";
+import { replayEpisode } from "./episode.js";
+
+export interface BrainPackPersistence {
+  insertPack(params: { nodeCount: number; edgeCount: number; healthJson: string }): Pack;
+  promotePack(version: number): void;
+  rollbackPack(version: number): void;
+}
+
+export class PackManager {
+  constructor(
+    private persistence: BrainPackPersistence,
+    private graph: BrainGraph,
+    private log: { info: (msg: string) => void; warn: (msg: string) => void },
+  ) {}
+
+  /**
+   * Build a candidate pack from current graph state.
+   */
+  buildCandidate(health: HealthMetrics): Pack {
+    return this.persistence.insertPack({
+      nodeCount: health.nodeCount,
+      edgeCount: health.edgeCount,
+      healthJson: JSON.stringify(health),
+    });
+  }
+
+  /**
+   * Replay gate: test candidate against recent episodes.
+   * Returns whether the candidate should be promoted.
+   */
+  replayGate(
+    recentEpisodes: Episode[],
+    config: { minFiredPerQuery: number; maxDormantPercent: number; maxOrphanCount: number },
+    candidateGraph: BrainGraph = this.graph,
+  ): { passed: boolean; reason: string; health: HealthMetrics } {
+    if (recentEpisodes.length === 0) {
+      return {
+        passed: true,
+        reason: "no episodes to replay",
+        health: computeHealth(candidateGraph, recentEpisodes, 0),
+      };
+    }
+
+    const health = computeHealth(candidateGraph, recentEpisodes, 0);
+
+    if (health.firedPerQuery < config.minFiredPerQuery) {
+      return {
+        passed: false,
+        reason: `firedPerQuery ${health.firedPerQuery.toFixed(2)} < ${config.minFiredPerQuery}`,
+        health,
+      };
+    }
+    if (health.dormantPercent > config.maxDormantPercent) {
+      return {
+        passed: false,
+        reason: `dormantPercent ${(health.dormantPercent * 100).toFixed(1)}% > ${(config.maxDormantPercent * 100).toFixed(1)}%`,
+        health,
+      };
+    }
+    if (health.orphanCount > config.maxOrphanCount) {
+      return {
+        passed: false,
+        reason: `orphanCount ${health.orphanCount} > ${config.maxOrphanCount}`,
+        health,
+      };
+    }
+
+    // Check no human-labeled episodes regressed
+    const humanEpisodes = recentEpisodes.filter((ep) => ep.rewardSource === "human" && ep.reward !== null);
+    for (const ep of humanEpisodes) {
+      const replay = replayEpisode(ep, candidateGraph);
+      if (replay.wouldChange && ep.reward! > 0) {
+        return {
+          passed: false,
+          reason: `human-positive episode ${ep.id} would change routing`,
+          health,
+        };
+      }
+    }
+
+    const selfNegativeEpisodes = recentEpisodes.filter(
+      (ep) => ep.rewardSource === "self" && ep.reward !== null && ep.reward < 0,
+    );
+    for (const ep of selfNegativeEpisodes) {
+      const replay = replayEpisode(ep, candidateGraph);
+      if (!replay.wouldChange) {
+        return {
+          passed: false,
+          reason: `self-negative episode ${ep.id} did not change routing`,
+          health,
+        };
+      }
+    }
+
+    return { passed: true, reason: "all gates passed", health };
+  }
+
+  promote(version: number): void {
+    this.persistence.promotePack(version);
+    this.log.info(`[brain] Pack v${version} promoted`);
+  }
+
+  rollback(version: number): void {
+    this.persistence.rollbackPack(version);
+    this.log.warn(`[brain] Pack v${version} rolled back`);
+  }
+}

package/src/brain-core/policy.ts

@@ -0,0 +1,153 @@
+/**
+ * Softmax routing policy over action sets.
+ *
+ * Implements P_ρ(a|s) from the paper:
+ * P_ρ(a_j | s_t) = exp(score(a_j) / τ) / Σ_k exp(score(a_k) / τ)
+ *
+ * Policy is ALWAYS stochastic (samples from softmax, never argmax).
+ * Temperature τ controls exploration vs exploitation:
+ *   - Learning: τ = 1.0 (explore)
+ *   - Serving:  τ = 0.1 (exploit, nearly deterministic)
+ */
+
+import type {
+  TraversalAction,
+  TraversalState,
+  PolicyParams,
+} from "./types.js";
+import { DEFAULT_POLICY_PARAMS } from "./types.js";
+import { BrainGraph, cosineSimilarity } from "./graph.js";
+
+/**
+ * Score a single action given current state and graph.
+ *
+ * For STOP: score increases with budget depletion and hop count.
+ * For traverse: score = edge.weight * edge.prior + cos(query, target) + bias
+ */
+export function scoreAction(
+  action: TraversalAction,
+  state: TraversalState,
+  graph: BrainGraph,
+  params: PolicyParams = DEFAULT_POLICY_PARAMS,
+): number {
+  if (action.type === "stop") {
+    const totalBudget = state.budgetRemaining + state.fired.reduce((sum, id) => {
+      const node = graph.getNode(id);
+      return sum + (node?.tokenCount ?? 0);
+    }, 0);
+    const budgetUsedFraction = totalBudget > 0 ? 1 - state.budgetRemaining / totalBudget : 0;
+    const hopFraction = state.maxHops > 0 ? state.hopCount / state.maxHops : 0;
+    return params.stopBias
+      + params.budgetPressure * budgetUsedFraction
+      + params.hopPressure * hopFraction;
+  }
+
+  // Traverse action
+  const targetNode = graph.getNode(action.targetNodeId);
+  if (!targetNode) return -Infinity;
+
+  if (state.currentNodeId === null) {
+    const seedPrior = action.seedScore ?? 0;
+    const learnedSeedWeight = graph.getSeedWeight(action.targetNodeId);
+    return seedPrior + learnedSeedWeight;
+  }
+
+  // Find edge from current position to target
+  const edge = graph.getEdge(state.currentNodeId, action.targetNodeId);
+
+  // Base score from edge weight and prior
+  const edgeScore = edge ? edge.weight * edge.prior : 0;
+
+  // Query relevance via embedding cosine similarity
+  let relevance = 0;
+  if (targetNode.embedding && state.queryEmbedding.length > 0) {
+    relevance = cosineSimilarity(state.queryEmbedding, targetNode.embedding);
+  }
+
+  // Edge kind bias
+  const kindBias = edge ? (params.edgeKindBias[edge.kind] ?? 0) : 0;
+
+  return edgeScore + relevance + kindBias;
+}
+
+/**
+ * Compute softmax distribution over the full action set.
+ *
+ * Returns sorted candidates with their scores and probabilities.
+ * Numerically stable: subtract max score before exp.
+ */
+export function softmaxPolicy(
+  actions: TraversalAction[],
+  state: TraversalState,
+  graph: BrainGraph,
+  params: PolicyParams = DEFAULT_POLICY_PARAMS,
+): Array<{ action: TraversalAction; score: number; probability: number }> {
+  if (actions.length === 0) return [];
+
+  const scored = actions.map((action) => ({
+    action,
+    score: scoreAction(action, state, graph, params),
+  }));
+
+  // Numerically stable softmax
+  const maxScore = Math.max(...scored.map((s) => s.score));
+  const tau = params.temperature;
+
+  const expScores = scored.map((s) => ({
+    ...s,
+    expScore: Math.exp((s.score - maxScore) / tau),
+  }));
+
+  const sumExp = expScores.reduce((sum, s) => sum + s.expScore, 0);
+
+  return expScores.map((s) => ({
+    action: s.action,
+    score: s.score,
+    probability: sumExp > 0 ? s.expScore / sumExp : 1 / actions.length,
+  }));
+}
+
+/**
+ * Sample an action from the softmax distribution.
+ *
+ * Stochastic — NEVER argmax. Even at low temperature, this samples
+ * from the distribution. This is required for the paper's REINFORCE
+ * update to have valid gradients.
+ */
+export function sampleAction(
+  distribution: Array<{ action: TraversalAction; probability: number }>,
+): { action: TraversalAction; probability: number; index: number } {
+  if (distribution.length === 0) {
+    return { action: { type: "stop" }, probability: 1.0, index: 0 };
+  }
+
+  const r = Math.random();
+  let cumulative = 0;
+
+  for (let i = 0; i < distribution.length; i++) {
+    cumulative += distribution[i].probability;
+    if (r <= cumulative) {
+      return {
+        action: distribution[i].action,
+        probability: distribution[i].probability,
+        index: i,
+      };
+    }
+  }
+
+  // Fallback: numerical precision edge case
+  const last = distribution.length - 1;
+  return {
+    action: distribution[last].action,
+    probability: distribution[last].probability,
+    index: last,
+  };
+}
+
+/**
+ * Compute log probability of a chosen action.
+ * Used in REINFORCE gradient: ∂logP_ρ(a|s)/∂ρ
+ */
+export function logProbability(probability: number): number {
+  return Math.log(Math.max(probability, 1e-10));
+}

package/src/brain-core/replay.ts

@@ -0,0 +1 @@
+export { replayEpisode } from "./episode.js";

package/src/brain-core/teacher.ts

@@ -0,0 +1,105 @@
+/**
+ * Off-path async teacher for episode evaluation.
+ *
+ * CRITICAL RULE: Teacher sees ONLY what the router saw.
+ * It evaluates the routing decision, not the overall task outcome.
+ * No cheating with extra context.
+ */
+
+import type { Episode } from "./types.js";
+import type { BrainGraph } from "./graph.js";
+
+export type BrainTeacherCompletion = (params: {
+  provider?: string;
+  model: string;
+  apiKey?: string;
+  messages: Array<{ role: string; content: unknown }>;
+  system?: string;
+  maxTokens: number;
+  temperature?: number;
+}) => Promise<{ content?: Array<{ text?: string }> }>;
+
+export type BrainTeacherResolveModel = () => { provider: string; model: string };
+export type BrainTeacherGetApiKey = (provider: string, model: string) => Promise<string | undefined>;
+
+const TEACHER_SYSTEM_PROMPT =
+  "You are evaluating a context routing decision. Score the quality of the selected context for the given query. Return ONLY a JSON object: {\"score\": <number from -1.0 to 1.0>, \"reason\": \"<brief explanation>\"}";
+
+export class BrainTeacher {
+  constructor(
+    private complete: BrainTeacherCompletion,
+    private resolveModel: BrainTeacherResolveModel,
+    private getApiKey: BrainTeacherGetApiKey,
+    private graph: BrainGraph,
+    private log: { info: (msg: string) => void; error: (msg: string) => void },
+  ) {}
+
+  async evaluate(episode: Episode): Promise<{ score: number; reason: string }> {
+    // Build prompt showing only what the router saw
+    const candidateDescriptions: string[] = [];
+    for (const step of episode.trajectory) {
+      for (const candidate of step.candidates) {
+        if (candidate.action.type === "traverse") {
+          const node = this.graph.getNode(candidate.action.targetNodeId);
+          if (node) {
+            candidateDescriptions.push(
+              `- [${node.kind}] ${node.content.slice(0, 200)}${node.content.length > 200 ? "..." : ""} (prob: ${candidate.probability.toFixed(3)})`,
+            );
+          }
+        }
+      }
+    }
+
+    const firedDescriptions = episode.firedNodes.map((id) => {
+      const node = this.graph.getNode(id);
+      return node
+        ? `- [${node.kind}] ${node.content.slice(0, 200)}${node.content.length > 200 ? "..." : ""}`
+        : `- ${id} (not found)`;
+    });
+
+    const prompt = `Query: "${episode.queryText}"
+
+Candidate nodes the router could have chosen:
+${candidateDescriptions.join("\n") || "(none)"}
+
+Nodes actually selected (fired):
+${firedDescriptions.join("\n") || "(none)"}
+
+Was this the right context for the query? Consider relevance, completeness, and conciseness.`;
+
+    try {
+      const { provider, model } = this.resolveModel();
+      const apiKey = await this.getApiKey(provider, model);
+
+      const result = await this.complete({
+        provider,
+        model,
+        apiKey,
+        system: TEACHER_SYSTEM_PROMPT,
+        messages: [{ role: "user", content: prompt }],
+        maxTokens: 200,
+        temperature: 0.1,
+      });
+
+      const text = result.content
+        ?.map((b: { text?: string }) => b.text ?? "")
+        .join("") ?? "";
+
+      const jsonMatch = text.match(/\{[\s\S]*"score"[\s\S]*\}/);
+      if (!jsonMatch) {
+        this.log.error(`[brain] Teacher returned non-JSON: ${text.slice(0, 100)}`);
+        return { score: 0, reason: "failed to parse teacher response" };
+      }
+
+      const parsed = JSON.parse(jsonMatch[0]);
+      const score = Math.max(-1, Math.min(1, Number(parsed.score) || 0));
+      const reason = String(parsed.reason || "teacher evaluation");
+
+      this.log.info(`[brain] Teacher scored episode ${episode.id}: ${score.toFixed(2)} (${reason})`);
+      return { score, reason };
+    } catch (err) {
+      this.log.error(`[brain] Teacher evaluation failed: ${(err as Error).message}`);
+      return { score: 0, reason: "teacher evaluation failed" };
+    }
+  }
+}

package/src/brain-core/trace.ts

@@ -0,0 +1,40 @@
+/**
+ * Decision trace recording and footer generation.
+ */
+
+import { randomUUID } from "node:crypto";
+import type { DecisionTrace } from "./types.js";
+import type { TraverseResult } from "./traverse.js";
+
+export function recordTrace(params: {
+  traversalResult: TraverseResult;
+  queryText: string;
+  episodeId: string | null;
+  packVersion: number | null;
+}): DecisionTrace {
+  return {
+    id: `bt_${randomUUID().slice(0, 8)}`,
+    episodeId: params.episodeId,
+    packVersion: params.packVersion,
+    queryText: params.queryText,
+    seedScores: params.traversalResult.seedScores,
+    trajectory: params.traversalResult.trajectory,
+    firedNodes: params.traversalResult.firedNodes.map((n) => n.nodeId),
+    vetoedNodes: params.traversalResult.vetoedNodes.map((v) => v.nodeId),
+    contextChars: params.traversalResult.contextChars,
+    footer: params.traversalResult.footer,
+    createdAt: Date.now(),
+  };
+}
+
+export function generateFooter(params: {
+  packVersion: number;
+  seedCount: number;
+  hopCount: number;
+  firedCount: number;
+  vetoCount: number;
+  contextChars: number;
+  traceId: string;
+}): string {
+  return `Brain v${params.packVersion} · ${params.seedCount} seeds · ${params.hopCount} hops · ${params.firedCount} fired · ${params.vetoCount} veto · ${params.contextChars} chars · trace ${params.traceId}`;
+}

package/src/brain-core/traverse.ts

@@ -0,0 +1,230 @@
+/**
+ * Full traversal loop implementing the paper's finite-time game.
+ *
+ * Algorithm:
+ * 1. Seed phase: select start nodes by embedding similarity
+ * 2. Loop: expand candidates → compute softmax policy → sample → fire/veto
+ * 3. Terminal: STOP chosen, budget exhausted, max hops, or dead end
+ *
+ * Paper assumptions honored:
+ * - Assumption 1: game ends in finite time (maxHops bound)
+ * - Assumption 2: reward only at terminal state (not intermediate)
+ * - Stochastic policy P_ρ(a|s) via softmax sampling
+ */
+
+import type {
+  TraversalState,
+  TraversalAction,
+  TrajectoryStep,
+  PolicyParams,
+  NodeKind,
+  SeedScore,
+} from "./types.js";
+import { DEFAULT_POLICY_PARAMS } from "./types.js";
+import type { BrainGraph } from "./graph.js";
+import { softmaxPolicy, sampleAction } from "./policy.js";
+
+export interface TraverseOptions {
+  graph: BrainGraph;
+  queryEmbedding: Float32Array;
+  queryText: string;
+  maxHops: number;
+  budgetChars: number;
+  temperature: number;
+  maxSeeds: number;
+  semanticThreshold: number;
+  policyParams?: Partial<PolicyParams>;
+}
+
+export interface TraverseResult {
+  firedNodes: Array<{ nodeId: string; kind: NodeKind; content: string; tokenCount: number }>;
+  vetoedNodes: Array<{ nodeId: string; reason: string }>;
+  trajectory: TrajectoryStep[];
+  seedScores: SeedScore[];
+  contextChars: number;
+  footer: string;
+}
+
+/**
+ * Execute a full graph traversal.
+ *
+ * Returns the fired nodes, vetoed nodes, full trajectory (for REINFORCE),
+ * seed scores, and a human-readable footer.
+ */
+export function traverse(options: TraverseOptions): TraverseResult {
+  const {
+    graph,
+    queryEmbedding,
+    maxHops,
+    budgetChars,
+    temperature,
+    maxSeeds,
+    semanticThreshold,
+  } = options;
+
+  const params: PolicyParams = {
+    ...DEFAULT_POLICY_PARAMS,
+    temperature,
+    ...options.policyParams,
+  };
+
+  // Step 1: Seed selection
+  const seedCandidates = graph.seedByEmbedding(queryEmbedding, maxSeeds, semanticThreshold);
+
+  if (seedCandidates.length === 0) {
+    return {
+      firedNodes: [],
+      vetoedNodes: [],
+      trajectory: [],
+      seedScores: [],
+      contextChars: 0,
+      footer: "Brain · 0 seeds · no traversal",
+    };
+  }
+
+  // Initialize traversal state
+  const state: TraversalState = {
+    currentNodeId: null,
+    queryEmbedding,
+    visited: new Set(),
+    fired: [],
+    budgetRemaining: budgetChars,
+    hopCount: 0,
+    maxHops,
+  };
+
+  const trajectory: TrajectoryStep[] = [];
+  const firedNodes: Array<{ nodeId: string; kind: NodeKind; content: string; tokenCount: number }> = [];
+  const vetoedNodes: Array<{ nodeId: string; reason: string }> = [];
+  let seedScores: SeedScore[] = [];
+
+  // Step 2: Traversal loop
+  for (let hop = 0; hop < maxHops; hop++) {
+    // Compute action set
+    const actions = graph.getActionSet(
+      state.currentNodeId,
+      state.visited,
+      state.currentNodeId === null ? seedCandidates : undefined,
+    );
+
+    if (actions.length === 0) break; // Dead end
+    if (actions.length === 1 && actions[0].type === "stop") {
+      // Only STOP available — record step and terminate
+      const step: TrajectoryStep = {
+        stateSnapshot: {
+          currentNodeId: state.currentNodeId,
+          hopCount: state.hopCount,
+          budgetRemaining: state.budgetRemaining,
+          visitedCount: state.visited.size,
+          firedCount: state.fired.length,
+        },
+        candidates: [{ action: { type: "stop" }, score: 0, probability: 1.0 }],
+        chosenAction: { type: "stop" },
+        chosenActionProbability: 1.0,
+        stopProbability: 1.0,
+      };
+      trajectory.push(step);
+      break;
+    }
+
+    // Compute softmax distribution
+    const distribution = softmaxPolicy(actions, state, graph, params);
+
+    // Sample action (stochastic)
+    const sampled = sampleAction(distribution);
+
+    if (state.currentNodeId === null) {
+      seedScores = seedCandidates.map((seed) => {
+        const traverseEntry = distribution.find(
+          (entry) => entry.action.type === "traverse" && entry.action.targetNodeId === seed.nodeId,
+        );
+        return {
+          nodeId: seed.nodeId,
+          priorScore: seed.score,
+          learnedSeedWeight: graph.getSeedWeight(seed.nodeId),
+          policyScore: traverseEntry?.score ?? seed.score,
+          probability: traverseEntry?.probability ?? 0,
+          chosen: sampled.action.type === "traverse" && sampled.action.targetNodeId === seed.nodeId,
+        };
+      });
+    }
+
+    // Find STOP probability for trace
+    const stopEntry = distribution.find((d) => d.action.type === "stop");
+    const stopProb = stopEntry?.probability ?? 0;
+
+    // Record trajectory step
+    const step: TrajectoryStep = {
+      stateSnapshot: {
+        currentNodeId: state.currentNodeId,
+        hopCount: state.hopCount,
+        budgetRemaining: state.budgetRemaining,
+        visitedCount: state.visited.size,
+        firedCount: state.fired.length,
+      },
+      candidates: distribution.map((d) => ({
+        action: d.action,
+        score: d.score,
+        probability: d.probability,
+        priorScore: d.action.type === "traverse" && state.currentNodeId === null ? d.action.seedScore : undefined,
+        learnedSeedWeight: d.action.type === "traverse" && state.currentNodeId === null
+          ? graph.getSeedWeight(d.action.targetNodeId)
+          : undefined,
+      })),
+      chosenAction: sampled.action,
+      chosenActionProbability: sampled.probability,
+      stopProbability: stopProb,
+    };
+    trajectory.push(step);
+
+    // Execute action
+    if (sampled.action.type === "stop") {
+      break; // Terminal: STOP chosen
+    }
+
+    const targetNodeId = sampled.action.targetNodeId;
+    state.visited.add(targetNodeId);
+    state.hopCount++;
+
+    // Inhibitory veto check
+    if (state.currentNodeId && graph.isVetoed(state.currentNodeId, targetNodeId)) {
+      const reason = graph.getVetoReason(state.currentNodeId, targetNodeId) ?? "inhibitory";
+      vetoedNodes.push({ nodeId: targetNodeId, reason });
+      state.currentNodeId = targetNodeId; // Still move to the node, but don't fire it
+      continue;
+    }
+
+    // Fire node: add to context
+    const targetNode = graph.getNode(targetNodeId);
+    if (targetNode) {
+      state.fired.push(targetNodeId);
+      state.budgetRemaining -= targetNode.tokenCount;
+
+      firedNodes.push({
+        nodeId: targetNode.id,
+        kind: targetNode.kind,
+        content: targetNode.content,
+        tokenCount: targetNode.tokenCount,
+      });
+    }
+
+    state.currentNodeId = targetNodeId;
+
+    // Terminal: budget exhausted
+    if (state.budgetRemaining <= 0) break;
+  }
+
+  const contextChars = firedNodes.reduce((sum, n) => sum + n.content.length, 0);
+
+  const chosenSeed = seedScores.find((seed) => seed.chosen)?.nodeId ?? "none";
+  const footer = `Brain · ${seedScores.length} seeds · start ${chosenSeed} · ${state.hopCount} hops · ${firedNodes.length} fired · ${vetoedNodes.length} veto · ${contextChars} chars`;
+
+  return {
+    firedNodes,
+    vetoedNodes,
+    trajectory,
+    seedScores,
+    contextChars,
+    footer,
+  };
+}