@jonathangu/openclawbrain 0.3.0

package/src/brain-core/types.ts

@@ -0,0 +1,405 @@
+/**
+ * Core type definitions for the OpenClawBrain learning layer.
+ *
+ * These types define the learned retrieval graph, traversal MDP,
+ * episodes, labels, packs, mutations, and traces.
+ *
+ * Paper reference: Gu (2016), "Reinforcement Learning"
+ * Lemma 6.1: ∂/∂ρ v_ρ(s_t) = E[z · Σ_{l=t}^{T} ∂logP_ρ(a_l|s_l)/∂ρ]
+ */
+
+// ═══════════════════════════════════════════
+// Node & Edge Types
+// ═══════════════════════════════════════════
+
+export type NodeKind =
+  | "chunk"            // Document/code fragment
+  | "workflow"         // Multi-step procedure
+  | "correction"       // Human-authored fix ("use X not Y")
+  | "toolcard"         // When/how to use a specific tool
+  | "episode_anchor"   // Pointer to a prior successful episode
+  | "summary_bridge";  // Bridge to LCM summary DAG
+
+export type EdgeKind =
+  | "sibling"          // Same-document adjacency (prior = 1.0)
+  | "semantic"         // Embedding cosine similarity (prior = cosine)
+  | "learned"          // Created by learning (prior = 0.5)
+  | "seed"             // Learned seed-head parameter from __START__
+  | "inhibitory"       // Suppresses traversal (weight < 0)
+  | "bridge";          // Links brain node to LCM summary
+
+export const START_NODE_ID = "__START__";
+
+export interface SeedWeightUpdate {
+  kind: "seed";
+  nodeId: string;
+  delta: number;
+}
+
+export interface EdgeWeightUpdate {
+  kind: "edge";
+  source: string;
+  target: string;
+  delta: number;
+}
+
+export type PolicyWeightUpdate = SeedWeightUpdate | EdgeWeightUpdate;
+
+export type TrustLevel = "human" | "scanner" | "teacher" | "self";
+
+export type RewardSource = TrustLevel;
+
+export interface BrainNode {
+  id: string;
+  kind: NodeKind;
+  content: string;
+  embedding: Float32Array | null;
+  sourceUri: string | null;
+  trust: TrustLevel;
+  tags: string[];
+  tokenCount: number;
+  metadata: Record<string, unknown>;
+  createdAt: number;
+  updatedAt: number;
+}
+
+export interface BrainEdge {
+  source: string;
+  target: string;
+  kind: EdgeKind;
+  weight: number;       // Learned parameter ρ (signed; negative = suppress)
+  prior: number;        // Immutable structural baseline
+  metadata: Record<string, unknown>;
+  decayedAt: number;
+  createdAt: number;
+}
+
+// ═══════════════════════════════════════════
+// Traversal MDP (paper-faithful)
+// ═══════════════════════════════════════════
+
+/**
+ * State s_t in the MDP.
+ * Paper: S = {s_0, s_1, s_2, ...}
+ */
+export interface TraversalState {
+  currentNodeId: string | null;   // null at seed phase (t=0)
+  queryEmbedding: Float32Array;
+  visited: Set<string>;
+  fired: string[];
+  budgetRemaining: number;
+  hopCount: number;
+  maxHops: number;
+}
+
+/**
+ * Action a_t in the MDP.
+ * Paper: A(s) ⊂ {a_0, a_1, a_2, ...}
+ * Our action set: A(s) = { traverse(neighbor) } ∪ { STOP }
+ */
+export type TraversalAction =
+  | { type: "traverse"; targetNodeId: string; seedScore?: number }
+  | { type: "stop" };
+
+export interface SeedScore {
+  nodeId: string;
+  priorScore: number;
+  learnedSeedWeight: number;
+  policyScore: number;
+  probability: number;
+  chosen: boolean;
+}
+
+export interface SeedWeight {
+  nodeId: string;
+  weight: number;
+  updatedAt: number;
+}
+
+/**
+ * One step of a recorded trajectory.
+ * Captures the full candidate distribution for REINFORCE gradient computation.
+ */
+export interface TrajectoryStep {
+  stateSnapshot: {
+    currentNodeId: string | null;
+    hopCount: number;
+    budgetRemaining: number;
+    visitedCount: number;
+    firedCount: number;
+  };
+  candidates: Array<{
+    action: TraversalAction;
+    score: number;
+    probability: number;
+    priorScore?: number;
+    learnedSeedWeight?: number;
+  }>;
+  chosenAction: TraversalAction;
+  chosenActionProbability: number;
+  stopProbability: number;
+}
+
+// ═══════════════════════════════════════════
+// Episodes & Labels
+// ═══════════════════════════════════════════
+
+/**
+ * A complete episode: one traversal from seed to terminal state.
+ * Paper: A game that ends in finite time with terminal reward z.
+ */
+export interface Episode {
+  id: string;
+  conversationId: number | null;
+  queryText: string;
+  queryEmbedding: Float32Array | null;
+  trajectory: TrajectoryStep[];
+  firedNodes: string[];
+  vetoedNodes: string[];
+  contextChars: number;
+  reward: number | null;          // Terminal reward z ∈ [-1, +1]
+  rewardSource: RewardSource | null;
+  packVersion: number | null;
+  createdAt: number;
+}
+
+/**
+ * A reward label from one of four sources.
+ * Human > self > scanner > teacher by trust ranking.
+ */
+export interface Label {
+  id: string;
+  episodeId: string;
+  source: RewardSource;
+  value: number;                  // z ∈ [-1, +1]
+  confidence: number;             // [0, 1]
+  reason: string | null;
+  applied: boolean;
+  createdAt: number;
+}
+
+export type BrainEvidenceKind =
+  | "human_feedback"
+  | "self_result"
+  | "scanner_signal"
+  | "teacher_review"
+  | "teach_correction";
+
+export interface BrainEvidence {
+  id: string;
+  episodeId: string;
+  conversationId: number | null;
+  source: RewardSource;
+  kind: BrainEvidenceKind;
+  value: number;
+  confidence: number;
+  reason: string | null;
+  contentSnippet: string | null;
+  metadata: Record<string, unknown>;
+  resolved: boolean;
+  createdAt: number;
+}
+
+export type BrainEvidenceResolution =
+  | "promoted_to_label"
+  | "discarded_missing_episode"
+  | "discarded_lower_trust"
+  | "discarded_duplicate";
+
+export interface ResolvedLabel {
+  id: string;
+  evidenceId: string;
+  episodeId: string;
+  source: RewardSource;
+  value: number;
+  confidence: number;
+  resolution: BrainEvidenceResolution;
+  labelId: string | null;
+  note: string | null;
+  createdAt: number;
+}
+
+// ═══════════════════════════════════════════
+// Packs & Mutations
+// ═══════════════════════════════════════════
+
+export interface Pack {
+  version: number;
+  nodeCount: number;
+  edgeCount: number;
+  healthJson: string;
+  promotedAt: number | null;
+  rolledBack: boolean;
+  createdAt: number;
+}
+
+export type MutationKind = "split" | "merge" | "prune" | "connect" | "inject";
+export type MutationStatus = "pending" | "validated" | "promoted" | "rejected";
+
+export interface MutationProposal {
+  id: string;
+  kind: MutationKind;
+  proposal: unknown;
+  evidence: unknown | null;
+  expectedGain: number | null;
+  status: MutationStatus;
+  createdAt: number;
+  resolvedAt: number | null;
+}
+
+// ═══════════════════════════════════════════
+// Health Metrics
+// ═══════════════════════════════════════════
+
+export interface HealthMetrics {
+  nodeCount: number;
+  edgeCount: number;
+  nodesByKind: Record<NodeKind, number>;
+  edgesByKind: Record<EdgeKind, number>;
+  firedPerQuery: number;
+  dormantPercent: number;
+  inhibitoryPercent: number;
+  orphanCount: number;
+  avgPathLength: number;
+  avgReward: number;
+  crossFileEdgePercent: number;
+  churn: number;
+  packVersion: number;
+  lastUpdateAt: number;
+  totalEpisodes: number;
+}
+
+// ═══════════════════════════════════════════
+// Decision Traces
+// ═══════════════════════════════════════════
+
+export interface DecisionTrace {
+  id: string;
+  episodeId: string | null;
+  packVersion: number | null;
+  queryText: string;
+  seedScores: SeedScore[];
+  trajectory: TrajectoryStep[];
+  firedNodes: string[];
+  vetoedNodes: string[];
+  contextChars: number;
+  footer: string;
+  createdAt: number;
+}
+
+// ═══════════════════════════════════════════
+// Configuration
+// ═══════════════════════════════════════════
+
+export interface BrainConfig {
+  enabled: boolean;
+  root: string;
+  maxHops: number;
+  servingTemperature: number;
+  learningTemperature: number;
+  budgetFraction: number;
+  maxSeeds: number;
+  semanticThreshold: number;
+  learningRate: number;
+  baselineAlpha: number;
+  decayRate: number;
+  trainerIntervalMs: number;
+  workerMode: "child" | "in_process";
+  workerHeartbeatTimeoutMs: number;
+  workerRestartDelayMs: number;
+  teacherEnabled: boolean;
+  mutationsEnabled: boolean;
+  replayEpisodeCount: number;
+  minFiredPerQuery: number;
+  maxDormantPercent: number;
+  maxOrphanCount: number;
+  shadowMode: boolean;
+  embeddingProvider: string;
+  embeddingModel: string;
+  embeddingBaseUrl: string;
+  teacherProvider: string;
+  teacherModel: string;
+}
+
+export const DEFAULT_BRAIN_CONFIG: BrainConfig = {
+  enabled: true,
+  root: "",
+  maxHops: 8,
+  servingTemperature: 0.1,
+  learningTemperature: 1.0,
+  budgetFraction: 0.3,
+  maxSeeds: 10,
+  semanticThreshold: 0.7,
+  learningRate: 0.01,
+  baselineAlpha: 0.1,
+  decayRate: 0.995,
+  trainerIntervalMs: 30_000,
+  workerMode: "child",
+  workerHeartbeatTimeoutMs: 90_000,
+  workerRestartDelayMs: 5_000,
+  teacherEnabled: true,
+  mutationsEnabled: true,
+  replayEpisodeCount: 100,
+  minFiredPerQuery: 1.0,
+  maxDormantPercent: 0.3,
+  maxOrphanCount: 10,
+  shadowMode: false,
+  embeddingProvider: "openai",
+  embeddingModel: "",
+  embeddingBaseUrl: "",
+  teacherProvider: "",
+  teacherModel: "",
+};
+
+// ═══════════════════════════════════════════
+// Traversal Result (returned to assembler)
+// ═══════════════════════════════════════════
+
+export interface TraversalResult {
+  fired: Array<{ nodeId: string; kind: NodeKind; content: string; tokenCount: number }>;
+  vetoed: Array<{ nodeId: string; reason: string }>;
+  episode: Episode;
+  trace: DecisionTrace;
+}
+
+/**
+ * The route_fn interface: query + candidate IDs → selected subset.
+ */
+export type RouteFn = (query: string, candidateIds: string[]) => Promise<string[]>;
+
+// ═══════════════════════════════════════════
+// Policy Parameters
+// ═══════════════════════════════════════════
+
+export interface PolicyParams {
+  temperature: number;
+  stopBias: number;
+  budgetPressure: number;
+  hopPressure: number;
+  edgeKindBias: Record<EdgeKind, number>;
+}
+
+export const DEFAULT_POLICY_PARAMS: PolicyParams = {
+  temperature: 1.0,
+  stopBias: -2.0,
+  budgetPressure: 3.0,
+  hopPressure: 2.0,
+  edgeKindBias: {
+    sibling: 0.0,
+    semantic: 0.1,
+    learned: 0.2,
+    seed: 0.15,
+    inhibitory: -10.0,
+    bridge: 0.0,
+  },
+};
+
+// Trust rank ordering: human > self > scanner > teacher
+export function trustRank(source: RewardSource): number {
+  switch (source) {
+    case "human": return 4;
+    case "self": return 3;
+    case "scanner": return 2;
+    case "teacher": return 1;
+  }
+}

package/src/brain-core/update.ts

@@ -0,0 +1,123 @@
+/**
+ * REINFORCE update rule implementing Lemma 6.1 from the paper.
+ *
+ * Paper's correct update direction:
+ *   ∂/∂ρ v_ρ(s_t) = E[z · Σ_{l=t}^{T} ∂logP_ρ(a_l|s_l)/∂ρ]
+ *
+ * Key insight: The sum goes over the ENTIRE trajectory from t to T.
+ * This assigns credit to every routing decision that led to the outcome,
+ * not just the last step. Williams (1992) equation (1) is one-step only.
+ *
+ * For the softmax policy over edge weights:
+ *   ∂logP(a_j|s)/∂w_j = 1 - P(a_j|s)     (for the chosen action's weight)
+ *   ∂logP(a_j|s)/∂w_k = -P(a_k|s)         (for other actions' weights)
+ *
+ * We update only the chosen edge's weight at each step, using:
+ *   Δw_j = learningRate × (z - baseline) × (1 - P(a_j|s))
+ *
+ * The full-trajectory sum is achieved by accumulating updates across all steps.
+ */
+
+import type { Episode, PolicyWeightUpdate } from "./types.js";
+import { START_NODE_ID } from "./types.js";
+import type { BrainGraph } from "./graph.js";
+
+/**
+ * Compute REINFORCE weight updates from a completed episode.
+ *
+ * Implements Lemma 6.1: full-trajectory credit assignment.
+ * Every step l from 0 to T contributes to the gradient.
+ */
+export function computeReinforceUpdates(
+  episode: Episode,
+  learningRate: number,
+  baseline: number,
+): PolicyWeightUpdate[] {
+  if (episode.reward === null) return [];
+
+  const advantage = episode.reward - baseline;
+  if (Math.abs(advantage) < 1e-8) return [];
+
+  const updates: Map<string, PolicyWeightUpdate> = new Map();
+
+  // Sum over all trajectory steps l = 0 to T (full-trajectory, not one-step)
+  for (const step of episode.trajectory) {
+    if (step.chosenAction.type !== "traverse") continue;
+
+    const sourceId = step.stateSnapshot.currentNodeId ?? START_NODE_ID;
+    const targetId = step.chosenAction.targetNodeId;
+
+    // ∂logP(a_l|s_l)/∂ρ for the softmax = (1 - P(a_l|s_l))
+    const gradLogP = 1 - step.chosenActionProbability;
+
+    // Δρ ∝ (z - baseline) × ∂logP/∂ρ
+    const delta = learningRate * advantage * gradLogP;
+
+    if (sourceId === START_NODE_ID) {
+      const key = `seed→${targetId}`;
+      const existing = updates.get(key);
+      if (existing && existing.kind === "seed") {
+        existing.delta += delta;
+      } else {
+        updates.set(key, { kind: "seed", nodeId: targetId, delta });
+      }
+      continue;
+    }
+
+    // Accumulate: the full-trajectory sum means each step adds to the gradient
+    const key = `${sourceId}→${targetId}`;
+
+    const existing = updates.get(key);
+    if (existing && existing.kind === "edge") {
+      existing.delta += delta;
+    } else {
+      updates.set(key, { kind: "edge", source: sourceId, target: targetId, delta });
+    }
+  }
+
+  return [...updates.values()];
+}
+
+/**
+ * Update running baseline via exponential moving average.
+ *
+ * baseline_{n+1} = α × z_n + (1 - α) × baseline_n
+ *
+ * The baseline reduces variance in the REINFORCE estimate
+ * without introducing bias (standard variance reduction technique).
+ */
+export function updateBaseline(
+  currentBaseline: number,
+  newReward: number,
+  alpha: number,
+): number {
+  return alpha * newReward + (1 - alpha) * currentBaseline;
+}
+
+/**
+ * Apply computed weight updates to graph edges.
+ *
+ * After applying, edge weights may become negative (inhibitory).
+ * This is intentional — the paper allows signed outcomes.
+ */
+export function applyWeightUpdates(
+  graph: BrainGraph,
+  updates: PolicyWeightUpdate[],
+): void {
+  for (const update of updates) {
+    if (update.kind === "seed") {
+      const nextWeight = Math.max(-10, Math.min(10, graph.getSeedWeight(update.nodeId) + update.delta));
+      graph.setSeedWeight(update.nodeId, nextWeight);
+      continue;
+    }
+
+    const edge = graph.getEdge(update.source, update.target);
+    if (!edge) continue;
+
+    // Update weight: w_new = w_old + Δw
+    edge.weight += update.delta;
+
+    // Clamp to prevent numerical explosion
+    edge.weight = Math.max(-10, Math.min(10, edge.weight));
+  }
+}

package/src/brain-harvest/human.ts

@@ -0,0 +1,46 @@
+import type { HarvestResult } from "../brain-runtime/evidence-detectors.js";
+
+const NEGATIVE_HUMAN_PATTERNS = [
+  /\bno[,.]?\s+(that'?s?\s+)?(not|wrong|incorrect)/i,
+  /\bdon'?t\s+(use|do|try)/i,
+  /\binstead\s+(use|do|try)/i,
+  /\bactually[,]?\s+(it'?s|the|you\s+should)/i,
+  /\bthat'?s\s+not\s+(right|correct|what)/i,
+  /\bwrong\s+(file|path|approach|answer|tool)/i,
+  /\bnot\s+what\s+i\s+(asked|wanted|meant)/i,
+];
+
+const POSITIVE_HUMAN_PATTERNS = [
+  /\b(perfect|exactly|correct)\b/i,
+  /\bthat('?s|\s+is)\s+(exactly\s+)?(right|correct|what\s+i)/i,
+  /\bgreat[,!]\s+(that|this)\s+(work|help)/i,
+  /\byes[,!.]\s+(that'?s?|exactly)/i,
+];
+
+export function detectHumanEvidence(content: string): HarvestResult | null {
+  for (const pattern of NEGATIVE_HUMAN_PATTERNS) {
+    if (pattern.test(content)) {
+      return {
+        value: -0.8,
+        source: "human",
+        reason: `negative pattern: ${pattern.source}`,
+        confidence: 0.9,
+        kind: "human_feedback",
+        extractor: "human_pattern",
+      };
+    }
+  }
+  for (const pattern of POSITIVE_HUMAN_PATTERNS) {
+    if (pattern.test(content)) {
+      return {
+        value: 0.8,
+        source: "human",
+        reason: `positive pattern: ${pattern.source}`,
+        confidence: 0.9,
+        kind: "human_feedback",
+        extractor: "human_pattern",
+      };
+    }
+  }
+  return null;
+}

package/src/brain-harvest/scanner.ts

@@ -0,0 +1,98 @@
+import type { HarvestResult } from "../brain-runtime/evidence-detectors.js";
+
+const EXPLICIT_SCANNER_PATTERNS = [
+  /\bexpand for details about\b/i,
+  /\bsummary bridge\b/i,
+];
+
+const DOC_MARKER_PATTERNS = [
+  /\b(runbook|workflow|playbook|checklist|troubleshooting|procedure)\b/i,
+  /\b(deploy|deployment|incident|recovery|rollback|pull request|release)\b/i,
+];
+
+const COMMAND_LINE_PATTERN = /^\s*(?:[-*]\s+|\d+\.\s+)?`?(gh|git|pnpm|npm|node|openclaw|ollama|curl|python3?|bash)\b.*$/gim;
+const NUMBERED_STEP_PATTERN = /^\s*\d+\.\s+\S.+$/gm;
+const BULLET_PATTERN = /^\s*[-*]\s+\S.+$/gm;
+const HEADING_PATTERN = /^\s{0,3}#{1,6}\s+\S.+$/m;
+const FILE_REF_PATTERN = /(?:^|[\s(])(?:\.?\/)?[\w./-]+\.(?:md|txt|ts|tsx|js|jsx|json|yaml|yml|sh|mjs)(?=$|[\s):,])/gim;
+const IMPERATIVE_STEP_PATTERN = /^\s*(?:[-*]\s+|\d+\.\s+)?(?:inspect|check|retry|run|use|open|read|edit|verify|restart|re-?run|apply|deploy|create|install|record|compare|promote|rollback)\b/gim;
+
+function countMatches(pattern: RegExp, content: string): number {
+  const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`;
+  const matcher = new RegExp(pattern.source, flags);
+  return Array.from(content.matchAll(matcher)).length;
+}
+
+export function detectScannerEvidence(content: string): HarvestResult | null {
+  for (const pattern of EXPLICIT_SCANNER_PATTERNS) {
+    if (pattern.test(content)) {
+      return {
+        value: 0.25,
+        source: "scanner",
+        reason: `scanner marker: ${pattern.source}`,
+        confidence: 0.7,
+        kind: "scanner_signal",
+        extractor: "scanner_marker",
+      };
+    }
+  }
+
+  const signals: string[] = [];
+  let score = 0;
+
+  for (const pattern of DOC_MARKER_PATTERNS) {
+    if (pattern.test(content)) {
+      signals.push(`doc:${pattern.source}`);
+      score += 1.0;
+      break;
+    }
+  }
+
+  const numberedSteps = countMatches(NUMBERED_STEP_PATTERN, content);
+  if (numberedSteps >= 2) {
+    signals.push(`numbered_steps:${numberedSteps}`);
+    score += 1.0;
+  }
+
+  const bulletLines = countMatches(BULLET_PATTERN, content);
+  if (bulletLines >= 3) {
+    signals.push(`bullets:${bulletLines}`);
+    score += 0.5;
+  }
+
+  const commandLines = countMatches(COMMAND_LINE_PATTERN, content);
+  if (commandLines >= 1) {
+    signals.push(`commands:${commandLines}`);
+    score += commandLines >= 2 ? 1.0 : 0.6;
+  }
+
+  const imperativeLines = countMatches(IMPERATIVE_STEP_PATTERN, content);
+  if (imperativeLines >= 2) {
+    signals.push(`imperatives:${imperativeLines}`);
+    score += 0.8;
+  }
+
+  if (HEADING_PATTERN.test(content) && (numberedSteps >= 1 || bulletLines >= 2)) {
+    signals.push("heading");
+    score += 0.4;
+  }
+
+  const fileRefs = countMatches(FILE_REF_PATTERN, content);
+  if (fileRefs >= 1) {
+    signals.push(`file_refs:${fileRefs}`);
+    score += 0.3;
+  }
+
+  if (score < 1.8) {
+    return null;
+  }
+
+  return {
+    value: 0.25,
+    source: "scanner",
+    reason: `scanner heuristic: ${signals.join(", ")}`,
+    confidence: Math.min(0.8, 0.5 + signals.length * 0.05),
+    kind: "scanner_signal",
+    extractor: "scanner_heuristic",
+  };
+}