@longarc/mdash 3.0.0

package/src/mcca/engine.ts

@@ -0,0 +1,778 @@
+/**
+ * mdash v3.0 - MCCA v3 (Manifold-Constrained Context Architecture)
+ * 
+ * Based on DeepSeek's research on constrained context windows.
+ * Implements manifold commitment proofs and influence budgets.
+ * 
+ * Core Concepts:
+ * - Manifold: A bounded region of valid agent behavior
+ * - Influence Budget: Maximum context influence per source
+ * - Commitment Proofs: Cryptographic proof of context integrity
+ * - Drift Detection: Identifies context manipulation attempts
+ * 
+ * @version 3.0.0
+ */
+
+import {
+  Hash,
+  Seal,
+  Timestamp,
+  FragmentId,
+  generateTimestamp,
+  sha256,
+  sha256Object,
+  hmacSeal,
+  deriveKey,
+  generateFragmentId,
+} from '../core/crypto.js';
+
+import { CommitmentEngine } from '../core/commitment.js';
+
+// ============================================================================
+// MCCA TYPES
+// ============================================================================
+
+export type SourceClass = 
+  | 'system'      // System prompts, policies
+  | 'user'        // User messages
+  | 'agent'       // Agent-generated content
+  | 'tool'        // Tool outputs
+  | 'memory'      // Retrieved memories
+  | 'external';   // External APIs, web content
+
+export type ManifoldRegion = 
+  | 'core'        // Core system behavior
+  | 'task'        // Current task context
+  | 'history'     // Conversation history
+  | 'reference'   // Reference materials
+  | 'ephemeral';  // Temporary working context
+
+export type DriftSeverity = 'none' | 'low' | 'medium' | 'high' | 'critical';
+
+/**
+ * Influence budget allocation per source class
+ * Total budget must sum to 1.0
+ */
+export interface InfluenceBudget {
+  system: number;    // Typically 0.30
+  user: number;      // Typically 0.35
+  agent: number;     // Typically 0.15
+  tool: number;      // Typically 0.10
+  memory: number;    // Typically 0.05
+  external: number;  // Typically 0.05
+}
+
+/**
+ * Default influence budget (conservative)
+ */
+export const DEFAULT_INFLUENCE_BUDGET: InfluenceBudget = {
+  system: 0.30,
+  user: 0.35,
+  agent: 0.15,
+  tool: 0.10,
+  memory: 0.05,
+  external: 0.05,
+};
+
+/**
+ * Context fragment with manifold positioning
+ */
+export interface ManifoldFragment {
+  /** Fragment ID */
+  id: FragmentId;
+  /** Content hash */
+  content_hash: Hash;
+  /** Source classification */
+  source_class: SourceClass;
+  /** Manifold region */
+  region: ManifoldRegion;
+  /** Position in manifold (normalized 0-1) */
+  position: {
+    x: number;  // Influence axis
+    y: number;  // Recency axis
+    z: number;  // Relevance axis
+  };
+  /** Influence weight */
+  influence: number;
+  /** Tokens in fragment */
+  token_count: number;
+  /** Creation time */
+  timestamp: Timestamp;
+  /** Seal */
+  seal: Seal;
+}
+
+/**
+ * Manifold commitment proof
+ * Proves the context state at a point in time
+ */
+export interface ManifoldCommitmentProof {
+  /** Proof ID */
+  id: string;
+  /** Timestamp of commitment */
+  timestamp: Timestamp;
+  /** Merkle root of all fragments */
+  merkle_root: Hash;
+  /** Fragment count */
+  fragment_count: number;
+  /** Total influence by source class */
+  influence_distribution: Record<SourceClass, number>;
+  /** Budget compliance */
+  budget_compliant: boolean;
+  /** Drift score (0 = no drift, 1 = maximum drift) */
+  drift_score: number;
+  /** L1 commitment reference */
+  commitment_id: string;
+  /** Seal */
+  seal: Seal;
+}
+
+/**
+ * Drift detection result
+ */
+export interface DriftAnalysis {
+  /** Overall drift score */
+  score: number;
+  /** Severity classification */
+  severity: DriftSeverity;
+  /** Drift by source class */
+  by_source: Record<SourceClass, number>;
+  /** Specific violations */
+  violations: Array<{
+    type: 'budget_exceeded' | 'region_overflow' | 'influence_spike' | 'position_anomaly';
+    source: SourceClass;
+    expected: number;
+    actual: number;
+    message: string;
+  }>;
+  /** Recommendations */
+  recommendations: string[];
+}
+
+// ============================================================================
+// MANIFOLD ENGINE
+// ============================================================================
+
+export interface MCCAConfig {
+  /** Influence budget allocation */
+  budget: InfluenceBudget;
+  /** Maximum fragments in manifold */
+  maxFragments: number;
+  /** Maximum total tokens */
+  maxTokens: number;
+  /** Drift threshold for alerts */
+  driftThreshold: number;
+  /** Enable automatic pruning */
+  autoPrune: boolean;
+  /** Region size limits (tokens) */
+  regionLimits: Record<ManifoldRegion, number>;
+}
+
+export const DEFAULT_CONFIG: MCCAConfig = {
+  budget: DEFAULT_INFLUENCE_BUDGET,
+  maxFragments: 1000,
+  maxTokens: 100000,
+  driftThreshold: 0.3,
+  autoPrune: true,
+  regionLimits: {
+    core: 10000,
+    task: 30000,
+    history: 40000,
+    reference: 15000,
+    ephemeral: 5000,
+  },
+};
+
+export class MCCAEngine {
+  private key: CryptoKey | null = null;
+  private config: MCCAConfig;
+  private commitmentEngine: CommitmentEngine;
+  
+  // Fragment storage
+  private fragments: Map<FragmentId, ManifoldFragment> = new Map();
+  private fragmentsByRegion: Map<ManifoldRegion, FragmentId[]> = new Map();
+  private fragmentsBySource: Map<SourceClass, FragmentId[]> = new Map();
+  
+  // Commitment proofs
+  private proofs: Map<string, ManifoldCommitmentProof> = new Map();
+  
+  // Running totals
+  private totalTokens: number = 0;
+  private influenceBySource: Record<SourceClass, number> = {
+    system: 0,
+    user: 0,
+    agent: 0,
+    tool: 0,
+    memory: 0,
+    external: 0,
+  };
+
+  constructor(commitmentEngine: CommitmentEngine, config: Partial<MCCAConfig> = {}) {
+    this.commitmentEngine = commitmentEngine;
+    this.config = { ...DEFAULT_CONFIG, ...config };
+    
+    // Initialize region maps
+    for (const region of Object.keys(this.config.regionLimits) as ManifoldRegion[]) {
+      this.fragmentsByRegion.set(region, []);
+    }
+    
+    // Initialize source maps
+    for (const source of Object.keys(this.config.budget) as SourceClass[]) {
+      this.fragmentsBySource.set(source, []);
+    }
+  }
+
+  /**
+   * Initialize the MCCA engine
+   */
+  async initialize(sealKey: string): Promise<void> {
+    this.key = await deriveKey(sealKey);
+  }
+
+  /**
+   * Add a fragment to the manifold
+   * Automatically positions it based on source and content
+   */
+  async addFragment(params: {
+    content: unknown;
+    source_class: SourceClass;
+    region: ManifoldRegion;
+    token_count: number;
+    influence?: number;
+  }): Promise<ManifoldFragment> {
+    if (!this.key) {
+      throw new Error('MCCA engine not initialized');
+    }
+
+    const startTime = performance.now();
+
+    // Check region capacity
+    const regionTokens = this.getRegionTokens(params.region);
+    if (regionTokens + params.token_count > this.config.regionLimits[params.region]) {
+      if (this.config.autoPrune) {
+        await this.pruneRegion(params.region, params.token_count);
+      } else {
+        throw new Error(`Region ${params.region} capacity exceeded`);
+      }
+    }
+
+    // Calculate influence
+    const baseInfluence = params.influence ?? this.calculateBaseInfluence(
+      params.source_class,
+      params.token_count
+    );
+
+    // Calculate position in manifold
+    const position = this.calculatePosition(
+      params.source_class,
+      params.region,
+      baseInfluence
+    );
+
+    // Create fragment
+    const id = generateFragmentId();
+    const contentHash = await sha256Object(params.content);
+    const now = generateTimestamp();
+
+    const fragmentData = {
+      id,
+      content_hash: contentHash,
+      source_class: params.source_class,
+      region: params.region,
+      position,
+      influence: baseInfluence,
+      token_count: params.token_count,
+      timestamp: now,
+    };
+
+    const seal = await hmacSeal(fragmentData, this.key);
+
+    const fragment: ManifoldFragment = {
+      ...fragmentData,
+      seal,
+    };
+
+    // Store fragment
+    this.fragments.set(id, fragment);
+    this.fragmentsByRegion.get(params.region)!.push(id);
+    this.fragmentsBySource.get(params.source_class)!.push(id);
+
+    // Update totals
+    this.totalTokens += params.token_count;
+    this.influenceBySource[params.source_class] += baseInfluence;
+
+    // Commit to L1
+    await this.commitmentEngine.commit(fragment, `mcca:${id}`);
+
+    const elapsed = performance.now() - startTime;
+    if (elapsed > 5) {
+      console.warn(`[MCCA] Fragment addition exceeded 5ms: ${elapsed.toFixed(2)}ms`);
+    }
+
+    return fragment;
+  }
+
+  /**
+   * Calculate base influence for a fragment
+   */
+  private calculateBaseInfluence(source: SourceClass, tokens: number): number {
+    const budgetShare = this.config.budget[source];
+    const currentUsage = this.influenceBySource[source];
+    const remaining = budgetShare - currentUsage;
+    
+    // Scale by token count (normalized)
+    const tokenFactor = Math.min(tokens / 1000, 1);
+    
+    return Math.max(0, Math.min(remaining * tokenFactor, remaining));
+  }
+
+  /**
+   * Calculate position in manifold
+   */
+  private calculatePosition(
+    source: SourceClass,
+    region: ManifoldRegion,
+    influence: number
+  ): { x: number; y: number; z: number } {
+    // X-axis: Influence (based on source class priority)
+    const sourcePriority: Record<SourceClass, number> = {
+      system: 0.9,
+      user: 0.8,
+      agent: 0.5,
+      tool: 0.6,
+      memory: 0.4,
+      external: 0.3,
+    };
+    const x = sourcePriority[source] * influence;
+
+    // Y-axis: Recency (newer = higher)
+    const y = this.fragments.size > 0
+      ? 1 - (this.fragments.size / this.config.maxFragments)
+      : 1;
+
+    // Z-axis: Region relevance
+    const regionRelevance: Record<ManifoldRegion, number> = {
+      core: 1.0,
+      task: 0.9,
+      history: 0.5,
+      reference: 0.6,
+      ephemeral: 0.2,
+    };
+    const z = regionRelevance[region];
+
+    return { x, y, z };
+  }
+
+  /**
+   * Prune a region to make space
+   */
+  private async pruneRegion(region: ManifoldRegion, needed: number): Promise<void> {
+    const fragmentIds = this.fragmentsByRegion.get(region) || [];
+    const fragments = fragmentIds
+      .map(id => this.fragments.get(id)!)
+      .sort((a, b) => {
+        // Prune by: low influence, old, low relevance
+        const scoreA = a.influence * a.position.y * a.position.z;
+        const scoreB = b.influence * b.position.y * b.position.z;
+        return scoreA - scoreB;
+      });
+
+    let freed = 0;
+    for (const fragment of fragments) {
+      if (freed >= needed) break;
+      
+      await this.removeFragment(fragment.id);
+      freed += fragment.token_count;
+    }
+  }
+
+  /**
+   * Remove a fragment from the manifold
+   */
+  async removeFragment(id: FragmentId): Promise<void> {
+    const fragment = this.fragments.get(id);
+    if (!fragment) return;
+
+    // Update totals
+    this.totalTokens -= fragment.token_count;
+    this.influenceBySource[fragment.source_class] -= fragment.influence;
+
+    // Remove from indices
+    const regionIds = this.fragmentsByRegion.get(fragment.region) || [];
+    this.fragmentsByRegion.set(
+      fragment.region,
+      regionIds.filter(fid => fid !== id)
+    );
+
+    const sourceIds = this.fragmentsBySource.get(fragment.source_class) || [];
+    this.fragmentsBySource.set(
+      fragment.source_class,
+      sourceIds.filter(fid => fid !== id)
+    );
+
+    this.fragments.delete(id);
+  }
+
+  /**
+   * Get region token count
+   */
+  private getRegionTokens(region: ManifoldRegion): number {
+    const fragmentIds = this.fragmentsByRegion.get(region) || [];
+    return fragmentIds.reduce((sum, id) => {
+      const fragment = this.fragments.get(id);
+      return sum + (fragment?.token_count || 0);
+    }, 0);
+  }
+
+  /**
+   * Generate a manifold commitment proof
+   */
+  async generateCommitmentProof(): Promise<ManifoldCommitmentProof> {
+    if (!this.key) {
+      throw new Error('MCCA engine not initialized');
+    }
+
+    // Calculate merkle root of all fragments
+    const fragmentHashes = Array.from(this.fragments.values())
+      .map(f => f.content_hash);
+    
+    const merkleRoot = fragmentHashes.length > 0
+      ? await this.computeMerkleRoot(fragmentHashes)
+      : await sha256('empty-manifold');
+
+    // Calculate total influence
+    const totalInfluence = Object.values(this.influenceBySource).reduce((a, b) => a + b, 0);
+    
+    // Calculate influence distribution (normalized)
+    const distribution: Record<SourceClass, number> = {} as Record<SourceClass, number>;
+    for (const [source, influence] of Object.entries(this.influenceBySource)) {
+      distribution[source as SourceClass] = totalInfluence > 0 
+        ? influence / totalInfluence 
+        : 0;
+    }
+
+    // Check budget compliance
+    const budgetCompliant = this.checkBudgetCompliance();
+
+    // Calculate drift score
+    const driftAnalysis = await this.analyzeDrift();
+
+    const now = generateTimestamp();
+    const id = `mcp-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+
+    const proofData = {
+      id,
+      timestamp: now,
+      merkle_root: merkleRoot,
+      fragment_count: this.fragments.size,
+      influence_distribution: distribution,
+      budget_compliant: budgetCompliant.compliant,
+      drift_score: driftAnalysis.score,
+      commitment_id: '',
+    };
+
+    // Commit to L1
+    const commitment = await this.commitmentEngine.commit(proofData, `mcca-proof:${id}`);
+    proofData.commitment_id = commitment.operationId;
+
+    const seal = await hmacSeal(proofData, this.key);
+
+    const proof: ManifoldCommitmentProof = {
+      ...proofData,
+      seal,
+    };
+
+    this.proofs.set(id, proof);
+
+    return proof;
+  }
+
+  /**
+   * Compute merkle root from hashes
+   */
+  private async computeMerkleRoot(hashes: Hash[]): Promise<Hash> {
+    if (hashes.length === 0) {
+      return sha256('empty');
+    }
+    if (hashes.length === 1) {
+      return hashes[0];
+    }
+
+    const pairs: Hash[] = [];
+    for (let i = 0; i < hashes.length; i += 2) {
+      const left = hashes[i];
+      const right = hashes[i + 1] || left; // Duplicate last if odd
+      const combined = await sha256(`${left}|${right}`);
+      pairs.push(combined as Hash);
+    }
+
+    return this.computeMerkleRoot(pairs);
+  }
+
+  /**
+   * Check if current state complies with budget
+   */
+  checkBudgetCompliance(): { compliant: boolean; violations: string[] } {
+    const violations: string[] = [];
+    const totalInfluence = Object.values(this.influenceBySource).reduce((a, b) => a + b, 0);
+
+    if (totalInfluence === 0) {
+      return { compliant: true, violations: [] };
+    }
+
+    for (const [source, budget] of Object.entries(this.config.budget)) {
+      const actual = this.influenceBySource[source as SourceClass] / totalInfluence;
+      const tolerance = 0.1; // 10% tolerance
+      
+      if (actual > budget + tolerance) {
+        violations.push(
+          `${source}: ${(actual * 100).toFixed(1)}% exceeds budget ${(budget * 100).toFixed(1)}%`
+        );
+      }
+    }
+
+    return {
+      compliant: violations.length === 0,
+      violations,
+    };
+  }
+
+  /**
+   * Analyze drift from expected manifold state
+   */
+  async analyzeDrift(): Promise<DriftAnalysis> {
+    const totalInfluence = Object.values(this.influenceBySource).reduce((a, b) => a + b, 0);
+    const violations: DriftAnalysis['violations'] = [];
+    const bySource: Record<SourceClass, number> = {} as Record<SourceClass, number>;
+
+    // Calculate drift per source
+    for (const [source, budget] of Object.entries(this.config.budget)) {
+      const actual = totalInfluence > 0 
+        ? this.influenceBySource[source as SourceClass] / totalInfluence 
+        : 0;
+      const drift = Math.abs(actual - budget);
+      bySource[source as SourceClass] = drift;
+
+      if (drift > 0.15) { // 15% threshold
+        violations.push({
+          type: 'budget_exceeded',
+          source: source as SourceClass,
+          expected: budget,
+          actual,
+          message: `${source} influence drifted ${(drift * 100).toFixed(1)}% from budget`,
+        });
+      }
+    }
+
+    // Check region overflow
+    for (const [region, limit] of Object.entries(this.config.regionLimits)) {
+      const tokens = this.getRegionTokens(region as ManifoldRegion);
+      if (tokens > limit * 0.9) { // 90% threshold
+        violations.push({
+          type: 'region_overflow',
+          source: 'system' as SourceClass,
+          expected: limit,
+          actual: tokens,
+          message: `Region ${region} at ${((tokens / limit) * 100).toFixed(1)}% capacity`,
+        });
+      }
+    }
+
+    // Calculate overall score
+    const avgDrift = Object.values(bySource).reduce((a, b) => a + b, 0) / 
+                     Object.values(bySource).length;
+    
+    const score = Math.min(avgDrift * 2, 1); // Scale to 0-1
+
+    // Classify severity
+    let severity: DriftSeverity;
+    if (score < 0.1) severity = 'none';
+    else if (score < 0.25) severity = 'low';
+    else if (score < 0.5) severity = 'medium';
+    else if (score < 0.75) severity = 'high';
+    else severity = 'critical';
+
+    // Generate recommendations
+    const recommendations: string[] = [];
+    if (violations.length > 0) {
+      if (bySource.external > 0.15) {
+        recommendations.push('Reduce external content influence');
+      }
+      if (bySource.agent > 0.15) {
+        recommendations.push('Agent self-influence exceeds threshold');
+      }
+      if (severity === 'high' || severity === 'critical') {
+        recommendations.push('Consider context reset');
+      }
+    }
+
+    return {
+      score,
+      severity,
+      by_source: bySource,
+      violations,
+      recommendations,
+    };
+  }
+
+  /**
+   * Get fragment by ID
+   */
+  getFragment(id: FragmentId): ManifoldFragment | null {
+    return this.fragments.get(id) || null;
+  }
+
+  /**
+   * Get all fragments in a region
+   */
+  getRegionFragments(region: ManifoldRegion): ManifoldFragment[] {
+    const ids = this.fragmentsByRegion.get(region) || [];
+    return ids.map(id => this.fragments.get(id)!).filter(Boolean);
+  }
+
+  /**
+   * Get fragments by source
+   */
+  getSourceFragments(source: SourceClass): ManifoldFragment[] {
+    const ids = this.fragmentsBySource.get(source) || [];
+    return ids.map(id => this.fragments.get(id)!).filter(Boolean);
+  }
+
+  /**
+   * Get current manifold state
+   */
+  getState(): {
+    totalFragments: number;
+    totalTokens: number;
+    influenceBySource: Record<SourceClass, number>;
+    tokensByRegion: Record<ManifoldRegion, number>;
+  } {
+    const tokensByRegion: Record<ManifoldRegion, number> = {} as Record<ManifoldRegion, number>;
+    for (const region of Object.keys(this.config.regionLimits) as ManifoldRegion[]) {
+      tokensByRegion[region] = this.getRegionTokens(region);
+    }
+
+    return {
+      totalFragments: this.fragments.size,
+      totalTokens: this.totalTokens,
+      influenceBySource: { ...this.influenceBySource },
+      tokensByRegion,
+    };
+  }
+
+  /**
+   * Get engine statistics
+   */
+  getStats(): {
+    fragments: number;
+    tokens: number;
+    proofs: number;
+    budgetCompliance: { compliant: boolean; violations: string[] };
+    config: MCCAConfig;
+  } {
+    return {
+      fragments: this.fragments.size,
+      tokens: this.totalTokens,
+      proofs: this.proofs.size,
+      budgetCompliance: this.checkBudgetCompliance(),
+      config: this.config,
+    };
+  }
+}
+
+// ============================================================================
+// CONTEXT WINDOW MANAGER
+// ============================================================================
+
+/**
+ * Higher-level context window management
+ * Uses MCCA engine underneath
+ */
+export class ContextWindowManager {
+  private mcca: MCCAEngine;
+  private maxTokens: number;
+
+  constructor(mcca: MCCAEngine, maxTokens: number = 100000) {
+    this.mcca = mcca;
+    this.maxTokens = maxTokens;
+  }
+
+  /**
+   * Add system prompt
+   */
+  async addSystemPrompt(content: string, tokens: number): Promise<ManifoldFragment> {
+    return this.mcca.addFragment({
+      content,
+      source_class: 'system',
+      region: 'core',
+      token_count: tokens,
+      influence: 0.3, // High influence for system
+    });
+  }
+
+  /**
+   * Add user message
+   */
+  async addUserMessage(content: string, tokens: number): Promise<ManifoldFragment> {
+    return this.mcca.addFragment({
+      content,
+      source_class: 'user',
+      region: 'task',
+      token_count: tokens,
+    });
+  }
+
+  /**
+   * Add agent response
+   */
+  async addAgentResponse(content: string, tokens: number): Promise<ManifoldFragment> {
+    return this.mcca.addFragment({
+      content,
+      source_class: 'agent',
+      region: 'history',
+      token_count: tokens,
+    });
+  }
+
+  /**
+   * Add tool output
+   */
+  async addToolOutput(content: unknown, tokens: number): Promise<ManifoldFragment> {
+    return this.mcca.addFragment({
+      content,
+      source_class: 'tool',
+      region: 'task',
+      token_count: tokens,
+    });
+  }
+
+  /**
+   * Add retrieved memory
+   */
+  async addMemory(content: unknown, tokens: number): Promise<ManifoldFragment> {
+    return this.mcca.addFragment({
+      content,
+      source_class: 'memory',
+      region: 'reference',
+      token_count: tokens,
+    });
+  }
+
+  /**
+   * Get context health
+   */
+  async getHealth(): Promise<{
+    healthy: boolean;
+    drift: DriftAnalysis;
+    utilization: number;
+  }> {
+    const state = this.mcca.getState();
+    const drift = await this.mcca.analyzeDrift();
+
+    return {
+      healthy: drift.severity === 'none' || drift.severity === 'low',
+      drift,
+      utilization: state.totalTokens / this.maxTokens,
+    };
+  }
+}