@ulrichc1/sparn 1.0.0 → 1.1.0

package/dist/index.d.cts

@@ -221,6 +221,27 @@ interface UIConfig {
     /** Verbose logging (default: false) */
     verbose: boolean;
 }
+/**
+ * Real-time optimization configuration.
+ */
+interface RealtimeConfig {
+    /** Target token budget for optimized context (default: 50000) */
+    tokenBudget: number;
+    /** Token threshold that triggers auto-optimization (default: 80000) */
+    autoOptimizeThreshold: number;
+    /** File patterns to watch for changes (default: ['**\/*.jsonl']) */
+    watchPatterns: string[];
+    /** Daemon PID file path (default: '.sparn/daemon.pid') */
+    pidFile: string;
+    /** Daemon log file path (default: '.sparn/daemon.log') */
+    logFile: string;
+    /** Debounce delay in milliseconds for file changes (default: 5000) */
+    debounceMs: number;
+    /** Enable incremental optimization (default: true) */
+    incremental: boolean;
+    /** Sliding window size for context entries (default: 500) */
+    windowSize: number;
+}
 /**
  * Complete Sparn configuration.
  */
@@ -232,6 +253,8 @@ interface SparnConfig {
     ui: UIConfig;
     /** Auto-consolidation interval in hours, or null for manual */
     autoConsolidate: number | null;
+    /** Real-time optimization settings */
+    realtime: RealtimeConfig;
 }
 /**
  * Default configuration values.
@@ -296,6 +319,91 @@ interface BTSPEmbedder {
  */
 declare function createBTSPEmbedder(): BTSPEmbedder;
 
+/**
+ * Sparse pruner types.
+ * Implements sparse coding principle from neuroscience.
+ */
+
+/**
+ * Result of a pruning operation.
+ */
+interface PruneResult {
+    /** Entries kept after pruning */
+    kept: MemoryEntry[];
+    /** Entries removed during pruning */
+    removed: MemoryEntry[];
+    /** Original token count before pruning */
+    originalTokens: number;
+    /** Token count after pruning */
+    prunedTokens: number;
+}
+
+/**
+ * Budget-Aware Pruner - Token budget optimization
+ *
+ * Unlike SparsePruner which keeps top N% entries, BudgetPruner fits entries
+ * within a target token budget using priority scoring that combines:
+ * - TF-IDF relevance
+ * - Engram decay
+ * - Confidence state multipliers
+ * - BTSP bypass (always included)
+ *
+ * Target use case: Real-time optimization for Opus model (~50K token budget)
+ */
+
+interface BudgetPrunerConfig {
+    /** Target token budget */
+    tokenBudget: number;
+    /** Decay configuration */
+    decay: {
+        defaultTTL: number;
+        decayThreshold: number;
+    };
+    /** State multipliers */
+    states: {
+        activeThreshold: number;
+        readyThreshold: number;
+    };
+}
+interface BudgetPruner {
+    /**
+     * Prune entries to fit within token budget
+     * @param entries - Memory entries to prune
+     * @param budget - Optional override budget (uses config default if not provided)
+     * @returns Result with kept/removed entries and budget utilization
+     */
+    pruneToFit(entries: MemoryEntry[], budget?: number): PruneResult & {
+        budgetUtilization: number;
+    };
+    /**
+     * Calculate priority score for an entry
+     * @param entry - Entry to score
+     * @param allEntries - All entries for TF-IDF calculation
+     * @returns Priority score (higher = more important)
+     */
+    priorityScore(entry: MemoryEntry, allEntries: MemoryEntry[]): number;
+}
+/**
+ * Create a budget-aware pruner instance
+ * @param config - Pruner configuration
+ * @returns BudgetPruner instance
+ */
+declare function createBudgetPruner(config: BudgetPrunerConfig): BudgetPruner;
+/**
+ * Helper to create budget pruner from RealtimeConfig
+ * @param realtimeConfig - Realtime configuration
+ * @param decayConfig - Decay configuration
+ * @param statesConfig - States configuration
+ * @returns BudgetPruner instance
+ */
+declare function createBudgetPrunerFromConfig(realtimeConfig: RealtimeConfig, decayConfig: {
+    defaultTTL: number;
+    decayThreshold: number;
+}, statesConfig: {
+    activeThreshold: number;
+    readyThreshold: number;
+}): BudgetPruner;
+
 /**
  * Confidence States - Implements multi-state synapses
  *
@@ -336,6 +444,158 @@ interface ConfidenceStates {
  */
 declare function createConfidenceStates(config: ConfidenceStatesConfig): ConfidenceStates;
 
+/**
+ * Incremental Optimizer - Cache-based delta processing
+ *
+ * Optimizes performance for real-time scenarios by:
+ * - Caching entry scores by content hash
+ * - Only recomputing scores for new/changed entries
+ * - Pre-computing and caching document frequency tables
+ * - Periodically forcing full re-optimization to prevent drift
+ *
+ * Target: <50ms for incremental updates on 100K token contexts
+ */
+
+interface IncrementalOptimizerConfig extends BudgetPrunerConfig {
+    /** Force full re-optimization every N incremental updates */
+    fullOptimizationInterval: number;
+}
+interface IncrementalOptimizerState {
+    /** Entry cache keyed by content hash */
+    entryCache: Map<string, {
+        entry: MemoryEntry;
+        score: number;
+        timestamp: number;
+    }>;
+    /** Document frequency table for IDF calculation */
+    documentFrequency: Map<string, number>;
+    /** Total document count for IDF */
+    totalDocuments: number;
+    /** Incremental update counter */
+    updateCount: number;
+    /** Last full optimization timestamp */
+    lastFullOptimization: number;
+}
+interface IncrementalOptimizer {
+    /**
+     * Optimize incrementally (only process new/changed entries)
+     * @param newEntries - New entries to add
+     * @param budget - Optional budget override
+     * @returns Prune result with budget utilization
+     */
+    optimizeIncremental(newEntries: MemoryEntry[], budget?: number): PruneResult & {
+        budgetUtilization: number;
+    };
+    /**
+     * Optimize fully (recompute all scores)
+     * @param allEntries - All entries to optimize
+     * @param budget - Optional budget override
+     * @returns Prune result with budget utilization
+     */
+    optimizeFull(allEntries: MemoryEntry[], budget?: number): PruneResult & {
+        budgetUtilization: number;
+    };
+    /**
+     * Get current optimizer state (for serialization)
+     * @returns Serializable state object
+     */
+    getState(): IncrementalOptimizerState;
+    /**
+     * Restore optimizer state (from serialization)
+     * @param state - State to restore
+     */
+    restoreState(state: IncrementalOptimizerState): void;
+    /**
+     * Reset optimizer state (clear all caches)
+     */
+    reset(): void;
+    /**
+     * Get cache statistics
+     * @returns Cache stats
+     */
+    getStats(): {
+        cachedEntries: number;
+        uniqueTerms: number;
+        totalDocuments: number;
+        updateCount: number;
+        lastFullOptimization: number;
+    };
+}
+/**
+ * Create an incremental optimizer instance
+ * @param config - Optimizer configuration
+ * @returns IncrementalOptimizer instance
+ */
+declare function createIncrementalOptimizer(config: IncrementalOptimizerConfig): IncrementalOptimizer;
+
+/**
+ * Streaming Context Pipeline - Real-time sliding window buffer
+ *
+ * Maintains an optimized context in real-time by:
+ * - Ingesting new content as it arrives
+ * - Storing entries by priority internally (for eviction decisions)
+ * - Outputting in chronological order (for conversation coherence)
+ * - Evicting lowest-priority entries when budget exceeded
+ * - Using IncrementalOptimizer for fast delta processing
+ */
+
+interface ContextPipelineConfig extends IncrementalOptimizerConfig {
+    /** Sliding window size (max entries to keep) */
+    windowSize: number;
+}
+interface ContextPipelineStats {
+    /** Total entries ingested */
+    totalIngested: number;
+    /** Current entry count */
+    currentEntries: number;
+    /** Current token count */
+    currentTokens: number;
+    /** Budget utilization (0.0-1.0) */
+    budgetUtilization: number;
+    /** Evicted entry count */
+    evictedEntries: number;
+    /** Optimizer stats */
+    optimizer: {
+        cachedEntries: number;
+        uniqueTerms: number;
+        updateCount: number;
+    };
+}
+interface ContextPipeline {
+    /**
+     * Ingest new content into the pipeline
+     * @param content - Raw content string
+     * @param metadata - Optional metadata to attach to entries
+     * @returns Number of entries ingested
+     */
+    ingest(content: string, metadata?: Record<string, unknown>): number;
+    /**
+     * Get current optimized context (chronologically ordered)
+     * @returns Optimized context string
+     */
+    getContext(): string;
+    /**
+     * Get current entries (chronologically ordered)
+     * @returns Array of memory entries
+     */
+    getEntries(): MemoryEntry[];
+    /**
+     * Get pipeline statistics
+     * @returns Pipeline stats
+     */
+    getStats(): ContextPipelineStats;
+    /**
+     * Clear all entries and reset state
+     */
+    clear(): void;
+}
+/**
+ * Create a context pipeline instance
+ * @param config - Pipeline configuration
+ * @returns ContextPipeline instance
+ */
+declare function createContextPipeline(config: ContextPipelineConfig): ContextPipeline;
+
 /**
  * Engram Scorer - Implements engram theory (memory decay)
  *
@@ -451,25 +711,6 @@ interface SleepCompressor {
  */
 declare function createSleepCompressor(): SleepCompressor;
 
-/**
- * Sparse pruner types.
- * Implements sparse coding principle from neuroscience.
- */
-
-/**
- * Result of a pruning operation.
- */
-interface PruneResult {
-    /** Entries kept after pruning */
-    kept: MemoryEntry[];
-    /** Entries removed during pruning */
-    removed: MemoryEntry[];
-    /** Original token count before pruning */
-    originalTokens: number;
-    /** Token count after pruning */
-    prunedTokens: number;
-}
-
 /**
  * Sparse Pruner - Implements sparse coding principle
  *
@@ -503,6 +744,204 @@ interface SparsePruner {
  */
 declare function createSparsePruner(config: SparsePrunerConfig): SparsePruner;
 
+/**
+ * Daemon Process Manager - Background process lifecycle management
+ *
+ * Handles:
+ * - Process forking and detachment
+ * - PID file management
+ * - Signal handling (SIGTERM, SIGINT)
+ * - Daemon start/stop/status commands
+ */
+
+interface DaemonCommand {
+    /** Start the daemon */
+    start(config: SparnConfig): Promise<DaemonStartResult>;
+    /** Stop the daemon */
+    stop(config: SparnConfig): Promise<DaemonStopResult>;
+    /** Get daemon status */
+    status(config: SparnConfig): Promise<DaemonStatusResult>;
+}
+interface DaemonStartResult {
+    success: boolean;
+    pid?: number;
+    message: string;
+    error?: string;
+}
+interface DaemonStopResult {
+    success: boolean;
+    message: string;
+    error?: string;
+}
+interface DaemonStatusResult {
+    running: boolean;
+    pid?: number;
+    uptime?: number;
+    sessionsWatched?: number;
+    tokensSaved?: number;
+    message: string;
+}
+/**
+ * Create daemon command interface
+ * @returns DaemonCommand instance
+ */
+declare function createDaemonCommand(): DaemonCommand;
+
+/**
+ * File Tracker - Incremental file reading with byte position tracking
+ *
+ * Tracks read positions for files to enable efficient incremental reading.
+ * Handles JSONL partial line buffering for incomplete writes.
+ *
+ * Use case: Monitor Claude Code session JSONL files and only read new lines
+ * as they're appended, without re-reading the entire file.
+ */
+interface FilePosition {
+    /** File path */
+    path: string;
+    /** Last read byte position */
+    position: number;
+    /** Partial line buffer (for JSONL incomplete writes) */
+    partialLine: string;
+    /** Last modification time */
+    lastModified: number;
+    /** File size at last read */
+    lastSize: number;
+}
+interface FileTracker {
+    /**
+     * Read new content from file since last read
+     * @param filePath - File to read
+     * @returns New content as array of lines (empty if no new content)
+     */
+    readNewLines(filePath: string): string[];
+    /**
+     * Get current position for a file
+     * @param filePath - File path
+     * @returns File position or null if not tracked
+     */
+    getPosition(filePath: string): FilePosition | null;
+    /**
+     * Reset position for a file (start from beginning on next read)
+     * @param filePath - File path
+     */
+    resetPosition(filePath: string): void;
+    /**
+     * Clear all tracked positions
+     */
+    clearAll(): void;
+    /**
+     * Get all tracked file paths
+     * @returns Array of tracked file paths
+     */
+    getTrackedFiles(): string[];
+}
+/**
+ * Create a file tracker instance
+ * @returns FileTracker instance
+ */
+declare function createFileTracker(): FileTracker;
+
+/**
+ * Session Watcher - Monitor Claude Code session files for changes
+ *
+ * Uses Node.js fs.watch to monitor ~/.claude/projects/**\/*.jsonl files.
+ * Debounces events and triggers optimization when token threshold exceeded.
+ * Maintains per-session ContextPipeline instances.
+ */
+
+interface SessionWatcherConfig {
+    /** Sparn configuration */
+    config: SparnConfig;
+    /** Callback when optimization triggered */
+    onOptimize?: (sessionId: string, stats: SessionStats) => void;
+    /** Callback on error */
+    onError?: (error: Error) => void;
+}
+interface SessionStats {
+    /** Session ID */
+    sessionId: string;
+    /** Total tokens ingested */
+    totalTokens: number;
+    /** Current optimized tokens */
+    optimizedTokens: number;
+    /** Reduction percentage */
+    reduction: number;
+    /** Entry count */
+    entryCount: number;
+    /** Budget utilization */
+    budgetUtilization: number;
+}
+interface SessionWatcher {
+    /**
+     * Start watching Claude Code session files
+     * @returns Promise that resolves when watcher is ready
+     */
+    start(): Promise<void>;
+    /**
+     * Stop watching and cleanup
+     */
+    stop(): void;
+    /**
+     * Get statistics for all sessions
+     * @returns Array of session stats
+     */
+    getStats(): SessionStats[];
+    /**
+     * Get statistics for a specific session
+     * @param sessionId - Session ID
+     * @returns Session stats or null if not found
+     */
+    getSessionStats(sessionId: string): SessionStats | null;
+    /**
+     * Manually trigger optimization for a session
+     * @param sessionId - Session ID
+     */
+    optimizeSession(sessionId: string): void;
+}
+/**
+ * Create a session watcher instance
+ * @param config - Watcher configuration
+ * @returns SessionWatcher instance
+ */
+declare function createSessionWatcher(config: SessionWatcherConfig): SessionWatcher;
+
+/**
+ * Context Parser - Shared utilities for parsing agent contexts into memory entries
+ *
+ * Extracted from claude-code adapter to enable reuse across:
+ * - Adapters (claude-code, generic)
+ * - Real-time pipeline (streaming context)
+ * - Hooks (pre-prompt, post-tool-result)
+ */
+
+/**
+ * Block type classification for Claude Code context
+ */
+type BlockType = 'conversation' | 'tool' | 'result' | 'other';
+/**
+ * Parse Claude Code context into memory entries
+ * Handles conversation turns, tool uses, and results
+ * @param context - Raw context string
+ * @returns Array of memory entries
+ */
+declare function parseClaudeCodeContext(context: string): MemoryEntry[];
+/**
+ * Create a memory entry from a content block
+ * @param content - Block content
+ * @param type - Block type
+ * @param baseTime - Base timestamp
+ * @returns Memory entry
+ */
+declare function createEntry(content: string, type: BlockType, baseTime: number): MemoryEntry;
+/**
+ * Parse generic context (fallback for non-Claude-Code agents)
+ * Splits on double newlines, treats as paragraphs
+ * @param context - Raw context string
+ * @returns Array of memory entries
+ */
+declare function parseGenericContext(context: string): MemoryEntry[];
+
 /**
  * Content hashing utilities.
  * Uses SHA-256 for deduplication.
@@ -564,4 +1003,4 @@ declare function createLogger(verbose?: boolean): Logger;
  */
 declare function estimateTokens(text: string): number;
 
-export { type AgentAdapter, type AgentType, type BTSPEmbedder, type ConfidenceState, type ConfidenceStates, type ConfidenceStatesConfig, type ConsolidateResult, DEFAULT_CONFIG, type DecayConfig, type DuplicateGroup, type EngramScorer, type EngramScorerConfig, type KVMemory, type LogLevel, type Logger, type MemoryEntry, type MemoryQueryFilters, type OptimizationResult, type OptimizeOptions, type PruneResult, type PruningConfig, type SleepCompressor, type SparnConfig, type SparsePruner, type SparsePrunerConfig, type StateDistribution, type StatesConfig, type UIConfig, createBTSPEmbedder, createClaudeCodeAdapter, createConfidenceStates, createEngramScorer, createGenericAdapter, createKVMemory, createLogger, createSleepCompressor, createSparsePruner, estimateTokens, hashContent };
+export { type AgentAdapter, type AgentType, type BTSPEmbedder, type BlockType, type BudgetPruner, type BudgetPrunerConfig, type ConfidenceState, type ConfidenceStates, type ConfidenceStatesConfig, type ConsolidateResult, type ContextPipeline, type ContextPipelineConfig, type ContextPipelineStats, DEFAULT_CONFIG, type DaemonCommand, type DaemonStartResult, type DaemonStatusResult, type DaemonStopResult, type DecayConfig, type DuplicateGroup, type EngramScorer, type EngramScorerConfig, type FilePosition, type FileTracker, type IncrementalOptimizer, type IncrementalOptimizerConfig, type IncrementalOptimizerState, type KVMemory, type LogLevel, type Logger, type MemoryEntry, type MemoryQueryFilters, type OptimizationResult, type OptimizeOptions, type PruneResult, type PruningConfig, type RealtimeConfig, type SessionStats, type SessionWatcher, type SessionWatcherConfig, type SleepCompressor, type SparnConfig, type SparsePruner, type SparsePrunerConfig, type StateDistribution, type StatesConfig, type UIConfig, createBTSPEmbedder, createBudgetPruner, createBudgetPrunerFromConfig, createClaudeCodeAdapter, createConfidenceStates, createContextPipeline, createDaemonCommand, createEngramScorer, createEntry, createFileTracker, createGenericAdapter, createIncrementalOptimizer, createKVMemory, createLogger, createSessionWatcher, createSleepCompressor, createSparsePruner, estimateTokens, hashContent, parseClaudeCodeContext, parseGenericContext };