htm 0.0.11 → 0.0.15

data/docs/guides/working-memory.md

@@ -51,15 +51,10 @@ htm = HTM.new(
 
 ### Adding Memories
 
-When you add a node, it goes to both working and long-term memory:
+When you remember content, it goes to both working and long-term memory:
 
 ```ruby
-htm.add_node(
-  "fact_001",
-  "User prefers Ruby for scripting",
-  type: :fact,
-  importance: 7.0
-)
+htm.remember("User prefers Ruby for scripting")
 
 # Internally:
 # 1. Calculate token count
@@ -74,8 +69,8 @@ When you recall, memories are added to working memory:
 
 ```ruby
 memories = htm.recall(
-  timeframe: "last week",
-  topic: "database design"
+  "database design",
+  timeframe: "last week"
 )
 
 # Internally:
@@ -123,7 +118,6 @@ class MemoryMonitor
 
   def report
     wm = @htm.working_memory
-    stats = @htm.memory_stats
 
     puts "=== Working Memory Report ==="
     puts "Capacity: #{wm.max_tokens} tokens"
@@ -134,8 +128,7 @@ class MemoryMonitor
     puts "Average tokens per node: #{wm.token_count / wm.node_count}" if wm.node_count > 0
     puts
     puts "=== Long-term Memory ==="
-    puts "Total nodes: #{stats[:total_nodes]}"
-    puts "Database size: #{(stats[:database_size] / 1024.0 / 1024.0).round(2)} MB"
+    puts "Total nodes: #{HTM::Models::Node.count}"
   end
 
   def health_check
@@ -189,29 +182,24 @@ htm = HTM.new(
   working_memory_size: 10_000  # Small for demo
 )
 
-# Add important fact (will stay)
-htm.add_node(
-  "critical",
+# Add important fact (stays longer due to higher access frequency)
+critical_id = htm.remember(
   "Critical system password",
-  importance: 10.0
+  metadata: { priority: "critical" }
 )
 
-# Add many low-importance items
+# Add many items
 100.times do |i|
-  htm.add_node(
-    "temp_#{i}",
-    "Temporary note #{i}",
-    importance: 1.0
-  )
+  htm.remember("Temporary note #{i}")
 end
 
-# Check what survived
+# Check what survived in working memory
 wm = htm.working_memory
 puts "Surviving nodes: #{wm.node_count}"
 
-# Critical fact should still be there
-critical = htm.retrieve("critical")
-puts "Critical fact present: #{!critical.nil?}"
+# Critical fact is still in long-term memory
+critical = htm.long_term_memory.retrieve(critical_id)
+puts "Critical fact present in LTM: #{!critical.nil?}"
 ```
 
 ### Manual Eviction
@@ -235,35 +223,31 @@ end
 
 ## Best Practices
 
-### 1. Set Appropriate Importance
+### 1. Use Metadata for Priority Tracking
 
 ```ruby
-# Critical data: Never evict
-htm.add_node(
-  "api_key",
-  "Production API key",
-  importance: 10.0
+# Critical data: Mark with priority metadata
+htm.remember(
+  "Production API key: secret123",
+  metadata: { priority: "critical", category: "credentials" }
 )
 
-# Important context: Retain longer
-htm.add_node(
-  "user_goal",
+# Important context: Mark appropriately
+htm.remember(
   "User wants to optimize database",
-  importance: 8.0
+  metadata: { priority: "high", category: "goal" }
 )
 
-# Temporary context: Evict when needed
-htm.add_node(
-  "current_topic",
+# Temporary context
+htm.remember(
   "Discussing query optimization",
-  importance: 5.0
+  metadata: { priority: "medium", category: "context" }
 )
 
-# Disposable notes: Evict first
-htm.add_node(
-  "scratch",
+# Disposable notes
+htm.remember(
   "Temporary calculation result",
-  importance: 1.0
+  metadata: { priority: "low", category: "scratch" }
 )
 ```
 
@@ -305,16 +289,16 @@ Don't load unnecessary data into working memory:
 ```ruby
 # Bad: Load everything
 all_memories = htm.recall(
+  "anything",
   timeframe: "all time",
-  topic: "anything",
   limit: 1000
 )
 # This fills working memory with potentially irrelevant data
 
 # Good: Load what you need
 relevant = htm.recall(
+  "current project",
   timeframe: "last week",
-  topic: "current project",
   limit: 20
 )
 # This keeps working memory focused
@@ -322,25 +306,27 @@ relevant = htm.recall(
 
 ### 4. Clean Up When Done
 
-Remove temporary memories:
+Remove temporary memories when no longer needed:
 
 ```ruby
-def with_temporary_context(htm, key, value)
+def with_temporary_context(htm, value)
   # Add temporary context
-  htm.add_node(key, value, type: :context, importance: 2.0)
+  node_id = htm.remember(value, metadata: { temporary: true })
 
-  yield
+  result = yield
 
-  # Clean up
-  htm.forget(key, confirm: :confirmed)
+  # Clean up - soft delete by default (recoverable)
+  htm.forget(node_id)
+
+  result
 end
 
-with_temporary_context(htm, "scratch_001", "Temp data") do
+with_temporary_context(htm, "Temp calculation data") do
   # Use the temporary context
-  context = htm.create_context(strategy: :recent)
-  # ... do work
+  context = htm.working_memory.assemble_context(strategy: :recent)
+  # ... do work with context
 end
-# Temp data is now removed
+# Temp data is now soft-deleted
 ```
 
 ### 5. Batch Operations Carefully
@@ -348,23 +334,9 @@ end
 Be mindful when adding many memories at once:
 
 ```ruby
-# Risky: Might fill working memory quickly
-1000.times do |i|
-  htm.add_node("item_#{i}", "Data #{i}", importance: 5.0)
-end
-
-# Better: Add with appropriate importance
-1000.times do |i|
-  htm.add_node(
-    "item_#{i}",
-    "Data #{i}",
-    importance: 3.0  # Lower importance for bulk data
-  )
-end
-
-# Or: Monitor during batch operations
+# Add batch data with monitoring
 batch_data.each_with_index do |data, i|
-  htm.add_node("item_#{i}", data, importance: 5.0)
+  htm.remember(data, metadata: { batch: "import_001", index: i })
 
   # Check capacity every 100 items
   if i % 100 == 0
@@ -378,58 +350,56 @@ end
 
 ### Strategy 1: Sliding Window
 
-Keep only recent memories:
+Keep only recent memories by tracking node IDs:
 
 ```ruby
 class SlidingWindow
   def initialize(htm, window_size: 50)
     @htm = htm
     @window_size = window_size
-    @keys = []
+    @node_ids = []
   end
 
-  def add(key, value, **opts)
-    @htm.add_node(key, value, **opts)
-    @keys << key
+  def add(value, **opts)
+    node_id = @htm.remember(value, **opts)
+    @node_ids << node_id
 
-    # Evict oldest if window exceeded
-    if @keys.length > @window_size
-      oldest = @keys.shift
-      @htm.forget(oldest, confirm: :confirmed) rescue nil
+    # Forget oldest if window exceeded
+    if @node_ids.length > @window_size
+      oldest_id = @node_ids.shift
+      @htm.forget(oldest_id) rescue nil
     end
+
+    node_id
   end
 end
 ```
 
-### Strategy 2: Importance Thresholding
+### Strategy 2: Priority-Based Management
 
-Only keep high-importance memories:
+Use metadata to track priority:
 
 ```ruby
-class ImportanceFilter
-  def initialize(htm, min_importance: 7.0)
+class PriorityManager
+  def initialize(htm)
     @htm = htm
-    @min_importance = min_importance
   end
 
-  def add(key, value, importance:, **opts)
-    @htm.add_node(key, value, importance: importance, **opts)
+  def add(value, priority: "medium", **opts)
+    metadata = (opts[:metadata] || {}).merge(priority: priority)
+    node_id = @htm.remember(value, **opts.merge(metadata: metadata))
 
-    # If low importance and memory is tight, evict immediately
-    if importance < @min_importance &&
-       @htm.working_memory.utilization_percentage > 80
+    # If low priority and memory is tight, it will evict naturally
+    # HTM uses LFU + LRU eviction based on access patterns
 
-      # Let it evict naturally or remove from working memory
-      # (Note: HTM doesn't expose direct working memory removal,
-      #  so we rely on natural eviction)
-    end
+    node_id
   end
 end
 ```
 
 ### Strategy 3: Topic-Based Management
 
-Group memories by topic and manage separately:
+Group memories by topic using tags:
 
 ```ruby
 class TopicManager
@@ -438,15 +408,17 @@ class TopicManager
     @topics = Hash.new { |h, k| h[k] = [] }
   end
 
-  def add(key, value, topic:, **opts)
-    @htm.add_node(key, value, **opts)
-    @topics[topic] << key
+  def add(value, topic:, **opts)
+    tags = (opts[:tags] || []) + ["topic:#{topic}"]
+    node_id = @htm.remember(value, **opts.merge(tags: tags))
+    @topics[topic] << node_id
+    node_id
   end
 
   def clear_topic(topic)
-    keys = @topics[topic] || []
-    keys.each do |key|
-      @htm.forget(key, confirm: :confirmed) rescue nil
+    node_ids = @topics[topic] || []
+    node_ids.each do |node_id|
+      @htm.forget(node_id) rescue nil
     end
     @topics.delete(topic)
   end
@@ -471,8 +443,7 @@ medium = "A" * 100     # ~25 tokens
 long = "word " * 1000  # ~1000 tokens
 
 # Check token count of a string
-embedding_service = HTM::EmbeddingService.new
-tokens = embedding_service.count_tokens(long)
+tokens = HTM.configuration.count_tokens(long)
 puts "Token count: #{tokens}"
 ```
 
@@ -512,18 +483,18 @@ htm = HTM.new(robot_name: "Perf Test")
 
 # Add 1000 memories
 1000.times do |i|
-  htm.add_node("key_#{i}", "Value #{i}", importance: 5.0)
+  htm.remember("Value #{i}")
 end
 
 # Benchmark working memory access
 Benchmark.bm do |x|
-  x.report("create_context:") do
-    1000.times { htm.create_context(strategy: :balanced) }
+  x.report("assemble_context:") do
+    1000.times { htm.working_memory.assemble_context(strategy: :balanced) }
   end
 end
 
 # Typical results:
-# create_context: ~1ms per call
+# assemble_context: ~1ms per call
 ```
 
 ### Optimization Tips
@@ -532,7 +503,7 @@ end
 # 1. Avoid frequent context assembly
 # Bad: Assemble context every message
 def process_message(message)
-  context = htm.create_context  # Slow if called frequently
+  context = htm.working_memory.assemble_context(strategy: :balanced)  # Slow if called frequently
   llm.chat(context + message)
 end
 
@@ -542,7 +513,7 @@ end
 
 def process_message(message)
   if @context_cache.nil? || @context_age > 10
-    @context_cache = htm.create_context
+    @context_cache = htm.working_memory.assemble_context(strategy: :balanced)
     @context_age = 0
   end
   @context_age += 1
@@ -552,7 +523,7 @@ end
 
 # 2. Use appropriate token limits
 # Don't request more than your LLM can handle
-context = htm.create_context(
+context = htm.working_memory.assemble_context(
   strategy: :balanced,
   max_tokens: 100_000  # Match LLM's context window
 )
@@ -605,29 +576,25 @@ end
 **Issue: Working memory always full**
 
 ```ruby
-# Check if you're adding too much
-stats = htm.memory_stats
-wm_util = stats[:working_memory][:utilization]
+# Check working memory utilization
+wm_util = htm.working_memory.utilization_percentage
 
 if wm_util > 95
   puts "Working memory consistently full"
   puts "Solutions:"
-  puts "1. Increase working_memory_size"
-  puts "2. Lower importance of bulk data"
-  puts "3. Reduce recall limit"
-  puts "4. Clean up temporary data more frequently"
+  puts "1. Increase working_memory_size when creating HTM"
+  puts "2. Reduce recall limit"
+  puts "3. Clean up temporary data more frequently with forget()"
 end
 ```
 
 **Issue: Important data getting evicted**
 
 ```ruby
-# Increase importance of critical data
-htm.add_node(
-  "critical_data",
-  "Important information",
-  importance: 9.5  # High enough to avoid eviction
-)
+# HTM evicts based on access frequency and recency
+# Access important data more frequently to keep it in working memory
+# Or query it from long-term memory when needed:
+critical = htm.long_term_memory.retrieve(critical_node_id)
 ```
 
 **Issue: Memory utilization too low**
@@ -640,8 +607,8 @@ if wm_util < 20
   puts "Working memory underutilized"
   puts "Consider:"
   puts "1. Reducing working_memory_size to save RAM"
-  puts "2. Recalling more context"
-  puts "3. Using larger token limits in create_context"
+  puts "2. Recalling more context with larger limit"
+  puts "3. Using larger token limits in assemble_context"
 end
 ```
 
@@ -678,37 +645,37 @@ end
 
 monitor = Monitor.new(htm)
 
-# Add memories with different importance
+# Add memories with different priorities via metadata
 puts "Adding critical data..."
-htm.add_node("critical", "Critical system data", importance: 10.0)
+critical_id = htm.remember("Critical system data", metadata: { priority: "critical" })
 monitor.report
 
 puts "\nAdding important data..."
 10.times do |i|
-  htm.add_node("important_#{i}", "Important item #{i}", importance: 8.0)
+  htm.remember("Important item #{i}", metadata: { priority: "high" })
 end
 monitor.report
 
 puts "\nAdding regular data..."
 50.times do |i|
-  htm.add_node("regular_#{i}", "Regular item #{i}", importance: 5.0)
+  htm.remember("Regular item #{i}", metadata: { priority: "medium" })
 end
 monitor.report
 
 puts "\nAdding temporary data..."
 100.times do |i|
-  htm.add_node("temp_#{i}", "Temporary item #{i}", importance: 2.0)
+  htm.remember("Temporary item #{i}", metadata: { priority: "low" })
 end
 monitor.report
 
-# Check what survived
+# Check that critical data is still in long-term memory
 puts "\n=== Survival Check ==="
-critical = htm.retrieve("critical")
-puts "Critical survived: #{!critical.nil?}"
+critical = htm.long_term_memory.retrieve(critical_id)
+puts "Critical in LTM: #{!critical.nil?}"
 
-# Create context
+# Create context from working memory
 puts "\nCreating context..."
-context = htm.create_context(strategy: :important, max_tokens: 50_000)
+context = htm.working_memory.assemble_context(strategy: :balanced, max_tokens: 50_000)
 puts "Context length: #{context.length} characters"
 
 # Final stats

data/docs/images/telemetry-architecture.svg

@@ -0,0 +1,153 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 500" style="background: transparent;">
+  <defs>
+    <!-- Gradients for visual depth -->
+    <linearGradient id="appGradient" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#3b82f6;stop-opacity:0.9"/>
+      <stop offset="100%" style="stop-color:#1d4ed8;stop-opacity:0.9"/>
+    </linearGradient>
+    <linearGradient id="telemetryGradient" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#8b5cf6;stop-opacity:0.9"/>
+      <stop offset="100%" style="stop-color:#6d28d9;stop-opacity:0.9"/>
+    </linearGradient>
+    <linearGradient id="nullGradient" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#6b7280;stop-opacity:0.8"/>
+      <stop offset="100%" style="stop-color:#4b5563;stop-opacity:0.8"/>
+    </linearGradient>
+    <linearGradient id="sdkGradient" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#10b981;stop-opacity:0.9"/>
+      <stop offset="100%" style="stop-color:#059669;stop-opacity:0.9"/>
+    </linearGradient>
+    <linearGradient id="backendGradient" x1="0%" y1="0%" x2="0%" y2="100%">
+      <stop offset="0%" style="stop-color:#f59e0b;stop-opacity:0.9"/>
+      <stop offset="100%" style="stop-color:#d97706;stop-opacity:0.9"/>
+    </linearGradient>
+
+    <!-- Arrow marker -->
+    <marker id="arrowhead" markerWidth="10" markerHeight="7" refX="9" refY="3.5" orient="auto">
+      <polygon points="0 0, 10 3.5, 0 7" fill="#94a3b8"/>
+    </marker>
+    <marker id="arrowheadGreen" markerWidth="10" markerHeight="7" refX="9" refY="3.5" orient="auto">
+      <polygon points="0 0, 10 3.5, 0 7" fill="#10b981"/>
+    </marker>
+  </defs>
+
+  <!-- HTM Application Container -->
+  <rect x="50" y="30" width="700" height="280" rx="12" ry="12"
+        fill="none" stroke="#3b82f6" stroke-width="2" stroke-dasharray="5,5"/>
+  <text x="400" y="55" text-anchor="middle" fill="#e2e8f0" font-family="system-ui, sans-serif" font-size="16" font-weight="600">
+    HTM Application
+  </text>
+
+  <!-- Job boxes at top -->
+  <rect x="80" y="75" width="180" height="50" rx="8" ry="8" fill="url(#appGradient)"/>
+  <text x="170" y="105" text-anchor="middle" fill="#ffffff" font-family="system-ui, sans-serif" font-size="13" font-weight="500">
+    GenerateEmbeddingJob
+  </text>
+
+  <rect x="310" y="75" width="180" height="50" rx="8" ry="8" fill="url(#appGradient)"/>
+  <text x="400" y="105" text-anchor="middle" fill="#ffffff" font-family="system-ui, sans-serif" font-size="13" font-weight="500">
+    GenerateTagsJob
+  </text>
+
+  <rect x="540" y="75" width="180" height="50" rx="8" ry="8" fill="url(#appGradient)"/>
+  <text x="630" y="105" text-anchor="middle" fill="#ffffff" font-family="system-ui, sans-serif" font-size="13" font-weight="500">
+    Search Methods
+  </text>
+
+  <!-- Arrows from jobs to telemetry -->
+  <line x1="170" y1="125" x2="170" y2="155" stroke="#94a3b8" stroke-width="2" marker-end="url(#arrowhead)"/>
+  <line x1="400" y1="125" x2="400" y2="155" stroke="#94a3b8" stroke-width="2" marker-end="url(#arrowhead)"/>
+  <line x1="630" y1="125" x2="630" y2="155" stroke="#94a3b8" stroke-width="2" marker-end="url(#arrowhead)"/>
+
+  <!-- Horizontal connector line -->
+  <line x1="170" y1="155" x2="630" y2="155" stroke="#94a3b8" stroke-width="2"/>
+  <line x1="400" y1="155" x2="400" y2="170" stroke="#94a3b8" stroke-width="2" marker-end="url(#arrowhead)"/>
+
+  <!-- HTM::Telemetry box -->
+  <rect x="280" y="175" width="240" height="45" rx="8" ry="8" fill="url(#telemetryGradient)"/>
+  <text x="400" y="203" text-anchor="middle" fill="#ffffff" font-family="system-ui, sans-serif" font-size="14" font-weight="600">
+    HTM::Telemetry
+  </text>
+
+  <!-- Three branches from telemetry -->
+  <line x1="400" y1="220" x2="400" y2="240" stroke="#94a3b8" stroke-width="2"/>
+
+  <!-- Branch lines -->
+  <line x1="150" y1="240" x2="650" y2="240" stroke="#94a3b8" stroke-width="2"/>
+  <line x1="150" y1="240" x2="150" y2="255" stroke="#94a3b8" stroke-width="2" marker-end="url(#arrowhead)"/>
+  <line x1="400" y1="240" x2="400" y2="255" stroke="#94a3b8" stroke-width="2" marker-end="url(#arrowhead)"/>
+  <line x1="650" y1="240" x2="650" y2="255" stroke="#94a3b8" stroke-width="2" marker-end="url(#arrowhead)"/>
+
+  <!-- Three meter options -->
+  <rect x="70" y="260" width="160" height="55" rx="8" ry="8" fill="url(#nullGradient)"/>
+  <text x="150" y="283" text-anchor="middle" fill="#ffffff" font-family="system-ui, sans-serif" font-size="12" font-weight="500">
+    NullMeter
+  </text>
+  <text x="150" y="300" text-anchor="middle" fill="#d1d5db" font-family="system-ui, sans-serif" font-size="10">
+    (when disabled)
+  </text>
+
+  <rect x="320" y="260" width="160" height="55" rx="8" ry="8" fill="url(#sdkGradient)"/>
+  <text x="400" y="283" text-anchor="middle" fill="#ffffff" font-family="system-ui, sans-serif" font-size="12" font-weight="500">
+    OpenTelemetry
+  </text>
+  <text x="400" y="300" text-anchor="middle" fill="#d1fae5" font-family="system-ui, sans-serif" font-size="10">
+    SDK Meter
+  </text>
+
+  <rect x="570" y="260" width="160" height="55" rx="8" ry="8" fill="url(#nullGradient)"/>
+  <text x="650" y="283" text-anchor="middle" fill="#ffffff" font-family="system-ui, sans-serif" font-size="12" font-weight="500">
+    NullMeter
+  </text>
+  <text x="650" y="300" text-anchor="middle" fill="#d1d5db" font-family="system-ui, sans-serif" font-size="10">
+    (SDK missing)
+  </text>
+
+  <!-- Arrows to outcomes -->
+  <line x1="150" y1="315" x2="150" y2="345" stroke="#6b7280" stroke-width="2" stroke-dasharray="4,4"/>
+  <line x1="400" y1="315" x2="400" y2="345" stroke="#10b981" stroke-width="2" marker-end="url(#arrowheadGreen)"/>
+  <line x1="650" y1="315" x2="650" y2="345" stroke="#6b7280" stroke-width="2" stroke-dasharray="4,4"/>
+
+  <!-- Outcome labels -->
+  <text x="150" y="365" text-anchor="middle" fill="#9ca3af" font-family="system-ui, sans-serif" font-size="12" font-style="italic">
+    No-op
+  </text>
+
+  <text x="400" y="365" text-anchor="middle" fill="#10b981" font-family="system-ui, sans-serif" font-size="12" font-weight="500">
+    OTLP Export
+  </text>
+
+  <text x="650" y="365" text-anchor="middle" fill="#9ca3af" font-family="system-ui, sans-serif" font-size="12" font-style="italic">
+    No-op
+  </text>
+
+  <!-- Arrow to backend -->
+  <line x1="400" y1="380" x2="400" y2="410" stroke="#10b981" stroke-width="2" marker-end="url(#arrowheadGreen)"/>
+
+  <!-- Backend box -->
+  <rect x="250" y="420" width="300" height="60" rx="10" ry="10" fill="url(#backendGradient)"/>
+  <text x="400" y="448" text-anchor="middle" fill="#ffffff" font-family="system-ui, sans-serif" font-size="14" font-weight="600">
+    Observability Backend
+  </text>
+  <text x="400" y="468" text-anchor="middle" fill="#fef3c7" font-family="system-ui, sans-serif" font-size="11">
+    (Jaeger, Datadog, Prometheus, etc.)
+  </text>
+
+  <!-- Legend -->
+  <rect x="600" y="380" width="140" height="80" rx="6" ry="6" fill="#1e293b" fill-opacity="0.7" stroke="#475569" stroke-width="1"/>
+  <text x="670" y="398" text-anchor="middle" fill="#94a3b8" font-family="system-ui, sans-serif" font-size="10" font-weight="600">
+    LEGEND
+  </text>
+  <line x1="615" y1="410" x2="645" y2="410" stroke="#10b981" stroke-width="2"/>
+  <text x="720" y="414" text-anchor="end" fill="#d1d5db" font-family="system-ui, sans-serif" font-size="10">
+    Active flow
+  </text>
+  <line x1="615" y1="430" x2="645" y2="430" stroke="#6b7280" stroke-width="2" stroke-dasharray="4,4"/>
+  <text x="720" y="434" text-anchor="end" fill="#d1d5db" font-family="system-ui, sans-serif" font-size="10">
+    Disabled path
+  </text>
+  <circle cx="630" cy="450" r="6" fill="url(#telemetryGradient)"/>
+  <text x="720" y="454" text-anchor="end" fill="#d1d5db" font-family="system-ui, sans-serif" font-size="10">
+    HTM component
+  </text>
+</svg>