htm 0.0.2 → 0.0.11

data/examples/robot_groups/multi_process.rb

@@ -0,0 +1,286 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Multi-Process Robot Group Demo
+#
+# Demonstrates real-time synchronization across SEPARATE PROCESSES
+# using PostgreSQL LISTEN/NOTIFY. Spawns robot_worker.rb as child processes.
+#
+# Key concepts:
+# 1. Cross-process sync via PostgreSQL LISTEN/NOTIFY
+# 2. Each robot runs as an independent process
+# 3. Failover when a process dies
+# 4. Dynamic scaling by spawning new processes
+#
+# Prerequisites:
+# 1. Set HTM_DBURL environment variable
+# 2. Initialize database schema: rake db_setup
+
+require_relative '../../lib/htm'
+require 'json'
+require 'timeout'
+require 'open3'
+
+# =============================================================================
+# Robot Process Manager
+# =============================================================================
+
+class RobotProcess
+  WORKER_SCRIPT = File.expand_path('robot_worker.rb', __dir__)
+
+  attr_reader :name, :pid
+
+  def initialize(name, group_name)
+    @name = name
+    @group_name = group_name
+
+    # Spawn the worker process
+    @stdin, @stdout, @stderr, @wait_thread = Open3.popen3(
+      { 'HTM_DBURL' => ENV['HTM_DBURL'] },
+      'ruby', WORKER_SCRIPT, name, group_name
+    )
+    @pid = @wait_thread.pid
+
+    # Start thread to read stderr (logs)
+    @log_thread = Thread.new do
+      @stderr.each_line { |line| print line }
+    rescue IOError
+      # Pipe closed
+    end
+  end
+
+  def send_command(cmd, **params)
+    command = { cmd: cmd }.merge(params)
+    @stdin.puts(command.to_json)
+    @stdin.flush
+
+    # Read response, skipping non-JSON lines
+    Timeout.timeout(10) do
+      loop do
+        line = @stdout.gets
+        return nil unless line
+
+        line = line.strip
+        next if line.empty?
+        next unless line.start_with?('{')
+
+        return JSON.parse(line, symbolize_names: true)
+      end
+    end
+  rescue Timeout::Error
+    { status: 'error', message: 'timeout' }
+  rescue Errno::EPIPE, IOError
+    { status: 'error', message: 'pipe closed' }
+  rescue JSON::ParserError => e
+    { status: 'error', message: "JSON parse error: #{e.message}" }
+  end
+
+  def shutdown
+    send_command('shutdown')
+    sleep 0.2
+    cleanup_pipes
+    @wait_thread.value
+  rescue Errno::EPIPE, IOError
+    # Already closed
+  end
+
+  def kill!
+    Process.kill('TERM', @pid)
+    cleanup_pipes
+    Process.wait(@pid)
+  rescue Errno::ESRCH, Errno::ECHILD
+    # Already dead
+  end
+
+  def alive?
+    @wait_thread && !@wait_thread.stop?
+  end
+
+  private
+
+  def cleanup_pipes
+    @stdin.close rescue nil
+    @stdout.close rescue nil
+    @stderr.close rescue nil
+    @log_thread.kill rescue nil
+  end
+end
+
+# =============================================================================
+# Demo
+# =============================================================================
+
+def run_demo
+  puts <<~BANNER
+    ╔══════════════════════════════════════════════════════════════╗
+    ║     HTM Multi-Process Robot Group Demo                       ║
+    ║     Real-time Sync via PostgreSQL LISTEN/NOTIFY              ║
+    ╚══════════════════════════════════════════════════════════════╝
+
+  BANNER
+
+  unless ENV['HTM_DBURL']
+    puts 'ERROR: HTM_DBURL not set.'
+    puts '  export HTM_DBURL="postgresql://user@localhost:5432/htm_development"'
+    exit 1
+  end
+
+  group_name = "demo-#{Time.now.to_i}"
+  robots = []
+
+  begin
+    # =========================================================================
+    # Scenario 1: Start robot processes
+    # =========================================================================
+    puts '━' * 60
+    puts "SCENARIO 1: Starting Robot Processes"
+    puts '━' * 60
+    puts
+
+    %w[robot-alpha robot-beta robot-gamma].each do |name|
+      robots << RobotProcess.new(name, group_name)
+    end
+
+    sleep 1.5
+
+    robots.each do |robot|
+      result = robot.send_command('ping')
+      status = result&.dig(:status) == 'ok' ? '✓' : '✗'
+      puts "  #{status} #{robot.name} (PID #{robot.pid})"
+    end
+    puts
+
+    # =========================================================================
+    # Scenario 2: Cross-process memory sharing
+    # =========================================================================
+    puts '━' * 60
+    puts "SCENARIO 2: Cross-Process Memory Sharing"
+    puts '━' * 60
+    puts
+    puts "  Alpha adds memories, others receive notifications..."
+    puts
+
+    alpha, beta, gamma = robots
+
+    alpha.send_command('remember', content: 'Customer John Smith prefers morning appointments.')
+    sleep 0.5
+    alpha.send_command('remember', content: 'Account #A-789 has a pending refund for $150.')
+    sleep 0.5
+    alpha.send_command('remember', content: 'Escalate billing issues to finance team.')
+    sleep 1.0
+
+    puts
+    puts "  Working memory status:"
+    robots.each do |robot|
+      status = robot.send_command('status')
+      next unless status&.dig(:status) == 'ok'
+
+      puts "    #{robot.name}: #{status[:working_memory_nodes]} nodes, #{status[:notifications_received]} notifications"
+    end
+    puts
+
+    # =========================================================================
+    # Scenario 3: Collaborative memory
+    # =========================================================================
+    puts '━' * 60
+    puts "SCENARIO 3: Collaborative Memory"
+    puts '━' * 60
+    puts
+
+    beta.send_command('remember', content: 'Customer confirmed refund was processed.')
+    sleep 0.5
+    gamma.send_command('remember', content: 'Customer wants email confirmation.')
+    sleep 1.0
+
+    puts "  Each robot recalls 'refund':"
+    robots.each do |robot|
+      result = robot.send_command('recall', query: 'refund', limit: 3)
+      next unless result&.dig(:status) == 'ok'
+
+      puts "    #{robot.name}: found #{result[:count]} memories"
+    end
+    puts
+
+    # =========================================================================
+    # Scenario 4: Failover
+    # =========================================================================
+    puts '━' * 60
+    puts "SCENARIO 4: Simulated Failover"
+    puts '━' * 60
+    puts
+
+    puts "  Killing robot-alpha..."
+    alpha.kill!
+    robots.delete(alpha)
+    sleep 0.5
+    puts "  ⚠ robot-alpha terminated"
+    puts
+
+    puts "  Remaining robots retain context:"
+    robots.each do |robot|
+      status = robot.send_command('status')
+      result = robot.send_command('recall', query: 'customer', limit: 5)
+      next unless status && result
+
+      puts "    #{robot.name}: #{status[:working_memory_nodes]} nodes, recalls #{result[:count]}"
+    end
+    puts
+    puts "  ✓ Failover successful"
+    puts
+
+    # =========================================================================
+    # Scenario 5: Dynamic scaling
+    # =========================================================================
+    puts '━' * 60
+    puts "SCENARIO 5: Dynamic Scaling"
+    puts '━' * 60
+    puts
+
+    puts "  Adding robot-delta..."
+    delta = RobotProcess.new('robot-delta', group_name)
+    robots << delta
+    sleep 1.5
+
+    result = delta.send_command('ping')
+    puts "  ✓ robot-delta (PID #{delta.pid}) joined" if result&.dig(:status) == 'ok'
+
+    delta.send_command('remember', content: 'New robot ready to assist.')
+    sleep 1.0
+
+    puts
+    puts "  Notifications received:"
+    robots.each do |robot|
+      status = robot.send_command('status')
+      next unless status&.dig(:status) == 'ok'
+
+      puts "    #{robot.name}: #{status[:notifications_received]}"
+    end
+    puts
+
+    # =========================================================================
+    # Summary
+    # =========================================================================
+    puts '━' * 60
+    puts "DEMO COMPLETE"
+    puts '━' * 60
+    puts
+    puts "  Demonstrated:"
+    puts "    • Real-time sync across #{robots.length} processes"
+    puts "    • PostgreSQL LISTEN/NOTIFY pub/sub"
+    puts "    • Failover with context preservation"
+    puts "    • Dynamic scaling"
+    puts
+
+  ensure
+    puts "Cleaning up..."
+    robots.each do |robot|
+      next unless robot.alive?
+
+      robot.shutdown
+      puts "  Stopped #{robot.name}"
+    end
+    puts "Done."
+  end
+end
+
+run_demo if __FILE__ == $PROGRAM_NAME

data/examples/robot_groups/robot_worker.rb

@@ -0,0 +1,136 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Robot Worker - Standalone process that participates in a robot group
+#
+# Usage: ruby robot_worker.rb <robot_name> <group_name>
+#
+# Communication:
+#   - Receives JSON commands via stdin
+#   - Sends JSON responses via stdout
+#   - Logs to stderr
+#
+# Commands:
+#   { "cmd": "ping" }
+#   { "cmd": "remember", "content": "..." }
+#   { "cmd": "recall", "query": "...", "limit": 5 }
+#   { "cmd": "status" }
+#   { "cmd": "shutdown" }
+
+require 'logger'
+require 'json'
+require_relative '../../lib/htm'
+require_relative 'lib/working_memory_channel'
+
+robot_name = ARGV[0]
+group_name = ARGV[1]
+
+unless robot_name && group_name
+  $stderr.puts "Usage: ruby robot_worker.rb <robot_name> <group_name>"
+  exit 1
+end
+
+def log(robot_name, message)
+  timestamp = Time.now.strftime('%H:%M:%S.%L')
+  $stderr.puts "[#{timestamp}] [#{robot_name}] #{message}"
+  $stderr.flush
+end
+
+# Configure HTM with logger to stderr (keep stdout clean for JSON)
+HTM.configure do |config|
+  config.embedding_provider = :ollama
+  config.embedding_model = 'nomic-embed-text:latest'
+  config.embedding_dimensions = 768
+  config.tag_provider = :ollama
+  config.tag_model = 'gemma3:latest'
+  config.logger = Logger.new($stderr, level: Logger::WARN)
+end
+
+log(robot_name, 'Starting up...')
+
+# Create HTM instance for this robot
+htm = HTM.new(robot_name: robot_name, working_memory_size: 8000)
+db_config = HTM::Database.default_config
+
+# Setup channel for cross-process notifications
+channel = WorkingMemoryChannel.new(group_name, db_config)
+
+# Track notifications received
+notifications_count = 0
+channel.on_change do |event, node_id, origin_robot_id|
+  next if origin_robot_id == htm.robot_id
+
+  notifications_count += 1
+  log(robot_name, "Received #{event} for node #{node_id}")
+
+  case event
+  when :added
+    node = HTM::Models::Node.find_by(id: node_id)
+    if node
+      htm.working_memory.add_from_sync(
+        id: node.id,
+        content: node.content,
+        token_count: node.token_count || 0,
+        created_at: node.created_at
+      )
+    end
+  when :evicted
+    htm.working_memory.remove_from_sync(node_id)
+  when :cleared
+    htm.working_memory.clear_from_sync
+  end
+end
+
+channel.start_listening
+log(robot_name, "Listening on channel: #{channel.channel_name}")
+
+# Process commands from stdin
+$stdin.each_line do |line|
+  begin
+    command = JSON.parse(line.strip, symbolize_names: true)
+
+    result = case command[:cmd]
+    when 'remember'
+      log(robot_name, "Remembering: #{command[:content][0..40]}...")
+      node_id = htm.remember(command[:content])
+      channel.notify(:added, node_id: node_id, robot_id: htm.robot_id)
+      log(robot_name, "Sent notification for node #{node_id}")
+      { status: 'ok', node_id: node_id }
+
+    when 'recall'
+      log(robot_name, "Recalling: #{command[:query]}")
+      results = htm.recall(command[:query], limit: command[:limit] || 5, strategy: :fulltext, raw: true)
+      { status: 'ok', count: results.length }
+
+    when 'status'
+      {
+        status: 'ok',
+        robot_id: htm.robot_id,
+        working_memory_nodes: htm.working_memory.node_count,
+        working_memory_tokens: htm.working_memory.token_count,
+        notifications_received: notifications_count
+      }
+
+    when 'ping'
+      { status: 'ok', message: 'pong', robot: robot_name }
+
+    when 'shutdown'
+      channel.stop_listening
+      log(robot_name, 'Shutting down.')
+      puts({ status: 'ok', message: 'bye' }.to_json)
+      $stdout.flush
+      exit 0
+
+    else
+      { status: 'error', message: "Unknown command: #{command[:cmd]}" }
+    end
+
+    puts result.to_json
+    $stdout.flush
+  rescue StandardError => e
+    puts({ status: 'error', message: e.message }.to_json)
+    $stdout.flush
+  end
+end
+
+channel.stop_listening

data/examples/robot_groups/same_process.rb

@@ -0,0 +1,229 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Robot Group Demo - Shared Working Memory & Failover
+#
+# This example demonstrates an application-level pattern for coordinating
+# multiple robots with shared working memory. Key concepts:
+#
+# 1. **Shared Working Memory**: Multiple robots can share the same working
+#    memory by having working_memory=true for the same nodes in robot_nodes.
+#
+# 2. **Active/Passive Roles**: Active robots participate in conversations;
+#    passive robots maintain synchronized context for instant failover.
+#
+# 3. **Failover**: When an active robot fails, a passive robot can take over
+#    with full context already loaded (warm standby).
+#
+# 4. **Real-time Sync**: PostgreSQL LISTEN/NOTIFY enables real-time
+#    synchronization of in-memory working memory across robots.
+#
+# Prerequisites:
+# 1. Set HTM_DBURL environment variable
+# 2. Initialize database schema: rake db_setup
+# 3. Install dependencies: bundle install
+
+require_relative '../../lib/htm'
+require 'json'
+
+require_relative 'lib/working_memory_channel'
+require_relative 'lib/robot_group'
+
+# =============================================================================
+# Demo Script
+# =============================================================================
+
+puts 'HTM Robot Group Demo - Shared Working Memory & Failover'
+puts '=' * 60
+
+unless ENV['HTM_DBURL']
+  puts 'ERROR: HTM_DBURL not set. Please set it:'
+  puts '  export HTM_DBURL="postgresql://postgres@localhost:5432/htm_development"'
+  exit 1
+end
+
+begin
+  # Configure HTM
+  puts "\n1. Configuring HTM..."
+  HTM.configure do |config|
+    config.embedding_provider = :ollama
+    config.embedding_model = 'nomic-embed-text:latest'
+    config.embedding_dimensions = 768
+    config.tag_provider = :ollama
+    config.tag_model = 'gemma3:latest'
+  end
+  puts '✓ HTM configured'
+
+  # ---------------------------------------------------------------------------
+  # Scenario 1: Create a high-availability robot group
+  # ---------------------------------------------------------------------------
+  puts "\n2. Creating robot group with primary + standby..."
+
+  group = RobotGroup.new(
+    name: 'customer-support-ha',
+    active: ['support-primary'],
+    passive: ['support-standby'],
+    max_tokens: 8000
+  )
+
+  status = group.status
+  puts "✓ Group created: #{status[:name]}"
+  puts "  Active:  #{status[:active].join(', ')}"
+  puts "  Passive: #{status[:passive].join(', ')}"
+
+  # ---------------------------------------------------------------------------
+  # Scenario 2: Add shared memories
+  # ---------------------------------------------------------------------------
+  puts "\n3. Adding memories to shared working memory..."
+
+  group.remember(
+    'Customer account #12345 prefers email communication over phone calls.',
+    originator: 'support-primary'
+  )
+  puts '  ✓ Remembered customer preference'
+
+  group.remember(
+    'Open ticket #789: Customer reported billing discrepancy on invoice dated Nov 15.',
+    originator: 'support-primary'
+  )
+  puts '  ✓ Remembered open ticket'
+
+  group.remember(
+    'Customer has been with us for 5 years and has premium tier subscription.',
+    originator: 'support-primary'
+  )
+  puts '  ✓ Remembered customer status'
+
+  # Brief pause for async jobs
+  sleep 0.3
+
+  # ---------------------------------------------------------------------------
+  # Scenario 3: Verify synchronization
+  # ---------------------------------------------------------------------------
+  puts "\n4. Verifying working memory synchronization..."
+
+  status = group.status
+  puts "  Working memory nodes: #{status[:working_memory_nodes]}"
+  puts "  Token utilization: #{(status[:token_utilization] * 100).round(1)}%"
+  puts "  In sync: #{status[:in_sync] ? '✓ Yes' : '✗ No'}"
+
+  # Force sync if needed
+  unless status[:in_sync]
+    result = group.sync_all
+    puts "  Synced #{result[:synced_nodes]} nodes to #{result[:members_updated]} members"
+  end
+
+  # ---------------------------------------------------------------------------
+  # Scenario 4: Simulate primary robot failure and failover
+  # ---------------------------------------------------------------------------
+  puts "\n5. Simulating failover scenario..."
+  puts "  ⚠ Primary robot 'support-primary' has stopped responding!"
+
+  # Failover to standby
+  group.failover!
+
+  status = group.status
+  puts "  Active robots now: #{status[:active].join(', ')}"
+  puts "  Passive robots now: #{status[:passive].join(', ') || '(none)'}"
+
+  # ---------------------------------------------------------------------------
+  # Scenario 5: Verify standby has full context
+  # ---------------------------------------------------------------------------
+  puts "\n6. Verifying standby has full context after failover..."
+
+  # Use fulltext search (doesn't require embeddings)
+  memories = group.recall('customer', limit: 5, strategy: :fulltext, raw: true)
+  puts "  ✓ Standby recalled #{memories.length} memories about 'customer'"
+  memories.each do |memory|
+    content = memory['content'] || memory[:content]
+    puts "    - #{content[0..55]}..."
+  end
+
+  # ---------------------------------------------------------------------------
+  # Scenario 6: Add a new active robot (scaling up)
+  # ---------------------------------------------------------------------------
+  puts "\n7. Adding a second active robot (scaling up)..."
+
+  group.add_active('support-secondary')
+  group.sync_robot('support-secondary')
+
+  status = group.status
+  puts "  ✓ Now running with #{status[:active].length} active robots"
+  puts "  Active: #{status[:active].join(', ')}"
+  puts "  In sync: #{status[:in_sync] ? '✓ Yes' : '✗ No'}"
+
+  # ---------------------------------------------------------------------------
+  # Scenario 7: Collaborative memory - both robots can add
+  # ---------------------------------------------------------------------------
+  puts "\n8. Demonstrating collaborative memory..."
+
+  # This memory is added through the group and synced to all
+  group.remember(
+    'Customer called again - issue escalated to billing department.',
+    originator: 'support-secondary'
+  )
+  puts '  ✓ Secondary robot added memory, synced to all'
+
+  status = group.status
+  puts "  Total shared memories: #{status[:working_memory_nodes]}"
+
+  # ---------------------------------------------------------------------------
+  # Scenario 8: Real-time sync via PostgreSQL LISTEN/NOTIFY
+  # ---------------------------------------------------------------------------
+  puts "\n9. Demonstrating real-time sync via PostgreSQL LISTEN/NOTIFY..."
+
+  # Check initial in-memory state of each robot
+  puts '  Initial in-memory working memory state:'
+  puts "    support-standby: #{group.instance_variable_get(:@active_robots)['support-standby']&.working_memory&.node_count || 0} nodes"
+  puts "    support-secondary: #{group.instance_variable_get(:@active_robots)['support-secondary']&.working_memory&.node_count || 0} nodes"
+
+  # Add a memory from secondary robot
+  puts "\n  Adding memory from support-secondary..."
+  group.remember(
+    'Resolution: Refund issued for $47.50 - billing error confirmed.',
+    originator: 'support-secondary'
+  )
+
+  # Give the LISTEN/NOTIFY a moment to propagate
+  sleep 0.2
+
+  # Check sync stats
+  sync_stats = group.sync_stats
+  puts "\n  Real-time sync statistics:"
+  puts "    Nodes synced via NOTIFY: #{sync_stats[:nodes_synced]}"
+  puts "    Evictions synced: #{sync_stats[:evictions_synced]}"
+  puts "    Channel notifications received: #{group.channel.notifications_received}"
+  puts "    Listener active: #{group.channel.listening? ? '✓ Yes' : '✗ No'}"
+
+  # Verify both robots have the memory in their in-memory cache
+  puts "\n  In-memory working memory after sync:"
+  group.instance_variable_get(:@active_robots).each do |name, htm|
+    puts "    #{name}: #{htm.working_memory.node_count} nodes"
+  end
+
+  # ---------------------------------------------------------------------------
+  # Summary
+  # ---------------------------------------------------------------------------
+  puts "\n" + '=' * 60
+  puts 'Demo Complete!'
+  puts "\nRobotGroup enables:"
+  puts '  • Shared working memory across multiple robots'
+  puts '  • Instant failover with warm standby (passive robots)'
+  puts '  • Collaborative context building'
+  puts '  • Dynamic scaling (add/remove robots)'
+  puts '  • Real-time sync via PostgreSQL LISTEN/NOTIFY'
+  puts "\nFinal group status:"
+  status = group.status
+  status.each { |k, v| puts "  #{k}: #{v}" }
+
+  # Cleanup
+  puts "\nCleaning up..."
+  group.clear_working_memory
+  group.shutdown
+  puts '✓ Cleared shared working memory and stopped listener'
+rescue StandardError => e
+  puts "\n✗ Error: #{e.message}"
+  puts e.backtrace.first(5).join("\n")
+  group&.shutdown # Ensure we clean up on error too
+  exit 1
+end