htm 0.0.14 → 0.0.15

data/lib/htm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class HTM
-  VERSION = '0.0.14'
+  VERSION = '0.0.15'
 end

data/lib/htm/working_memory_channel.rb

@@ -0,0 +1,250 @@
+# examples/robot_groups/lib/htm/working_memory_channel.rb
+# frozen_string_literal: true
+
+class HTM
+  # Provides real-time synchronization of working memory changes across multiple
+  # robots using PostgreSQL LISTEN/NOTIFY pub/sub mechanism.
+  #
+  # This class enables distributed robots to maintain synchronized working memory
+  # by broadcasting change notifications through PostgreSQL channels. When one robot
+  # adds, evicts, or clears working memory, all other robots in the group receive
+  # immediate notification.
+  #
+  # @example Basic usage
+  #   channel = HTM::WorkingMemoryChannel.new('support-team', db_config)
+  #
+  #   # Subscribe to changes
+  #   channel.on_change do |event, node_id, robot_id|
+  #     case event
+  #     when :added   then puts "Node #{node_id} added by robot #{robot_id}"
+  #     when :evicted then puts "Node #{node_id} evicted by robot #{robot_id}"
+  #     when :cleared then puts "Working memory cleared by robot #{robot_id}"
+  #     end
+  #   end
+  #
+  #   # Start listening in background thread
+  #   channel.start_listening
+  #
+  #   # Publish a change
+  #   channel.notify(:added, node_id: 123, robot_id: 456)
+  #
+  #   # Cleanup when done
+  #   channel.stop_listening
+  #
+  # @see HTM::RobotGroup Higher-level coordination using this channel
+  #
+  class WorkingMemoryChannel
+    # Prefix used for all PostgreSQL channel names
+    # @return [String]
+    CHANNEL_PREFIX = 'htm_wm'
+
+    # Number of notifications received since channel was created
+    # @return [Integer]
+    attr_reader :notifications_received
+
+    # Creates a new working memory channel for a robot group.
+    #
+    # The channel name is derived from the group name with non-alphanumeric
+    # characters replaced by underscores to ensure PostgreSQL compatibility.
+    #
+    # @param group_name [String] Name of the robot group (used to create unique channel)
+    # @param db_config [Hash] PostgreSQL connection configuration hash
+    # @option db_config [String] :host Database host
+    # @option db_config [Integer] :port Database port
+    # @option db_config [String] :dbname Database name
+    # @option db_config [String] :user Database user
+    # @option db_config [String] :password Database password (optional)
+    #
+    # @example
+    #   db_config = { host: 'localhost', port: 5432, dbname: 'htm_dev', user: 'postgres' }
+    #   channel = HTM::WorkingMemoryChannel.new('customer-support', db_config)
+    #
+    def initialize(group_name, db_config)
+      @group_name             = group_name
+      @channel                = "#{CHANNEL_PREFIX}_#{group_name.gsub(/[^a-z0-9_]/i, '_')}"
+      @db_config              = db_config
+      @listeners              = []
+      @listen_thread          = nil
+      @stop_requested         = false
+      @notifications_received = 0
+      @mutex                  = Mutex.new
+    end
+
+    # @!group Publishing
+
+    # Broadcasts a working memory change notification to all listeners.
+    #
+    # Uses PostgreSQL's pg_notify function to send a JSON payload containing
+    # the event type, affected node ID, originating robot ID, and timestamp.
+    #
+    # @param event [Symbol] Type of change (:added, :evicted, or :cleared)
+    # @param node_id [Integer, nil] ID of the affected node (nil for :cleared events)
+    # @param robot_id [Integer] ID of the robot that triggered the change
+    # @return [void]
+    #
+    # @example Notify that a node was added
+    #   channel.notify(:added, node_id: 123, robot_id: 1)
+    #
+    # @example Notify that working memory was cleared
+    #   channel.notify(:cleared, node_id: nil, robot_id: 1)
+    #
+    def notify(event, node_id:, robot_id:)
+      payload = {
+        event: event,
+        node_id: node_id,
+        robot_id: robot_id,
+        timestamp: Time.now.iso8601
+      }.to_json
+
+      with_connection do |conn|
+        conn.exec_params('SELECT pg_notify($1, $2)', [@channel, payload])
+      end
+    end
+
+    # @!endgroup
+
+    # @!group Subscribing
+
+    # Registers a callback to be invoked when working memory changes occur.
+    #
+    # Multiple callbacks can be registered; all will be called for each event.
+    # Callbacks are invoked synchronously within the listener thread.
+    #
+    # @yield [event, node_id, robot_id] Block called for each notification
+    # @yieldparam event [Symbol] Type of change (:added, :evicted, or :cleared)
+    # @yieldparam node_id [Integer, nil] ID of the affected node
+    # @yieldparam robot_id [Integer] ID of the robot that triggered the change
+    # @return [void]
+    #
+    # @example Register a change handler
+    #   channel.on_change do |event, node_id, robot_id|
+    #     puts "Received #{event} event for node #{node_id}"
+    #   end
+    #
+    def on_change(&callback)
+      @mutex.synchronize { @listeners << callback }
+    end
+
+    # Starts listening for notifications in a background thread.
+    #
+    # Creates a dedicated PostgreSQL connection that uses LISTEN to receive
+    # notifications. The thread polls every 0.5 seconds, allowing for clean
+    # shutdown via {#stop_listening}.
+    #
+    # @return [Thread] The background listener thread
+    #
+    # @example Start and verify listening
+    #   thread = channel.start_listening
+    #   puts "Listening: #{channel.listening?}"  # => true
+    #
+    def start_listening
+      @stop_requested = false
+      @listen_thread  = Thread.new do
+        listen_loop
+      end
+      @listen_thread.abort_on_exception = true
+      @listen_thread
+    end
+
+    # Stops the background listener thread.
+    #
+    # Signals the listener to stop, waits up to 0.5 seconds for clean exit,
+    # then forcefully terminates if still running. The PostgreSQL connection
+    # is closed automatically.
+    #
+    # @return [void]
+    #
+    # @example Clean shutdown
+    #   channel.stop_listening
+    #   puts "Listening: #{channel.listening?}"  # => false
+    #
+    def stop_listening
+      @stop_requested = true
+      # Give the thread a moment to exit cleanly
+      @listen_thread&.join(0.5)
+      @listen_thread&.kill if @listen_thread&.alive?
+      @listen_thread = nil
+    end
+
+    # @!endgroup
+
+    # @!group Status
+
+    # Checks if the listener thread is currently active.
+    #
+    # @return [Boolean] true if listening for notifications, false otherwise
+    #
+    # @example
+    #   channel.start_listening
+    #   channel.listening?  # => true
+    #   channel.stop_listening
+    #   channel.listening?  # => false
+    #
+    def listening?
+      @listen_thread&.alive? || false
+    end
+
+    # Returns the PostgreSQL channel name used for notifications.
+    #
+    # The channel name is derived from the group name with a prefix and
+    # sanitization of special characters.
+    #
+    # @return [String] The PostgreSQL LISTEN/NOTIFY channel name
+    #
+    # @example
+    #   channel = HTM::WorkingMemoryChannel.new('my-group', db_config)
+    #   channel.channel_name  # => "htm_wm_my_group"
+    #
+    def channel_name
+      @channel
+    end
+
+    # @!endgroup
+
+    private
+
+    def listen_loop
+      conn = PG.connect(@db_config)
+      conn.exec("LISTEN #{conn.escape_identifier(@channel)}")
+
+      until @stop_requested
+        # Wait for notification with timeout (allows checking @stop_requested)
+        conn.wait_for_notify(0.5) do |_channel, _pid, payload|
+          handle_notification(payload)
+        end
+      end
+    rescue PG::Error => e
+      unless @stop_requested
+        HTM.logger.error "WorkingMemoryChannel error: #{e.message}"
+        sleep 1
+        retry
+      end
+    ensure
+      conn&.close
+    end
+
+    def handle_notification(payload)
+      data = JSON.parse(payload, symbolize_names: true)
+
+      @mutex.synchronize do
+        @notifications_received += 1
+        @listeners.each do |callback|
+          callback.call(
+            data[:event].to_sym,
+            data[:node_id],
+            data[:robot_id]
+          )
+        end
+      end
+    rescue JSON::ParserError => e
+      HTM.logger.error "Invalid notification payload: #{e.message}"
+    end
+
+    def with_connection
+      conn = PG.connect(@db_config)
+      yield conn
+    ensure
+      conn&.close
+    end
+  end
+end

data/lib/htm.rb

@@ -21,6 +21,8 @@ require_relative "htm/loaders/markdown_chunker"
 require_relative "htm/loaders/markdown_loader"
 require_relative "htm/observability"
 require_relative "htm/telemetry"
+require_relative "htm/working_memory_channel"
+require_relative "htm/robot_group"
 
 require "pg"
 require "securerandom"

data/mkdocs.yml

@@ -199,6 +199,7 @@ nav:
       - Multi-Robot Usage: guides/multi-robot.md
       - Search Strategies: guides/search-strategies.md
       - Context Assembly: guides/context-assembly.md
+      - MCP Server: guides/mcp-server.md
   - API Reference:
       - api/index.md
       - HTM Class: api/htm.md
@@ -245,6 +246,7 @@ nav:
       - Setup: development/setup.md
       - Testing: development/testing.md
       - Contributing: development/contributing.md
+      - Rake Tasks Reference: development/rake-tasks.md
       - Database Schema: development/schema.md
       - Database Tables:
           - database/README.md

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: htm
 version: !ruby/object:Gem::Version
-  version: 0.0.14
+  version: 0.0.15
 platform: ruby
 authors:
 - Dewayne VanHoozer
@@ -299,11 +299,6 @@ files:
 - db/migrate/00005_create_tags.rb
 - db/migrate/00006_create_node_tags.rb
 - db/migrate/00007_create_robot_nodes.rb
-- db/migrate/00009_add_working_memory_to_robot_nodes.rb
-- db/migrate/00010_add_soft_delete_to_associations.rb
-- db/migrate/00011_add_performance_indexes.rb
-- db/migrate/00012_add_tags_trigram_index.rb
-- db/migrate/00013_enable_lz4_compression.rb
 - db/schema.sql
 - db/seed_data/README.md
 - db/seed_data/presidents.md
@@ -331,17 +326,26 @@ files:
 - docs/api/yard/HTM/LongTermMemory.md
 - docs/api/yard/HTM/NotFoundError.md
 - docs/api/yard/HTM/Observability.md
+- docs/api/yard/HTM/PropositionError.md
+- docs/api/yard/HTM/PropositionService.md
+- docs/api/yard/HTM/QueryCache.md
 - docs/api/yard/HTM/QueryTimeoutError.md
 - docs/api/yard/HTM/Railtie.md
 - docs/api/yard/HTM/ResourceExhaustedError.md
+- docs/api/yard/HTM/RobotGroup.md
+- docs/api/yard/HTM/SqlBuilder.md
 - docs/api/yard/HTM/TagError.md
 - docs/api/yard/HTM/TagService.md
+- docs/api/yard/HTM/Telemetry.md
+- docs/api/yard/HTM/Telemetry/NullInstrument.md
+- docs/api/yard/HTM/Telemetry/NullMeter.md
 - docs/api/yard/HTM/Timeframe.md
 - docs/api/yard/HTM/Timeframe/Result.md
 - docs/api/yard/HTM/TimeframeExtractor.md
 - docs/api/yard/HTM/TimeframeExtractor/Result.md
 - docs/api/yard/HTM/ValidationError.md
 - docs/api/yard/HTM/WorkingMemory.md
+- docs/api/yard/HTM/WorkingMemoryChannel.md
 - docs/api/yard/index.csv
 - docs/architecture/adrs/001-postgresql-timescaledb.md
 - docs/architecture/adrs/002-two-tier-memory.md
@@ -381,7 +385,9 @@ files:
 - docs/assets/images/htm-working-memory-architecture.svg
 - docs/assets/images/htm.jpg
 - docs/assets/images/htm_demo.gif
+- docs/assets/images/multi-provider-failover.svg
 - docs/assets/images/project-structure.svg
+- docs/assets/images/robot-group-architecture.svg
 - docs/assets/images/test-directory-structure.svg
 - docs/assets/js/mathjax.js
 - docs/assets/videos/htm_video.mp4
@@ -419,6 +425,7 @@ files:
 - docs/database_rake_tasks.md
 - docs/development/contributing.md
 - docs/development/index.md
+- docs/development/rake-tasks.md
 - docs/development/schema.md
 - docs/development/setup.md
 - docs/development/testing.md
@@ -430,8 +437,10 @@ files:
 - docs/guides/getting-started.md
 - docs/guides/index.md
 - docs/guides/long-term-memory.md
+- docs/guides/mcp-server.md
 - docs/guides/multi-robot.md
 - docs/guides/recalling-memories.md
+- docs/guides/robot-groups.md
 - docs/guides/search-strategies.md
 - docs/guides/working-memory.md
 - docs/images/htm-er-diagram.svg
@@ -450,8 +459,6 @@ files:
 - examples/example_app/app.rb
 - examples/file_loader_usage.rb
 - examples/mcp_client.rb
-- examples/robot_groups/lib/robot_group.rb
-- examples/robot_groups/lib/working_memory_channel.rb
 - examples/robot_groups/multi_process.rb
 - examples/robot_groups/robot_worker.rb
 - examples/robot_groups/same_process.rb
@@ -495,6 +502,7 @@ files:
 - lib/htm/proposition_service.rb
 - lib/htm/query_cache.rb
 - lib/htm/railtie.rb
+- lib/htm/robot_group.rb
 - lib/htm/sql_builder.rb
 - lib/htm/tag_service.rb
 - lib/htm/tasks.rb
@@ -503,6 +511,7 @@ files:
 - lib/htm/timeframe_extractor.rb
 - lib/htm/version.rb
 - lib/htm/working_memory.rb
+- lib/htm/working_memory_channel.rb
 - lib/tasks/db.rake
 - lib/tasks/doc.rake
 - lib/tasks/files.rake
@@ -532,7 +541,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.0.dev
+rubygems_version: 4.0.0
 specification_version: 4
 summary: Hierarchical Temporal Memory for LLM robots
 test_files: []

data/db/migrate/00009_add_working_memory_to_robot_nodes.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-class AddWorkingMemoryToRobotNodes < ActiveRecord::Migration[7.1]
-  def change
-    add_column :robot_nodes, :working_memory, :boolean, default: false, null: false,
-               comment: 'True if this node is currently in the robot working memory'
-
-    add_index :robot_nodes, [:robot_id, :working_memory],
-              where: 'working_memory = true',
-              name: 'idx_robot_nodes_working_memory'
-  end
-end

data/db/migrate/00010_add_soft_delete_to_associations.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-class AddSoftDeleteToAssociations < ActiveRecord::Migration[7.0]
-  def change
-    # Add deleted_at to robot_nodes for soft delete support
-    unless column_exists?(:robot_nodes, :deleted_at)
-      add_column :robot_nodes, :deleted_at, :datetime, null: true
-    end
-    unless index_exists?(:robot_nodes, :deleted_at)
-      add_index :robot_nodes, :deleted_at
-    end
-
-    # Add deleted_at to node_tags for soft delete support
-    unless column_exists?(:node_tags, :deleted_at)
-      add_column :node_tags, :deleted_at, :datetime, null: true
-    end
-    unless index_exists?(:node_tags, :deleted_at)
-      add_index :node_tags, :deleted_at
-    end
-
-    # Add deleted_at to tags for soft delete support
-    unless column_exists?(:tags, :deleted_at)
-      add_column :tags, :deleted_at, :datetime, null: true
-    end
-    unless index_exists?(:tags, :deleted_at)
-      add_index :tags, :deleted_at
-    end
-  end
-end

data/db/migrate/00011_add_performance_indexes.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-class AddPerformanceIndexes < ActiveRecord::Migration[7.1]
-  def change
-    # Partial index for soft-delete filter (used in almost every query)
-    # This complements idx_nodes_not_deleted_created_at for queries that
-    # don't need created_at ordering but still filter by deleted_at IS NULL
-    add_index :nodes, :id,
-              name: 'idx_nodes_active',
-              where: 'deleted_at IS NULL',
-              comment: 'Partial index for active (non-deleted) node queries'
-
-    # Composite index for embedding-based searches on active nodes
-    # Helps vector_search and hybrid_search which filter by deleted_at IS NULL
-    # and embedding IS NOT NULL
-    execute <<-SQL
-      CREATE INDEX idx_nodes_active_with_embedding ON nodes (id)
-        WHERE deleted_at IS NULL AND embedding IS NOT NULL
-    SQL
-  end
-end

data/db/migrate/00012_add_tags_trigram_index.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-class AddTagsTrigramIndex < ActiveRecord::Migration[7.0]
-  def up
-    # Add GIN trigram index on tags.name for fuzzy search
-    # Enables queries like: WHERE name % 'postgrsql' (typo-tolerant)
-    # Also speeds up LIKE '%pattern%' queries
-    execute <<~SQL
-      CREATE INDEX idx_tags_name_trgm ON tags USING gin(name gin_trgm_ops);
-    SQL
-  end
-
-  def down
-    execute <<~SQL
-      DROP INDEX IF EXISTS idx_tags_name_trgm;
-    SQL
-  end
-end

data/db/migrate/00013_enable_lz4_compression.rb

@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-class EnableLz4Compression < ActiveRecord::Migration[7.0]
-  def up
-    # Switch TOAST compression from pglz to lz4 for better read performance
-    # LZ4 decompression is ~32% faster than pglz with only marginally lower compression ratio
-    # See: https://www.depesz.com/2025/11/29/using-json-json-vs-jsonb-pglz-vs-lz4-key-optimization-parsing-speed/
-
-    # nodes.metadata - JSONB column for flexible key-value storage
-    execute <<~SQL
-      ALTER TABLE nodes ALTER COLUMN metadata SET COMPRESSION lz4;
-    SQL
-
-    # nodes.content - TEXT column containing memory content
-    execute <<~SQL
-      ALTER TABLE nodes ALTER COLUMN content SET COMPRESSION lz4;
-    SQL
-
-    # file_sources.frontmatter - JSONB column for parsed YAML frontmatter
-    execute <<~SQL
-      ALTER TABLE file_sources ALTER COLUMN frontmatter SET COMPRESSION lz4;
-    SQL
-
-    # Note: Existing rows retain their original compression until rewritten.
-    # To recompress existing data, run: VACUUM FULL nodes; VACUUM FULL file_sources;
-    # This is optional and can be done during a maintenance window.
-  end
-
-  def down
-    # Revert to default pglz compression
-    execute <<~SQL
-      ALTER TABLE nodes ALTER COLUMN metadata SET COMPRESSION pglz;
-    SQL
-
-    execute <<~SQL
-      ALTER TABLE nodes ALTER COLUMN content SET COMPRESSION pglz;
-    SQL
-
-    execute <<~SQL
-      ALTER TABLE file_sources ALTER COLUMN frontmatter SET COMPRESSION pglz;
-    SQL
-  end
-end