htm 0.0.11 → 0.0.15

data/lib/htm/working_memory_channel.rb

@@ -0,0 +1,250 @@
+# examples/robot_groups/lib/htm/working_memory_channel.rb
+# frozen_string_literal: true
+
+class HTM
+  # Provides real-time synchronization of working memory changes across multiple
+  # robots using PostgreSQL LISTEN/NOTIFY pub/sub mechanism.
+  #
+  # This class enables distributed robots to maintain synchronized working memory
+  # by broadcasting change notifications through PostgreSQL channels. When one robot
+  # adds, evicts, or clears working memory, all other robots in the group receive
+  # immediate notification.
+  #
+  # @example Basic usage
+  #   channel = HTM::WorkingMemoryChannel.new('support-team', db_config)
+  #
+  #   # Subscribe to changes
+  #   channel.on_change do |event, node_id, robot_id|
+  #     case event
+  #     when :added   then puts "Node #{node_id} added by robot #{robot_id}"
+  #     when :evicted then puts "Node #{node_id} evicted by robot #{robot_id}"
+  #     when :cleared then puts "Working memory cleared by robot #{robot_id}"
+  #     end
+  #   end
+  #
+  #   # Start listening in background thread
+  #   channel.start_listening
+  #
+  #   # Publish a change
+  #   channel.notify(:added, node_id: 123, robot_id: 456)
+  #
+  #   # Cleanup when done
+  #   channel.stop_listening
+  #
+  # @see HTM::RobotGroup Higher-level coordination using this channel
+  #
+  class WorkingMemoryChannel
+    # Prefix used for all PostgreSQL channel names
+    # @return [String]
+    CHANNEL_PREFIX = 'htm_wm'
+
+    # Number of notifications received since channel was created
+    # @return [Integer]
+    attr_reader :notifications_received
+
+    # Creates a new working memory channel for a robot group.
+    #
+    # The channel name is derived from the group name with non-alphanumeric
+    # characters replaced by underscores to ensure PostgreSQL compatibility.
+    #
+    # @param group_name [String] Name of the robot group (used to create unique channel)
+    # @param db_config [Hash] PostgreSQL connection configuration hash
+    # @option db_config [String] :host Database host
+    # @option db_config [Integer] :port Database port
+    # @option db_config [String] :dbname Database name
+    # @option db_config [String] :user Database user
+    # @option db_config [String] :password Database password (optional)
+    #
+    # @example
+    #   db_config = { host: 'localhost', port: 5432, dbname: 'htm_dev', user: 'postgres' }
+    #   channel = HTM::WorkingMemoryChannel.new('customer-support', db_config)
+    #
+    def initialize(group_name, db_config)
+      @group_name             = group_name
+      @channel                = "#{CHANNEL_PREFIX}_#{group_name.gsub(/[^a-z0-9_]/i, '_')}"
+      @db_config              = db_config
+      @listeners              = []
+      @listen_thread          = nil
+      @stop_requested         = false
+      @notifications_received = 0
+      @mutex                  = Mutex.new
+    end
+
+    # @!group Publishing
+
+    # Broadcasts a working memory change notification to all listeners.
+    #
+    # Uses PostgreSQL's pg_notify function to send a JSON payload containing
+    # the event type, affected node ID, originating robot ID, and timestamp.
+    #
+    # @param event [Symbol] Type of change (:added, :evicted, or :cleared)
+    # @param node_id [Integer, nil] ID of the affected node (nil for :cleared events)
+    # @param robot_id [Integer] ID of the robot that triggered the change
+    # @return [void]
+    #
+    # @example Notify that a node was added
+    #   channel.notify(:added, node_id: 123, robot_id: 1)
+    #
+    # @example Notify that working memory was cleared
+    #   channel.notify(:cleared, node_id: nil, robot_id: 1)
+    #
+    def notify(event, node_id:, robot_id:)
+      payload = {
+        event: event,
+        node_id: node_id,
+        robot_id: robot_id,
+        timestamp: Time.now.iso8601
+      }.to_json
+
+      with_connection do |conn|
+        conn.exec_params('SELECT pg_notify($1, $2)', [@channel, payload])
+      end
+    end
+
+    # @!endgroup
+
+    # @!group Subscribing
+
+    # Registers a callback to be invoked when working memory changes occur.
+    #
+    # Multiple callbacks can be registered; all will be called for each event.
+    # Callbacks are invoked synchronously within the listener thread.
+    #
+    # @yield [event, node_id, robot_id] Block called for each notification
+    # @yieldparam event [Symbol] Type of change (:added, :evicted, or :cleared)
+    # @yieldparam node_id [Integer, nil] ID of the affected node
+    # @yieldparam robot_id [Integer] ID of the robot that triggered the change
+    # @return [void]
+    #
+    # @example Register a change handler
+    #   channel.on_change do |event, node_id, robot_id|
+    #     puts "Received #{event} event for node #{node_id}"
+    #   end
+    #
+    def on_change(&callback)
+      @mutex.synchronize { @listeners << callback }
+    end
+
+    # Starts listening for notifications in a background thread.
+    #
+    # Creates a dedicated PostgreSQL connection that uses LISTEN to receive
+    # notifications. The thread polls every 0.5 seconds, allowing for clean
+    # shutdown via {#stop_listening}.
+    #
+    # @return [Thread] The background listener thread
+    #
+    # @example Start and verify listening
+    #   thread = channel.start_listening
+    #   puts "Listening: #{channel.listening?}"  # => true
+    #
+    def start_listening
+      @stop_requested = false
+      @listen_thread  = Thread.new do
+        listen_loop
+      end
+      @listen_thread.abort_on_exception = true
+      @listen_thread
+    end
+
+    # Stops the background listener thread.
+    #
+    # Signals the listener to stop, waits up to 0.5 seconds for clean exit,
+    # then forcefully terminates if still running. The PostgreSQL connection
+    # is closed automatically.
+    #
+    # @return [void]
+    #
+    # @example Clean shutdown
+    #   channel.stop_listening
+    #   puts "Listening: #{channel.listening?}"  # => false
+    #
+    def stop_listening
+      @stop_requested = true
+      # Give the thread a moment to exit cleanly
+      @listen_thread&.join(0.5)
+      @listen_thread&.kill if @listen_thread&.alive?
+      @listen_thread = nil
+    end
+
+    # @!endgroup
+
+    # @!group Status
+
+    # Checks if the listener thread is currently active.
+    #
+    # @return [Boolean] true if listening for notifications, false otherwise
+    #
+    # @example
+    #   channel.start_listening
+    #   channel.listening?  # => true
+    #   channel.stop_listening
+    #   channel.listening?  # => false
+    #
+    def listening?
+      @listen_thread&.alive? || false
+    end
+
+    # Returns the PostgreSQL channel name used for notifications.
+    #
+    # The channel name is derived from the group name with a prefix and
+    # sanitization of special characters.
+    #
+    # @return [String] The PostgreSQL LISTEN/NOTIFY channel name
+    #
+    # @example
+    #   channel = HTM::WorkingMemoryChannel.new('my-group', db_config)
+    #   channel.channel_name  # => "htm_wm_my_group"
+    #
+    def channel_name
+      @channel
+    end
+
+    # @!endgroup
+
+    private
+
+    def listen_loop
+      conn = PG.connect(@db_config)
+      conn.exec("LISTEN #{conn.escape_identifier(@channel)}")
+
+      until @stop_requested
+        # Wait for notification with timeout (allows checking @stop_requested)
+        conn.wait_for_notify(0.5) do |_channel, _pid, payload|
+          handle_notification(payload)
+        end
+      end
+    rescue PG::Error => e
+      unless @stop_requested
+        HTM.logger.error "WorkingMemoryChannel error: #{e.message}"
+        sleep 1
+        retry
+      end
+    ensure
+      conn&.close
+    end
+
+    def handle_notification(payload)
+      data = JSON.parse(payload, symbolize_names: true)
+
+      @mutex.synchronize do
+        @notifications_received += 1
+        @listeners.each do |callback|
+          callback.call(
+            data[:event].to_sym,
+            data[:node_id],
+            data[:robot_id]
+          )
+        end
+      end
+    rescue JSON::ParserError => e
+      HTM.logger.error "Invalid notification payload: #{e.message}"
+    end
+
+    def with_connection
+      conn = PG.connect(@db_config)
+      yield conn
+    ensure
+      conn&.close
+    end
+  end
+end

data/lib/htm.rb

@@ -10,14 +10,19 @@ require_relative "htm/long_term_memory"
 require_relative "htm/working_memory"
 require_relative "htm/embedding_service"
 require_relative "htm/tag_service"
+require_relative "htm/proposition_service"
 require_relative "htm/timeframe_extractor"
 require_relative "htm/timeframe"
 require_relative "htm/job_adapter"
 require_relative "htm/jobs/generate_embedding_job"
 require_relative "htm/jobs/generate_tags_job"
-require_relative "htm/loaders/paragraph_chunker"
+require_relative "htm/jobs/generate_propositions_job"
+require_relative "htm/loaders/markdown_chunker"
 require_relative "htm/loaders/markdown_loader"
 require_relative "htm/observability"
+require_relative "htm/telemetry"
+require_relative "htm/working_memory_channel"
+require_relative "htm/robot_group"
 
 require "pg"
 require "securerandom"
@@ -26,7 +31,7 @@ require "uri"
 # Load Rails integration if Rails is defined
 require_relative "htm/railtie" if defined?(Rails::Railtie)
 
-# HTM (Hierarchical Temporary Memory) - Intelligent memory management for LLM robots
+# HTM (Hierarchical Temporal Memory) - Intelligent memory management for LLM robots
 #
 # HTM implements a two-tier memory system:
 # - Working Memory: Token-limited, active context for immediate LLM use
@@ -163,6 +168,11 @@ class HTM
       # Only for NEW nodes - existing nodes already have embeddings/tags
       enqueue_embedding_job(node_id)
       enqueue_tags_job(node_id, manual_tags: tags)
+
+      # Enqueue proposition extraction if enabled and not already a proposition
+      if HTM.configuration.extract_propositions && !metadata[:is_proposition]
+        enqueue_propositions_job(node_id)
+      end
     else
       HTM.logger.info "Node #{node_id} already exists, linked to robot #{@robot_name} (remember_count: #{result[:robot_node].remember_count})"
 
@@ -228,7 +238,7 @@ class HTM
   #   memories = htm.recall("preferences", metadata: { source: "user" })
   #   memories = htm.recall("decisions", metadata: { confidence: 0.9, type: "architectural" })
   #
-  def recall(topic, timeframe: nil, limit: 20, strategy: :vector, with_relevance: false, query_tags: [], raw: false, metadata: {})
+  def recall(topic, timeframe: nil, limit: 20, strategy: :fulltext, with_relevance: false, query_tags: [], raw: false, metadata: {})
     # Validate inputs
     validate_timeframe!(timeframe)
     validate_positive_integer!(limit, "limit")
@@ -350,6 +360,51 @@ class HTM
     true
   end
 
+  # Forget all nodes whose content includes the given string
+  #
+  # Performs a soft delete on all matching nodes. The nodes remain in the
+  # database but are excluded from queries. Use case-insensitive LIKE matching.
+  #
+  # @param content_substring [String] Substring to search for in node content
+  # @param soft [Boolean] If true (default), soft delete; if false, permanent delete
+  # @param confirm [Symbol] Must be :confirmed to proceed with permanent deletion
+  # @return [Array<Integer>] Array of node IDs that were deleted
+  # @raise [ArgumentError] if content_substring is blank
+  # @raise [ArgumentError] if permanent deletion requested without confirmation
+  #
+  # @example Soft delete all nodes containing "deprecated"
+  #   htm.forget_content("deprecated")
+  #   # => [42, 56, 78]  # IDs of deleted nodes
+  #
+  # @example Permanent delete all nodes containing "test data"
+  #   htm.forget_content("test data", soft: false, confirm: :confirmed)
+  #
+  def forget_content(content_substring, soft: true, confirm: false)
+    raise ArgumentError, "Content substring cannot be blank" if content_substring.to_s.strip.empty?
+
+    # Permanent delete requires confirmation
+    if !soft && confirm != :confirmed
+      raise ArgumentError, "Permanent deletion requires confirm: :confirmed"
+    end
+
+    # Find all nodes containing the substring (case-insensitive)
+    matching_nodes = HTM::Models::Node.where("content ILIKE ?", "%#{content_substring}%")
+    node_ids = matching_nodes.pluck(:id)
+
+    if node_ids.empty?
+      HTM.logger.info "No nodes found containing: #{content_substring}"
+      return []
+    end
+
+    # Delete each matching node
+    node_ids.each do |node_id|
+      forget(node_id, soft: soft, confirm: confirm)
+    end
+
+    HTM.logger.info "Forgot #{node_ids.length} nodes containing: #{content_substring}"
+    node_ids
+  end
+
   # Restore a soft-deleted memory node
   #
   # @param node_id [Integer] ID of the soft-deleted node to restore
@@ -550,6 +605,14 @@ class HTM
     HTM.logger.error "Failed to enqueue tags job for node #{node_id}: #{e.message}"
   end
 
+  def enqueue_propositions_job(node_id)
+    # Enqueue proposition extraction using configured job backend
+    # Job will use HTM.extract_propositions which delegates to configured proposition_extractor
+    HTM::JobAdapter.enqueue(HTM::Jobs::GeneratePropositionsJob, node_id: node_id, robot_id: @robot_id)
+  rescue StandardError => e
+    HTM.logger.error "Failed to enqueue propositions job for node #{node_id}: #{e.message}"
+  end
+
   def add_to_working_memory(node)
     # Convert token_count to integer (may be String from database/cache)
     token_count = node['token_count'].to_i

data/lib/tasks/doc.rake

@@ -22,7 +22,7 @@ namespace :htm do
         "--plugin markdown",
         "--output-dir #{output_dir}",
         "--format markdown",
-        "--title 'HTM - Hierarchical Temporary Memory API'",
+        "--title 'HTM - Hierarchical Temporal Memory API'",
         "--markup markdown",
         "--charset utf-8",
         "--protected",

data/lib/tasks/htm.rake

@@ -68,21 +68,28 @@ namespace :htm do
       end
     end
 
-    desc "Test database connection"
-    task :test do
+    desc "Verify database connection (respects RAILS_ENV)"
+    task :verify do
       require 'htm'
-      config = HTM::Database.default_config
-      raise "Database not configured. Set HTM_DBURL environment variable." unless config
 
-      puts "Testing HTM database connection..."
+      env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+      config = HTM::ActiveRecordConfig.load_database_config
+
+      puts "Verifying HTM database connection (#{env})..."
       puts "  Host: #{config[:host]}"
       puts "  Port: #{config[:port]}"
-      puts "  Database: #{config[:dbname]}"
-      puts "  User: #{config[:user]}"
+      puts "  Database: #{config[:database]}"
+      puts "  User: #{config[:username]}"
 
       begin
         require 'pg'
-        conn = PG.connect(config)
+        conn = PG.connect(
+          host: config[:host],
+          port: config[:port],
+          dbname: config[:database],
+          user: config[:username],
+          password: config[:password]
+        )
 
         # Check pgvector
         pgvector = conn.exec("SELECT extversion FROM pg_extension WHERE extname='vector'").first
@@ -100,16 +107,18 @@ namespace :htm do
       end
     end
 
-    desc "Open PostgreSQL console"
+    desc "Open PostgreSQL console (respects RAILS_ENV)"
     task :console do
       require 'htm'
-      config = HTM::Database.default_config
-      raise "Database not configured. Set HTM_DBURL environment variable." unless config
 
+      env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+      config = HTM::ActiveRecordConfig.load_database_config
+
+      puts "Connecting to #{config[:database]} (#{env})..."
       exec "psql", "-h", config[:host],
                    "-p", config[:port].to_s,
-                   "-U", config[:user],
-                   "-d", config[:dbname]
+                   "-U", config[:username],
+                   "-d", config[:database]
     end
 
     desc "Seed database with sample data"
@@ -277,6 +286,120 @@ namespace :htm do
         puts "  Errors: #{errors}"
         puts "  Nodes with embeddings: #{final_with_embeddings}"
       end
+
+      desc "Rebuild propositions for all non-proposition nodes. Extracts atomic facts and creates new nodes."
+      task :propositions do
+        require 'htm'
+        require 'ruby-progressbar'
+
+        # Ensure database connection
+        HTM::ActiveRecordConfig.establish_connection!
+
+        # Find all non-proposition nodes (nodes that haven't been extracted from)
+        source_nodes = HTM::Models::Node.non_propositions
+        source_count = source_nodes.count
+
+        # Count existing proposition nodes
+        existing_propositions = HTM::Models::Node.propositions.count
+
+        puts "\nHTM Propositions Rebuild"
+        puts "=" * 50
+        puts "Current state:"
+        puts "  Source nodes (non-propositions): #{source_count}"
+        puts "  Existing proposition nodes: #{existing_propositions}"
+        puts "\nThis will extract propositions from ALL #{source_count} source nodes."
+        puts "Existing proposition nodes will be deleted and regenerated."
+        puts "This operation may take a long time depending on your LLM provider."
+        print "\nType 'yes' to confirm: "
+
+        confirmation = $stdin.gets&.strip
+        unless confirmation == 'yes'
+          puts "Aborted."
+          next
+        end
+
+        # Delete existing proposition nodes
+        if existing_propositions > 0
+          puts "\nDeleting #{existing_propositions} existing proposition nodes..."
+          deleted = HTM::Models::Node.propositions.delete_all
+          puts "  Deleted #{deleted} proposition nodes"
+        end
+
+        puts "\nExtracting propositions from #{source_count} nodes..."
+        puts "(This may take a while depending on your LLM provider)\n"
+
+        # Get a robot ID for linking proposition nodes
+        # Use the first robot or create a system robot
+        robot = HTM::Models::Robot.first || HTM::Models::Robot.create!(name: 'proposition_rebuilder')
+
+        # Create progress bar with ETA
+        progressbar = ProgressBar.create(
+          total: source_count,
+          format: '%t: |%B| %c/%C (%p%%) %e',
+          title: 'Extracting',
+          output: $stdout,
+          smoothing: 0.5
+        )
+
+        # Track stats
+        errors = 0
+        nodes_processed = 0
+        propositions_created = 0
+
+        source_nodes.find_each do |node|
+          begin
+            # Extract propositions
+            propositions = HTM::PropositionService.extract(node.content)
+
+            if propositions.any?
+              propositions.each do |proposition_text|
+                token_count = HTM.count_tokens(proposition_text)
+
+                # Create proposition node
+                prop_node = HTM::Models::Node.create!(
+                  content: proposition_text,
+                  token_count: token_count,
+                  metadata: { is_proposition: true, source_node_id: node.id }
+                )
+
+                # Link to robot
+                HTM::Models::RobotNode.find_or_create_by!(
+                  robot_id: robot.id,
+                  node_id: prop_node.id
+                )
+
+                # Generate embedding for proposition node
+                begin
+                  result = HTM::EmbeddingService.generate(proposition_text)
+                  prop_node.update!(embedding: result[:storage_embedding])
+                rescue StandardError => e
+                  progressbar.log "  Warning: Embedding failed for proposition: #{e.message}"
+                end
+
+                propositions_created += 1
+              end
+            end
+
+            nodes_processed += 1
+          rescue StandardError => e
+            errors += 1
+            progressbar.log "  Error on node #{node.id}: #{e.message}"
+          end
+
+          progressbar.increment
+        end
+
+        progressbar.finish
+
+        # Final stats
+        final_proposition_count = HTM::Models::Node.propositions.count
+
+        puts "\nRebuild complete!"
+        puts "  Source nodes processed: #{nodes_processed}"
+        puts "  Propositions created: #{propositions_created}"
+        puts "  Errors: #{errors}"
+        puts "  Total proposition nodes: #{final_proposition_count}"
+      end
     end
 
     namespace :schema do
@@ -293,6 +416,129 @@ namespace :htm do
       end
     end
 
+    desc "Create database if it doesn't exist (respects RAILS_ENV)"
+    task :create do
+      require 'htm'
+
+      env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+      config = HTM::ActiveRecordConfig.load_database_config
+      db_name = config[:database]
+
+      puts "Creating database: #{db_name} (#{env})"
+
+      admin_config = config.dup
+      admin_config[:database] = 'postgres'
+
+      begin
+        require 'pg'
+        admin_conn = PG.connect(
+          host: admin_config[:host],
+          port: admin_config[:port],
+          dbname: admin_config[:database],
+          user: admin_config[:username],
+          password: admin_config[:password]
+        )
+
+        result = admin_conn.exec_params(
+          "SELECT 1 FROM pg_database WHERE datname = $1",
+          [db_name]
+        )
+
+        if result.ntuples == 0
+          admin_conn.exec("CREATE DATABASE #{PG::Connection.quote_ident(db_name)}")
+          puts "✓ Database created: #{db_name}"
+
+          # Connect to new database and enable extensions
+          db_conn = PG.connect(
+            host: config[:host],
+            port: config[:port],
+            dbname: db_name,
+            user: config[:username],
+            password: config[:password]
+          )
+          %w[vector pg_trgm].each do |ext|
+            db_conn.exec("CREATE EXTENSION IF NOT EXISTS #{ext}")
+          end
+          db_conn.close
+          puts "✓ Extensions enabled (pgvector, pg_trgm)"
+        else
+          puts "✓ Database already exists: #{db_name}"
+        end
+
+        admin_conn.close
+      rescue PG::Error => e
+        puts "✗ Error: #{e.message}"
+        exit 1
+      end
+    end
+
+    namespace :tags do
+      desc "Soft delete orphaned tags and stale node_tags entries"
+      task :cleanup do
+        require 'htm'
+
+        # Ensure database connection
+        HTM::ActiveRecordConfig.establish_connection!
+
+        puts "\nHTM Tag Cleanup"
+        puts "=" * 50
+
+        # Step 1: Find active node_tags pointing to soft-deleted or missing nodes
+        stale_node_tags = HTM::Models::NodeTag
+          .joins("LEFT JOIN nodes ON nodes.id = node_tags.node_id")
+          .where("nodes.id IS NULL OR nodes.deleted_at IS NOT NULL")
+
+        stale_count = stale_node_tags.count
+
+        # Step 2: Find orphaned tags using the Tag.orphaned scope
+        orphaned_tags = HTM::Models::Tag.orphaned
+        orphan_count = orphaned_tags.count
+
+        if stale_count == 0 && orphan_count == 0
+          puts "No cleanup needed."
+          puts "  Stale node_tags entries: 0"
+          puts "  Orphaned tags: 0"
+          next
+        end
+
+        puts "Found:"
+        puts "  Stale node_tags entries: #{stale_count} (pointing to deleted/missing nodes)"
+        puts "  Orphaned tags: #{orphan_count} (no active nodes)"
+
+        if orphan_count > 0
+          puts "\nOrphaned tags:"
+          orphaned_tags.limit(20).pluck(:name).each do |name|
+            puts "  - #{name}"
+          end
+          puts "  ... and #{orphan_count - 20} more" if orphan_count > 20
+        end
+
+        print "\nSoft delete these entries? (yes/no): "
+        confirmation = $stdin.gets&.strip
+
+        unless confirmation == 'yes'
+          puts "Cancelled."
+          next
+        end
+
+        now = Time.current
+
+        # Soft delete stale node_tags first
+        if stale_count > 0
+          soft_deleted_node_tags = stale_node_tags.update_all(deleted_at: now)
+          puts "\nSoft deleted #{soft_deleted_node_tags} stale node_tags entries."
+        end
+
+        # Then soft delete orphaned tags
+        if orphan_count > 0
+          soft_deleted_tags = orphaned_tags.update_all(deleted_at: now)
+          puts "Soft deleted #{soft_deleted_tags} orphaned tags."
+        end
+
+        puts "\nCleanup complete (soft delete)."
+      end
+    end
+
   end
 
   namespace :doc do