htm 0.0.1 → 0.0.2

data/lib/htm/long_term_memory.rb

@@ -39,44 +39,96 @@ class HTM
       end
     end
 
-    # Add a node to long-term memory
+    # Add a node to long-term memory (with deduplication)
     #
-    # Embeddings should be generated client-side and provided via the embedding parameter.
+    # If content already exists (by content_hash), links the robot to the existing
+    # node and updates timestamps. Otherwise creates a new node.
     #
     # @param content [String] Conversation message/utterance
-    # @param speaker [String] Who said it: 'user' or robot name
     # @param token_count [Integer] Token count
-    # @param robot_id [String] Robot identifier
+    # @param robot_id [Integer] Robot identifier
     # @param embedding [Array<Float>, nil] Pre-generated embedding vector
-    # @return [Integer] Node database ID
-    #
-    def add(content:, source:, token_count: 0, robot_id:, embedding: nil)
-      # Prepare embedding if provided
-      if embedding
-        # Pad embedding to 2000 dimensions if needed
-        actual_dimension = embedding.length
-        if actual_dimension < 2000
-          padded_embedding = embedding + Array.new(2000 - actual_dimension, 0.0)
-        else
-          padded_embedding = embedding
+    # @return [Hash] { node_id:, is_new:, robot_node: }
+    #
+    def add(content:, token_count: 0, robot_id:, embedding: nil)
+      content_hash = HTM::Models::Node.generate_content_hash(content)
+
+      # Check for existing node with same content
+      existing_node = HTM::Models::Node.find_by(content_hash: content_hash)
+
+      if existing_node
+        # Link robot to existing node (or update if already linked)
+        robot_node = link_robot_to_node(robot_id: robot_id, node: existing_node)
+
+        # Update the node's updated_at timestamp
+        existing_node.touch
+
+        {
+          node_id: existing_node.id,
+          is_new: false,
+          robot_node: robot_node
+        }
+      else
+        # Prepare embedding if provided
+        embedding_str = nil
+        if embedding
+          # Pad embedding to 2000 dimensions if needed
+          actual_dimension = embedding.length
+          padded_embedding = if actual_dimension < 2000
+            embedding + Array.new(2000 - actual_dimension, 0.0)
+          else
+            embedding
+          end
+          embedding_str = "[#{padded_embedding.join(',')}]"
         end
-        embedding_str = "[#{padded_embedding.join(',')}]"
+
+        # Create new node
+        node = HTM::Models::Node.create!(
+          content: content,
+          content_hash: content_hash,
+          token_count: token_count,
+          embedding: embedding_str,
+          embedding_dimension: embedding&.length
+        )
+
+        # Link robot to new node
+        robot_node = link_robot_to_node(robot_id: robot_id, node: node)
+
+        # Invalidate cache since database content changed
+        invalidate_cache!
+
+        {
+          node_id: node.id,
+          is_new: true,
+          robot_node: robot_node
+        }
       end
+    end
 
-      # Create node using ActiveRecord
-      node = HTM::Models::Node.create!(
-        content: content,
-        source: source,
-        token_count: token_count,
-        robot_id: robot_id,
-        embedding: embedding ? embedding_str : nil,
-        embedding_dimension: embedding ? embedding.length : nil
-      )
+    # Link a robot to a node (create or update robot_node record)
+    #
+    # @param robot_id [Integer] Robot ID
+    # @param node [HTM::Models::Node] Node to link
+    # @return [HTM::Models::RobotNode] The robot_node link record
+    #
+    def link_robot_to_node(robot_id:, node:)
+      robot_node = HTM::Models::RobotNode.find_by(robot_id: robot_id, node_id: node.id)
 
-      # Invalidate cache since database content changed
-      invalidate_cache!
+      if robot_node
+        # Existing link - record that robot remembered this again
+        robot_node.record_remember!
+      else
+        # New link
+        robot_node = HTM::Models::RobotNode.create!(
+          robot_id: robot_id,
+          node_id: node.id,
+          first_remembered_at: Time.current,
+          last_remembered_at: Time.current,
+          remember_count: 1
+        )
+      end
 
-      node.id
+      robot_node
     end
 
     # Retrieve a node by ID
@@ -240,13 +292,15 @@ class HTM
 
     # Mark nodes as evicted from working memory
     #
-    # @param node_ids [Array<Integer>] Node IDs
+    # Working memory state is now tracked per-robot in the working_memories table
+    # (optional persistence). The in-memory WorkingMemory class handles eviction
+    # tracking. This method is retained for API compatibility but is a no-op.
+    #
+    # @param node_ids [Array<Integer>] Node IDs (ignored)
     # @return [void]
     #
     def mark_evicted(node_ids)
-      return if node_ids.empty?
-
-      HTM::Models::Node.where(id: node_ids).update_all(in_working_memory: false)
+      # No-op: working memory is tracked in-memory or via WorkingMemoryEntry model
     end
 
     # Track access for multiple nodes (bulk operation)
@@ -294,7 +348,7 @@ class HTM
     def stats
       base_stats = {
         total_nodes: HTM::Models::Node.count,
-        nodes_by_robot: HTM::Models::Node.group(:robot_id).count,
+        nodes_by_robot: HTM::Models::RobotNode.group(:robot_id).count,
         total_tags: HTM::Models::Tag.count,
         oldest_memory: HTM::Models::Node.minimum(:created_at),
         newest_memory: HTM::Models::Node.maximum(:created_at),
@@ -574,6 +628,32 @@ class HTM
         .map { |tag| { name: tag.name, usage_count: tag.usage_count } }
     end
 
+    # Find tags that match terms in the query
+    #
+    # Searches the tags table for tags where any hierarchy level matches
+    # query words. For example, query "PostgreSQL database" would match
+    # tags like "database:postgresql", "database:sql", etc.
+    #
+    # @param query [String] Search query
+    # @return [Array<String>] Matching tag names
+    #
+    def find_query_matching_tags(query)
+      return [] if query.nil? || query.strip.empty?
+
+      # Extract words from query (lowercase, 3+ chars)
+      words = query.downcase.split(/\s+/).select { |w| w.length >= 3 }
+      return [] if words.empty?
+
+      # Build LIKE conditions for each word
+      # Match tags where any part of the hierarchy contains the word
+      conditions = words.map { |w| "name ILIKE ?" }
+      values = words.map { |w| "%#{w}%" }
+
+      HTM::Models::Tag
+        .where(conditions.join(' OR '), *values)
+        .pluck(:name)
+    end
+
     private
 
     # Generate cache key for query
@@ -682,7 +762,7 @@ class HTM
 
       result = ActiveRecord::Base.connection.select_all(
         <<~SQL,
-          SELECT id, content, source, access_count, created_at, robot_id, token_count,
+          SELECT id, content, access_count, created_at, token_count,
                  1 - (embedding <=> '#{embedding_str}'::vector) as similarity
           FROM nodes
           WHERE created_at BETWEEN '#{timeframe.begin.iso8601}' AND '#{timeframe.end.iso8601}'
@@ -710,7 +790,7 @@ class HTM
       result = ActiveRecord::Base.connection.select_all(
         ActiveRecord::Base.sanitize_sql_array([
           <<~SQL,
-            SELECT id, content, source, access_count, created_at, robot_id, token_count,
+            SELECT id, content, access_count, created_at, token_count,
                    ts_rank(to_tsvector('english', content), plainto_tsquery('english', ?)) as rank
             FROM nodes
             WHERE created_at BETWEEN ? AND ?
@@ -731,15 +811,17 @@ class HTM
 
     # Uncached hybrid search
     #
-    # Generates query embedding client-side, then combines full-text search for
-    # candidate selection with vector similarity for ranking.
+    # Generates query embedding client-side, then combines:
+    # 1. Full-text search for content matching
+    # 2. Tag matching for categorical relevance
+    # 3. Vector similarity for semantic ranking
     #
     # @param timeframe [Range] Time range to search
     # @param query [String] Search query
     # @param limit [Integer] Maximum results
     # @param embedding_service [Object] Service to generate query embedding
     # @param prefilter_limit [Integer] Candidates to consider
-    # @return [Array<Hash>] Matching nodes
+    # @return [Array<Hash>] Matching nodes with similarity and tag_boost scores
     #
     def search_hybrid_uncached(timeframe:, query:, limit:, embedding_service:, prefilter_limit:)
       # Generate query embedding client-side
@@ -753,26 +835,107 @@ class HTM
       # Convert to PostgreSQL vector format
       embedding_str = "[#{query_embedding.join(',')}]"
 
-      result = ActiveRecord::Base.connection.select_all(
-        ActiveRecord::Base.sanitize_sql_array([
-          <<~SQL,
-            WITH candidates AS (
-              SELECT id, content, source, access_count, created_at, robot_id, token_count, embedding
-              FROM nodes
-              WHERE created_at BETWEEN ? AND ?
-              AND to_tsvector('english', content) @@ plainto_tsquery('english', ?)
-              AND embedding IS NOT NULL
+      # Find tags that match query terms
+      matching_tags = find_query_matching_tags(query)
+
+      # Build the hybrid query
+      # If we have matching tags, include nodes with those tags in the candidate pool
+      # NOTE: Hybrid search includes nodes without embeddings using a default
+      # similarity score of 0.5. This allows newly created nodes to appear in
+      # search results immediately (via fulltext matching) before their embeddings
+      # are generated by background jobs. Useful for demos with short timeframes
+      # (seconds) where async embedding generation hasn't completed yet.
+      # In production with longer timeframes, embeddings are typically ready
+      # within 1-5 seconds, so this fallback is rarely used.
+
+      if matching_tags.any?
+        # Escape tag names for SQL
+        tag_list = matching_tags.map { |t| ActiveRecord::Base.connection.quote(t) }.join(', ')
+        result = ActiveRecord::Base.connection.select_all(
+          ActiveRecord::Base.sanitize_sql_array([
+            <<~SQL,
+              WITH fulltext_candidates AS (
+                -- Nodes matching full-text search (with or without embeddings)
+                SELECT DISTINCT n.id, n.content, n.access_count, n.created_at, n.token_count, n.embedding
+                FROM nodes n
+                WHERE n.created_at BETWEEN ? AND ?
+                AND to_tsvector('english', n.content) @@ plainto_tsquery('english', ?)
+                LIMIT ?
+              ),
+              tag_candidates AS (
+                -- Nodes matching relevant tags (with or without embeddings)
+                SELECT DISTINCT n.id, n.content, n.access_count, n.created_at, n.token_count, n.embedding
+                FROM nodes n
+                JOIN node_tags nt ON nt.node_id = n.id
+                JOIN tags t ON t.id = nt.tag_id
+                WHERE n.created_at BETWEEN ? AND ?
+                AND t.name IN (#{tag_list})
+                LIMIT ?
+              ),
+              all_candidates AS (
+                SELECT * FROM fulltext_candidates
+                UNION
+                SELECT * FROM tag_candidates
+              ),
+              scored AS (
+                SELECT
+                  ac.id, ac.content, ac.access_count, ac.created_at, ac.token_count,
+                  CASE
+                    WHEN ac.embedding IS NOT NULL THEN 1 - (ac.embedding <=> '#{embedding_str}'::vector)
+                    ELSE 0.5  -- Default similarity for nodes without embeddings
+                  END as similarity,
+                  COALESCE((
+                    SELECT COUNT(DISTINCT t.name)::float / ?
+                    FROM node_tags nt
+                    JOIN tags t ON t.id = nt.tag_id
+                    WHERE nt.node_id = ac.id AND t.name IN (#{tag_list})
+                  ), 0) as tag_boost
+                FROM all_candidates ac
+              )
+              SELECT id, content, access_count, created_at, token_count,
+                     similarity, tag_boost,
+                     (similarity * 0.7 + tag_boost * 0.3) as combined_score
+              FROM scored
+              ORDER BY combined_score DESC
               LIMIT ?
-            )
-            SELECT id, content, source, access_count, created_at, robot_id, token_count,
-                   1 - (embedding <=> '#{embedding_str}'::vector) as similarity
-            FROM candidates
-            ORDER BY embedding <=> '#{embedding_str}'::vector
-            LIMIT ?
-          SQL
-          timeframe.begin, timeframe.end, query, prefilter_limit, limit
-        ])
-      )
+            SQL
+            timeframe.begin, timeframe.end, query, prefilter_limit,
+            timeframe.begin, timeframe.end, prefilter_limit,
+            matching_tags.length.to_f,
+            limit
+          ])
+        )
+      else
+        # No matching tags, fall back to standard hybrid (fulltext + vector)
+        # Include nodes without embeddings with a default similarity score
+        result = ActiveRecord::Base.connection.select_all(
+          ActiveRecord::Base.sanitize_sql_array([
+            <<~SQL,
+              WITH candidates AS (
+                SELECT id, content, access_count, created_at, token_count, embedding
+                FROM nodes
+                WHERE created_at BETWEEN ? AND ?
+                AND to_tsvector('english', content) @@ plainto_tsquery('english', ?)
+                LIMIT ?
+              )
+              SELECT id, content, access_count, created_at, token_count,
+                     CASE
+                       WHEN embedding IS NOT NULL THEN 1 - (embedding <=> '#{embedding_str}'::vector)
+                       ELSE 0.5  -- Default similarity for nodes without embeddings
+                     END as similarity,
+                     0.0 as tag_boost,
+                     CASE
+                       WHEN embedding IS NOT NULL THEN 1 - (embedding <=> '#{embedding_str}'::vector)
+                       ELSE 0.5  -- Default score for nodes without embeddings (fulltext matched)
+                     END as combined_score
+              FROM candidates
+              ORDER BY combined_score DESC
+              LIMIT ?
+            SQL
+            timeframe.begin, timeframe.end, query, prefilter_limit, limit
+          ])
+        )
+      end
 
       # Track access for retrieved nodes
       node_ids = result.map { |r| r['id'] }

data/lib/htm/models/node.rb

@@ -1,9 +1,14 @@
 # frozen_string_literal: true
 
+require 'digest'
+
 class HTM
   module Models
     # Node model - represents a memory node (conversation message)
     #
+    # Nodes are globally unique by content (via content_hash) and can be
+    # linked to multiple robots through the robot_nodes join table.
+    #
     # Nearest Neighbor Search (via neighbor gem):
     #   # Find 5 nearest neighbors by cosine distance
     #   neighbors = Node.nearest_neighbors(:embedding, query_vector, distance: "cosine").limit(5)
@@ -18,8 +23,9 @@ class HTM
     class Node < ActiveRecord::Base
       self.table_name = 'nodes'
 
-      # Associations
-      belongs_to :robot, class_name: 'HTM::Models::Robot', foreign_key: 'robot_id', primary_key: 'id'
+      # Associations - Many-to-many with robots via robot_nodes
+      has_many :robot_nodes, class_name: 'HTM::Models::RobotNode', dependent: :destroy
+      has_many :robots, through: :robot_nodes, class_name: 'HTM::Models::Robot'
       has_many :node_tags, class_name: 'HTM::Models::NodeTag', dependent: :destroy
       has_many :tags, through: :node_tags, class_name: 'HTM::Models::Tag'
 
@@ -28,21 +34,41 @@ class HTM
 
       # Validations
       validates :content, presence: true
-      validates :robot_id, presence: true
+      validates :content_hash, presence: true, uniqueness: true
       validates :embedding_dimension, numericality: { greater_than: 0, less_than_or_equal_to: 2000 }, allow_nil: true
 
       # Callbacks
+      before_validation :set_content_hash, if: -> { content_hash.blank? && content.present? }
       before_create :set_defaults
       before_save :update_timestamps
 
       # Scopes
-      scope :by_robot, ->(robot_id) { where(robot_id: robot_id) }
-      scope :by_source, ->(source) { where(source: source) }
-      scope :in_working_memory, -> { where(in_working_memory: true) }
+      scope :by_robot, ->(robot_id) { joins(:robot_nodes).where(robot_nodes: { robot_id: robot_id }) }
       scope :recent, -> { order(created_at: :desc) }
       scope :in_timeframe, ->(start_time, end_time) { where(created_at: start_time..end_time) }
       scope :with_embeddings, -> { where.not(embedding: nil) }
 
+      # Class methods
+
+      # Find a node by content hash, or return nil
+      #
+      # @param content [String] The content to search for
+      # @return [Node, nil] The existing node or nil
+      #
+      def self.find_by_content(content)
+        hash = generate_content_hash(content)
+        find_by(content_hash: hash)
+      end
+
+      # Generate SHA-256 hash for content
+      #
+      # @param content [String] Content to hash
+      # @return [String] 64-character hex hash
+      #
+      def self.generate_content_hash(content)
+        Digest::SHA256.hexdigest(content.to_s)
+      end
+
       # Instance methods
 
       # Find nearest neighbors to this node's embedding
@@ -94,8 +120,11 @@ class HTM
 
       private
 
+      def set_content_hash
+        self.content_hash = self.class.generate_content_hash(content)
+      end
+
       def set_defaults
-        self.in_working_memory ||= false
         self.created_at ||= Time.current
         self.updated_at ||= Time.current
         self.last_accessed ||= Time.current

data/lib/htm/models/robot.rb

@@ -3,12 +3,18 @@
 class HTM
   module Models
     # Robot model - represents an LLM agent using the HTM system
+    #
+    # Robots can share memories through the many-to-many relationship with nodes.
+    # When a robot is deleted, only the robot_nodes links are removed; shared
+    # nodes remain in the database for other robots.
+    #
     class Robot < ActiveRecord::Base
       self.table_name = 'robots'
 
-      # Associations
-      has_many :nodes, class_name: 'HTM::Models::Node', dependent: :destroy
-      has_many :operation_logs, class_name: 'HTM::Models::OperationLog', dependent: :destroy
+      # Associations - Many-to-many with nodes via robot_nodes
+      # dependent: :destroy removes links only, NOT the shared nodes
+      has_many :robot_nodes, class_name: 'HTM::Models::RobotNode', dependent: :destroy
+      has_many :nodes, through: :robot_nodes, class_name: 'HTM::Models::Node'
 
       # Validations
       validates :name, presence: true
@@ -34,10 +40,30 @@ class HTM
         nodes.recent.limit(limit)
       end
 
+      # Get nodes with their remember metadata for this robot
+      #
+      # @param limit [Integer] Max nodes to return
+      # @return [Array<Hash>] Nodes with remember_count, first/last_remembered_at
+      #
+      def nodes_with_metadata(limit = 10)
+        robot_nodes
+          .includes(:node)
+          .order(last_remembered_at: :desc)
+          .limit(limit)
+          .map do |rn|
+            {
+              node: rn.node,
+              remember_count: rn.remember_count,
+              first_remembered_at: rn.first_remembered_at,
+              last_remembered_at: rn.last_remembered_at
+            }
+          end
+      end
+
       def memory_summary
         {
           total_nodes: nodes.count,
-          in_working_memory: nodes.in_working_memory.count,
+          in_working_memory: HTM::Models::WorkingMemoryEntry.where(robot_id: id).count,
           with_embeddings: nodes.with_embeddings.count
         }
       end

data/lib/htm/models/robot_node.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+class HTM
+  module Models
+    # RobotNode Join Model - Links robots to nodes (many-to-many)
+    #
+    # This model represents the relationship between a robot and a node,
+    # tracking when and how many times a robot has "remembered" a piece of content.
+    #
+    # @example Find all robots that remember a node
+    #   node.robots
+    #
+    # @example Find all nodes a robot remembers
+    #   robot.nodes
+    #
+    # @example Track remember activity
+    #   link = RobotNode.find_by(robot: robot, node: node)
+    #   link.remember_count  # => 3
+    #   link.first_remembered_at
+    #   link.last_remembered_at
+    #
+    class RobotNode < ActiveRecord::Base
+      self.table_name = 'robot_nodes'
+
+      belongs_to :robot, class_name: 'HTM::Models::Robot'
+      belongs_to :node, class_name: 'HTM::Models::Node'
+
+      validates :robot_id, presence: true
+      validates :node_id, presence: true
+      validates :robot_id, uniqueness: { scope: :node_id, message: 'already linked to this node' }
+
+      # Scopes
+      scope :recent, -> { order(last_remembered_at: :desc) }
+      scope :by_robot, ->(robot_id) { where(robot_id: robot_id) }
+      scope :by_node, ->(node_id) { where(node_id: node_id) }
+      scope :frequently_remembered, -> { where('remember_count > 1').order(remember_count: :desc) }
+
+      # Record that a robot remembered this content again
+      #
+      # @return [RobotNode] Updated record
+      #
+      def record_remember!
+        self.remember_count += 1
+        self.last_remembered_at = Time.current
+        save!
+        self
+      end
+    end
+  end
+end

data/lib/htm/models/tag.rb

@@ -45,6 +45,58 @@ class HTM
         find_or_create_by(name: name)
       end
 
+      # Returns a nested hash tree structure from the current scope
+      # Example: Tag.all.tree => { "database" => { "postgresql" => {} } }
+      # Example: Tag.with_prefix("database").tree => { "database" => { "postgresql" => {} } }
+      def self.tree
+        tree = {}
+
+        all.order(:name).pluck(:name).each do |tag_name|
+          parts = tag_name.split(':')
+          current = tree
+
+          parts.each do |part|
+            current[part] ||= {}
+            current = current[part]
+          end
+        end
+
+        tree
+      end
+
+      # Returns a formatted string representation of the tag tree
+      # Uses directory-style formatting with ├── and └── characters
+      # Example: puts Tag.all.tree_string
+      # Example: puts Tag.with_prefix("database").tree_string
+      def self.tree_string
+        format_tree_branch(tree)
+      end
+
+      # Format a tree branch recursively (internal helper)
+      def self.format_tree_branch(node, is_last_array = [])
+        result = ''
+        sorted_keys = node.keys.sort
+
+        sorted_keys.each_with_index do |key, index|
+          is_last = (index == sorted_keys.size - 1)
+
+          # Build prefix from parent branches
+          line_prefix = is_last_array.map { |was_last| was_last ? '    ' : '│   ' }.join
+
+          # Add branch character and key
+          branch = is_last ? '└── ' : '├── '
+          result += "#{line_prefix}#{branch}#{key}\n"
+
+          # Recurse into children
+          children = node[key]
+          unless children.empty?
+            result += format_tree_branch(children, is_last_array + [is_last])
+          end
+        end
+
+        result
+      end
+
       # Instance methods
       def root_topic
         name.split(':').first