htm 0.0.1 → 0.0.10

data/lib/htm/models/file_source.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+class HTM
+  module Models
+    # FileSource model - tracks loaded source files
+    #
+    # Represents a file that has been loaded into HTM with its metadata.
+    # Each file can have multiple associated nodes (chunks).
+    #
+    # @example Find source by path
+    #   source = FileSource.by_path('/path/to/doc.md').first
+    #   source.chunks  # => [Node, Node, ...]
+    #
+    # @example Check if re-sync needed
+    #   current_mtime = File.mtime('/path/to/doc.md')
+    #   source.needs_sync?(current_mtime)  # => true/false
+    #
+    class FileSource < ActiveRecord::Base
+      self.table_name = 'file_sources'
+
+      # Tolerance for mtime comparison to avoid false positives from
+      # precision differences between filesystem and database timestamps
+      DELTA_TIME = 5  # seconds
+
+      # Associations
+      has_many :nodes, class_name: 'HTM::Models::Node',
+               foreign_key: :source_id, dependent: :nullify
+
+      # Validations
+      validates :file_path, presence: true, uniqueness: true
+
+      # Scopes
+      scope :by_path, ->(path) { where(file_path: File.expand_path(path)) }
+      scope :stale, -> { where('mtime < last_synced_at') }
+      scope :recently_synced, -> { order(last_synced_at: :desc) }
+
+      # Check if file needs re-sync based on mtime
+      #
+      # Uses DELTA_TIME tolerance to avoid false positives from:
+      # - Nanosecond/microsecond precision differences (filesystem vs PostgreSQL)
+      # - Floating-point rounding errors
+      # - Minor timestamp discrepancies across systems
+      #
+      # @param current_mtime [Time] Current file modification time
+      # @return [Boolean] true if file modification time differs by more than DELTA_TIME
+      #
+      def needs_sync?(current_mtime)
+        return true if mtime.nil?
+
+        (current_mtime.to_i - mtime.to_i).abs > DELTA_TIME
+      end
+
+      # Get ordered chunks from this file
+      #
+      # @return [ActiveRecord::Relation] Nodes ordered by chunk_position
+      #
+      def chunks
+        nodes.order(:chunk_position)
+      end
+
+      # Extract tags from frontmatter
+      #
+      # @return [Array<String>] Tag names from frontmatter 'tags' field
+      #
+      def frontmatter_tags
+        return [] unless frontmatter.is_a?(Hash)
+
+        tags = frontmatter['tags'] || frontmatter[:tags] || []
+        Array(tags).map(&:to_s)
+      end
+
+      # Get title from frontmatter
+      #
+      # @return [String, nil] Title from frontmatter
+      #
+      def title
+        return nil unless frontmatter.is_a?(Hash)
+        frontmatter['title'] || frontmatter[:title]
+      end
+
+      # Get author from frontmatter
+      #
+      # @return [String, nil] Author from frontmatter
+      #
+      def author
+        return nil unless frontmatter.is_a?(Hash)
+        frontmatter['author'] || frontmatter[:author]
+      end
+
+      # Soft delete all chunks from this file
+      #
+      # @return [Integer] Number of chunks soft-deleted
+      #
+      def soft_delete_chunks!
+        nodes.update_all(deleted_at: Time.current)
+      end
+    end
+  end
+end

data/lib/htm/models/node.rb

@@ -1,9 +1,14 @@
 # frozen_string_literal: true
 
+require 'digest'
+
 class HTM
   module Models
     # Node model - represents a memory node (conversation message)
     #
+    # Nodes are globally unique by content (via content_hash) and can be
+    # linked to multiple robots through the robot_nodes join table.
+    #
     # Nearest Neighbor Search (via neighbor gem):
     #   # Find 5 nearest neighbors by cosine distance
     #   neighbors = Node.nearest_neighbors(:embedding, query_vector, distance: "cosine").limit(5)
@@ -18,30 +23,73 @@ class HTM
     class Node < ActiveRecord::Base
       self.table_name = 'nodes'
 
-      # Associations
-      belongs_to :robot, class_name: 'HTM::Models::Robot', foreign_key: 'robot_id', primary_key: 'id'
+      # Associations - Many-to-many with robots via robot_nodes
+      has_many :robot_nodes, class_name: 'HTM::Models::RobotNode', dependent: :destroy
+      has_many :robots, through: :robot_nodes, class_name: 'HTM::Models::Robot'
       has_many :node_tags, class_name: 'HTM::Models::NodeTag', dependent: :destroy
       has_many :tags, through: :node_tags, class_name: 'HTM::Models::Tag'
 
+      # Optional source file association (for nodes loaded from files)
+      belongs_to :file_source, class_name: 'HTM::Models::FileSource',
+                 foreign_key: :source_id, optional: true
+
       # Neighbor - vector similarity search
       has_neighbors :embedding
 
       # Validations
       validates :content, presence: true
-      validates :robot_id, presence: true
-      validates :embedding_dimension, numericality: { greater_than: 0, less_than_or_equal_to: 2000 }, allow_nil: true
+      validates :content_hash, presence: true, uniqueness: true
 
       # Callbacks
+      before_validation :set_content_hash, if: -> { content_hash.blank? && content.present? }
       before_create :set_defaults
       before_save :update_timestamps
 
       # Scopes
-      scope :by_robot, ->(robot_id) { where(robot_id: robot_id) }
-      scope :by_source, ->(source) { where(source: source) }
-      scope :in_working_memory, -> { where(in_working_memory: true) }
+      # Soft delete - by default, only show non-deleted nodes
+      default_scope { where(deleted_at: nil) }
+
+      scope :by_robot, ->(robot_id) { joins(:robot_nodes).where(robot_nodes: { robot_id: robot_id }) }
       scope :recent, -> { order(created_at: :desc) }
       scope :in_timeframe, ->(start_time, end_time) { where(created_at: start_time..end_time) }
       scope :with_embeddings, -> { where.not(embedding: nil) }
+      scope :from_source, ->(source_id) { where(source_id: source_id).order(:chunk_position) }
+
+      # Soft delete scopes
+      scope :deleted, -> { unscoped.where.not(deleted_at: nil) }
+      scope :with_deleted, -> { unscoped }
+      scope :deleted_before, ->(time) { deleted.where('deleted_at < ?', time) }
+
+      # Class methods
+
+      # Permanently delete all soft-deleted nodes older than the specified time
+      #
+      # @param older_than [Time, ActiveSupport::Duration] Delete nodes soft-deleted before this time
+      #   Can be a Time object or a duration like 30.days.ago
+      # @return [Integer] Number of nodes permanently deleted
+      #
+      def self.purge_deleted(older_than:)
+        deleted_before(older_than).destroy_all.count
+      end
+
+      # Find a node by content hash, or return nil
+      #
+      # @param content [String] The content to search for
+      # @return [Node, nil] The existing node or nil
+      #
+      def self.find_by_content(content)
+        hash = generate_content_hash(content)
+        find_by(content_hash: hash)
+      end
+
+      # Generate SHA-256 hash for content
+      #
+      # @param content [String] Content to hash
+      # @return [String] 64-character hex hash
+      #
+      def self.generate_content_hash(content)
+        Digest::SHA256.hexdigest(content.to_s)
+      end
 
       # Instance methods
 
@@ -65,19 +113,43 @@ class HTM
         query_embedding = other.is_a?(Node) ? other.embedding : other
         return nil unless embedding.present? && query_embedding.present?
 
+        # Validate embedding is an array of finite numeric values
+        unless query_embedding.is_a?(Array) && query_embedding.all? { |v| v.is_a?(Numeric) && v.finite? }
+          return nil
+        end
+
         # Calculate cosine similarity: 1 - (embedding <=> query_embedding)
-        # Format the array as a PostgreSQL vector literal: '[0.1,0.2,0.3]'
-        vector_str = "[#{query_embedding.join(',')}]"
-        result = self.class.connection.select_value(
-          "SELECT 1 - (embedding <=> '#{vector_str}'::vector) FROM nodes WHERE id = #{id}"
+        # Safely format the array as a PostgreSQL vector literal
+        vector_str = "[#{query_embedding.map { |v| v.to_f }.join(',')}]"
+        conn = self.class.connection
+        quoted_vector = conn.quote(vector_str)
+        quoted_id = conn.quote(id)
+
+        result = conn.select_value(
+          "SELECT 1 - (embedding <=> #{quoted_vector}::vector) FROM nodes WHERE id = #{quoted_id}"
         )
         result&.to_f
       end
 
+      # Get all tag names associated with this node
+      #
+      # @return [Array<String>] Array of hierarchical tag names (e.g., ["database:postgresql", "ai:llm"])
+      #
       def tag_names
         tags.pluck(:name)
       end
 
+      # Add tags to this node (creates tags if they don't exist)
+      #
+      # @param tag_names [Array<String>, String] Tag name(s) to add
+      # @return [void]
+      #
+      # @example Add a single tag
+      #   node.add_tags("database:postgresql")
+      #
+      # @example Add multiple tags
+      #   node.add_tags(["database:postgresql", "ai:embeddings"])
+      #
       def add_tags(tag_names)
         Array(tag_names).each do |tag_name|
           tag = HTM::Models::Tag.find_or_create_by(name: tag_name)
@@ -85,6 +157,11 @@ class HTM
         end
       end
 
+      # Remove a tag from this node
+      #
+      # @param tag_name [String] Tag name to remove
+      # @return [void]
+      #
       def remove_tag(tag_name)
         tag = HTM::Models::Tag.find_by(name: tag_name)
         return unless tag
@@ -92,10 +169,37 @@ class HTM
         node_tags.where(tag_id: tag.id).destroy_all
       end
 
+      # Soft delete - mark node as deleted without removing from database
+      #
+      # @return [Boolean] true if soft deleted successfully
+      #
+      def soft_delete!
+        update!(deleted_at: Time.current)
+      end
+
+      # Restore a soft-deleted node
+      #
+      # @return [Boolean] true if restored successfully
+      #
+      def restore!
+        update!(deleted_at: nil)
+      end
+
+      # Check if node is soft-deleted
+      #
+      # @return [Boolean] true if deleted_at is set
+      #
+      def deleted?
+        deleted_at.present?
+      end
+
       private
 
+      def set_content_hash
+        self.content_hash = self.class.generate_content_hash(content)
+      end
+
       def set_defaults
-        self.in_working_memory ||= false
         self.created_at ||= Time.current
         self.updated_at ||= Time.current
         self.last_accessed ||= Time.current

data/lib/htm/models/robot.rb

@@ -3,12 +3,18 @@
 class HTM
   module Models
     # Robot model - represents an LLM agent using the HTM system
+    #
+    # Robots can share memories through the many-to-many relationship with nodes.
+    # When a robot is deleted, only the robot_nodes links are removed; shared
+    # nodes remain in the database for other robots.
+    #
     class Robot < ActiveRecord::Base
       self.table_name = 'robots'
 
-      # Associations
-      has_many :nodes, class_name: 'HTM::Models::Node', dependent: :destroy
-      has_many :operation_logs, class_name: 'HTM::Models::OperationLog', dependent: :destroy
+      # Associations - Many-to-many with nodes via robot_nodes
+      # dependent: :destroy removes links only, NOT the shared nodes
+      has_many :robot_nodes, class_name: 'HTM::Models::RobotNode', dependent: :destroy
+      has_many :nodes, through: :robot_nodes, class_name: 'HTM::Models::Node'
 
       # Validations
       validates :name, presence: true
@@ -21,23 +27,66 @@ class HTM
       scope :by_name, ->(name) { where(name: name) }
 
       # Class methods
+
+      # Find or create a robot by name
+      #
+      # @param robot_name [String] Name of the robot
+      # @return [Robot] The found or created robot
+      #
       def self.find_or_create_by_name(robot_name)
         find_or_create_by(name: robot_name)
       end
 
       # Instance methods
+
+      # Get the total number of nodes associated with this robot
+      #
+      # @return [Integer] Number of nodes
+      #
       def node_count
         nodes.count
       end
 
+      # Get the most recent nodes for this robot
+      #
+      # @param limit [Integer] Maximum number of nodes to return (default: 10)
+      # @return [ActiveRecord::Relation] Recent nodes ordered by created_at desc
+      #
       def recent_nodes(limit = 10)
         nodes.recent.limit(limit)
       end
 
+      # Get nodes with their remember metadata for this robot
+      #
+      # @param limit [Integer] Max nodes to return
+      # @return [Array<Hash>] Nodes with remember_count, first/last_remembered_at
+      #
+      def nodes_with_metadata(limit = 10)
+        robot_nodes
+          .includes(:node)
+          .order(last_remembered_at: :desc)
+          .limit(limit)
+          .map do |rn|
+            {
+              node: rn.node,
+              remember_count: rn.remember_count,
+              first_remembered_at: rn.first_remembered_at,
+              last_remembered_at: rn.last_remembered_at
+            }
+          end
+      end
+
+      # Get a summary of this robot's memory state
+      #
+      # @return [Hash] Summary including:
+      #   - :total_nodes [Integer] Total nodes associated with this robot
+      #   - :in_working_memory [Integer] Nodes currently in working memory
+      #   - :with_embeddings [Integer] Nodes that have embeddings generated
+      #
       def memory_summary
         {
           total_nodes: nodes.count,
-          in_working_memory: nodes.in_working_memory.count,
+          in_working_memory: robot_nodes.in_working_memory.count,
           with_embeddings: nodes.with_embeddings.count
         }
       end

data/lib/htm/models/robot_node.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+class HTM
+  module Models
+    # RobotNode Join Model - Links robots to nodes (many-to-many)
+    #
+    # This model represents the relationship between a robot and a node,
+    # tracking when and how many times a robot has "remembered" a piece of content.
+    #
+    # @example Find all robots that remember a node
+    #   node.robots
+    #
+    # @example Find all nodes a robot remembers
+    #   robot.nodes
+    #
+    # @example Track remember activity
+    #   link = RobotNode.find_by(robot: robot, node: node)
+    #   link.remember_count  # => 3
+    #   link.first_remembered_at
+    #   link.last_remembered_at
+    #
+    class RobotNode < ActiveRecord::Base
+      self.table_name = 'robot_nodes'
+
+      belongs_to :robot, class_name: 'HTM::Models::Robot'
+      belongs_to :node, class_name: 'HTM::Models::Node'
+
+      validates :robot_id, presence: true
+      validates :node_id, presence: true
+      validates :robot_id, uniqueness: { scope: :node_id, message: 'already linked to this node' }
+
+      # Scopes
+      scope :recent, -> { order(last_remembered_at: :desc) }
+      scope :by_robot, ->(robot_id) { where(robot_id: robot_id) }
+      scope :by_node, ->(node_id) { where(node_id: node_id) }
+      scope :frequently_remembered, -> { where('remember_count > 1').order(remember_count: :desc) }
+      scope :in_working_memory, -> { where(working_memory: true) }
+
+      # Record that a robot remembered this content again
+      #
+      # @return [RobotNode] Updated record
+      #
+      def record_remember!
+        self.remember_count += 1
+        self.last_remembered_at = Time.current
+        save!
+        self
+      end
+    end
+  end
+end