htm 0.0.2 → 0.0.10

data/lib/htm/models/file_source.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+class HTM
+  module Models
+    # FileSource model - tracks loaded source files
+    #
+    # Represents a file that has been loaded into HTM with its metadata.
+    # Each file can have multiple associated nodes (chunks).
+    #
+    # @example Find source by path
+    #   source = FileSource.by_path('/path/to/doc.md').first
+    #   source.chunks  # => [Node, Node, ...]
+    #
+    # @example Check if re-sync needed
+    #   current_mtime = File.mtime('/path/to/doc.md')
+    #   source.needs_sync?(current_mtime)  # => true/false
+    #
+    class FileSource < ActiveRecord::Base
+      self.table_name = 'file_sources'
+
+      # Tolerance for mtime comparison to avoid false positives from
+      # precision differences between filesystem and database timestamps
+      DELTA_TIME = 5  # seconds
+
+      # Associations
+      has_many :nodes, class_name: 'HTM::Models::Node',
+               foreign_key: :source_id, dependent: :nullify
+
+      # Validations
+      validates :file_path, presence: true, uniqueness: true
+
+      # Scopes
+      scope :by_path, ->(path) { where(file_path: File.expand_path(path)) }
+      scope :stale, -> { where('mtime < last_synced_at') }
+      scope :recently_synced, -> { order(last_synced_at: :desc) }
+
+      # Check if file needs re-sync based on mtime
+      #
+      # Uses DELTA_TIME tolerance to avoid false positives from:
+      # - Nanosecond/microsecond precision differences (filesystem vs PostgreSQL)
+      # - Floating-point rounding errors
+      # - Minor timestamp discrepancies across systems
+      #
+      # @param current_mtime [Time] Current file modification time
+      # @return [Boolean] true if file modification time differs by more than DELTA_TIME
+      #
+      def needs_sync?(current_mtime)
+        return true if mtime.nil?
+
+        (current_mtime.to_i - mtime.to_i).abs > DELTA_TIME
+      end
+
+      # Get ordered chunks from this file
+      #
+      # @return [ActiveRecord::Relation] Nodes ordered by chunk_position
+      #
+      def chunks
+        nodes.order(:chunk_position)
+      end
+
+      # Extract tags from frontmatter
+      #
+      # @return [Array<String>] Tag names from frontmatter 'tags' field
+      #
+      def frontmatter_tags
+        return [] unless frontmatter.is_a?(Hash)
+
+        tags = frontmatter['tags'] || frontmatter[:tags] || []
+        Array(tags).map(&:to_s)
+      end
+
+      # Get title from frontmatter
+      #
+      # @return [String, nil] Title from frontmatter
+      #
+      def title
+        return nil unless frontmatter.is_a?(Hash)
+        frontmatter['title'] || frontmatter[:title]
+      end
+
+      # Get author from frontmatter
+      #
+      # @return [String, nil] Author from frontmatter
+      #
+      def author
+        return nil unless frontmatter.is_a?(Hash)
+        frontmatter['author'] || frontmatter[:author]
+      end
+
+      # Soft delete all chunks from this file
+      #
+      # @return [Integer] Number of chunks soft-deleted
+      #
+      def soft_delete_chunks!
+        nodes.update_all(deleted_at: Time.current)
+      end
+    end
+  end
+end

data/lib/htm/models/node.rb

@@ -29,13 +29,16 @@ class HTM
       has_many :node_tags, class_name: 'HTM::Models::NodeTag', dependent: :destroy
       has_many :tags, through: :node_tags, class_name: 'HTM::Models::Tag'
 
+      # Optional source file association (for nodes loaded from files)
+      belongs_to :file_source, class_name: 'HTM::Models::FileSource',
+                 foreign_key: :source_id, optional: true
+
       # Neighbor - vector similarity search
       has_neighbors :embedding
 
       # Validations
       validates :content, presence: true
       validates :content_hash, presence: true, uniqueness: true
-      validates :embedding_dimension, numericality: { greater_than: 0, less_than_or_equal_to: 2000 }, allow_nil: true
 
       # Callbacks
       before_validation :set_content_hash, if: -> { content_hash.blank? && content.present? }
@@ -43,13 +46,32 @@ class HTM
       before_save :update_timestamps
 
       # Scopes
+      # Soft delete - by default, only show non-deleted nodes
+      default_scope { where(deleted_at: nil) }
+
       scope :by_robot, ->(robot_id) { joins(:robot_nodes).where(robot_nodes: { robot_id: robot_id }) }
       scope :recent, -> { order(created_at: :desc) }
       scope :in_timeframe, ->(start_time, end_time) { where(created_at: start_time..end_time) }
       scope :with_embeddings, -> { where.not(embedding: nil) }
+      scope :from_source, ->(source_id) { where(source_id: source_id).order(:chunk_position) }
+
+      # Soft delete scopes
+      scope :deleted, -> { unscoped.where.not(deleted_at: nil) }
+      scope :with_deleted, -> { unscoped }
+      scope :deleted_before, ->(time) { deleted.where('deleted_at < ?', time) }
 
       # Class methods
 
+      # Permanently delete all soft-deleted nodes older than the specified time
+      #
+      # @param older_than [Time, ActiveSupport::Duration] Delete nodes soft-deleted before this time
+      #   Can be a Time object or a duration like 30.days.ago
+      # @return [Integer] Number of nodes permanently deleted
+      #
+      def self.purge_deleted(older_than:)
+        deleted_before(older_than).destroy_all.count
+      end
+
       # Find a node by content hash, or return nil
       #
       # @param content [String] The content to search for
@@ -91,19 +113,43 @@ class HTM
         query_embedding = other.is_a?(Node) ? other.embedding : other
         return nil unless embedding.present? && query_embedding.present?
 
+        # Validate embedding is an array of finite numeric values
+        unless query_embedding.is_a?(Array) && query_embedding.all? { |v| v.is_a?(Numeric) && v.finite? }
+          return nil
+        end
+
         # Calculate cosine similarity: 1 - (embedding <=> query_embedding)
-        # Format the array as a PostgreSQL vector literal: '[0.1,0.2,0.3]'
-        vector_str = "[#{query_embedding.join(',')}]"
-        result = self.class.connection.select_value(
-          "SELECT 1 - (embedding <=> '#{vector_str}'::vector) FROM nodes WHERE id = #{id}"
+        # Safely format the array as a PostgreSQL vector literal
+        vector_str = "[#{query_embedding.map { |v| v.to_f }.join(',')}]"
+        conn = self.class.connection
+        quoted_vector = conn.quote(vector_str)
+        quoted_id = conn.quote(id)
+
+        result = conn.select_value(
+          "SELECT 1 - (embedding <=> #{quoted_vector}::vector) FROM nodes WHERE id = #{quoted_id}"
         )
         result&.to_f
       end
 
+      # Get all tag names associated with this node
+      #
+      # @return [Array<String>] Array of hierarchical tag names (e.g., ["database:postgresql", "ai:llm"])
+      #
       def tag_names
         tags.pluck(:name)
       end
 
+      # Add tags to this node (creates tags if they don't exist)
+      #
+      # @param tag_names [Array<String>, String] Tag name(s) to add
+      # @return [void]
+      #
+      # @example Add a single tag
+      #   node.add_tags("database:postgresql")
+      #
+      # @example Add multiple tags
+      #   node.add_tags(["database:postgresql", "ai:embeddings"])
+      #
       def add_tags(tag_names)
         Array(tag_names).each do |tag_name|
           tag = HTM::Models::Tag.find_or_create_by(name: tag_name)
@@ -111,6 +157,11 @@ class HTM
         end
       end
 
+      # Remove a tag from this node
+      #
+      # @param tag_name [String] Tag name to remove
+      # @return [void]
+      #
       def remove_tag(tag_name)
         tag = HTM::Models::Tag.find_by(name: tag_name)
         return unless tag
@@ -118,6 +169,30 @@ class HTM
         node_tags.where(tag_id: tag.id).destroy_all
       end
 
+      # Soft delete - mark node as deleted without removing from database
+      #
+      # @return [Boolean] true if soft deleted successfully
+      #
+      def soft_delete!
+        update!(deleted_at: Time.current)
+      end
+
+      # Restore a soft-deleted node
+      #
+      # @return [Boolean] true if restored successfully
+      #
+      def restore!
+        update!(deleted_at: nil)
+      end
+
+      # Check if node is soft-deleted
+      #
+      # @return [Boolean] true if deleted_at is set
+      #
+      def deleted?
+        deleted_at.present?
+      end
+
       private
 
       def set_content_hash

data/lib/htm/models/robot.rb

@@ -27,15 +27,31 @@ class HTM
       scope :by_name, ->(name) { where(name: name) }
 
       # Class methods
+
+      # Find or create a robot by name
+      #
+      # @param robot_name [String] Name of the robot
+      # @return [Robot] The found or created robot
+      #
       def self.find_or_create_by_name(robot_name)
         find_or_create_by(name: robot_name)
       end
 
       # Instance methods
+
+      # Get the total number of nodes associated with this robot
+      #
+      # @return [Integer] Number of nodes
+      #
       def node_count
         nodes.count
       end
 
+      # Get the most recent nodes for this robot
+      #
+      # @param limit [Integer] Maximum number of nodes to return (default: 10)
+      # @return [ActiveRecord::Relation] Recent nodes ordered by created_at desc
+      #
       def recent_nodes(limit = 10)
         nodes.recent.limit(limit)
       end
@@ -60,10 +76,17 @@ class HTM
           end
       end
 
+      # Get a summary of this robot's memory state
+      #
+      # @return [Hash] Summary including:
+      #   - :total_nodes [Integer] Total nodes associated with this robot
+      #   - :in_working_memory [Integer] Nodes currently in working memory
+      #   - :with_embeddings [Integer] Nodes that have embeddings generated
+      #
       def memory_summary
         {
           total_nodes: nodes.count,
-          in_working_memory: HTM::Models::WorkingMemoryEntry.where(robot_id: id).count,
+          in_working_memory: robot_nodes.in_working_memory.count,
           with_embeddings: nodes.with_embeddings.count
         }
       end

data/lib/htm/models/robot_node.rb

@@ -34,6 +34,7 @@ class HTM
       scope :by_robot, ->(robot_id) { where(robot_id: robot_id) }
       scope :by_node, ->(node_id) { where(node_id: node_id) }
       scope :frequently_remembered, -> { where('remember_count > 1').order(remember_count: :desc) }
+      scope :in_working_memory, -> { where(working_memory: true) }
 
       # Record that a robot remembered this content again
       #

data/lib/htm/models/tag.rb

@@ -29,10 +29,28 @@ class HTM
       scope :root_level, -> { where("name NOT LIKE '%:%'") }
 
       # Class methods
+
+      # Find tags with a given prefix (hierarchical query)
+      #
+      # @param prefix [String] Tag prefix to match (e.g., "database" matches "database:postgresql")
+      # @return [ActiveRecord::Relation] Tags matching the prefix
+      #
+      # @example Find all database-related tags
+      #   Tag.find_by_topic_prefix("database")
+      #   # => [#<Tag name: "database:postgresql">, #<Tag name: "database:mysql">]
+      #
       def self.find_by_topic_prefix(prefix)
         where("name LIKE ?", "#{prefix}%")
       end
 
+      # Get the most frequently used tags
+      #
+      # @param limit [Integer] Maximum number of tags to return (default: 10)
+      # @return [ActiveRecord::Relation] Tags with usage_count attribute
+      #
+      # @example Get top 5 most used tags
+      #   Tag.popular_tags(5).each { |t| puts "#{t.name}: #{t.usage_count}" }
+      #
       def self.popular_tags(limit = 10)
         joins(:node_tags)
           .select('tags.*, COUNT(node_tags.id) as usage_count')
@@ -41,13 +59,27 @@ class HTM
           .limit(limit)
       end
 
+      # Find or create a tag by name
+      #
+      # @param name [String] Hierarchical tag name (e.g., "database:postgresql")
+      # @return [Tag] The found or created tag
+      #
       def self.find_or_create_by_name(name)
         find_or_create_by(name: name)
       end
 
       # Returns a nested hash tree structure from the current scope
-      # Example: Tag.all.tree => { "database" => { "postgresql" => {} } }
-      # Example: Tag.with_prefix("database").tree => { "database" => { "postgresql" => {} } }
+      #
+      # @return [Hash] Nested hash representing the tag hierarchy
+      #
+      # @example Get all tags as a tree
+      #   Tag.all.tree
+      #   # => { "database" => { "postgresql" => {}, "mysql" => {} }, "ai" => { "llm" => {} } }
+      #
+      # @example Get filtered tags as a tree
+      #   Tag.with_prefix("database").tree
+      #   # => { "database" => { "postgresql" => {} } }
+      #
       def self.tree
         tree = {}
 
@@ -65,13 +97,79 @@ class HTM
       end
 
       # Returns a formatted string representation of the tag tree
+      #
       # Uses directory-style formatting with ├── and └── characters
-      # Example: puts Tag.all.tree_string
-      # Example: puts Tag.with_prefix("database").tree_string
+      #
+      # @return [String] Formatted tree string
+      #
+      # @example Display all tags as a tree
+      #   puts Tag.all.tree_string
+      #   # ├── ai
+      #   # │   └── llm
+      #   # └── database
+      #   #     └── postgresql
+      #
       def self.tree_string
         format_tree_branch(tree)
       end
 
+      # Returns a Mermaid flowchart representation of the tag tree
+      # Example: puts Tag.all.tree_mermaid
+      # Example: Tag.all.tree_mermaid(direction: 'LR') # Left to right
+      #
+      # @param direction [String] Flow direction: 'TD' (top-down), 'LR' (left-right), 'BT', 'RL'
+      # @return [String] Mermaid flowchart syntax
+      #
+      def self.tree_mermaid(direction: 'TD')
+        tree_data = tree
+        return "flowchart #{direction}\n  empty[No tags]" if tree_data.empty?
+
+        lines = ["flowchart #{direction}"]
+        node_id = 0
+        node_ids = {}
+
+        # Generate Mermaid nodes and connections
+        generate_mermaid_nodes(tree_data, nil, lines, node_ids, node_id)
+
+        lines.join("\n")
+      end
+
+      # Returns an SVG representation of the tag tree
+      # Uses dark theme with transparent background
+      # Example: File.write('tags.svg', Tag.all.tree_svg)
+      #
+      # @param title [String] Optional title for the SVG
+      # @return [String] SVG markup
+      #
+      def self.tree_svg(title: 'HTM Tag Hierarchy')
+        tree_data = tree
+        return empty_tree_svg(title) if tree_data.empty?
+
+        # Calculate dimensions based on tree structure
+        stats = calculate_tree_stats(tree_data)
+        node_count = stats[:total_nodes]
+        max_depth = stats[:max_depth]
+
+        # Layout constants
+        node_width = 140
+        node_height = 30
+        h_spacing = 180
+        v_spacing = 50
+        padding = 40
+
+        # Calculate positions for all nodes
+        positions = {}
+        y_offset = [0]  # Use array to allow mutation in closure
+        calculate_node_positions(tree_data, 0, positions, y_offset, h_spacing, v_spacing)
+
+        # Calculate SVG dimensions
+        width = (max_depth * h_spacing) + node_width + (padding * 2)
+        height = (y_offset[0] * v_spacing) + node_height + (padding * 2)
+
+        # Generate SVG
+        generate_tree_svg(tree_data, positions, width, height, padding, node_width, node_height, title)
+      end
+
       # Format a tree branch recursively (internal helper)
       def self.format_tree_branch(node, is_last_array = [])
         result = ''
@@ -97,23 +195,175 @@ class HTM
         result
       end
 
+      # Generate Mermaid nodes recursively (internal helper)
+      def self.generate_mermaid_nodes(node, parent_path, lines, node_ids, counter)
+        node.keys.sort.each do |key|
+          current_path = parent_path ? "#{parent_path}:#{key}" : key
+
+          # Create unique node ID
+          node_id = "n#{counter}"
+          node_ids[current_path] = node_id
+          counter += 1
+
+          # Add node definition with styling
+          lines << "  #{node_id}[\"#{key}\"]"
+
+          # Add connection from parent
+          if parent_path && node_ids[parent_path]
+            lines << "  #{node_ids[parent_path]} --> #{node_id}"
+          end
+
+          # Recurse into children
+          children = node[key]
+          counter = generate_mermaid_nodes(children, current_path, lines, node_ids, counter) unless children.empty?
+        end
+
+        counter
+      end
+
+      # Calculate tree statistics (internal helper)
+      def self.calculate_tree_stats(node, depth = 0)
+        return { total_nodes: 0, max_depth: depth } if node.empty?
+
+        total = node.keys.size
+        max = depth + 1
+
+        node.each_value do |children|
+          child_stats = calculate_tree_stats(children, depth + 1)
+          total += child_stats[:total_nodes]
+          max = [max, child_stats[:max_depth]].max
+        end
+
+        { total_nodes: total, max_depth: max }
+      end
+
+      # Calculate node positions for SVG layout (internal helper)
+      def self.calculate_node_positions(node, depth, positions, y_offset, h_spacing, v_spacing, parent_path = nil)
+        node.keys.sort.each do |key|
+          current_path = parent_path ? "#{parent_path}:#{key}" : key
+
+          positions[current_path] = {
+            x: depth,
+            y: y_offset[0],
+            label: key
+          }
+          y_offset[0] += 1
+
+          children = node[key]
+          calculate_node_positions(children, depth + 1, positions, y_offset, h_spacing, v_spacing, current_path) unless children.empty?
+        end
+      end
+
+      # Generate SVG for empty tree (internal helper)
+      def self.empty_tree_svg(title)
+        <<~SVG
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 300 100">
+            <rect width="100%" height="100%" fill="transparent"/>
+            <text x="150" y="30" text-anchor="middle" fill="#9CA3AF" font-family="system-ui, sans-serif" font-size="14" font-weight="bold">#{title}</text>
+            <text x="150" y="60" text-anchor="middle" fill="#6B7280" font-family="system-ui, sans-serif" font-size="12">No tags found</text>
+          </svg>
+        SVG
+      end
+
+      # Generate SVG tree visualization (internal helper)
+      def self.generate_tree_svg(tree_data, positions, width, height, padding, node_width, node_height, title)
+        # Color palette for different depths (dark theme)
+        colors = ['#3B82F6', '#8B5CF6', '#EC4899', '#F59E0B', '#10B981', '#6366F1']
+
+        svg_lines = []
+        svg_lines << %(<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 #{width} #{height + 40}">)
+        svg_lines << '  <rect width="100%" height="100%" fill="transparent"/>'
+
+        # Title
+        svg_lines << %Q(  <text x="#{width / 2}" y="25" text-anchor="middle" fill="#F3F4F6" font-family="system-ui, sans-serif" font-size="16" font-weight="bold">#{title}</text>)
+
+        # Draw connections first (so they appear behind nodes)
+        positions.each do |path, pos|
+          parent_path = path.include?(':') ? path.split(':')[0..-2].join(':') : nil
+          next unless parent_path && positions[parent_path]
+
+          parent_pos = positions[parent_path]
+          x1 = padding + (parent_pos[:x] * (node_width + 40)) + node_width
+          y1 = 40 + padding + (parent_pos[:y] * (node_height + 20)) + (node_height / 2)
+          x2 = padding + (pos[:x] * (node_width + 40))
+          y2 = 40 + padding + (pos[:y] * (node_height + 20)) + (node_height / 2)
+
+          # Curved connection line
+          mid_x = (x1 + x2) / 2
+          svg_lines << %Q(  <path d="M#{x1},#{y1} C#{mid_x},#{y1} #{mid_x},#{y2} #{x2},#{y2}" stroke="#4B5563" stroke-width="2" fill="none"/>)
+        end
+
+        # Draw nodes
+        positions.each do |path, pos|
+          depth = path.count(':')
+          color = colors[depth % colors.size]
+
+          x = padding + (pos[:x] * (node_width + 40))
+          y = 40 + padding + (pos[:y] * (node_height + 20))
+
+          # Node rectangle with rounded corners
+          svg_lines << %Q(  <rect x="#{x}" y="#{y}" width="#{node_width}" height="#{node_height}" rx="6" fill="#{color}" opacity="0.9"/>)
+
+          # Node label
+          text_x = x + (node_width / 2)
+          text_y = y + (node_height / 2) + 4
+          svg_lines << %Q(  <text x="#{text_x}" y="#{text_y}" text-anchor="middle" fill="#FFFFFF" font-family="system-ui, sans-serif" font-size="11" font-weight="500">#{pos[:label]}</text>)
+        end
+
+        svg_lines << '</svg>'
+        svg_lines.join("\n")
+      end
+
       # Instance methods
+
+      # Get the root (top-level) topic of this tag
+      #
+      # @return [String] The first segment of the hierarchical tag
+      #
+      # @example
+      #   tag = Tag.find_by(name: "database:postgresql:extensions")
+      #   tag.root_topic  # => "database"
+      #
       def root_topic
         name.split(':').first
       end
 
+      # Get all hierarchy levels of this tag
+      #
+      # @return [Array<String>] Array of topic segments
+      #
+      # @example
+      #   tag = Tag.find_by(name: "database:postgresql:extensions")
+      #   tag.topic_levels  # => ["database", "postgresql", "extensions"]
+      #
       def topic_levels
         name.split(':')
       end
 
+      # Get the depth (number of levels) of this tag
+      #
+      # @return [Integer] Number of hierarchy levels
+      #
+      # @example
+      #   Tag.find_by(name: "database").depth           # => 1
+      #   Tag.find_by(name: "database:postgresql").depth  # => 2
+      #
       def depth
         topic_levels.length
       end
 
+      # Check if this tag is hierarchical (has child levels)
+      #
+      # @return [Boolean] True if tag contains colons (hierarchy separators)
+      #
       def hierarchical?
         name.include?(':')
       end
 
+      # Get the number of nodes using this tag
+      #
+      # @return [Integer] Count of nodes with this tag
+      #
       def usage_count
         node_tags.count
       end