htm 0.0.1

data/lib/htm/models/node.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+class HTM
+  module Models
+    # Node model - represents a memory node (conversation message)
+    #
+    # Nearest Neighbor Search (via neighbor gem):
+    #   # Find 5 nearest neighbors by cosine distance
+    #   neighbors = Node.nearest_neighbors(:embedding, query_vector, distance: "cosine").limit(5)
+    #
+    #   # Get distance to query for each result
+    #   neighbors.each do |node|
+    #     puts "Node #{node.id}: distance = #{node.neighbor_distance}"
+    #   end
+    #
+    # Distance metrics: "cosine", "euclidean", "inner_product", "taxicab"
+    #
+    class Node < ActiveRecord::Base
+      self.table_name = 'nodes'
+
+      # Associations
+      belongs_to :robot, class_name: 'HTM::Models::Robot', foreign_key: 'robot_id', primary_key: 'id'
+      has_many :node_tags, class_name: 'HTM::Models::NodeTag', dependent: :destroy
+      has_many :tags, through: :node_tags, class_name: 'HTM::Models::Tag'
+
+      # Neighbor - vector similarity search
+      has_neighbors :embedding
+
+      # Validations
+      validates :content, presence: true
+      validates :robot_id, presence: true
+      validates :embedding_dimension, numericality: { greater_than: 0, less_than_or_equal_to: 2000 }, allow_nil: true
+
+      # Callbacks
+      before_create :set_defaults
+      before_save :update_timestamps
+
+      # Scopes
+      scope :by_robot, ->(robot_id) { where(robot_id: robot_id) }
+      scope :by_source, ->(source) { where(source: source) }
+      scope :in_working_memory, -> { where(in_working_memory: true) }
+      scope :recent, -> { order(created_at: :desc) }
+      scope :in_timeframe, ->(start_time, end_time) { where(created_at: start_time..end_time) }
+      scope :with_embeddings, -> { where.not(embedding: nil) }
+
+      # Instance methods
+
+      # Find nearest neighbors to this node's embedding
+      # @param limit [Integer] number of neighbors to return (default: 10)
+      # @param distance [String] distance metric: "cosine", "euclidean", "inner_product", "taxicab" (default: "cosine")
+      # @return [ActiveRecord::Relation] ordered by distance (closest first)
+      def nearest_neighbors(limit: 10, distance: "cosine")
+        return self.class.none unless embedding.present?
+
+        self.class.with_embeddings
+          .where.not(id: id)  # Exclude self
+          .nearest_neighbors(:embedding, embedding, distance: distance)
+          .limit(limit)
+      end
+
+      # Calculate cosine similarity to another embedding or node
+      # @param other [Array, Node] query embedding vector or another Node
+      # @return [Float] similarity score (0.0 to 1.0, higher is more similar)
+      def similarity_to(other)
+        query_embedding = other.is_a?(Node) ? other.embedding : other
+        return nil unless embedding.present? && query_embedding.present?
+
+        # Calculate cosine similarity: 1 - (embedding <=> query_embedding)
+        # Format the array as a PostgreSQL vector literal: '[0.1,0.2,0.3]'
+        vector_str = "[#{query_embedding.join(',')}]"
+        result = self.class.connection.select_value(
+          "SELECT 1 - (embedding <=> '#{vector_str}'::vector) FROM nodes WHERE id = #{id}"
+        )
+        result&.to_f
+      end
+
+      def tag_names
+        tags.pluck(:name)
+      end
+
+      def add_tags(tag_names)
+        Array(tag_names).each do |tag_name|
+          tag = HTM::Models::Tag.find_or_create_by(name: tag_name)
+          node_tags.find_or_create_by(tag_id: tag.id)
+        end
+      end
+
+      def remove_tag(tag_name)
+        tag = HTM::Models::Tag.find_by(name: tag_name)
+        return unless tag
+
+        node_tags.where(tag_id: tag.id).destroy_all
+      end
+
+      private
+
+      def set_defaults
+        self.in_working_memory ||= false
+        self.created_at ||= Time.current
+        self.updated_at ||= Time.current
+        self.last_accessed ||= Time.current
+      end
+
+      def update_timestamps
+        self.updated_at = Time.current if changed?
+      end
+    end
+  end
+end

data/lib/htm/models/node_tag.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class HTM
+  module Models
+    # NodeTag model - join table for many-to-many relationship between nodes and tags
+    class NodeTag < ActiveRecord::Base
+      self.table_name = 'node_tags'
+
+      # Associations
+      belongs_to :node, class_name: 'HTM::Models::Node'
+      belongs_to :tag, class_name: 'HTM::Models::Tag'
+
+      # Validations
+      validates :node_id, presence: true
+      validates :tag_id, presence: true
+      validates :tag_id, uniqueness: { scope: :node_id, message: "already associated with this node" }
+
+      # Callbacks
+      before_create :set_created_at
+
+      # Scopes
+      scope :for_node, ->(node_id) { where(node_id: node_id) }
+      scope :for_tag, ->(tag_id) { where(tag_id: tag_id) }
+      scope :recent, -> { order(created_at: :desc) }
+
+      private
+
+      def set_created_at
+        self.created_at ||= Time.current
+      end
+    end
+  end
+end

data/lib/htm/models/robot.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+class HTM
+  module Models
+    # Robot model - represents an LLM agent using the HTM system
+    class Robot < ActiveRecord::Base
+      self.table_name = 'robots'
+
+      # Associations
+      has_many :nodes, class_name: 'HTM::Models::Node', dependent: :destroy
+      has_many :operation_logs, class_name: 'HTM::Models::OperationLog', dependent: :destroy
+
+      # Validations
+      validates :name, presence: true
+
+      # Callbacks
+      before_create :set_created_at
+
+      # Scopes
+      scope :recent, -> { order(created_at: :desc) }
+      scope :by_name, ->(name) { where(name: name) }
+
+      # Class methods
+      def self.find_or_create_by_name(robot_name)
+        find_or_create_by(name: robot_name)
+      end
+
+      # Instance methods
+      def node_count
+        nodes.count
+      end
+
+      def recent_nodes(limit = 10)
+        nodes.recent.limit(limit)
+      end
+
+      def memory_summary
+        {
+          total_nodes: nodes.count,
+          in_working_memory: nodes.in_working_memory.count,
+          with_embeddings: nodes.with_embeddings.count
+        }
+      end
+
+      private
+
+      def set_created_at
+        self.created_at ||= Time.current
+      end
+    end
+  end
+end

data/lib/htm/models/tag.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+class HTM
+  module Models
+    # Tag model - represents unique tag names
+    # Tags have a many-to-many relationship with nodes through node_tags
+    class Tag < ActiveRecord::Base
+      self.table_name = 'tags'
+
+      # Associations
+      has_many :node_tags, class_name: 'HTM::Models::NodeTag', dependent: :destroy
+      has_many :nodes, through: :node_tags, class_name: 'HTM::Models::Node'
+
+      # Validations
+      validates :name, presence: true
+      validates :name, format: {
+        with: /\A[a-z0-9\-]+(:[a-z0-9\-]+)*\z/,
+        message: "must be lowercase with hyphens, using colons for hierarchy (e.g., 'database:postgresql:performance')"
+      }
+      validates :name, uniqueness: { message: "already exists" }
+
+      # Callbacks
+      before_create :set_created_at
+
+      # Scopes
+      scope :by_name, ->(name) { where(name: name) }
+      scope :with_prefix, ->(prefix) { where("name LIKE ?", "#{prefix}%") }
+      scope :hierarchical, -> { where("name LIKE '%:%'") }
+      scope :root_level, -> { where("name NOT LIKE '%:%'") }
+
+      # Class methods
+      def self.find_by_topic_prefix(prefix)
+        where("name LIKE ?", "#{prefix}%")
+      end
+
+      def self.popular_tags(limit = 10)
+        joins(:node_tags)
+          .select('tags.*, COUNT(node_tags.id) as usage_count')
+          .group('tags.id')
+          .order('usage_count DESC')
+          .limit(limit)
+      end
+
+      def self.find_or_create_by_name(name)
+        find_or_create_by(name: name)
+      end
+
+      # Instance methods
+      def root_topic
+        name.split(':').first
+      end
+
+      def topic_levels
+        name.split(':')
+      end
+
+      def depth
+        topic_levels.length
+      end
+
+      def hierarchical?
+        name.include?(':')
+      end
+
+      def usage_count
+        node_tags.count
+      end
+
+      private
+
+      def set_created_at
+        self.created_at ||= Time.current
+      end
+    end
+  end
+end

data/lib/htm/railtie.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'rails/railtie'
+
+class HTM
+  # Rails Railtie for automatic HTM configuration in Rails applications
+  #
+  # This railtie automatically configures HTM when Rails boots:
+  # - Sets logger to Rails.logger
+  # - Sets job backend to :active_job
+  # - Loads Rake tasks
+  # - Configures test environment for synchronous jobs
+  #
+  # @example Rails application
+  #   # HTM is automatically configured on Rails boot
+  #   # No additional setup required
+  #
+  # @example Custom configuration
+  #   # config/initializers/htm.rb
+  #   HTM.configure do |config|
+  #     config.embedding_model = 'custom-model'
+  #     config.tag_model = 'custom-tag-model'
+  #   end
+  #
+  class Railtie < Rails::Railtie
+    railtie_name :htm
+
+    # Configure HTM before Rails initializers run
+    initializer "htm.configure" do |app|
+      HTM.configure do |config|
+        # Use Rails logger
+        config.logger = Rails.logger
+
+        # Use ActiveJob for background jobs in Rails
+        config.job_backend = :active_job unless Rails.env.test?
+
+        # Use inline execution in test environment for synchronous behavior
+        config.job_backend = :inline if Rails.env.test?
+      end
+
+      HTM.logger.info "HTM initialized for Rails application"
+      HTM.logger.debug "HTM job backend: #{HTM.configuration.job_backend}"
+    end
+
+    # Load Rake tasks
+    rake_tasks do
+      load File.expand_path('../tasks/htm.rake', __dir__)
+      load File.expand_path('../tasks/jobs.rake', __dir__)
+    end
+
+    # Add middleware for connection management (if needed)
+    initializer "htm.middleware" do |app|
+      # Middleware can be added here if needed for connection cleanup
+      # app.middleware.use HTM::Middleware
+    end
+
+    # Optionally verify database connection on boot (development only)
+    config.after_initialize do
+      if Rails.env.development?
+        begin
+          HTM::ActiveRecordConfig.establish_connection! unless HTM::ActiveRecordConfig.connected?
+          HTM::ActiveRecordConfig.verify_extensions!
+          HTM.logger.info "HTM database connection verified"
+        rescue StandardError => e
+          HTM.logger.warn "HTM database connection check failed: #{e.message}"
+          HTM.logger.warn "Set HTM_DBURL environment variable or configure database.yml"
+        end
+      end
+    end
+
+    # Add generators path
+    config.generators do |g|
+      g.templates.unshift File.expand_path('../generators/templates', __dir__)
+    end
+  end
+end

data/lib/htm/sinatra.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require 'sinatra/base'
+
+class HTM
+  # Sinatra helpers for HTM integration
+  #
+  # Provides convenient helper methods for using HTM in Sinatra applications.
+  #
+  # @example Basic usage
+  #   class MyApp < Sinatra::Base
+  #     helpers HTM::Sinatra::Helpers
+  #
+  #     before do
+  #       init_htm(robot_name: session[:user_id] || 'guest')
+  #     end
+  #
+  #     post '/remember' do
+  #       node_id = htm.remember(params[:content], source: 'user')
+  #       json status: 'ok', node_id: node_id
+  #     end
+  #
+  #     get '/recall' do
+  #       memories = htm.recall(params[:topic], limit: 10)
+  #       json memories: memories
+  #     end
+  #   end
+  #
+  module Sinatra
+    module Helpers
+      # Initialize HTM instance for current request
+      #
+      # @param robot_name [String] Robot identifier (typically user/session ID)
+      # @param working_memory_size [Integer] Token limit for working memory
+      # @return [HTM] HTM instance for this request
+      #
+      def init_htm(robot_name: 'guest', working_memory_size: 128_000)
+        @htm = HTM.new(
+          robot_name: robot_name,
+          working_memory_size: working_memory_size
+        )
+      end
+
+      # Get current HTM instance
+      #
+      # @return [HTM] HTM instance for this request
+      # @raise [RuntimeError] If HTM not initialized (call init_htm first)
+      #
+      def htm
+        @htm || raise("HTM not initialized. Call init_htm in a before filter.")
+      end
+
+      # Remember information (convenience method)
+      #
+      # @param content [String] Content to remember
+      # @param source [String] Source identifier (default: 'user')
+      # @return [Integer] Node ID
+      #
+      def remember(content, source: 'user')
+        htm.remember(content, source: source)
+      end
+
+      # Recall memories (convenience method)
+      #
+      # @param topic [String] Topic to search for
+      # @param options [Hash] Recall options (timeframe, limit, strategy, etc.)
+      # @return [Array<Hash>] Matching memories
+      #
+      def recall(topic, **options)
+        htm.recall(topic, **options)
+      end
+
+      # JSON response helper
+      #
+      # @param data [Hash] Data to convert to JSON
+      # @return [String] JSON response
+      #
+      def json(data)
+        content_type :json
+        data.to_json
+      end
+    end
+
+    # Rack middleware for HTM connection management
+    #
+    # Ensures database connections are properly managed across requests.
+    #
+    # @example Use in Sinatra app
+    #   class MyApp < Sinatra::Base
+    #     use HTM::Sinatra::Middleware
+    #   end
+    #
+    class Middleware
+      def initialize(app, options = {})
+        @app = app
+        @options = options
+      end
+
+      def call(env)
+        # Establish connection if needed
+        unless HTM::ActiveRecordConfig.connected?
+          HTM::ActiveRecordConfig.establish_connection!
+        end
+
+        # Process request
+        status, headers, body = @app.call(env)
+
+        # Return response
+        [status, headers, body]
+      ensure
+        # Return connections to pool
+        ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord)
+      end
+    end
+  end
+end
+
+# Extend Sinatra::Base with HTM registration helper
+module ::Sinatra
+  class Base
+    # Register HTM with Sinatra application
+    #
+    # Automatically configures HTM for Sinatra apps:
+    # - Adds helpers
+    # - Adds middleware
+    # - Configures logger
+    #
+    # @example
+    #   class MyApp < Sinatra::Base
+    #     register_htm
+    #
+    #     post '/remember' do
+    #       remember(params[:content])
+    #     end
+    #   end
+    #
+    def self.register_htm
+      helpers HTM::Sinatra::Helpers
+      use HTM::Sinatra::Middleware
+
+      # Configure HTM with Sinatra logger
+      HTM.configure do |config|
+        config.logger = logger if respond_to?(:logger)
+
+        # Use Sidekiq if available, otherwise thread-based
+        if defined?(::Sidekiq)
+          config.job_backend = :sidekiq
+        else
+          config.job_backend = :thread
+        end
+      end
+
+      HTM.logger.info "HTM registered with Sinatra application"
+      HTM.logger.debug "HTM job backend: #{HTM.configuration.job_backend}"
+    end
+  end
+end

data/lib/htm/tag_service.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require_relative 'errors'
+
+class HTM
+  # Tag Service - Processes and validates hierarchical tags
+  #
+  # This service wraps the configured tag extractor and provides:
+  # - Response parsing (string or array)
+  # - Format validation (lowercase, alphanumeric, hyphens, colons)
+  # - Depth validation (max 5 levels)
+  # - Ontology consistency
+  #
+  # The actual LLM call is delegated to HTM.configuration.tag_extractor
+  #
+  class TagService
+    MAX_DEPTH = 5  # Maximum hierarchy depth (4 colons)
+    TAG_FORMAT = /^[a-z0-9\-]+(:[a-z0-9\-]+)*$/  # Validation regex
+
+    # Extract tags with validation and processing
+    #
+    # @param content [String] Text to analyze
+    # @param existing_ontology [Array<String>] Sample of existing tags for context
+    # @return [Array<String>] Validated tag names
+    #
+    def self.extract(content, existing_ontology: [])
+      HTM.logger.debug "TagService: Extracting tags from #{content.length} chars"
+      HTM.logger.debug "TagService: Using ontology with #{existing_ontology.size} existing tags"
+
+      # Call configured tag extractor
+      raw_tags = HTM.configuration.tag_extractor.call(content, existing_ontology)
+
+      # Parse response (may be string or array)
+      parsed_tags = parse_tags(raw_tags)
+
+      # Validate and filter tags
+      valid_tags = validate_and_filter_tags(parsed_tags)
+
+      HTM.logger.debug "TagService: Extracted #{valid_tags.length} valid tags: #{valid_tags.join(', ')}"
+
+      valid_tags
+
+    rescue HTM::TagError
+      raise
+    rescue StandardError => e
+      HTM.logger.error "TagService: Failed to extract tags: #{e.message}"
+      raise HTM::TagError, "Tag extraction failed: #{e.message}"
+    end
+
+    # Parse tag response (handles string or array input)
+    #
+    # @param raw_tags [String, Array] Raw response from extractor
+    # @return [Array<String>] Parsed tag strings
+    #
+    def self.parse_tags(raw_tags)
+      case raw_tags
+      when Array
+        # Already an array, return as-is
+        raw_tags.map(&:to_s).map(&:strip).reject(&:empty?)
+      when String
+        # String response - split by newlines
+        raw_tags.split("\n").map(&:strip).reject(&:empty?)
+      else
+        raise HTM::TagError, "Tag response must be Array or String, got #{raw_tags.class}"
+      end
+    end
+
+    # Validate and filter tags
+    #
+    # @param tags [Array<String>] Parsed tags
+    # @return [Array<String>] Valid tags only
+    #
+    def self.validate_and_filter_tags(tags)
+      valid_tags = []
+
+      tags.each do |tag|
+        # Check format
+        unless tag.match?(TAG_FORMAT)
+          HTM.logger.warn "TagService: Invalid tag format, skipping: #{tag}"
+          next
+        end
+
+        # Check depth
+        depth = tag.count(':')
+        if depth >= MAX_DEPTH
+          HTM.logger.warn "TagService: Tag depth #{depth + 1} exceeds max #{MAX_DEPTH}, skipping: #{tag}"
+          next
+        end
+
+        # Tag is valid
+        valid_tags << tag
+      end
+
+      valid_tags.uniq
+    end
+
+    # Validate single tag format
+    #
+    # @param tag [String] Tag to validate
+    # @return [Boolean] True if valid
+    #
+    def self.valid_tag?(tag)
+      return false unless tag.is_a?(String)
+      return false if tag.empty?
+      return false unless tag.match?(TAG_FORMAT)
+      return false if tag.count(':') >= MAX_DEPTH
+
+      true
+    end
+
+    # Parse hierarchical structure of a tag
+    #
+    # @param tag [String] Hierarchical tag (e.g., "ai:llm:embedding")
+    # @return [Hash] Hierarchy structure
+    #   {
+    #     full: "ai:llm:embedding",
+    #     root: "ai",
+    #     parent: "ai:llm",
+    #     levels: ["ai", "llm", "embedding"],
+    #     depth: 3
+    #   }
+    #
+    def self.parse_hierarchy(tag)
+      levels = tag.split(':')
+
+      {
+        full: tag,
+        root: levels.first,
+        parent: levels.size > 1 ? levels[0..-2].join(':') : nil,
+        levels: levels,
+        depth: levels.size
+      }
+    end
+  end
+end

data/lib/htm/tasks.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# HTM Rake Tasks Loader
+#
+# Load HTM database management tasks into your application's Rakefile:
+#
+#   require 'htm/tasks'
+#
+# This will make the following tasks available:
+#
+# Database tasks:
+#   rake htm:db:setup      # Set up HTM database schema and run migrations
+#   rake htm:db:migrate    # Run pending database migrations
+#   rake htm:db:status     # Show migration status
+#   rake htm:db:info       # Show database info
+#   rake htm:db:test       # Test database connection
+#   rake htm:db:console    # Open PostgreSQL console
+#   rake htm:db:seed       # Seed database with sample data
+#   rake htm:db:drop       # Drop all HTM tables (destructive!)
+#   rake htm:db:reset      # Drop and recreate database (destructive!)
+#
+# Async job tasks:
+#   rake htm:jobs:stats              # Show async job statistics
+#   rake htm:jobs:process_embeddings # Process pending embedding jobs
+#   rake htm:jobs:process_tags       # Process pending tag extraction jobs
+#   rake htm:jobs:process_all        # Process all pending jobs
+#   rake htm:jobs:reprocess_embeddings # Force regenerate all embeddings
+#   rake htm:jobs:failed             # Show nodes with processing issues
+#   rake htm:jobs:clear_all          # Clear all embeddings and tags (testing)
+#
+
+if defined?(Rake)
+  # Load the rake tasks
+  load File.expand_path('../tasks/htm.rake', __dir__)
+  load File.expand_path('../tasks/jobs.rake', __dir__)
+else
+  warn "HTM tasks not loaded: Rake is not available"
+end

data/lib/htm/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class HTM
+  VERSION = "0.0.1"
+end