htm 0.0.31 → 0.0.32

data/lib/htm/workflows/remember_workflow.rb

@@ -79,134 +79,131 @@ class HTM
       private
 
       def build_pipeline
+        save_step  = save_node_step
+        embed_step = generate_embedding_step
+        tags_step  = generate_tags_step
+        props_step = generate_propositions_step
+        final_step = finalize_step
+
         SimpleFlow::Pipeline.new(concurrency: @concurrency) do
-          # Step 1: Save node to database (no dependencies)
-          step :save_node, ->(result) {
-            data = result.value
-            htm = data[:htm]
-
-            # Calculate token count
-            token_count = HTM.count_tokens(data[:content])
-
-            # Store in long-term memory
-            save_result = htm.long_term_memory.add(
-              content: data[:content],
-              token_count: token_count,
-              robot_id: data[:robot_id],
-              embedding: nil,
-              metadata: data[:metadata]
-            )
-
-            node_id = save_result[:node_id]
-            is_new = save_result[:is_new]
-
-            HTM.logger.info "RememberWorkflow: Node #{node_id} saved (new: #{is_new})"
-
-            result
-              .with_context(:node_id, node_id)
-              .with_context(:is_new, is_new)
-              .with_context(:token_count, token_count)
-              .with_context(:robot_node, save_result[:robot_node])
-              .continue(data)
-          }, depends_on: :none
-
-          # Step 2: Generate embedding (depends on save_node, runs in parallel with tags/propositions)
-          step :generate_embedding, ->(result) {
-            node_id = result.context[:node_id]
-            is_new = result.context[:is_new]
-
-            # Only generate for new nodes
-            if is_new
-              begin
-                HTM::Jobs::GenerateEmbeddingJob.perform(node_id: node_id)
-              rescue StandardError => e
-                HTM.logger.error "RememberWorkflow: Embedding generation failed: #{e.message}"
-                # Continue despite error - embedding is non-critical
-              end
-            end
+          step :save_node,             save_step,  depends_on: :none
+          step :generate_embedding,    embed_step, depends_on: [:save_node]
+          step :generate_tags,         tags_step,  depends_on: [:save_node]
+          step :generate_propositions, props_step, depends_on: [:save_node]
+          step :finalize,              final_step, depends_on: %i[generate_embedding generate_tags generate_propositions]
+        end
+      end
 
-            result.continue(result.value)
-          }, depends_on: [:save_node]
-
-          # Step 3: Generate tags (depends on save_node, runs in parallel with embedding/propositions)
-          step :generate_tags, ->(result) {
-            node_id = result.context[:node_id]
-            is_new = result.context[:is_new]
-            manual_tags = result.value[:tags] || []
-
-            if is_new
-              # Add manual tags immediately (including parent tags)
-              if manual_tags.any?
-                manual_tags.each do |tag_name|
-                  HTM::Models::Tag.find_or_create_with_ancestors(tag_name).each do |tag|
-                    HTM::Models::NodeTag.find_or_create(node_id: node_id, tag_id: tag.id)
-                  end
-                end
-              end
+      def save_node_step
+        lambda { |result|
+          data = result.value
+          htm = data[:htm]
+          token_count = HTM.count_tokens(data[:content])
+          save_result = htm.long_term_memory.add(
+            content: data[:content],
+            token_count: token_count,
+            robot_id: data[:robot_id],
+            embedding: nil,
+            metadata: data[:metadata]
+          )
+          node_id = save_result[:node_id]
+          is_new = save_result[:is_new]
+          HTM.logger.info "RememberWorkflow: Node #{node_id} saved (new: #{is_new})"
+          result
+            .with_context(:node_id, node_id)
+            .with_context(:is_new, is_new)
+            .with_context(:token_count, token_count)
+            .with_context(:robot_node, save_result[:robot_node])
+            .continue(data)
+        }
+      end
 
-              begin
-                HTM::Jobs::GenerateTagsJob.perform(node_id: node_id)
-              rescue StandardError => e
-                HTM.logger.error "RememberWorkflow: Tag generation failed: #{e.message}"
-                # Continue despite error - tags are non-critical
-              end
-            else
-              # For existing nodes, only add manual tags
-              if manual_tags.any?
-                node = HTM::Models::Node[node_id]
-                node.add_tags(manual_tags)
-              end
+      def generate_embedding_step
+        lambda { |result|
+          node_id = result.context[:node_id]
+          if result.context[:is_new]
+            begin
+              HTM::Jobs::GenerateEmbeddingJob.perform(node_id: node_id)
+            rescue StandardError => e
+              HTM.logger.error "RememberWorkflow: Embedding generation failed: #{e.message}"
             end
+          end
+          result.continue(result.value)
+        }
+      end
 
-            result.continue(result.value)
-          }, depends_on: [:save_node]
-
-          # Step 4: Generate propositions (depends on save_node, runs in parallel with embedding/tags)
-          step :generate_propositions, ->(result) {
-            node_id = result.context[:node_id]
-            is_new = result.context[:is_new]
-            metadata = result.value[:metadata] || {}
-            robot_id = result.value[:robot_id]
-
-            # Only extract propositions for new nodes that aren't already propositions
-            if is_new && HTM.config.extract_propositions && !metadata[:is_proposition]
-              begin
-                HTM::Jobs::GeneratePropositionsJob.perform(node_id: node_id, robot_id: robot_id)
-              rescue StandardError => e
-                HTM.logger.error "RememberWorkflow: Proposition extraction failed: #{e.message}"
-                # Continue despite error - propositions are non-critical
+      def generate_tags_step
+        lambda { |result|
+          node_id    = result.context[:node_id]
+          is_new     = result.context[:is_new]
+          manual_tags = result.value[:tags] || []
+
+          if is_new
+            manual_tags.each do |tag_name|
+              HTM::Models::Tag.find_or_create_with_ancestors(tag_name).each do |tag|
+                HTM::Models::NodeTag.find_or_create(node_id: node_id, tag_id: tag.id)
               end
             end
+            begin
+              HTM::Jobs::GenerateTagsJob.perform(node_id: node_id)
+            rescue StandardError => e
+              HTM.logger.error "RememberWorkflow: Tag generation failed: #{e.message}"
+            end
+          elsif manual_tags.any?
+            HTM::Models::Node[node_id].add_tags(manual_tags)
+          end
+
+          result.continue(result.value)
+        }
+      end
 
-            result.continue(result.value)
-          }, depends_on: [:save_node]
-
-          # Step 5: Finalize (depends on all enrichment steps)
-          step :finalize, ->(result) {
-            node_id = result.context[:node_id]
-            token_count = result.context[:token_count]
-            robot_node = result.context[:robot_node]
-            htm = result.value[:htm]
-
-            # Add to working memory
-            unless htm.working_memory.has_space?(token_count)
-              evicted = htm.working_memory.evict_to_make_space(token_count)
-              evicted_keys = evicted.map { |n| n[:key] }
-              htm.long_term_memory.mark_evicted(robot_id: result.value[:robot_id], node_ids: evicted_keys) if evicted_keys.any?
+      def generate_propositions_step
+        lambda { |result|
+          node_id  = result.context[:node_id]
+          is_new   = result.context[:is_new]
+          metadata = result.value[:metadata] || {}
+          robot_id = result.value[:robot_id]
+
+          if is_new && HTM.config.extract_propositions && !metadata[:is_proposition]
+            begin
+              HTM::Jobs::GeneratePropositionsJob.perform(node_id: node_id, robot_id: robot_id)
+            rescue StandardError => e
+              HTM.logger.error "RememberWorkflow: Proposition extraction failed: #{e.message}"
             end
-            htm.working_memory.add(node_id, result.value[:content], token_count: token_count, access_count: 0)
+          end
 
-            # Mark as in working memory
-            robot_node.update(working_memory: true)
+          result.continue(result.value)
+        }
+      end
 
-            # Update robot activity
-            htm.long_term_memory.update_robot_activity(result.value[:robot_id])
+      def finalize_step
+        lambda { |result|
+          ctx = result.context
+          finalize_node(
+            htm:         result.value[:htm],
+            node_id:     ctx[:node_id],
+            token_count: ctx[:token_count],
+            robot_node:  ctx[:robot_node],
+            content:     result.value[:content],
+            robot_id:    result.value[:robot_id]
+          )
+          result.continue(result.value)
+        }
+      end
 
-            HTM.logger.info "RememberWorkflow: Node #{node_id} finalized"
+      def finalize_node(htm:, node_id:, token_count:, robot_node:, content:, robot_id:)
+        evict_working_memory_if_needed(htm, token_count, robot_id)
+        htm.working_memory.add(node_id, content, token_count: token_count, access_count: 0)
+        robot_node.update(working_memory: true)
+        htm.long_term_memory.update_robot_activity(robot_id)
+        HTM.logger.info "RememberWorkflow: Node #{node_id} finalized"
+      end
 
-            result.continue(result.value)
-          }, depends_on: [:generate_embedding, :generate_tags, :generate_propositions]
-        end
+      def evict_working_memory_if_needed(htm, token_count, robot_id)
+        return if htm.working_memory.has_space?(token_count)
+        evicted      = htm.working_memory.evict_to_make_space(token_count)
+        evicted_keys = evicted.map { |n| n[:key] }
+        htm.long_term_memory.mark_evicted(robot_id: robot_id, node_ids: evicted_keys) if evicted_keys.any?
       end
     end
   end

data/lib/htm/working_memory.rb

@@ -84,7 +84,7 @@ class HTM
         tokens_freed = 0
 
         # Sort by access frequency + recency (lower score = more evictable)
-        candidates = @nodes.sort_by do |key, node|
+        candidates = @nodes.sort_by do |_key, node|
           access_frequency = node[:access_count] || 0
           time_since_accessed = Time.now - (node[:last_accessed] || node[:added_at])
 
@@ -92,7 +92,7 @@ class HTM
           # Frequently accessed = higher score (keep)
           # Recently accessed = higher score (keep)
           access_score = Math.log(1 + access_frequency)
-          recency_score = 1.0 / (1 + time_since_accessed / 3600.0)
+          recency_score = 1.0 / (1 + (time_since_accessed / 3600.0))
 
           -(access_score + recency_score)  # Negative for ascending sort
         end
@@ -122,33 +122,10 @@ class HTM
     def assemble_context(strategy:, max_tokens: nil)
       @mutex.synchronize do
         max = max_tokens || @max_tokens
+        nodes = sorted_nodes_by_strategy(strategy)
 
-        # Make defensive copies of nodes to prevent external mutation of internal state
-        nodes = case strategy
-        when :recent
-          # Most recently accessed (LRU)
-          @access_order.reverse.map { |k| @nodes[k]&.dup }.compact
-        when :frequent
-          # Most frequently accessed (LFU)
-          @nodes.sort_by { |k, v| -(v[:access_count] || 0) }.map { |_, v| v.dup }
-        when :balanced
-          # Combined frequency × recency
-          @nodes.sort_by { |k, v|
-            access_frequency = v[:access_count] || 0
-            time_since_accessed = Time.now - (v[:last_accessed] || v[:added_at])
-            recency_factor = 1.0 / (1 + time_since_accessed / 3600.0)
-
-            # Higher score = more relevant
-            -(Math.log(1 + access_frequency) * recency_factor)
-          }.map { |_, v| v.dup }
-        else
-          raise ArgumentError, "Unknown strategy: #{strategy}. Use :recent, :frequent, or :balanced"
-        end
-
-        # Build context up to token limit
         context_parts = []
         current_tokens = 0
-
         nodes.each do |node|
           break if current_tokens + node[:token_count] > max
           context_parts << node[:value]
@@ -263,6 +240,24 @@ class HTM
 
     private
 
+    def sorted_nodes_by_strategy(strategy)
+      case strategy
+      when :recent
+        @access_order.reverse.map { |k| @nodes[k]&.dup }.compact
+      when :frequent
+        @nodes.sort_by { |_k, v| -(v[:access_count] || 0) }.map { |_, v| v.dup }
+      when :balanced
+        sorted = @nodes.sort_by do |_k, v|
+          freq = v[:access_count] || 0
+          age  = Time.now - (v[:last_accessed] || v[:added_at])
+          -(Math.log(1 + freq) * (1.0 / (1 + (age / 3600.0))))
+        end
+        sorted.map { |_, v| v.dup }
+      else
+        raise ArgumentError, "Unknown strategy: #{strategy}. Use :recent, :frequent, or :balanced"
+      end
+    end
+
     # Internal unlocked version - must be called within @mutex.synchronize
     def current_tokens_unlocked
       @nodes.values.sum { |n| n[:token_count] }

data/lib/htm.rb

@@ -17,6 +17,7 @@ require_relative "htm/job_adapter"
 require_relative "htm/jobs/generate_embedding_job"
 require_relative "htm/jobs/generate_tags_job"
 require_relative "htm/jobs/generate_propositions_job"
+require_relative "htm/jobs/generate_relationships_job"
 require_relative "htm/loaders/markdown_chunker"
 require_relative "htm/loaders/markdown_loader"
 require_relative "htm/workflows/remember_workflow"
@@ -49,13 +50,13 @@ require_relative "htm/railtie" if defined?(Rails::Railtie)
 #   htm = HTM.new(robot_name: "Code Helper")
 #
 #   # Remember information
-#   htm.remember("We decided to use PostgreSQL for HTM", source: "architect")
+#   htm.remember("We decided to use PostgreSQL for HTM", metadata: { source: "architect" })
 #
 #   # Recall from the past
-#   memories = htm.recall(timeframe: "last week", topic: "PostgreSQL")
+#   memories = htm.recall("PostgreSQL", timeframe: "last week")
 #
 #   # Create context for LLM
-#   context = htm.create_context(strategy: :balanced)
+#   context = htm.working_memory.assemble_context(strategy: :balanced)
 #
 class HTM
   attr_reader :robot_id, :robot_name, :working_memory, :long_term_memory
@@ -65,7 +66,7 @@ class HTM
   MAX_VALUE_LENGTH = 1_000_000  # 1MB
   MAX_ARRAY_SIZE = 1000
 
-  VALID_RECALL_STRATEGIES = [:vector, :fulltext, :hybrid].freeze
+  VALID_RECALL_STRATEGIES = %i[vector fulltext hybrid].freeze
 
   # Initialize a new HTM instance
   #
@@ -125,77 +126,30 @@ class HTM
   #   node_id = htm.remember("User prefers dark mode", metadata: { source: "user", confidence: 0.95 })
   #
   def remember(content, tags: [], metadata: {})
-    # Validate inputs
     raise ValidationError, "Content cannot be nil" if content.nil?
-
-    content_str = content.to_s.strip
-    raise ValidationError, "Content cannot be empty" if content_str.empty?
-
-    if content_str.bytesize > MAX_VALUE_LENGTH
-      raise ValidationError, "Content exceeds maximum size (#{MAX_VALUE_LENGTH} bytes)"
-    end
-
-    validate_array!(tags, "tags")
-    tags.each do |tag|
-      unless tag.is_a?(String) && tag.match?(/\A[a-z0-9\-]+(:[a-z0-9\-]+)*\z/)
-        raise ValidationError, "Invalid tag format: #{tag.inspect}. Tags must be lowercase alphanumeric with hyphens, separated by colons."
-      end
-    end
-
+    content = content.to_s.strip
+    raise ValidationError, "Content cannot be empty" if content.empty?
+    validate_remember_content!(content)
+    validate_remember_tags!(tags)
     validate_metadata!(metadata)
 
-    content = content_str
-
-    # Calculate token count using configured counter
     token_count = HTM.count_tokens(content)
-
-    # Store in long-term memory (with deduplication)
-    # Returns { node_id:, is_new:, robot_node: }
     result = @long_term_memory.add(
-      content: content,
-      token_count: token_count,
-      robot_id: @robot_id,
-      embedding: nil,  # Will be generated in background
-      metadata: metadata
+      content: content, token_count: token_count, robot_id: @robot_id,
+      embedding: nil, metadata: metadata
     )
-
     node_id = result[:node_id]
-    is_new = result[:is_new]
 
-    if is_new
+    if result[:is_new]
       HTM.logger.info "Node #{node_id} created for robot #{@robot_name} (#{token_count} tokens)"
-
-      # Enqueue background jobs for embedding and tag generation
-      # Only for NEW nodes - existing nodes already have embeddings/tags
-      enqueue_embedding_job(node_id)
-      enqueue_tags_job(node_id, manual_tags: tags)
-
-      # Enqueue proposition extraction if enabled and not already a proposition
-      if HTM.config.extract_propositions && !metadata[:is_proposition]
-        enqueue_propositions_job(node_id)
-      end
+      enqueue_background_jobs(node_id, tags: tags, metadata: metadata)
     else
-      HTM.logger.info "Node #{node_id} already exists, linked to robot #{@robot_name} (remember_count: #{result[:robot_node].remember_count})"
-
-      # For existing nodes, only add manual tags if provided
-      if tags.any?
-        node = HTM::Models::Node[node_id]
-        node.add_tags(tags)
-        HTM.logger.info "Added #{tags.length} manual tags to existing node #{node_id}"
-      end
+      rc = result[:robot_node].remember_count
+      HTM.logger.info "Node #{node_id} already exists, linked to robot #{@robot_name} (remember_count: #{rc})"
+      handle_existing_node_tags(node_id, tags)
     end
 
-    # Add to working memory (evict if needed, access_count starts at 0)
-    unless @working_memory.has_space?(token_count)
-      evicted = @working_memory.evict_to_make_space(token_count)
-      evicted_keys = evicted.map { |n| n[:key] }
-      @long_term_memory.mark_evicted(robot_id: @robot_id, node_ids: evicted_keys) if evicted_keys.any?
-    end
-    @working_memory.add(node_id, content, token_count: token_count, access_count: 0)
-
-    # Mark node as in working memory in the robot_nodes join table
-    result[:robot_node].update(working_memory: true)
-
+    store_in_working_memory(node_id, content, token_count: token_count, robot_node: result[:robot_node])
     update_robot_activity
     node_id
   end
@@ -250,53 +204,53 @@ class HTM
     # Normalize timeframe and potentially extract from query
     search_query = topic
     normalized_timeframe = if timeframe == :auto
-      result = HTM::Timeframe.normalize(:auto, query: topic)
-      search_query = result.query  # Use cleaned query for search
-      result.timeframe
-    else
-      HTM::Timeframe.normalize(timeframe)
-    end
+                             result = HTM::Timeframe.normalize(:auto, query: topic)
+                             search_query = result.query  # Use cleaned query for search
+                             result.timeframe
+                           else
+                             HTM::Timeframe.normalize(timeframe)
+                           end
 
     # Use relevance-based search if requested
-    if with_relevance
-      nodes = @long_term_memory.search_with_relevance(
-        timeframe: normalized_timeframe,
-        query: search_query,
-        query_tags: query_tags,
-        limit: limit,
-        embedding_service: (strategy == :vector || strategy == :hybrid) ? HTM : nil,
-        metadata: metadata
-      )
-    else
-      # Perform standard RAG-based retrieval
-      nodes = case strategy
-      when :vector
-        # Vector search using query embedding
-        @long_term_memory.search(
-          timeframe: normalized_timeframe,
-          query: search_query,
-          limit: limit,
-          embedding_service: HTM,
-          metadata: metadata
-        )
-      when :fulltext
-        @long_term_memory.search_fulltext(
-          timeframe: normalized_timeframe,
-          query: search_query,
-          limit: limit,
-          metadata: metadata
-        )
-      when :hybrid
-        # Hybrid search combining vector + fulltext
-        @long_term_memory.search_hybrid(
-          timeframe: normalized_timeframe,
-          query: search_query,
-          limit: limit,
-          embedding_service: HTM,
-          metadata: metadata
-        )
-      end
-    end
+    nodes = if with_relevance
+              @long_term_memory.search_with_relevance(
+                timeframe: normalized_timeframe,
+                query: search_query,
+                query_tags: query_tags,
+                limit: limit,
+                embedding_service: %i[vector hybrid].include?(strategy) ? HTM : nil,
+                metadata: metadata
+              )
+            else
+              # Perform standard RAG-based retrieval
+              case strategy
+              when :vector
+                # Vector search using query embedding
+                @long_term_memory.search(
+                  timeframe: normalized_timeframe,
+                  query: search_query,
+                  limit: limit,
+                  embedding_service: HTM,
+                  metadata: metadata
+                )
+              when :fulltext
+                @long_term_memory.search_fulltext(
+                  timeframe: normalized_timeframe,
+                  query: search_query,
+                  limit: limit,
+                  metadata: metadata
+                )
+              when :hybrid
+                # Hybrid search combining vector + fulltext
+                @long_term_memory.search_hybrid(
+                  timeframe: normalized_timeframe,
+                  query: search_query,
+                  limit: limit,
+                  embedding_service: HTM,
+                  metadata: metadata
+                )
+              end
+            end
 
     # Add to working memory (evict if needed)
     nodes.each do |node|
@@ -471,8 +425,8 @@ class HTM
 
     # Update database: mark all as evicted from working memory
     count = HTM::Models::RobotNode
-      .where(robot_id: @robot_id, working_memory: true)
-      .update(working_memory: false)
+            .where(robot_id: @robot_id, working_memory: true)
+            .update(working_memory: false)
 
     HTM.logger.info "Cleared #{count} nodes from working memory"
     count
@@ -580,6 +534,41 @@ class HTM
     @long_term_memory.update_robot_activity(@robot_id)
   end
 
+  def validate_remember_content!(content)
+    return unless content.bytesize > MAX_VALUE_LENGTH
+    raise ValidationError, "Content exceeds maximum size (#{MAX_VALUE_LENGTH} bytes)"
+  end
+
+  def validate_remember_tags!(tags)
+    validate_array!(tags, "tags")
+    tags.each do |tag|
+      next if tag.is_a?(String) && tag.match?(/\A[a-z0-9-]+(:[a-z0-9-]+)*\z/)
+      raise ValidationError, "Invalid tag format: #{tag.inspect}. Tags must be lowercase alphanumeric with hyphens, separated by colons."
+    end
+  end
+
+  def enqueue_background_jobs(node_id, tags:, metadata:)
+    enqueue_embedding_job(node_id)
+    enqueue_tags_job(node_id, manual_tags: tags)
+    enqueue_propositions_job(node_id) if HTM.config.extract_propositions && !metadata[:is_proposition]
+  end
+
+  def handle_existing_node_tags(node_id, tags)
+    return unless tags.any?
+    HTM::Models::Node[node_id].add_tags(tags)
+    HTM.logger.info "Added #{tags.length} manual tags to existing node #{node_id}"
+  end
+
+  def store_in_working_memory(node_id, content, token_count:, robot_node:)
+    unless @working_memory.has_space?(token_count)
+      evicted      = @working_memory.evict_to_make_space(token_count)
+      evicted_keys = evicted.map { |n| n[:key] }
+      @long_term_memory.mark_evicted(robot_id: @robot_id, node_ids: evicted_keys) if evicted_keys.any?
+    end
+    @working_memory.add(node_id, content, token_count: token_count, access_count: 0)
+    robot_node.update(working_memory: true)
+  end
+
   def enqueue_embedding_job(node_id)
     # Enqueue embedding generation using configured job backend
     # Job will use HTM.embed which delegates to configured embedding_generator
@@ -655,20 +644,18 @@ class HTM
 
   def validate_recall_strategy!(strategy)
     raise ValidationError, "Strategy must be a Symbol" unless strategy.is_a?(Symbol)
-    unless VALID_RECALL_STRATEGIES.include?(strategy)
-      raise ValidationError, "Invalid strategy: #{strategy}. Must be one of #{VALID_RECALL_STRATEGIES.join(', ')}"
-    end
+    return if VALID_RECALL_STRATEGIES.include?(strategy)
+    raise ValidationError, "Invalid strategy: #{strategy}. Must be one of #{VALID_RECALL_STRATEGIES.join(', ')}"
   end
 
-
   def validate_timeframe!(timeframe)
     return if HTM::Timeframe.valid?(timeframe)
     raise ValidationError, "Invalid timeframe type: #{timeframe.class}. " \
-      "Expected nil, Range, Array<Range>, Date, DateTime, Time, String, or :auto"
+                           "Expected nil, Range, Array<Range>, Date, DateTime, Time, String, or :auto"
   end
 
   def validate_positive_integer!(value, name)
-    raise ValidationError, "#{name} must be a positive Integer" unless value.is_a?(Integer) && value > 0
+    raise ValidationError, "#{name} must be a positive Integer" unless value.is_a?(Integer) && value.positive?
   end
 
   def validate_metadata!(metadata)

data/lib/tasks/db.rake

@@ -3,14 +3,12 @@
 namespace :db do
   desc "Run database migrations"
   task :migrate do
-
     HTM::Database.migrate
     puts "Database migrations completed successfully"
   end
 
   desc "Setup database schema (includes migrations)"
   task :setup do
-
     HTM::Database.setup
     puts "Database setup completed successfully"
   end