htm 0.0.18 → 0.0.30

data/lib/htm/sql_builder.rb

@@ -105,29 +105,28 @@ class HTM
 
         prefix = table_alias ? "#{table_alias}." : ""
         full_column = "#{prefix}#{column}"
-        conn = ActiveRecord::Base.connection
 
         case timeframe
         when Range
-          begin_quoted = conn.quote(timeframe.begin.iso8601)
-          end_quoted = conn.quote(timeframe.end.iso8601)
+          begin_quoted = HTM.db.literal(timeframe.begin.iso8601)
+          end_quoted = HTM.db.literal(timeframe.end.iso8601)
           "(#{full_column} BETWEEN #{begin_quoted} AND #{end_quoted})"
         when Array
           conditions = timeframe.map do |range|
-            begin_quoted = conn.quote(range.begin.iso8601)
-            end_quoted = conn.quote(range.end.iso8601)
+            begin_quoted = HTM.db.literal(range.begin.iso8601)
+            end_quoted = HTM.db.literal(range.end.iso8601)
             "(#{full_column} BETWEEN #{begin_quoted} AND #{end_quoted})"
           end
           "(#{conditions.join(' OR ')})"
         end
       end
 
-      # Apply timeframe filter to ActiveRecord scope
+      # Apply timeframe filter to Sequel dataset
       #
-      # @param scope [ActiveRecord::Relation] Base scope
+      # @param scope [Sequel::Dataset] Base dataset
       # @param timeframe [nil, Range, Array<Range>] Time range(s)
       # @param column [Symbol] Column name (default: :created_at)
-      # @return [ActiveRecord::Relation] Scoped query
+      # @return [Sequel::Dataset] Filtered dataset
       #
       def apply_timeframe(scope, timeframe, column: :created_at)
         return scope if timeframe.nil?
@@ -136,8 +135,8 @@ class HTM
         when Range
           scope.where(column => timeframe)
         when Array
-          conditions = timeframe.map { |range| scope.where(column => range) }
-          conditions.reduce { |result, condition| result.or(condition) }
+          conditions = timeframe.map { |range| Sequel.expr(column => range) }
+          scope.where(Sequel.|(*conditions))
         else
           scope
         end
@@ -155,23 +154,22 @@ class HTM
 
         prefix = table_alias ? "#{table_alias}." : ""
         full_column = "#{prefix}#{column}"
-        conn = ActiveRecord::Base.connection
 
-        quoted_metadata = conn.quote(metadata.to_json)
+        quoted_metadata = HTM.db.literal(metadata.to_json)
         "(#{full_column} @> #{quoted_metadata}::jsonb)"
       end
 
-      # Apply metadata filter to ActiveRecord scope
+      # Apply metadata filter to Sequel dataset
       #
-      # @param scope [ActiveRecord::Relation] Base scope
+      # @param scope [Sequel::Dataset] Base dataset
       # @param metadata [Hash] Metadata to filter by
       # @param column [String] Column name (default: "metadata")
-      # @return [ActiveRecord::Relation] Scoped query
+      # @return [Sequel::Dataset] Filtered dataset
       #
       def apply_metadata(scope, metadata, column: "metadata")
         return scope if metadata.nil? || metadata.empty?
 
-        scope.where("#{column} @> ?::jsonb", metadata.to_json)
+        scope.where(Sequel.lit("#{column} @> ?::jsonb", metadata.to_json))
       end
     end
   end

data/lib/htm/tag_service.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'errors'
+require 'active_support/core_ext/string/inflections'
 
 class HTM
   # Tag Service - Processes and validates hierarchical tags
@@ -113,6 +114,8 @@ class HTM
       valid_tags = []
 
       tags.each do |tag|
+        # Normalize: convert plural levels to singular
+        tag = singularize_tag_levels(tag)
         # Check format
         unless tag.match?(TAG_FORMAT)
           HTM.logger.warn "TagService: Invalid tag format, skipping: #{tag}"
@@ -191,5 +194,80 @@ class HTM
         depth: levels.size
       }
     end
+
+    # Words that should NOT be singularized (proper nouns, technical terms, etc.)
+    SINGULARIZE_SKIP_LIST = %w[
+      rails kubernetes aws gcp azure s3 ios macos redis postgres
+      postgresql mysql jenkins travis github gitlab mkdocs devops
+      analytics statistics mathematics physics ethics dynamics
+      graphics linguistics economics robotics
+      pages windows
+    ].freeze
+
+    # Normalize tag levels to singular form
+    #
+    # Converts plural levels to singular using ActiveSupport's singularize.
+    # This ensures taxonomy consistency (e.g., "users" -> "user").
+    #
+    # Skips:
+    # - Proper nouns and technical terms (Rails, MkDocs, etc.)
+    # - Words ending in -ics (analytics, robotics, etc.)
+    # - Words that don't end in common plural patterns
+    #
+    # @param tag [String] Tag with potentially plural levels
+    # @return [String] Tag with all levels singularized
+    #
+    def self.singularize_tag_levels(tag)
+      levels = tag.split(':')
+      singularized = levels.map do |level|
+        singularize_level(level)
+      end
+      singularized.join(':')
+    rescue NoMethodError
+      # singularize not available (ActiveSupport not loaded)
+      tag
+    end
+
+    # Singularize a single tag level with safety checks
+    #
+    # @param level [String] Single tag level
+    # @return [String] Singularized level or original if skipped
+    #
+    def self.singularize_level(level)
+      # Skip if in the skip list
+      return level if SINGULARIZE_SKIP_LIST.include?(level.downcase)
+
+      # Skip words ending in -ics (usually singular: analytics, robotics, etc.)
+      return level if level.end_with?('ics')
+
+      # Skip words ending in -ous (adjectives: victorious, precious, etc.)
+      return level if level.end_with?('ous')
+
+      # Skip words ending in -ss (class, access, etc.)
+      return level if level.end_with?('ss')
+
+      # Skip single-letter or very short words
+      return level if level.length <= 2
+
+      # Only singularize if it looks like a regular plural
+      # (ends in s but not ss, ics, ous)
+      unless level.end_with?('s')
+        return level
+      end
+
+      singular = level.singularize
+
+      # Sanity check: if singularize made it weird, keep original
+      # (e.g., "pages" -> "page" is fine, but "bus" -> "bu" is not)
+      if singular.length < level.length - 2
+        return level
+      end
+
+      if singular != level
+        HTM.logger.debug "TagService: Normalized '#{level}' to '#{singular}'"
+      end
+
+      singular
+    end
   end
 end

data/lib/htm/tasks.rb

@@ -65,6 +65,9 @@
 #
 
 if defined?(Rake)
+  # Load HTM before loading rake tasks
+  require_relative '../htm'
+
   # Load the rake tasks
   load File.expand_path('../tasks/htm.rake', __dir__)
   load File.expand_path('../tasks/jobs.rake', __dir__)

data/lib/htm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class HTM
-  VERSION = '0.0.18'
+  VERSION = '0.0.30'
 end

data/lib/htm/workflows/remember_workflow.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+require 'simple_flow'
+
+class HTM
+  module Workflows
+    # RememberWorkflow orchestrates the parallel processing of node enrichment
+    #
+    # Uses simple_flow to manage the dependency graph and parallel execution
+    # of embedding generation, tag extraction, and proposition extraction.
+    #
+    # The workflow structure:
+    #   save_node (no deps) -> embedding, tags, propositions (parallel)
+    #
+    # @example Basic usage with fiber concurrency
+    #   workflow = HTM::Workflows::RememberWorkflow.new(htm_instance)
+    #   node_id = workflow.call(content: "PostgreSQL is great", tags: ["database"])
+    #
+    # @example With inline execution (for testing)
+    #   workflow = HTM::Workflows::RememberWorkflow.new(htm_instance, concurrency: :threads)
+    #   node_id = workflow.call(content: "Test content")
+    #
+    class RememberWorkflow
+      attr_reader :htm, :pipeline
+
+      # Initialize the remember workflow
+      #
+      # @param htm [HTM] HTM instance for the robot
+      # @param concurrency [Symbol] Concurrency model (:auto, :threads, :async)
+      #
+      def initialize(htm, concurrency: :auto)
+        @htm = htm
+        @concurrency = concurrency
+        @pipeline = build_pipeline
+      end
+
+      # Execute the remember workflow
+      #
+      # @param content [String] Content to remember
+      # @param tags [Array<String>] Manual tags to assign
+      # @param metadata [Hash] Metadata for the node
+      # @return [Integer] Node ID of the created memory
+      #
+      def call(content:, tags: [], metadata: {})
+        initial_data = {
+          content: content,
+          tags: tags,
+          metadata: metadata,
+          robot_id: @htm.robot_id,
+          htm: @htm
+        }
+
+        result = @pipeline.call_parallel(SimpleFlow::Result.new(initial_data))
+
+        if result.continue?
+          result.context[:node_id]
+        else
+          HTM.logger.error "RememberWorkflow failed: #{result.errors.inspect}"
+          raise HTM::Error, "Remember workflow failed: #{result.errors.values.flatten.join(', ')}"
+        end
+      end
+
+      # Get visualization of the workflow as Mermaid diagram
+      #
+      # @return [String] Mermaid diagram source
+      #
+      def to_mermaid
+        @pipeline.visualize_mermaid
+      end
+
+      # Get execution plan
+      #
+      # @return [String] Execution plan description
+      #
+      def execution_plan
+        @pipeline.execution_plan
+      end
+
+      private
+
+      def build_pipeline
+        SimpleFlow::Pipeline.new(concurrency: @concurrency) do
+          # Step 1: Save node to database (no dependencies)
+          step :save_node, ->(result) {
+            data = result.value
+            htm = data[:htm]
+
+            # Calculate token count
+            token_count = HTM.count_tokens(data[:content])
+
+            # Store in long-term memory
+            save_result = htm.long_term_memory.add(
+              content: data[:content],
+              token_count: token_count,
+              robot_id: data[:robot_id],
+              embedding: nil,
+              metadata: data[:metadata]
+            )
+
+            node_id = save_result[:node_id]
+            is_new = save_result[:is_new]
+
+            HTM.logger.info "RememberWorkflow: Node #{node_id} saved (new: #{is_new})"
+
+            result
+              .with_context(:node_id, node_id)
+              .with_context(:is_new, is_new)
+              .with_context(:token_count, token_count)
+              .with_context(:robot_node, save_result[:robot_node])
+              .continue(data)
+          }, depends_on: :none
+
+          # Step 2: Generate embedding (depends on save_node, runs in parallel with tags/propositions)
+          step :generate_embedding, ->(result) {
+            node_id = result.context[:node_id]
+            is_new = result.context[:is_new]
+
+            # Only generate for new nodes
+            if is_new
+              begin
+                HTM::Jobs::GenerateEmbeddingJob.perform(node_id: node_id)
+              rescue StandardError => e
+                HTM.logger.error "RememberWorkflow: Embedding generation failed: #{e.message}"
+                # Continue despite error - embedding is non-critical
+              end
+            end
+
+            result.continue(result.value)
+          }, depends_on: [:save_node]
+
+          # Step 3: Generate tags (depends on save_node, runs in parallel with embedding/propositions)
+          step :generate_tags, ->(result) {
+            node_id = result.context[:node_id]
+            is_new = result.context[:is_new]
+            manual_tags = result.value[:tags] || []
+
+            if is_new
+              # Add manual tags immediately (including parent tags)
+              if manual_tags.any?
+                manual_tags.each do |tag_name|
+                  HTM::Models::Tag.find_or_create_with_ancestors(tag_name).each do |tag|
+                    HTM::Models::NodeTag.find_or_create(node_id: node_id, tag_id: tag.id)
+                  end
+                end
+              end
+
+              begin
+                HTM::Jobs::GenerateTagsJob.perform(node_id: node_id)
+              rescue StandardError => e
+                HTM.logger.error "RememberWorkflow: Tag generation failed: #{e.message}"
+                # Continue despite error - tags are non-critical
+              end
+            else
+              # For existing nodes, only add manual tags
+              if manual_tags.any?
+                node = HTM::Models::Node[node_id]
+                node.add_tags(manual_tags)
+              end
+            end
+
+            result.continue(result.value)
+          }, depends_on: [:save_node]
+
+          # Step 4: Generate propositions (depends on save_node, runs in parallel with embedding/tags)
+          step :generate_propositions, ->(result) {
+            node_id = result.context[:node_id]
+            is_new = result.context[:is_new]
+            metadata = result.value[:metadata] || {}
+            robot_id = result.value[:robot_id]
+
+            # Only extract propositions for new nodes that aren't already propositions
+            if is_new && HTM.config.extract_propositions && !metadata[:is_proposition]
+              begin
+                HTM::Jobs::GeneratePropositionsJob.perform(node_id: node_id, robot_id: robot_id)
+              rescue StandardError => e
+                HTM.logger.error "RememberWorkflow: Proposition extraction failed: #{e.message}"
+                # Continue despite error - propositions are non-critical
+              end
+            end
+
+            result.continue(result.value)
+          }, depends_on: [:save_node]
+
+          # Step 5: Finalize (depends on all enrichment steps)
+          step :finalize, ->(result) {
+            node_id = result.context[:node_id]
+            token_count = result.context[:token_count]
+            robot_node = result.context[:robot_node]
+            htm = result.value[:htm]
+
+            # Add to working memory
+            unless htm.working_memory.has_space?(token_count)
+              evicted = htm.working_memory.evict_to_make_space(token_count)
+              evicted_keys = evicted.map { |n| n[:key] }
+              htm.long_term_memory.mark_evicted(robot_id: result.value[:robot_id], node_ids: evicted_keys) if evicted_keys.any?
+            end
+            htm.working_memory.add(node_id, result.value[:content], token_count: token_count, access_count: 0)
+
+            # Mark as in working memory
+            robot_node.update(working_memory: true)
+
+            # Update robot activity
+            htm.long_term_memory.update_robot_activity(result.value[:robot_id])
+
+            HTM.logger.info "RememberWorkflow: Node #{node_id} finalized"
+
+            result.continue(result.value)
+          }, depends_on: [:generate_embedding, :generate_tags, :generate_propositions]
+        end
+      end
+    end
+  end
+end

data/lib/htm.rb

@@ -4,7 +4,7 @@ require_relative "htm/version"
 require_relative "htm/errors"
 require_relative "htm/config"
 require_relative "htm/circuit_breaker"
-require_relative "htm/active_record_config"
+require_relative "htm/sequel_config"
 require_relative "htm/database"
 require_relative "htm/long_term_memory"
 require_relative "htm/working_memory"
@@ -19,6 +19,7 @@ require_relative "htm/jobs/generate_tags_job"
 require_relative "htm/jobs/generate_propositions_job"
 require_relative "htm/loaders/markdown_chunker"
 require_relative "htm/loaders/markdown_loader"
+require_relative "htm/workflows/remember_workflow"
 require_relative "htm/observability"
 require_relative "htm/telemetry"
 require_relative "htm/working_memory_channel"
@@ -85,8 +86,8 @@ class HTM
     db_cache_size: 1000,
     db_cache_ttl: 300
   )
-    # Establish ActiveRecord connection if not already connected
-    HTM::ActiveRecordConfig.establish_connection! unless HTM::ActiveRecordConfig.connected?
+    # Establish Sequel connection if not already connected
+    HTM::SequelConfig.establish_connection! unless HTM::SequelConfig.db
 
     @robot_name = robot_name || "robot_#{SecureRandom.uuid[0..7]}"
 
@@ -178,7 +179,7 @@ class HTM
 
       # For existing nodes, only add manual tags if provided
       if tags.any?
-        node = HTM::Models::Node.find(node_id)
+        node = HTM::Models::Node[node_id]
         node.add_tags(tags)
         HTM.logger.info "Added #{tags.length} manual tags to existing node #{node_id}"
       end
@@ -193,7 +194,7 @@ class HTM
     @working_memory.add(node_id, content, token_count: token_count, access_count: 0)
 
     # Mark node as in working memory in the robot_nodes join table
-    result[:robot_node].update!(working_memory: true)
+    result[:robot_node].update(working_memory: true)
 
     update_robot_activity
     node_id
@@ -338,7 +339,7 @@ class HTM
     end
 
     # Verify node exists (including soft-deleted for restore scenarios)
-    node = HTM::Models::Node.with_deleted.find_by(id: node_id)
+    node = HTM::Models::Node.with_deleted.first(id: node_id)
     raise HTM::NotFoundError, "Node not found: #{node_id}" unless node
 
     if soft
@@ -387,8 +388,8 @@ class HTM
     end
 
     # Find all nodes containing the substring (case-insensitive)
-    matching_nodes = HTM::Models::Node.where("content ILIKE ?", "%#{content_substring}%")
-    node_ids = matching_nodes.pluck(:id)
+    matching_nodes = HTM::Models::Node.where(Sequel.ilike(:content, "%#{content_substring}%"))
+    node_ids = matching_nodes.select_map(:id)
 
     if node_ids.empty?
       HTM.logger.info "No nodes found containing: #{content_substring}"
@@ -418,7 +419,7 @@ class HTM
     raise ArgumentError, "Node ID cannot be nil" if node_id.nil?
 
     # Find including soft-deleted nodes
-    node = HTM::Models::Node.with_deleted.find_by(id: node_id)
+    node = HTM::Models::Node.with_deleted.first(id: node_id)
     raise HTM::NotFoundError, "Node not found: #{node_id}" unless node
 
     unless node.deleted?
@@ -471,7 +472,7 @@ class HTM
     # Update database: mark all as evicted from working memory
     count = HTM::Models::RobotNode
       .where(robot_id: @robot_id, working_memory: true)
-      .update_all(working_memory: false)
+      .update(working_memory: false)
 
     HTM.logger.info "Cleared #{count} nodes from working memory"
     count
@@ -535,17 +536,17 @@ class HTM
   # Get all nodes loaded from a specific file
   #
   # @param file_path [String] Path to the source file
-  # @return [ActiveRecord::Relation] Nodes from this file, ordered by chunk_position
+  # @return [Array<HTM::Models::Node>] Nodes from this file, ordered by chunk_position
   #
   # @example
   #   nodes = htm.nodes_from_file('/path/to/doc.md')
   #   nodes.each { |node| puts node.content }
   #
   def nodes_from_file(file_path)
-    source = HTM::Models::FileSource.find_by(file_path: File.expand_path(file_path))
-    return HTM::Models::Node.none unless source
+    source = HTM::Models::FileSource.first(file_path: File.expand_path(file_path))
+    return [] unless source
 
-    HTM::Models::Node.from_source(source.id)
+    HTM::Models::Node.from_source(source.id).all
   end
 
   # Unload a file (soft-delete all its chunks and remove source record)
@@ -558,12 +559,12 @@ class HTM
   #   puts "Unloaded #{count} chunks"
   #
   def unload_file(file_path)
-    source = HTM::Models::FileSource.find_by(file_path: File.expand_path(file_path))
+    source = HTM::Models::FileSource.first(file_path: File.expand_path(file_path))
     return 0 unless source
 
     count = source.soft_delete_chunks!
     @long_term_memory.clear_cache!
-    source.destroy
+    source.delete
 
     update_robot_activity
     count
@@ -588,11 +589,16 @@ class HTM
   end
 
   def enqueue_tags_job(node_id, manual_tags: [])
-    # Add manual tags immediately if provided
+    # Add manual tags immediately if provided (including all parent tags)
+    # For "database:postgresql:extensions", this creates and associates:
+    # - "database"
+    # - "database:postgresql"
+    # - "database:postgresql:extensions"
     if manual_tags.any?
       manual_tags.each do |tag_name|
-        tag = HTM::Models::Tag.find_or_create_by!(name: tag_name)
-        HTM::Models::NodeTag.find_or_create_by!(node_id: node_id, tag_id: tag.id)
+        HTM::Models::Tag.find_or_create_with_ancestors(tag_name).each do |tag|
+          HTM::Models::NodeTag.find_or_create(node_id: node_id, tag_id: tag.id)
+        end
       end
     end
 
@@ -636,9 +642,8 @@ class HTM
     )
 
     # Mark node as in working memory in the robot_nodes join table
-    HTM::Models::RobotNode
-      .find_by(robot_id: @robot_id, node_id: node_id)
-      &.update!(working_memory: true)
+    robot_node = HTM::Models::RobotNode.first(robot_id: @robot_id, node_id: node_id)
+    robot_node&.update(working_memory: true)
   end
 
   # Validation helper methods

data/lib/tasks/db.rake

@@ -3,7 +3,6 @@
 namespace :db do
   desc "Run database migrations"
   task :migrate do
-    require_relative '../htm'
 
     HTM::Database.migrate
     puts "Database migrations completed successfully"
@@ -11,7 +10,6 @@ namespace :db do
 
   desc "Setup database schema (includes migrations)"
   task :setup do
-    require_relative '../htm'
 
     HTM::Database.setup
     puts "Database setup completed successfully"

data/lib/tasks/doc.rake

@@ -206,7 +206,7 @@ namespace :htm do
         # Skip error classes and internal classes
         next if class_name.end_with?("Error")
         next if class_name.include?("Railtie")
-        next if class_name.include?("ActiveRecordConfig")
+        next if class_name.include?("SequelConfig")
 
         # Get description
         simple_name = basename
@@ -228,7 +228,7 @@ namespace :htm do
             basename = File.basename(file, ".html")
             next if basename.end_with?("Error")
             next if basename == "Railtie"
-            next if basename == "ActiveRecordConfig"
+            next if basename == "SequelConfig"
 
             desc = descriptions[basename] || "#{basename} class"
             classes << ["HTM::#{basename}", desc, "yard/HTM/#{basename}.html"]