htm 0.0.18 → 0.0.30

data/lib/htm/job_adapter.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'async'
+require 'async/barrier'
+
 class HTM
   # Job adapter for pluggable background job backends
   #
@@ -11,6 +14,7 @@ class HTM
   # - :sidekiq - Direct Sidekiq integration (recommended for Sinatra apps)
   # - :inline - Synchronous execution (recommended for CLI and tests)
   # - :thread - Background thread (legacy, for standalone apps)
+  # - :fiber - Fiber-based concurrency using async gem (recommended for I/O-bound jobs)
   #
   # @example Configure job backend
   #   HTM.configure do |config|
@@ -44,8 +48,10 @@ class HTM
           enqueue_inline(job_class, **params)
         when :thread
           enqueue_thread(job_class, **params)
+        when :fiber
+          enqueue_fiber(job_class, **params)
         else
-          raise HTM::Error, "Unknown job backend: #{backend}. Supported backends: :active_job, :sidekiq, :inline, :thread"
+          raise HTM::Error, "Unknown job backend: #{backend}. Supported backends: :active_job, :sidekiq, :inline, :thread, :fiber"
         end
       end
 
@@ -98,6 +104,74 @@ class HTM
         HTM.logger.error "Failed to start thread for #{job_class.name}: #{e.message}"
       end
 
+      # Execute job using async gem (fiber-based concurrency)
+      # Non-blocking for I/O-bound operations like LLM API calls
+      def enqueue_fiber(job_class, **params)
+        Async do
+          begin
+            job_class.perform(**params)
+          rescue StandardError => e
+            HTM.logger.error "Fiber job #{job_class.name} failed: #{e.class.name} - #{e.message}"
+          end
+        end
+      rescue StandardError => e
+        HTM.logger.error "Failed to start fiber for #{job_class.name}: #{e.message}"
+      end
+
+      public
+
+      # Execute multiple jobs in parallel using fibers
+      # Best for I/O-bound jobs like LLM API calls
+      #
+      # @param jobs [Array<Array>] Array of [job_class, params] pairs
+      # @return [void]
+      #
+      # @example Run embedding and tags jobs in parallel
+      #   JobAdapter.enqueue_parallel([
+      #     [GenerateEmbeddingJob, { node_id: 123 }],
+      #     [GenerateTagsJob, { node_id: 123 }]
+      #   ])
+      #
+      def enqueue_parallel(jobs)
+        return if jobs.empty?
+
+        backend = HTM.configuration.job_backend
+
+        case backend
+        when :fiber
+          enqueue_parallel_fiber(jobs)
+        when :inline
+          # Run sequentially for inline backend
+          jobs.each { |job_class, params| enqueue_inline(job_class, **params) }
+        else
+          # For other backends, enqueue each job separately
+          jobs.each { |job_class, params| enqueue(job_class, **params) }
+        end
+      end
+
+      private
+
+      # Execute multiple jobs in parallel using async fibers
+      def enqueue_parallel_fiber(jobs)
+        Async do |task|
+          barrier = Async::Barrier.new
+
+          jobs.each do |job_class, params|
+            barrier.async do
+              begin
+                job_class.perform(**params)
+              rescue StandardError => e
+                HTM.logger.error "Parallel fiber job #{job_class.name} failed: #{e.class.name} - #{e.message}"
+              end
+            end
+          end
+
+          barrier.wait
+        end
+      rescue StandardError => e
+        HTM.logger.error "Failed to start parallel fibers: #{e.message}"
+      end
+
       # Convert HTM job class to ActiveJob class
       def to_active_job_class(job_class)
         # If it's already an ActiveJob, return it

data/lib/htm/jobs/generate_embedding_job.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative '../errors'
-require_relative '../models/node'
 require_relative '../embedding_service'
 
 class HTM
@@ -23,7 +22,7 @@ class HTM
       # @param node_id [Integer] ID of the node to process
       #
       def self.perform(node_id:)
-        node = HTM::Models::Node.find_by(id: node_id)
+        node = HTM::Models::Node.first(id: node_id)
 
         unless node
           HTM.logger.warn "GenerateEmbeddingJob: Node #{node_id} not found"
@@ -31,7 +30,7 @@ class HTM
         end
 
         # Skip if already has embedding
-        return if node.embedding.present?
+        return if node.embedding
 
         provider = HTM.configuration.embedding_provider.to_s
         start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -41,7 +40,7 @@ class HTM
           result = HTM::EmbeddingService.generate(node.content)
 
           # Update node with processed embedding
-          node.update!(embedding: result[:storage_embedding])
+          node.update(embedding: result[:storage_embedding])
 
           # Record success metrics
           elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round

data/lib/htm/jobs/generate_propositions_job.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative '../errors'
-require_relative '../models/node'
 require_relative '../proposition_service'
 
 class HTM
@@ -25,7 +24,7 @@ class HTM
       # @param robot_id [Integer] ID of the robot that owns this node
       #
       def self.perform(node_id:, robot_id:)
-        node = HTM::Models::Node.find_by(id: node_id)
+        node = HTM::Models::Node.first(id: node_id)
 
         unless node
           HTM.logger.warn "GeneratePropositionsJob: Node #{node_id} not found"
@@ -49,14 +48,14 @@ class HTM
             token_count = HTM.count_tokens(proposition_text)
 
             # Create proposition node with is_proposition marker
-            proposition_node = HTM::Models::Node.create!(
+            proposition_node = HTM::Models::Node.create(
               content: proposition_text,
               token_count: token_count,
               metadata: { is_proposition: true, source_node_id: node_id }
             )
 
             # Link to robot via RobotNode
-            HTM::Models::RobotNode.find_or_create_by!(
+            HTM::Models::RobotNode.find_or_create(
               robot_id: robot_id,
               node_id: proposition_node.id
             )
@@ -79,7 +78,7 @@ class HTM
           # Log proposition-specific errors
           HTM.logger.error "GeneratePropositionsJob: Proposition extraction failed for node #{node_id}: #{e.message}"
 
-        rescue ActiveRecord::RecordInvalid => e
+        rescue Sequel::ValidationFailed => e
           # Log validation errors
           HTM.logger.error "GeneratePropositionsJob: Database validation failed for node #{node_id}: #{e.message}"
 

data/lib/htm/jobs/generate_tags_job.rb

@@ -1,9 +1,6 @@
 # frozen_string_literal: true
 
 require_relative '../errors'
-require_relative '../models/node'
-require_relative '../models/tag'
-require_relative '../models/node_tag'
 require_relative '../tag_service'
 
 class HTM
@@ -26,7 +23,7 @@ class HTM
       # @param node_id [Integer] ID of the node to process
       #
       def self.perform(node_id:)
-        node = HTM::Models::Node.find_by(id: node_id)
+        node = HTM::Models::Node.first(id: node_id)
 
         unless node
           HTM.logger.warn "GenerateTagsJob: Node #{node_id} not found"
@@ -39,23 +36,27 @@ class HTM
         begin
           # Get existing ontology for context (sample of recent tags)
           existing_ontology = HTM::Models::Tag
-            .order(created_at: :desc)
+            .order(Sequel.desc(:created_at))
             .limit(100)
-            .pluck(:name)
+            .select_map(:name)
 
           # Extract and validate tags using TagService
           tag_names = HTM::TagService.extract(node.content, existing_ontology: existing_ontology)
           return if tag_names.empty?
 
-          # Create or find tags and associate with node
+          # Create or find tags (including all parent tags) and associate with node
+          # For "database:postgresql:extensions", this creates and associates:
+          # - "database"
+          # - "database:postgresql"
+          # - "database:postgresql:extensions"
           tag_names.each do |tag_name|
-            tag = HTM::Models::Tag.find_or_create_by!(name: tag_name)
-
-            # Create association if it doesn't exist
-            HTM::Models::NodeTag.find_or_create_by!(
-              node_id: node.id,
-              tag_id: tag.id
-            )
+            HTM::Models::Tag.find_or_create_with_ancestors(tag_name).each do |tag|
+              # Create association if it doesn't exist
+              HTM::Models::NodeTag.find_or_create(
+                node_id: node.id,
+                tag_id: tag.id
+              )
+            end
           end
 
           # Record success metrics
@@ -79,7 +80,7 @@ class HTM
           # Log tag-specific errors
           HTM.logger.error "GenerateTagsJob: Tag generation failed for node #{node_id}: #{e.message}"
 
-        rescue ActiveRecord::RecordInvalid => e
+        rescue Sequel::ValidationFailed => e
           # Record failure metrics
           elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
           HTM::Telemetry.tag_latency.record(elapsed_ms, attributes: { 'provider' => provider, 'status' => 'error' })

data/lib/htm/loaders/defaults_loader.rb

@@ -79,6 +79,29 @@ class HTM
           raw = load_raw_yaml
           raw[:defaults] || {}
         end
+
+        # Returns valid environment names from the config file
+        #
+        # Valid environments are top-level keys in defaults.yml excluding 'defaults'.
+        # For example, if defaults.yml has keys: defaults, development, test, production
+        # this returns [:development, :test, :production]
+        #
+        # @return [Array<Symbol>] list of valid environment names
+        def valid_environments
+          raw = load_raw_yaml
+          raw.keys.reject { |k| k == :defaults }.sort
+        end
+
+        # Check if a given environment name is valid
+        #
+        # @param env [String, Symbol] environment name to check
+        # @return [Boolean] true if environment is valid
+        def valid_environment?(env)
+          return false if env.nil? || env.to_s.empty?
+          return false if env.to_s == 'defaults'
+
+          valid_environments.include?(env.to_sym)
+        end
       end
 
       def call(name:, **_options)

data/lib/htm/loaders/markdown_loader.rb

@@ -78,10 +78,12 @@ class HTM
         file_hash = Digest::SHA256.hexdigest(content)
 
         # Find or create source record
-        source = HTM::Models::FileSource.find_or_initialize_by(file_path: expanded_path)
+        source = HTM::Models::FileSource.first(file_path: expanded_path)
+        is_new = source.nil?
+        source ||= HTM::Models::FileSource.new(file_path: expanded_path)
 
         # Check if sync needed
-        unless force || source.new_record? || source.needs_sync?(stat.mtime)
+        unless force || is_new || source.needs_sync?(stat.mtime)
           return {
             file_path: expanded_path,
             chunks_created: 0,
@@ -104,18 +106,18 @@ class HTM
         end
 
         # Save source first (need ID for node association)
-        source.save! if source.new_record?
+        source.save if is_new
 
         # Sync chunks to database (chunks now include cursor positions)
         result = sync_chunks(source, chunks)
 
         # Update source record
-        source.update!(
+        source.update(
           file_hash: file_hash,
           mtime: stat.mtime,
           file_size: stat.size,
           frontmatter: frontmatter,
-          last_synced_at: Time.current
+          last_synced_at: Time.now
         )
 
         result.merge(
@@ -195,9 +197,9 @@ class HTM
         deleted = 0
 
         # Get existing nodes for this source (include soft-deleted for potential restore)
-        existing_nodes = source.persisted? ?
-          HTM::Models::Node.unscoped.where(source_id: source.id).to_a : []
-        existing_by_hash = existing_nodes.index_by(&:content_hash)
+        existing_nodes = source.id ?
+          HTM::Models::Node.with_deleted.where(source_id: source.id).all : []
+        existing_by_hash = existing_nodes.each_with_object({}) { |n, h| h[n.content_hash] = n }
 
         # Track which existing nodes we've matched
         matched_hashes = Set.new
@@ -217,7 +219,7 @@ class HTM
 
             changes = {}
             changes[:chunk_position] = position if node.chunk_position != position
-            changes[:deleted_at] = nil if node.deleted_at.present?
+            changes[:deleted_at] = nil if node.deleted_at
 
             # Update cursor in metadata if changed
             current_cursor = node.metadata&.dig('cursor')
@@ -227,7 +229,7 @@ class HTM
             end
 
             if changes.any?
-              node.update!(changes)
+              node.update(changes)
               updated += 1
             end
           else
@@ -240,7 +242,7 @@ class HTM
         # Soft-delete chunks that no longer exist in file
         existing_by_hash.each do |hash, node|
           next if matched_hashes.include?(hash)
-          next if node.deleted_at.present?  # Already deleted
+          next if node.deleted_at  # Already deleted
 
           node.soft_delete!
           deleted += 1
@@ -265,18 +267,18 @@ class HTM
         node_id = @htm.remember(content, metadata: chunk_metadata)
 
         # Update with source reference
-        node = HTM::Models::Node.find(node_id)
-        node.update!(source_id: source.id, chunk_position: position)
+        node = HTM::Models::Node[node_id]
+        node.update(source_id: source.id, chunk_position: position)
 
         node
-      rescue ActiveRecord::RecordNotUnique
+      rescue Sequel::UniqueConstraintViolation
         # Duplicate content exists (different source or no source)
         # Find and link to this source
         existing = HTM::Models::Node.find_by_content(content)
         if existing && existing.source_id.nil?
           # Merge cursor into existing metadata
           new_metadata = (existing.metadata || {}).merge('cursor' => cursor) if cursor
-          existing.update!(
+          existing.update(
             source_id: source.id,
             chunk_position: position,
             metadata: new_metadata || existing.metadata

data/lib/htm/loaders/xdg_config_loader.rb

@@ -28,24 +28,24 @@ class HTM
       class << self
         # Returns all XDG config paths to check, in order of priority (lowest first)
         #
+        # Per XDG spec: If $XDG_CONFIG_HOME is set, use it; otherwise use ~/.config
+        #
         # @return [Array<String>] list of potential config file paths
         def config_paths
           paths = []
 
-          # macOS Application Support (lowest priority for XDG loader)
-          if macos?
+          # macOS Application Support (lowest priority, only when XDG_CONFIG_HOME is not set)
+          if macos? && (!ENV['XDG_CONFIG_HOME'] || ENV['XDG_CONFIG_HOME'].empty?)
             macos_path = File.expand_path('~/Library/Application Support/htm')
             paths << macos_path if Dir.exist?(File.dirname(macos_path))
           end
 
-          # XDG default: ~/.config/htm
-          xdg_default = File.expand_path('~/.config/htm')
-          paths << xdg_default
-
-          # XDG_CONFIG_HOME override (highest priority for XDG loader)
+          # XDG_CONFIG_HOME takes precedence over default
           if ENV['XDG_CONFIG_HOME'] && !ENV['XDG_CONFIG_HOME'].empty?
-            xdg_home = File.join(ENV['XDG_CONFIG_HOME'], 'htm')
-            paths << xdg_home unless xdg_home == xdg_default
+            paths << File.join(ENV['XDG_CONFIG_HOME'], 'htm')
+          else
+            # XDG default: ~/.config/htm (only when XDG_CONFIG_HOME is not set)
+            paths << File.expand_path('~/.config/htm')
           end
 
           paths

data/lib/htm/long_term_memory/fulltext_search.rb

@@ -79,6 +79,7 @@ class HTM
 
         # Combined tsvector + trigram search
         # tsvector matches get boosted score, trigram provides fuzzy fallback
+        # Note: Using ? placeholders for Sequel compatibility
         sql = <<~SQL
           WITH tsvector_matches AS (
             -- Primary: tsvector full-text search (stemmed word matching)
@@ -114,24 +115,23 @@ class HTM
           LIMIT ?
         SQL
 
-        result = ActiveRecord::Base.connection.select_all(
-          ActiveRecord::Base.sanitize_sql_array([
-            sql,
-            TSVECTOR_SCORE_BOOST,  # boost for tsvector
-            query,                  # ts_rank query
-            query,                  # tsvector match query
-            query,                  # trigram similarity query
-            query,                  # trigram match query
-            TRIGRAM_SIMILARITY_THRESHOLD,
-            limit
-          ])
-        )
+        result = HTM.db.fetch(
+          sql,
+          TSVECTOR_SCORE_BOOST,           # boost for tsvector
+          query,                           # query for ts_rank
+          query,                           # query for plainto_tsquery
+          query,                           # query for similarity (trigram)
+          query,                           # query for similarity condition
+          TRIGRAM_SIMILARITY_THRESHOLD,    # similarity threshold
+          limit                            # limit
+        ).all
 
         # Track access for retrieved nodes
-        node_ids = result.map { |r| r['id'] }
+        node_ids = result.map { |r| r[:id] }
         track_access(node_ids)
 
-        result.to_a
+        # Convert to hash with string keys for compatibility
+        result.map { |r| r.transform_keys(&:to_s) }
       end
     end
   end