htm 0.0.20 → 0.0.30

data/lib/tasks/jobs.rake

@@ -4,10 +4,9 @@ namespace :htm do
   namespace :jobs do
     desc "Show statistics for nodes and async job processing"
     task :stats => :environment do
-      require 'htm'
 
       # Establish connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       puts "HTM Async Job Statistics"
       puts "=" * 60
@@ -17,7 +16,7 @@ namespace :htm do
       puts "Total nodes: #{total_nodes}"
 
       # Nodes with embeddings
-      with_embeddings = HTM::Models::Node.where.not(embedding: nil).count
+      with_embeddings = HTM::Models::Node.exclude(embedding: nil).count
       puts "Nodes with embeddings: #{with_embeddings} (#{percentage(with_embeddings, total_nodes)}%)"
 
       # Nodes without embeddings (pending embedding jobs)
@@ -26,7 +25,7 @@ namespace :htm do
 
       # Nodes with tags
       nodes_with_tags = HTM::Models::Node
-        .joins(:node_tags)
+        .join(:node_tags, node_id: :id)
         .distinct
         .count
       puts "Nodes with tags: #{nodes_with_tags} (#{percentage(nodes_with_tags, total_nodes)}%)"
@@ -43,7 +42,7 @@ namespace :htm do
       if total_tags > 0
         puts "\nTag hierarchy breakdown:"
         depth_counts = Hash.new(0)
-        HTM::Models::Tag.pluck(:name).each do |name|
+        HTM::Models::Tag.select_map(:name).each do |name|
           depth = name.count(':')
           depth_counts[depth] += 1
         end
@@ -58,15 +57,15 @@ namespace :htm do
         puts "\nAverage tags per node: #{avg_tags.round(2)}"
       end
 
-      HTM::ActiveRecordConfig.disconnect!
+      HTM::SequelConfig.disconnect!
     end
 
     desc "Process pending embedding jobs for nodes without embeddings"
     task :process_embeddings => :environment do
-      require 'htm'
+      require 'ruby-progressbar'
 
       # Establish connection and configure HTM
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
       HTM.configure  # Use default configuration
 
       # Find nodes without embeddings
@@ -75,95 +74,222 @@ namespace :htm do
 
       if total.zero?
         puts "No pending embedding jobs"
-        HTM::ActiveRecordConfig.disconnect!
+        HTM::SequelConfig.disconnect!
         exit 0
       end
 
-      puts "Processing #{total} pending embedding jobs..."
+      puts "Processing #{total} pending embedding jobs...\n"
+
+      # Create progress bar with ETA
+      progressbar = ProgressBar.create(
+        total: total,
+        format: '%t: |%B| %c/%C (%p%%) %e',
+        title: 'Embeddings',
+        output: $stdout,
+        smoothing: 0.5
+      )
 
       processed = 0
       failed = 0
 
-      pending_nodes.find_each do |node|
+      pending_nodes.paged_each do |node|
         begin
           # Use the service class directly (same as job)
           result = HTM::EmbeddingService.generate(node.content)
-          node.update!(embedding: result[:storage_embedding])
+          node.update(embedding: result[:storage_embedding])
           processed += 1
-          print "\rProcessed: #{processed}/#{total}"
         rescue StandardError => e
           failed += 1
-          HTM.logger.error "Failed to process node #{node.id}: #{e.message}"
+          progressbar.log "  Error on node #{node.id}: #{e.message}"
         end
+
+        progressbar.increment
       end
 
-      puts "\n\nCompleted:"
+      progressbar.finish
+
+      puts "\nCompleted:"
       puts "  Processed: #{processed}"
       puts "  Failed: #{failed}"
 
-      HTM::ActiveRecordConfig.disconnect!
+      HTM::SequelConfig.disconnect!
     end
 
     desc "Process pending tag extraction jobs for nodes without tags"
     task :process_tags => :environment do
-      require 'htm'
+      require 'ruby-progressbar'
 
       # Establish connection and configure HTM
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
       HTM.configure  # Use default configuration
 
-      # Find nodes without any tags
+      # Find nodes without any tags (using NOT EXISTS subquery)
       nodes_without_tags = HTM::Models::Node
-        .left_joins(:node_tags)
-        .where(node_tags: { id: nil })
+        .where(Sequel.~(Sequel.exists(
+          HTM::Models::NodeTag.where(Sequel[:node_tags][:node_id] => Sequel[:nodes][:id]).select(1)
+        )))
 
       total = nodes_without_tags.count
 
       if total.zero?
         puts "No pending tag extraction jobs"
-        HTM::ActiveRecordConfig.disconnect!
+        HTM::SequelConfig.disconnect!
         exit 0
       end
 
-      puts "Processing #{total} pending tag extraction jobs..."
+      puts "Processing #{total} pending tag extraction jobs...\n"
+
+      # Create progress bar with ETA
+      progressbar = ProgressBar.create(
+        total: total,
+        format: '%t: |%B| %c/%C (%p%%) %e',
+        title: 'Tags',
+        output: $stdout,
+        smoothing: 0.5
+      )
 
       processed = 0
       failed = 0
 
-      nodes_without_tags.find_each do |node|
+      nodes_without_tags.paged_each do |node|
         begin
           # Use the service class directly (same as job)
-          existing_ontology = HTM::Models::Tag.order(created_at: :desc).limit(100).pluck(:name)
+          existing_ontology = HTM::Models::Tag.order(Sequel.desc(:created_at)).limit(100).select_map(:name)
           tag_names = HTM::TagService.extract(node.content, existing_ontology: existing_ontology)
 
           tag_names.each do |tag_name|
-            tag = HTM::Models::Tag.find_or_create_by!(name: tag_name)
-            HTM::Models::NodeTag.find_or_create_by!(node_id: node.id, tag_id: tag.id)
+            tag = HTM::Models::Tag.find_or_create_by_name(tag_name)
+            HTM::Models::NodeTag.find_or_create(node_id: node.id, tag_id: tag.id)
           end
 
           processed += 1
-          print "\rProcessed: #{processed}/#{total}"
         rescue StandardError => e
           failed += 1
-          HTM.logger.error "Failed to process node #{node.id}: #{e.message}"
+          progressbar.log "  Error on node #{node.id}: #{e.message}"
         end
+
+        progressbar.increment
       end
 
-      puts "\n\nCompleted:"
+      progressbar.finish
+
+      puts "\nCompleted:"
       puts "  Processed: #{processed}"
       puts "  Failed: #{failed}"
 
-      HTM::ActiveRecordConfig.disconnect!
+      HTM::SequelConfig.disconnect!
+    end
+
+    desc "Process pending proposition extraction for nodes without propositions"
+    task :process_propositions => :environment do
+      require 'ruby-progressbar'
+
+      # Establish connection and configure HTM
+      HTM::SequelConfig.establish_connection!
+      HTM.configure  # Use default configuration
+
+      # Find non-proposition nodes that haven't been processed yet
+      # A node needs processing if:
+      # 1. It's not a proposition itself (is_proposition != true)
+      # 2. No proposition node references it as source_node_id
+      processed_source_ids = HTM::Models::Node
+        .where(Sequel.lit("metadata->>'is_proposition' = ?", 'true'))
+        .exclude(Sequel.lit("metadata->>'source_node_id' IS NULL"))
+        .select_map(Sequel.lit("metadata->>'source_node_id'"))
+        .map(&:to_i)
+        .uniq
+
+      pending_nodes = HTM::Models::Node
+        .non_propositions
+        .exclude(id: processed_source_ids)
+
+      total = pending_nodes.count
+
+      if total.zero?
+        puts "No pending proposition extraction jobs"
+        HTM::SequelConfig.disconnect!
+        exit 0
+      end
+
+      puts "Processing #{total} nodes for proposition extraction..."
+      puts "(Nodes already processed: #{processed_source_ids.size})\n"
+
+      # Get a robot for linking proposition nodes
+      robot = HTM::Models::Robot.first || HTM::Models::Robot.create(name: 'proposition_extractor')
+
+      # Create progress bar with ETA
+      progressbar = ProgressBar.create(
+        total: total,
+        format: '%t: |%B| %c/%C (%p%%) %e',
+        title: 'Extracting',
+        output: $stdout,
+        smoothing: 0.5
+      )
+
+      nodes_processed = 0
+      propositions_created = 0
+      failed = 0
+
+      pending_nodes.paged_each do |node|
+        begin
+          # Extract propositions using the service
+          propositions = HTM::PropositionService.extract(node.content)
+
+          if propositions.any?
+            propositions.each do |proposition_text|
+              token_count = HTM.count_tokens(proposition_text)
+
+              # Create proposition node
+              prop_node = HTM::Models::Node.create(
+                content: proposition_text,
+                token_count: token_count,
+                metadata: { is_proposition: true, source_node_id: node.id }
+              )
+
+              # Link to robot
+              HTM::Models::RobotNode.find_or_create(
+                robot_id: robot.id,
+                node_id: prop_node.id
+              )
+
+              # Generate embedding for proposition node
+              begin
+                result = HTM::EmbeddingService.generate(proposition_text)
+                prop_node.update(embedding: result[:storage_embedding])
+              rescue StandardError => e
+                progressbar.log "  Warning: Embedding failed for proposition #{prop_node.id}: #{e.message}"
+              end
+
+              propositions_created += 1
+            end
+          end
+
+          nodes_processed += 1
+        rescue StandardError => e
+          failed += 1
+          progressbar.log "  Error on node #{node.id}: #{e.message}"
+        end
+
+        progressbar.increment
+      end
+
+      progressbar.finish
+
+      puts "\nCompleted:"
+      puts "  Nodes processed: #{nodes_processed}"
+      puts "  Propositions created: #{propositions_created}"
+      puts "  Failed: #{failed}"
+
+      HTM::SequelConfig.disconnect!
     end
 
-    desc "Process all pending jobs (embeddings and tags)"
-    task :process_all => [:process_embeddings, :process_tags] do
+    desc "Process all pending jobs (embeddings, tags, and propositions)"
+    task :process_all => [:process_embeddings, :process_tags, :process_propositions] do
       puts "\nAll pending jobs processed!"
     end
 
     desc "Reprocess embeddings for all nodes (force regeneration)"
     task :reprocess_embeddings => :environment do
-      require 'htm'
 
       print "This will regenerate embeddings for ALL nodes. Are you sure? (yes/no): "
       confirmation = $stdin.gets.chomp
@@ -174,7 +300,7 @@ namespace :htm do
       end
 
       # Establish connection and configure HTM
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
       HTM.configure  # Use default configuration
 
       total = HTM::Models::Node.count
@@ -184,11 +310,11 @@ namespace :htm do
       processed = 0
       failed = 0
 
-      HTM::Models::Node.find_each do |node|
+      HTM::Models::Node.paged_each do |node|
         begin
           # Use the service class directly to regenerate
           result = HTM::EmbeddingService.generate(node.content)
-          node.update!(embedding: result[:storage_embedding])
+          node.update(embedding: result[:storage_embedding])
           processed += 1
           print "\rProcessed: #{processed}/#{total}"
         rescue StandardError => e
@@ -201,14 +327,13 @@ namespace :htm do
       puts "  Processed: #{processed}"
       puts "  Failed: #{failed}"
 
-      HTM::ActiveRecordConfig.disconnect!
+      HTM::SequelConfig.disconnect!
     end
 
     desc "Show nodes that failed async processing"
     task :failed => :environment do
-      require 'htm'
 
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       puts "Nodes with Processing Issues"
       puts "=" * 60
@@ -216,9 +341,9 @@ namespace :htm do
       # Old nodes without embeddings (created more than 1 hour ago)
       old_without_embeddings = HTM::Models::Node
         .where(embedding: nil)
-        .where('created_at < ?', 1.hour.ago)
+        .where(Sequel.lit('created_at < ?', Time.now - 3600))
 
-      if old_without_embeddings.any?
+      if old_without_embeddings.count > 0
         puts "\nNodes without embeddings (>1 hour old):"
         old_without_embeddings.limit(10).each do |node|
           puts "  Node #{node.id}: created #{time_ago(node.created_at)}"
@@ -231,13 +356,14 @@ namespace :htm do
         puts "\n✓ No old nodes without embeddings"
       end
 
-      # Old nodes without tags
+      # Old nodes without tags (using NOT EXISTS subquery)
       old_without_tags = HTM::Models::Node
-        .left_joins(:node_tags)
-        .where(node_tags: { id: nil })
-        .where('nodes.created_at < ?', 1.hour.ago)
+        .where(Sequel.~(Sequel.exists(
+          HTM::Models::NodeTag.where(Sequel[:node_tags][:node_id] => Sequel[:nodes][:id]).select(1)
+        )))
+        .where(Sequel.lit('created_at < ?', Time.now - 3600))
 
-      if old_without_tags.any?
+      if old_without_tags.count > 0
         puts "\nNodes without tags (>1 hour old):"
         old_without_tags.limit(10).each do |node|
           puts "  Node #{node.id}: created #{time_ago(node.created_at)}"
@@ -250,12 +376,11 @@ namespace :htm do
         puts "\n✓ No old nodes without tags"
       end
 
-      HTM::ActiveRecordConfig.disconnect!
+      HTM::SequelConfig.disconnect!
     end
 
     desc "Clear all embeddings and tags (for testing/development)"
     task :clear_all => :environment do
-      require 'htm'
 
       print "This will clear ALL embeddings and tags. Are you sure? (yes/no): "
       confirmation = $stdin.gets.chomp
@@ -265,18 +390,18 @@ namespace :htm do
         exit 0
       end
 
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       puts "Clearing embeddings..."
-      HTM::Models::Node.update_all(embedding: nil)
+      HTM::Models::Node.update(embedding: nil)
 
       puts "Clearing tags..."
-      HTM::Models::NodeTag.delete_all
-      HTM::Models::Tag.delete_all
+      HTM::Models::NodeTag.dataset.delete
+      HTM::Models::Tag.dataset.delete
 
       puts "Done! All embeddings and tags cleared."
 
-      HTM::ActiveRecordConfig.disconnect!
+      HTM::SequelConfig.disconnect!
     end
 
     # Helper methods

data/lib/tasks/tags.rake

@@ -12,10 +12,9 @@ namespace :htm do
   namespace :tags do
     desc "Display tags as a hierarchical tree (text format). Optional prefix filter."
     task :tree, [:prefix] do |_t, args|
-      require 'htm'
 
       # Ensure database connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       tags = args[:prefix] ? HTM::Models::Tag.with_prefix(args[:prefix]) : HTM::Models::Tag.all
       count = tags.count
@@ -34,10 +33,9 @@ namespace :htm do
 
     desc "Export tags as Mermaid flowchart to tags.md. Optional prefix filter."
     task :mermaid, [:prefix] do |_t, args|
-      require 'htm'
 
       # Ensure database connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       tags = args[:prefix] ? HTM::Models::Tag.with_prefix(args[:prefix]) : HTM::Models::Tag.all
       count = tags.count
@@ -56,10 +54,9 @@ namespace :htm do
 
     desc "Export tags as SVG visualization to tags.svg. Optional prefix filter."
     task :svg, [:prefix] do |_t, args|
-      require 'htm'
 
       # Ensure database connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       tags = args[:prefix] ? HTM::Models::Tag.with_prefix(args[:prefix]) : HTM::Models::Tag.all
       count = tags.count
@@ -79,10 +76,9 @@ namespace :htm do
 
     desc "Rebuild all tags from node content. Clears existing tags and regenerates using LLM."
     task :rebuild do
-      require 'htm'
 
       # Ensure database connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       # Node uses default_scope for active (non-deleted) nodes
       node_count = HTM::Models::Node.count
@@ -107,11 +103,11 @@ namespace :htm do
       puts "\nClearing existing tags..."
 
       # Clear join table first (foreign key constraint)
-      deleted_associations = HTM::Models::NodeTag.delete_all
+      deleted_associations = HTM::Models::NodeTag.dataset.delete
       puts "  Deleted #{deleted_associations} node-tag associations"
 
       # Clear tags table
-      deleted_tags = HTM::Models::Tag.delete_all
+      deleted_tags = HTM::Models::Tag.dataset.delete
       puts "  Deleted #{deleted_tags} tags"
 
       puts "\nRegenerating tags for #{node_count} nodes..."
@@ -131,7 +127,7 @@ namespace :htm do
       # Process each active node (default_scope excludes deleted)
       errors = 0
 
-      HTM::Models::Node.find_each do |node|
+      HTM::Models::Node.paged_each do |node|
         begin
           HTM::Jobs::GenerateTagsJob.perform(node_id: node.id)
         rescue StandardError => e
@@ -157,10 +153,9 @@ namespace :htm do
 
     desc "Export tags in all formats (tags.txt, tags.md, tags.svg). Optional prefix filter."
     task :export, [:prefix] do |_t, args|
-      require 'htm'
 
       # Ensure database connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       tags = args[:prefix] ? HTM::Models::Tag.with_prefix(args[:prefix]) : HTM::Models::Tag.all
       count = tags.count