htm 0.0.18 → 0.0.30

data/lib/tasks/files.rake

@@ -12,7 +12,6 @@ namespace :htm do
   namespace :files do
     desc "Load a markdown file into long-term memory. Usage: rake htm:files:load[path/to/file.md]"
     task :load, [:path] do |_t, args|
-      require 'htm'
 
       path = args[:path]
       unless path
@@ -27,7 +26,7 @@ namespace :htm do
       end
 
       # Ensure database connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       htm = HTM.new(robot_name: "FileLoader")
       force = ENV['FORCE'] == 'true'
@@ -49,7 +48,6 @@ namespace :htm do
 
     desc "Load all markdown files from a directory. Usage: rake htm:files:load_dir[path/to/dir]"
     task :load_dir, [:path, :pattern] do |_t, args|
-      require 'htm'
 
       path = args[:path]
       unless path
@@ -65,7 +63,7 @@ namespace :htm do
       end
 
       # Ensure database connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       htm = HTM.new(robot_name: "FileLoader")
       pattern = args[:pattern] || '**/*.md'
@@ -104,10 +102,9 @@ namespace :htm do
 
     desc "List all loaded file sources"
     task :list do
-      require 'htm'
 
       # Ensure database connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       sources = HTM::Models::FileSource.order(:file_path)
       count = sources.count
@@ -136,7 +133,6 @@ namespace :htm do
 
     desc "Show details for a loaded file. Usage: rake htm:files:info[path/to/file.md]"
     task :info, [:path] do |_t, args|
-      require 'htm'
 
       path = args[:path]
       unless path
@@ -146,11 +142,11 @@ namespace :htm do
       end
 
       # Ensure database connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       # Try to find by exact path or expanded path
-      source = HTM::Models::FileSource.find_by(file_path: path) ||
-               HTM::Models::FileSource.find_by(file_path: File.expand_path(path))
+      source = HTM::Models::FileSource.first(file_path: path) ||
+               HTM::Models::FileSource.first(file_path: File.expand_path(path))
 
       unless source
         puts "Error: File not loaded: #{path}"
@@ -192,7 +188,6 @@ namespace :htm do
 
     desc "Unload a file from memory. Usage: rake htm:files:unload[path/to/file.md]"
     task :unload, [:path] do |_t, args|
-      require 'htm'
 
       path = args[:path]
       unless path
@@ -202,7 +197,7 @@ namespace :htm do
       end
 
       # Ensure database connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       htm = HTM.new(robot_name: "FileLoader")
       result = htm.unload_file(path)
@@ -216,10 +211,9 @@ namespace :htm do
 
     desc "Sync all loaded files (reload changed files)"
     task :sync do
-      require 'htm'
 
       # Ensure database connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       htm = HTM.new(robot_name: "FileLoader")
       sources = HTM::Models::FileSource.all
@@ -263,18 +257,17 @@ namespace :htm do
 
     desc "Show file loading statistics"
     task :stats do
-      require 'htm'
 
       # Ensure database connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       total_sources = HTM::Models::FileSource.count
-      total_chunks = HTM::Models::Node.where.not(source_id: nil).count
+      total_chunks = HTM::Models::Node.exclude(source_id: nil).count
 
       # Count files needing sync (checking actual file mtime)
       needs_sync = 0
       missing = 0
-      HTM::Models::FileSource.find_each do |source|
+      HTM::Models::FileSource.paged_each do |source|
         if File.exist?(source.file_path)
           current_mtime = File.mtime(source.file_path)
           needs_sync += 1 if source.needs_sync?(current_mtime)

data/lib/tasks/htm.rake

@@ -8,35 +8,29 @@
 #   require 'htm/tasks'
 #
 
-# Add lib directory to load path for development
-# This allows the tasks to work both during gem development and when installed
-lib_path = File.expand_path('../../lib', __dir__)
-$LOAD_PATH.unshift(lib_path) unless $LOAD_PATH.include?(lib_path)
-
 namespace :htm do
   namespace :db do
+    # Note: Database configuration validation (environment, URL/component reconciliation,
+    # naming convention) happens automatically when HTM is required above.
+
     desc "Set up HTM database schema and run migrations (set DUMP_SCHEMA=true to auto-dump schema after)"
     task :setup do
-      require 'htm'
       dump_schema = ENV['DUMP_SCHEMA'] == 'true'
       HTM::Database.setup(dump_schema: dump_schema)
     end
 
     desc "Run pending database migrations"
     task :migrate do
-      require 'htm'
       HTM::Database.migrate
     end
 
     desc "Show migration status"
     task :status do
-      require 'htm'
       HTM::Database.migration_status
     end
 
     desc "Drop all HTM tables (WARNING: destructive! Set CONFIRM=yes to skip prompt)"
     task :drop do
-      require 'htm'
       if ENV['CONFIRM'] == 'yes'
         HTM::Database.drop
       else
@@ -52,7 +46,6 @@ namespace :htm do
 
     desc "Drop and recreate database (WARNING: destructive! Set CONFIRM=yes to skip prompt)"
     task :reset do
-      require 'htm'
       if ENV['CONFIRM'] == 'yes'
         HTM::Database.drop
         HTM::Database.setup(dump_schema: true)
@@ -70,9 +63,7 @@ namespace :htm do
 
     desc "Verify database connection (respects HTM_ENV/RAILS_ENV)"
     task :verify do
-      require 'htm'
-
-      config = HTM::ActiveRecordConfig.load_database_config
+      config = HTM::SequelConfig.load_database_config
 
       puts "Verifying HTM database connection (#{HTM.env})..."
       puts "  Host: #{config[:host]}"
@@ -108,9 +99,7 @@ namespace :htm do
 
     desc "Open PostgreSQL console (respects HTM_ENV/RAILS_ENV)"
     task :console do
-      require 'htm'
-
-      config = HTM::ActiveRecordConfig.load_database_config
+      config = HTM::SequelConfig.load_database_config
 
       puts "Connecting to #{config[:database]} (#{HTM.env})..."
       exec "psql", "-h", config[:host],
@@ -121,22 +110,18 @@ namespace :htm do
 
     desc "Seed database with sample data"
     task :seed do
-      require 'htm'
       HTM::Database.seed
     end
 
     desc "Show database info (size, tables, extensions)"
     task :info do
-      require 'htm'
       HTM::Database.info
     end
 
     desc "Show record counts for all HTM tables"
     task :stats do
-      require 'htm'
-
       # Ensure database connection
-      HTM::ActiveRecordConfig.establish_connection!
+      HTM::SequelConfig.establish_connection!
 
       puts "\nHTM Database Statistics"
       puts "=" * 50
@@ -148,7 +133,7 @@ namespace :htm do
           # Node uses default_scope for active nodes, so m.count is active count
           active = m.count
           deleted = m.deleted.count
-          with_embedding = m.where.not(embedding: nil).count
+          with_embedding = m.exclude(embedding: nil).count
           "  (active: #{active}, deleted: #{deleted}, with embeddings: #{with_embedding})"
         }},
         { name: 'tags', model: HTM::Models::Tag },
@@ -175,11 +160,12 @@ namespace :htm do
 
       # Nodes per robot (via robot_nodes join table)
       robot_counts = HTM::Models::RobotNode
-        .joins(:node)
-        .where(nodes: { deleted_at: nil })
-        .group(:robot_id)
-        .count
-        .transform_keys { |id| HTM::Models::Robot.find(id).name rescue "Unknown (#{id})" }
+        .join(:nodes, id: :node_id)
+        .where(Sequel[:nodes][:deleted_at] => nil)
+        .group_and_count(:robot_id)
+        .all
+        .to_h { |row| [row[:robot_id], row[:count]] }
+        .transform_keys { |id| HTM::Models::Robot[id]&.name || "Unknown (#{id})" }
         .sort_by { |_, count| -count }
         .first(5)
 
@@ -191,12 +177,13 @@ namespace :htm do
       end
 
       # Tag distribution
-      top_root_tags = HTM::Models::Tag
-        .select("split_part(name, ':', 1) as root, count(*) as cnt")
-        .group("split_part(name, ':', 1)")
-        .order("cnt DESC")
+      top_root_tags = HTM.db[:tags]
+        .select(Sequel.lit("split_part(name, ':', 1) as root"), Sequel.function(:count, Sequel.lit('*')).as(:cnt))
+        .group(Sequel.lit("split_part(name, ':', 1)"))
+        .order(Sequel.desc(:cnt))
         .limit(5)
-        .map { |t| [t.root, t.cnt] }
+        .all
+        .map { |t| [t[:root], t[:cnt]] }
 
       if top_root_tags.any?
         puts "  Top root tag categories:"
@@ -211,15 +198,14 @@ namespace :htm do
     namespace :rebuild do
       desc "Rebuild embeddings for all nodes. Clears existing embeddings and regenerates via LLM."
       task :embeddings do
-        require 'htm'
         require 'ruby-progressbar'
 
         # Ensure database connection
-        HTM::ActiveRecordConfig.establish_connection!
+        HTM::SequelConfig.establish_connection!
 
         # Node uses default_scope for active (non-deleted) nodes
         node_count = HTM::Models::Node.count
-        with_embeddings = HTM::Models::Node.where.not(embedding: nil).count
+        with_embeddings = HTM::Models::Node.exclude(embedding: nil).count
         without_embeddings = node_count - with_embeddings
 
         puts "\nHTM Embeddings Rebuild"
@@ -239,7 +225,7 @@ namespace :htm do
         end
 
         puts "\nClearing existing embeddings..."
-        cleared = HTM::Models::Node.where.not(embedding: nil).update_all(embedding: nil)
+        cleared = HTM::Models::Node.exclude(embedding: nil).update(embedding: nil)
         puts "  Cleared #{cleared} embeddings"
 
         puts "\nGenerating embeddings for #{node_count} nodes..."
@@ -258,12 +244,12 @@ namespace :htm do
         errors = 0
         success = 0
 
-        HTM::Models::Node.find_each do |node|
+        HTM::Models::Node.paged_each do |node|
           begin
             # Generate embedding directly (not via job since we cleared them)
             result = HTM::EmbeddingService.generate(node.content)
 
-            node.update!(embedding: result[:storage_embedding])
+            node.update(embedding: result[:storage_embedding])
             success += 1
           rescue StandardError => e
             errors += 1
@@ -276,7 +262,7 @@ namespace :htm do
         progressbar.finish
 
         # Final stats
-        final_with_embeddings = HTM::Models::Node.where.not(embedding: nil).count
+        final_with_embeddings = HTM::Models::Node.exclude(embedding: nil).count
 
         puts "\nRebuild complete!"
         puts "  Nodes processed: #{node_count}"
@@ -287,11 +273,10 @@ namespace :htm do
 
       desc "Rebuild propositions for all non-proposition nodes. Extracts atomic facts and creates new nodes."
       task :propositions do
-        require 'htm'
         require 'ruby-progressbar'
 
         # Ensure database connection
-        HTM::ActiveRecordConfig.establish_connection!
+        HTM::SequelConfig.establish_connection!
 
         # Find all non-proposition nodes (nodes that haven't been extracted from)
         source_nodes = HTM::Models::Node.non_propositions
@@ -319,7 +304,7 @@ namespace :htm do
         # Delete existing proposition nodes
         if existing_propositions > 0
           puts "\nDeleting #{existing_propositions} existing proposition nodes..."
-          deleted = HTM::Models::Node.propositions.delete_all
+          deleted = HTM::Models::Node.propositions.delete
           puts "  Deleted #{deleted} proposition nodes"
         end
 
@@ -328,7 +313,7 @@ namespace :htm do
 
         # Get a robot ID for linking proposition nodes
         # Use the first robot or create a system robot
-        robot = HTM::Models::Robot.first || HTM::Models::Robot.create!(name: 'proposition_rebuilder')
+        robot = HTM::Models::Robot.first || HTM::Models::Robot.create(name: 'proposition_rebuilder')
 
         # Create progress bar with ETA
         progressbar = ProgressBar.create(
@@ -344,7 +329,7 @@ namespace :htm do
         nodes_processed = 0
         propositions_created = 0
 
-        source_nodes.find_each do |node|
+        source_nodes.paged_each do |node|
           begin
             # Extract propositions
             propositions = HTM::PropositionService.extract(node.content)
@@ -354,14 +339,14 @@ namespace :htm do
                 token_count = HTM.count_tokens(proposition_text)
 
                 # Create proposition node
-                prop_node = HTM::Models::Node.create!(
+                prop_node = HTM::Models::Node.create(
                   content: proposition_text,
                   token_count: token_count,
                   metadata: { is_proposition: true, source_node_id: node.id }
                 )
 
                 # Link to robot
-                HTM::Models::RobotNode.find_or_create_by!(
+                HTM::Models::RobotNode.find_or_create(
                   robot_id: robot.id,
                   node_id: prop_node.id
                 )
@@ -369,7 +354,7 @@ namespace :htm do
                 # Generate embedding for proposition node
                 begin
                   result = HTM::EmbeddingService.generate(proposition_text)
-                  prop_node.update!(embedding: result[:storage_embedding])
+                  prop_node.update(embedding: result[:storage_embedding])
                 rescue StandardError => e
                   progressbar.log "  Warning: Embedding failed for proposition: #{e.message}"
                 end
@@ -403,22 +388,18 @@ namespace :htm do
     namespace :schema do
       desc "Dump current schema to db/schema.sql"
       task :dump do
-        require 'htm'
         HTM::Database.dump_schema
       end
 
       desc "Load schema from db/schema.sql"
       task :load do
-        require 'htm'
         HTM::Database.load_schema
       end
     end
 
     desc "Create database if it doesn't exist (respects HTM_ENV/RAILS_ENV)"
     task :create do
-      require 'htm'
-
-      config = HTM::ActiveRecordConfig.load_database_config
+      config = HTM::SequelConfig.load_database_config
       db_name = config[:database]
 
       puts "Creating database: #{db_name} (#{HTM.env})"
@@ -472,18 +453,16 @@ namespace :htm do
     namespace :tags do
       desc "Soft delete orphaned tags and stale node_tags entries"
       task :cleanup do
-        require 'htm'
-
         # Ensure database connection
-        HTM::ActiveRecordConfig.establish_connection!
+        HTM::SequelConfig.establish_connection!
 
         puts "\nHTM Tag Cleanup"
         puts "=" * 50
 
         # Step 1: Find active node_tags pointing to soft-deleted or missing nodes
         stale_node_tags = HTM::Models::NodeTag
-          .joins("LEFT JOIN nodes ON nodes.id = node_tags.node_id")
-          .where("nodes.id IS NULL OR nodes.deleted_at IS NOT NULL")
+          .left_join(:nodes, id: :node_id)
+          .where(Sequel.lit("nodes.id IS NULL OR nodes.deleted_at IS NOT NULL"))
 
         stale_count = stale_node_tags.count
 
@@ -504,7 +483,7 @@ namespace :htm do
 
         if orphan_count > 0
           puts "\nOrphaned tags:"
-          orphaned_tags.limit(20).pluck(:name).each do |name|
+          orphaned_tags.limit(20).select_map(:name).each do |name|
             puts "  - #{name}"
           end
           puts "  ... and #{orphan_count - 20} more" if orphan_count > 20
@@ -518,17 +497,17 @@ namespace :htm do
           next
         end
 
-        now = Time.current
+        now = Time.now
 
         # Soft delete stale node_tags first
         if stale_count > 0
-          soft_deleted_node_tags = stale_node_tags.update_all(deleted_at: now)
+          soft_deleted_node_tags = stale_node_tags.update(deleted_at: now)
           puts "\nSoft deleted #{soft_deleted_node_tags} stale node_tags entries."
         end
 
         # Then soft delete orphaned tags
         if orphan_count > 0
-          soft_deleted_tags = orphaned_tags.update_all(deleted_at: now)
+          soft_deleted_tags = orphaned_tags.update(deleted_at: now)
           puts "Soft deleted #{soft_deleted_tags} orphaned tags."
         end
 
@@ -536,6 +515,156 @@ namespace :htm do
       end
     end
 
+    desc "Permanently delete all soft-deleted records from all tables (WARNING: irreversible!)"
+    task :purge_all do
+      # Ensure database connection
+      HTM::SequelConfig.establish_connection!
+
+      puts "\nHTM Purge All Soft-Deleted Records"
+      puts "=" * 60
+
+      # Count soft-deleted records in each table
+      deleted_nodes = HTM::Models::Node.deleted.count
+      deleted_node_tags = HTM::Models::NodeTag.deleted.count
+      deleted_robot_nodes = HTM::Models::RobotNode.deleted.count
+
+      # Find orphaned propositions (source_node_id no longer exists)
+      # Get all source_node_ids from propositions
+      proposition_source_ids = HTM::Models::Node
+        .where(Sequel.lit("metadata->>'is_proposition' = ?", 'true'))
+        .exclude(Sequel.lit("metadata->>'source_node_id' IS NULL"))
+        .select_map(Sequel.lit("(metadata->>'source_node_id')::integer"))
+        .uniq
+
+      # Find which source nodes no longer exist (not even soft-deleted)
+      existing_node_ids = HTM::Models::Node.with_deleted
+        .where(id: proposition_source_ids)
+        .select_map(:id)
+
+      missing_source_ids = proposition_source_ids - existing_node_ids
+
+      orphaned_propositions = if missing_source_ids.any?
+        HTM::Models::Node
+          .where(Sequel.lit("metadata->>'is_proposition' = ?", 'true'))
+          .where(Sequel.lit("(metadata->>'source_node_id')::integer") => missing_source_ids)
+          .count
+      else
+        0
+      end
+
+      # Find orphaned join table entries (pointing to non-existent nodes)
+      orphaned_node_tags = HTM::Models::NodeTag.with_deleted
+        .left_join(:nodes, id: :node_id)
+        .where(Sequel[:nodes][:id] => nil)
+        .count
+
+      orphaned_robot_nodes = HTM::Models::RobotNode.with_deleted
+        .left_join(:nodes, id: :node_id)
+        .where(Sequel[:nodes][:id] => nil)
+        .count
+
+      # Find orphaned robots (no active memory nodes)
+      orphaned_robots = HTM::Models::Robot
+        .where(Sequel.~(Sequel.exists(
+          HTM::Models::RobotNode.where(Sequel[:robot_nodes][:robot_id] => Sequel[:robots][:id]).select(1)
+        )))
+        .count
+
+      # Display record counts by table
+      puts "\nSoft-deleted records by table:"
+      puts "  %-20s %8d" % ['nodes', deleted_nodes]
+      puts "  %-20s %8d" % ['node_tags', deleted_node_tags]
+      puts "  %-20s %8d" % ['robot_nodes', deleted_robot_nodes]
+
+      puts "\nOrphaned records:"
+      puts "  %-20s %8d  (source node no longer exists)" % ['propositions', orphaned_propositions]
+      puts "  %-20s %8d  (pointing to missing nodes)" % ['node_tags', orphaned_node_tags]
+      puts "  %-20s %8d  (pointing to missing nodes)" % ['robot_nodes', orphaned_robot_nodes]
+      puts "  %-20s %8d  (no associated memory nodes)" % ['robots', orphaned_robots]
+
+      total_to_delete = deleted_nodes + deleted_node_tags + deleted_robot_nodes +
+                        orphaned_propositions + orphaned_node_tags + orphaned_robot_nodes + orphaned_robots
+
+      puts "  " + "-" * 40
+      puts "  %-20s %8d" % ['Total', total_to_delete]
+
+      if total_to_delete == 0
+        puts "\nNo records to purge."
+        next
+      end
+
+      puts "\nWARNING: This permanently deletes records and cannot be undone!"
+      print "Type 'yes' to continue with hard delete: "
+      confirmation = $stdin.gets&.strip
+
+      unless confirmation == 'yes'
+        puts "Cancelled."
+        next
+      end
+
+      puts "\nPurging records..."
+
+      purged = {}
+
+      # Delete in correct order to maintain referential integrity:
+      # 1. Orphaned propositions first (creates orphaned join table entries)
+      # 2. Join tables (node_tags, robot_nodes)
+      # 3. Main tables last (nodes, robots)
+
+      # Step 1: Delete orphaned propositions (source_node_id no longer exists)
+      if missing_source_ids.any?
+        purged[:orphaned_propositions] = HTM::Models::Node
+          .where(Sequel.lit("metadata->>'is_proposition' = ?", 'true'))
+          .where(Sequel.lit("(metadata->>'source_node_id')::integer") => missing_source_ids)
+          .delete
+      else
+        purged[:orphaned_propositions] = 0
+      end
+
+      # Step 2: Delete orphaned node_tags (pointing to non-existent nodes)
+      # This now includes entries from deleted propositions
+      orphaned_nt_ids = HTM::Models::NodeTag.with_deleted
+        .left_join(:nodes, id: :node_id)
+        .where(Sequel[:nodes][:id] => nil)
+        .select_map(Sequel[:node_tags][:id])
+      purged[:orphaned_node_tags] = HTM::Models::NodeTag.with_deleted.where(id: orphaned_nt_ids).delete
+
+      # Step 3: Delete soft-deleted node_tags
+      purged[:deleted_node_tags] = HTM::Models::NodeTag.deleted.delete
+
+      # Step 4: Delete orphaned robot_nodes (pointing to non-existent nodes)
+      # This now includes entries from deleted propositions
+      orphaned_rn_ids = HTM::Models::RobotNode.with_deleted
+        .left_join(:nodes, id: :node_id)
+        .where(Sequel[:nodes][:id] => nil)
+        .select_map(Sequel[:robot_nodes][:id])
+      purged[:orphaned_robot_nodes] = HTM::Models::RobotNode.with_deleted.where(id: orphaned_rn_ids).delete
+
+      # Step 5: Delete soft-deleted robot_nodes
+      purged[:deleted_robot_nodes] = HTM::Models::RobotNode.deleted.delete
+
+      # Step 6: Delete soft-deleted nodes
+      purged[:deleted_nodes] = HTM::Models::Node.deleted.delete
+
+      # Step 7: Delete orphaned robots (no associated memory nodes)
+      purged[:orphaned_robots] = HTM::Models::Robot
+        .where(Sequel.~(Sequel.exists(
+          HTM::Models::RobotNode.where(Sequel[:robot_nodes][:robot_id] => Sequel[:robots][:id]).select(1)
+        )))
+        .delete
+
+      puts "\nPurge complete!"
+      puts "  Orphaned propositions purged: #{purged[:orphaned_propositions]}"
+      puts "  Orphaned node_tags purged:    #{purged[:orphaned_node_tags]}"
+      puts "  Deleted node_tags purged:     #{purged[:deleted_node_tags]}"
+      puts "  Orphaned robot_nodes purged:  #{purged[:orphaned_robot_nodes]}"
+      puts "  Deleted robot_nodes purged:   #{purged[:deleted_robot_nodes]}"
+      puts "  Deleted nodes purged:         #{purged[:deleted_nodes]}"
+      puts "  Orphaned robots purged:       #{purged[:orphaned_robots]}"
+      puts "  " + "-" * 40
+      puts "  Total records purged:         #{purged.values.sum}"
+    end
+
   end
 
   namespace :doc do
@@ -546,7 +675,6 @@ namespace :htm do
         puts "Install it with: brew install tbls"
         exit 1
       end
-      require 'htm'
       HTM::Database.generate_docs
     end