htm 0.0.10 → 0.0.14

data/lib/tasks/htm.rake

@@ -68,21 +68,28 @@ namespace :htm do
       end
     end
 
-    desc "Test database connection"
-    task :test do
+    desc "Verify database connection (respects RAILS_ENV)"
+    task :verify do
       require 'htm'
-      config = HTM::Database.default_config
-      raise "Database not configured. Set HTM_DBURL environment variable." unless config
 
-      puts "Testing HTM database connection..."
+      env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+      config = HTM::ActiveRecordConfig.load_database_config
+
+      puts "Verifying HTM database connection (#{env})..."
       puts "  Host: #{config[:host]}"
       puts "  Port: #{config[:port]}"
-      puts "  Database: #{config[:dbname]}"
-      puts "  User: #{config[:user]}"
+      puts "  Database: #{config[:database]}"
+      puts "  User: #{config[:username]}"
 
       begin
         require 'pg'
-        conn = PG.connect(config)
+        conn = PG.connect(
+          host: config[:host],
+          port: config[:port],
+          dbname: config[:database],
+          user: config[:username],
+          password: config[:password]
+        )
 
         # Check pgvector
         pgvector = conn.exec("SELECT extversion FROM pg_extension WHERE extname='vector'").first
@@ -100,16 +107,18 @@ namespace :htm do
       end
     end
 
-    desc "Open PostgreSQL console"
+    desc "Open PostgreSQL console (respects RAILS_ENV)"
     task :console do
       require 'htm'
-      config = HTM::Database.default_config
-      raise "Database not configured. Set HTM_DBURL environment variable." unless config
 
+      env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+      config = HTM::ActiveRecordConfig.load_database_config
+
+      puts "Connecting to #{config[:database]} (#{env})..."
       exec "psql", "-h", config[:host],
                    "-p", config[:port].to_s,
-                   "-U", config[:user],
-                   "-d", config[:dbname]
+                   "-U", config[:username],
+                   "-d", config[:database]
     end
 
     desc "Seed database with sample data"
@@ -277,6 +286,120 @@ namespace :htm do
         puts "  Errors: #{errors}"
         puts "  Nodes with embeddings: #{final_with_embeddings}"
       end
+
+      desc "Rebuild propositions for all non-proposition nodes. Extracts atomic facts and creates new nodes."
+      task :propositions do
+        require 'htm'
+        require 'ruby-progressbar'
+
+        # Ensure database connection
+        HTM::ActiveRecordConfig.establish_connection!
+
+        # Find all non-proposition nodes (nodes that haven't been extracted from)
+        source_nodes = HTM::Models::Node.non_propositions
+        source_count = source_nodes.count
+
+        # Count existing proposition nodes
+        existing_propositions = HTM::Models::Node.propositions.count
+
+        puts "\nHTM Propositions Rebuild"
+        puts "=" * 50
+        puts "Current state:"
+        puts "  Source nodes (non-propositions): #{source_count}"
+        puts "  Existing proposition nodes: #{existing_propositions}"
+        puts "\nThis will extract propositions from ALL #{source_count} source nodes."
+        puts "Existing proposition nodes will be deleted and regenerated."
+        puts "This operation may take a long time depending on your LLM provider."
+        print "\nType 'yes' to confirm: "
+
+        confirmation = $stdin.gets&.strip
+        unless confirmation == 'yes'
+          puts "Aborted."
+          next
+        end
+
+        # Delete existing proposition nodes
+        if existing_propositions > 0
+          puts "\nDeleting #{existing_propositions} existing proposition nodes..."
+          deleted = HTM::Models::Node.propositions.delete_all
+          puts "  Deleted #{deleted} proposition nodes"
+        end
+
+        puts "\nExtracting propositions from #{source_count} nodes..."
+        puts "(This may take a while depending on your LLM provider)\n"
+
+        # Get a robot ID for linking proposition nodes
+        # Use the first robot or create a system robot
+        robot = HTM::Models::Robot.first || HTM::Models::Robot.create!(name: 'proposition_rebuilder')
+
+        # Create progress bar with ETA
+        progressbar = ProgressBar.create(
+          total: source_count,
+          format: '%t: |%B| %c/%C (%p%%) %e',
+          title: 'Extracting',
+          output: $stdout,
+          smoothing: 0.5
+        )
+
+        # Track stats
+        errors = 0
+        nodes_processed = 0
+        propositions_created = 0
+
+        source_nodes.find_each do |node|
+          begin
+            # Extract propositions
+            propositions = HTM::PropositionService.extract(node.content)
+
+            if propositions.any?
+              propositions.each do |proposition_text|
+                token_count = HTM.count_tokens(proposition_text)
+
+                # Create proposition node
+                prop_node = HTM::Models::Node.create!(
+                  content: proposition_text,
+                  token_count: token_count,
+                  metadata: { is_proposition: true, source_node_id: node.id }
+                )
+
+                # Link to robot
+                HTM::Models::RobotNode.find_or_create_by!(
+                  robot_id: robot.id,
+                  node_id: prop_node.id
+                )
+
+                # Generate embedding for proposition node
+                begin
+                  result = HTM::EmbeddingService.generate(proposition_text)
+                  prop_node.update!(embedding: result[:storage_embedding])
+                rescue StandardError => e
+                  progressbar.log "  Warning: Embedding failed for proposition: #{e.message}"
+                end
+
+                propositions_created += 1
+              end
+            end
+
+            nodes_processed += 1
+          rescue StandardError => e
+            errors += 1
+            progressbar.log "  Error on node #{node.id}: #{e.message}"
+          end
+
+          progressbar.increment
+        end
+
+        progressbar.finish
+
+        # Final stats
+        final_proposition_count = HTM::Models::Node.propositions.count
+
+        puts "\nRebuild complete!"
+        puts "  Source nodes processed: #{nodes_processed}"
+        puts "  Propositions created: #{propositions_created}"
+        puts "  Errors: #{errors}"
+        puts "  Total proposition nodes: #{final_proposition_count}"
+      end
     end
 
     namespace :schema do
@@ -293,6 +416,129 @@ namespace :htm do
       end
     end
 
+    desc "Create database if it doesn't exist (respects RAILS_ENV)"
+    task :create do
+      require 'htm'
+
+      env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+      config = HTM::ActiveRecordConfig.load_database_config
+      db_name = config[:database]
+
+      puts "Creating database: #{db_name} (#{env})"
+
+      admin_config = config.dup
+      admin_config[:database] = 'postgres'
+
+      begin
+        require 'pg'
+        admin_conn = PG.connect(
+          host: admin_config[:host],
+          port: admin_config[:port],
+          dbname: admin_config[:database],
+          user: admin_config[:username],
+          password: admin_config[:password]
+        )
+
+        result = admin_conn.exec_params(
+          "SELECT 1 FROM pg_database WHERE datname = $1",
+          [db_name]
+        )
+
+        if result.ntuples == 0
+          admin_conn.exec("CREATE DATABASE #{PG::Connection.quote_ident(db_name)}")
+          puts "✓ Database created: #{db_name}"
+
+          # Connect to new database and enable extensions
+          db_conn = PG.connect(
+            host: config[:host],
+            port: config[:port],
+            dbname: db_name,
+            user: config[:username],
+            password: config[:password]
+          )
+          %w[vector pg_trgm].each do |ext|
+            db_conn.exec("CREATE EXTENSION IF NOT EXISTS #{ext}")
+          end
+          db_conn.close
+          puts "✓ Extensions enabled (pgvector, pg_trgm)"
+        else
+          puts "✓ Database already exists: #{db_name}"
+        end
+
+        admin_conn.close
+      rescue PG::Error => e
+        puts "✗ Error: #{e.message}"
+        exit 1
+      end
+    end
+
+    namespace :tags do
+      desc "Soft delete orphaned tags and stale node_tags entries"
+      task :cleanup do
+        require 'htm'
+
+        # Ensure database connection
+        HTM::ActiveRecordConfig.establish_connection!
+
+        puts "\nHTM Tag Cleanup"
+        puts "=" * 50
+
+        # Step 1: Find active node_tags pointing to soft-deleted or missing nodes
+        stale_node_tags = HTM::Models::NodeTag
+          .joins("LEFT JOIN nodes ON nodes.id = node_tags.node_id")
+          .where("nodes.id IS NULL OR nodes.deleted_at IS NOT NULL")
+
+        stale_count = stale_node_tags.count
+
+        # Step 2: Find orphaned tags using the Tag.orphaned scope
+        orphaned_tags = HTM::Models::Tag.orphaned
+        orphan_count = orphaned_tags.count
+
+        if stale_count == 0 && orphan_count == 0
+          puts "No cleanup needed."
+          puts "  Stale node_tags entries: 0"
+          puts "  Orphaned tags: 0"
+          next
+        end
+
+        puts "Found:"
+        puts "  Stale node_tags entries: #{stale_count} (pointing to deleted/missing nodes)"
+        puts "  Orphaned tags: #{orphan_count} (no active nodes)"
+
+        if orphan_count > 0
+          puts "\nOrphaned tags:"
+          orphaned_tags.limit(20).pluck(:name).each do |name|
+            puts "  - #{name}"
+          end
+          puts "  ... and #{orphan_count - 20} more" if orphan_count > 20
+        end
+
+        print "\nSoft delete these entries? (yes/no): "
+        confirmation = $stdin.gets&.strip
+
+        unless confirmation == 'yes'
+          puts "Cancelled."
+          next
+        end
+
+        now = Time.current
+
+        # Soft delete stale node_tags first
+        if stale_count > 0
+          soft_deleted_node_tags = stale_node_tags.update_all(deleted_at: now)
+          puts "\nSoft deleted #{soft_deleted_node_tags} stale node_tags entries."
+        end
+
+        # Then soft delete orphaned tags
+        if orphan_count > 0
+          soft_deleted_tags = orphaned_tags.update_all(deleted_at: now)
+          puts "Soft deleted #{soft_deleted_tags} orphaned tags."
+        end
+
+        puts "\nCleanup complete (soft delete)."
+      end
+    end
+
   end
 
   namespace :doc do

data/mkdocs.yml

@@ -1,5 +1,5 @@
 # MkDocs Configuration for HTM Documentation
-site_name: HTM - Hierarchical Temporary Memory
+site_name: HTM - Hierarchical Temporal Memory
 site_description: Intelligent memory management for LLM robots with two-tier architecture and RAG-based retrieval
 site_author: Dewayne VanHoozer
 site_url: https://madbomber.github.io/htm
@@ -172,101 +172,101 @@ nav:
       - Installation: getting-started/installation.md
       - Quick Start: getting-started/quick-start.md
   - Architecture:
-    - architecture/index.md
-    - Overview: architecture/overview.md
-    - Two-Tier Memory: architecture/two-tier-memory.md
-    - Hive Mind: architecture/hive-mind.md
-    - ADRs:
-      - architecture/adrs/index.md
-      - ADR-001 PostgreSQL & TimescaleDB: architecture/adrs/001-postgresql-timescaledb.md
-      - ADR-002 Two-Tier Memory: architecture/adrs/002-two-tier-memory.md
-      - ADR-003 Ollama Embeddings: architecture/adrs/003-ollama-embeddings.md
-      - ADR-004 Hive Mind: architecture/adrs/004-hive-mind.md
-      - ADR-005 RAG Retrieval: architecture/adrs/005-rag-retrieval.md
-      - ADR-006 Context Assembly: architecture/adrs/006-context-assembly.md
-      - ADR-007 Eviction Strategy: architecture/adrs/007-eviction-strategy.md
-      - ADR-008 Robot Identification: architecture/adrs/008-robot-identification.md
-      - ADR-009 Never-Forget Philosophy: architecture/adrs/009-never-forget.md
-      - ADR-010 Redis Working Memory (Rejected): architecture/adrs/010-redis-working-memory-rejected.md
-      - ADR-011 pgai Integration: architecture/adrs/011-pgai-integration.md
+      - architecture/index.md
+      - Overview: architecture/overview.md
+      - Two-Tier Memory: architecture/two-tier-memory.md
+      - Hive Mind: architecture/hive-mind.md
+      - ADRs:
+          - architecture/adrs/index.md
+          - ADR-001 PostgreSQL & TimescaleDB: architecture/adrs/001-postgresql-timescaledb.md
+          - ADR-002 Two-Tier Memory: architecture/adrs/002-two-tier-memory.md
+          - ADR-003 Ollama Embeddings: architecture/adrs/003-ollama-embeddings.md
+          - ADR-004 Hive Mind: architecture/adrs/004-hive-mind.md
+          - ADR-005 RAG Retrieval: architecture/adrs/005-rag-retrieval.md
+          - ADR-006 Context Assembly: architecture/adrs/006-context-assembly.md
+          - ADR-007 Eviction Strategy: architecture/adrs/007-eviction-strategy.md
+          - ADR-008 Robot Identification: architecture/adrs/008-robot-identification.md
+          - ADR-009 Never-Forget Philosophy: architecture/adrs/009-never-forget.md
+          - ADR-010 Redis Working Memory (Rejected): architecture/adrs/010-redis-working-memory-rejected.md
+          - ADR-011 pgai Integration: architecture/adrs/011-pgai-integration.md
   - Guides:
-    - guides/index.md
-    - Getting Started: guides/getting-started.md
-    - Adding Memories: guides/adding-memories.md
-    - Recalling Memories: guides/recalling-memories.md
-    - Working Memory: guides/working-memory.md
-    - Long-Term Memory: guides/long-term-memory.md
-    - Multi-Robot Usage: guides/multi-robot.md
-    - Search Strategies: guides/search-strategies.md
-    - Context Assembly: guides/context-assembly.md
+      - guides/index.md
+      - Getting Started: guides/getting-started.md
+      - Adding Memories: guides/adding-memories.md
+      - Recalling Memories: guides/recalling-memories.md
+      - Working Memory: guides/working-memory.md
+      - Long-Term Memory: guides/long-term-memory.md
+      - Multi-Robot Usage: guides/multi-robot.md
+      - Search Strategies: guides/search-strategies.md
+      - Context Assembly: guides/context-assembly.md
   - API Reference:
-    - api/index.md
-    - HTM Class: api/htm.md
-    - Working Memory: api/working-memory.md
-    - Long-Term Memory: api/long-term-memory.md
-    - Embedding Service: api/embedding-service.md
-    - Database: api/database.md
-    - YARD Reference:
-      - api/yard-reference.md
-      - HTM Module: api/yard/HTM.md
-      - Core Classes:
-        - Configuration: api/yard/HTM/Configuration.md
-        - Database: api/yard/HTM/Database.md
-        - LongTermMemory: api/yard/HTM/LongTermMemory.md
-        - WorkingMemory: api/yard/HTM/WorkingMemory.md
-      - Services:
-        - EmbeddingService: api/yard/HTM/EmbeddingService.md
-        - TagService: api/yard/HTM/TagService.md
-        - JobAdapter: api/yard/HTM/JobAdapter.md
-      - Utilities:
-        - CircuitBreaker: api/yard/HTM/CircuitBreaker.md
-        - Observability: api/yard/HTM/Observability.md
-        - Timeframe: api/yard/HTM/Timeframe.md
-        - TimeframeExtractor: api/yard/HTM/TimeframeExtractor.md
-      - Internal:
-        - ActiveRecordConfig: api/yard/HTM/ActiveRecordConfig.md
-        - Railtie: api/yard/HTM/Railtie.md
-      - Errors:
-        - Error: api/yard/HTM/Error.md
-        - AuthorizationError: api/yard/HTM/AuthorizationError.md
-        - CircuitBreakerOpenError: api/yard/HTM/CircuitBreakerOpenError.md
-        - DatabaseError: api/yard/HTM/DatabaseError.md
-        - EmbeddingError: api/yard/HTM/EmbeddingError.md
-        - NotFoundError: api/yard/HTM/NotFoundError.md
-        - QueryTimeoutError: api/yard/HTM/QueryTimeoutError.md
-        - ResourceExhaustedError: api/yard/HTM/ResourceExhaustedError.md
-        - TagError: api/yard/HTM/TagError.md
-        - ValidationError: api/yard/HTM/ValidationError.md
-      - Result Types:
-        - Timeframe Result: api/yard/HTM/Timeframe/Result.md
-        - TimeframeExtractor Result: api/yard/HTM/TimeframeExtractor/Result.md
+      - api/index.md
+      - HTM Class: api/htm.md
+      - Working Memory: api/working-memory.md
+      - Long-Term Memory: api/long-term-memory.md
+      - Embedding Service: api/embedding-service.md
+      - Database: api/database.md
+      - YARD Reference:
+          - api/yard-reference.md
+          - HTM Module: api/yard/HTM.md
+          - Core Classes:
+              - Configuration: api/yard/HTM/Configuration.md
+              - Database: api/yard/HTM/Database.md
+              - LongTermMemory: api/yard/HTM/LongTermMemory.md
+              - WorkingMemory: api/yard/HTM/WorkingMemory.md
+          - Services:
+              - EmbeddingService: api/yard/HTM/EmbeddingService.md
+              - TagService: api/yard/HTM/TagService.md
+              - JobAdapter: api/yard/HTM/JobAdapter.md
+          - Utilities:
+              - CircuitBreaker: api/yard/HTM/CircuitBreaker.md
+              - Observability: api/yard/HTM/Observability.md
+              - Timeframe: api/yard/HTM/Timeframe.md
+              - TimeframeExtractor: api/yard/HTM/TimeframeExtractor.md
+          - Internal:
+              - ActiveRecordConfig: api/yard/HTM/ActiveRecordConfig.md
+              - Railtie: api/yard/HTM/Railtie.md
+          - Errors:
+              - Error: api/yard/HTM/Error.md
+              - AuthorizationError: api/yard/HTM/AuthorizationError.md
+              - CircuitBreakerOpenError: api/yard/HTM/CircuitBreakerOpenError.md
+              - DatabaseError: api/yard/HTM/DatabaseError.md
+              - EmbeddingError: api/yard/HTM/EmbeddingError.md
+              - NotFoundError: api/yard/HTM/NotFoundError.md
+              - QueryTimeoutError: api/yard/HTM/QueryTimeoutError.md
+              - ResourceExhaustedError: api/yard/HTM/ResourceExhaustedError.md
+              - TagError: api/yard/HTM/TagError.md
+              - ValidationError: api/yard/HTM/ValidationError.md
+          - Result Types:
+              - Timeframe Result: api/yard/HTM/Timeframe/Result.md
+              - TimeframeExtractor Result: api/yard/HTM/TimeframeExtractor/Result.md
   - Development:
-    - development/index.md
-    - Setup: development/setup.md
-    - Testing: development/testing.md
-    - Contributing: development/contributing.md
-    - Database Schema: development/schema.md
-    - Database Tables:
-      - database/README.md
-      - Core Tables:
-        - Robots: database/public.robots.md
-        - Nodes: database/public.nodes.md
-        - Tags: database/public.tags.md
-        - File Sources: database/public.file_sources.md
-      - Join Tables:
-        - Node Tags: database/public.node_tags.md
-        - Nodes Tags (Legacy): database/public.nodes_tags.md
-        - Robot Nodes: database/public.robot_nodes.md
-      - Views & Stats:
-        - Node Stats: database/public.node_stats.md
-        - Robot Activity: database/public.robot_activity.md
-        - Topic Relationships: database/public.topic_relationships.md
-        - Ontology Structure: database/public.ontology_structure.md
-        - Relationships: database/public.relationships.md
-      - System:
-        - Operations Log: database/public.operations_log.md
-        - Schema Migrations: database/public.schema_migrations.md
-    - Multi-Framework Support: multi_framework_support.md
-    - Database Rake Tasks: database_rake_tasks.md
-    - Using Rake Tasks: using_rake_tasks_in_your_app.md
-    - Setup Local Database: setup_local_database.md
+      - development/index.md
+      - Setup: development/setup.md
+      - Testing: development/testing.md
+      - Contributing: development/contributing.md
+      - Database Schema: development/schema.md
+      - Database Tables:
+          - database/README.md
+          - Core Tables:
+              - Robots: database/public.robots.md
+              - Nodes: database/public.nodes.md
+              - Tags: database/public.tags.md
+              - File Sources: database/public.file_sources.md
+          - Join Tables:
+              - Node Tags: database/public.node_tags.md
+              - Nodes Tags (Legacy): database/public.nodes_tags.md
+              - Robot Nodes: database/public.robot_nodes.md
+          - Views & Stats:
+              - Node Stats: database/public.node_stats.md
+              - Robot Activity: database/public.robot_activity.md
+              - Topic Relationships: database/public.topic_relationships.md
+              - Ontology Structure: database/public.ontology_structure.md
+              - Relationships: database/public.relationships.md
+          - System:
+              - Operations Log: database/public.operations_log.md
+              - Schema Migrations: database/public.schema_migrations.md
+      - Multi-Framework Support: multi_framework_support.md
+      - Database Rake Tasks: database_rake_tasks.md
+      - Using Rake Tasks: using_rake_tasks_in_your_app.md
+      - Setup Local Database: setup_local_database.md