htm 0.0.1

data/lib/tasks/htm.rake

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+# HTM Database Management Tasks
+#
+# These tasks are available to any application using the HTM gem.
+# Add to your application's Rakefile:
+#
+#   require 'htm/tasks'
+#
+
+# Add lib directory to load path for development
+# This allows the tasks to work both during gem development and when installed
+lib_path = File.expand_path('../../lib', __dir__)
+$LOAD_PATH.unshift(lib_path) unless $LOAD_PATH.include?(lib_path)
+
+namespace :htm do
+  namespace :db do
+    desc "Set up HTM database schema and run migrations (set DUMP_SCHEMA=true to auto-dump schema after)"
+    task :setup do
+      require 'htm'
+      dump_schema = ENV['DUMP_SCHEMA'] == 'true'
+      HTM::Database.setup(dump_schema: dump_schema)
+    end
+
+    desc "Run pending database migrations"
+    task :migrate do
+      require 'htm'
+      HTM::Database.migrate
+    end
+
+    desc "Show migration status"
+    task :status do
+      require 'htm'
+      HTM::Database.migration_status
+    end
+
+    desc "Drop all HTM tables (WARNING: destructive! Set CONFIRM=yes to skip prompt)"
+    task :drop do
+      require 'htm'
+      if ENV['CONFIRM'] == 'yes'
+        HTM::Database.drop
+      else
+        print "Are you sure you want to drop all tables? This cannot be undone! (yes/no): "
+        response = STDIN.gets&.chomp
+        if response&.downcase == 'yes'
+          HTM::Database.drop
+        else
+          puts "Cancelled."
+        end
+      end
+    end
+
+    desc "Drop and recreate database (WARNING: destructive! Set CONFIRM=yes to skip prompt)"
+    task :reset do
+      require 'htm'
+      if ENV['CONFIRM'] == 'yes'
+        HTM::Database.drop
+        HTM::Database.setup(dump_schema: true)
+      else
+        print "Are you sure you want to drop all tables? This cannot be undone! (yes/no): "
+        response = STDIN.gets&.chomp
+        if response&.downcase == 'yes'
+          HTM::Database.drop
+          HTM::Database.setup(dump_schema: true)
+        else
+          puts "Cancelled."
+        end
+      end
+    end
+
+    desc "Test database connection"
+    task :test do
+      require 'htm'
+      config = HTM::Database.default_config
+      raise "Database not configured. Set HTM_DBURL environment variable." unless config
+
+      puts "Testing HTM database connection..."
+      puts "  Host: #{config[:host]}"
+      puts "  Port: #{config[:port]}"
+      puts "  Database: #{config[:dbname]}"
+      puts "  User: #{config[:user]}"
+
+      begin
+        require 'pg'
+        conn = PG.connect(config)
+
+        # Check pgvector
+        pgvector = conn.exec("SELECT extversion FROM pg_extension WHERE extname='vector'").first
+        if pgvector
+          puts "  ✓ pgvector version: #{pgvector['extversion']}"
+        else
+          puts "  ⚠ Warning: pgvector extension not found"
+        end
+
+        conn.close
+        puts "✓ Connection successful!"
+      rescue PG::Error => e
+        puts "✗ Connection failed: #{e.message}"
+        exit 1
+      end
+    end
+
+    desc "Open PostgreSQL console"
+    task :console do
+      require 'htm'
+      config = HTM::Database.default_config
+      raise "Database not configured. Set HTM_DBURL environment variable." unless config
+
+      exec "psql", "-h", config[:host],
+                   "-p", config[:port].to_s,
+                   "-U", config[:user],
+                   "-d", config[:dbname]
+    end
+
+    desc "Seed database with sample data"
+    task :seed do
+      require 'htm'
+      HTM::Database.seed
+    end
+
+    desc "Show database info (size, tables, extensions)"
+    task :info do
+      require 'htm'
+      HTM::Database.info
+    end
+
+    namespace :schema do
+      desc "Dump current schema to db/schema.sql"
+      task :dump do
+        require 'htm'
+        HTM::Database.dump_schema
+      end
+
+      desc "Load schema from db/schema.sql"
+      task :load do
+        require 'htm'
+        HTM::Database.load_schema
+      end
+    end
+
+    desc "Generate/update database documentation in dbdoc/"
+    task :doc do
+      require 'htm'
+      HTM::Database.generate_docs
+    end
+  end
+end

data/lib/tasks/jobs.rake

@@ -0,0 +1,312 @@
+# frozen_string_literal: true
+
+namespace :htm do
+  namespace :jobs do
+    desc "Show statistics for nodes and async job processing"
+    task :stats => :environment do
+      require 'htm'
+
+      # Establish connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      puts "HTM Async Job Statistics"
+      puts "=" * 60
+
+      # Total nodes
+      total_nodes = HTM::Models::Node.count
+      puts "Total nodes: #{total_nodes}"
+
+      # Nodes with embeddings
+      with_embeddings = HTM::Models::Node.where.not(embedding: nil).count
+      puts "Nodes with embeddings: #{with_embeddings} (#{percentage(with_embeddings, total_nodes)}%)"
+
+      # Nodes without embeddings (pending embedding jobs)
+      without_embeddings = HTM::Models::Node.where(embedding: nil).count
+      puts "Nodes without embeddings: #{without_embeddings} (#{percentage(without_embeddings, total_nodes)}%)"
+
+      # Nodes with tags
+      nodes_with_tags = HTM::Models::Node
+        .joins(:node_tags)
+        .distinct
+        .count
+      puts "Nodes with tags: #{nodes_with_tags} (#{percentage(nodes_with_tags, total_nodes)}%)"
+
+      # Nodes without tags
+      nodes_without_tags = total_nodes - nodes_with_tags
+      puts "Nodes without tags: #{nodes_without_tags} (#{percentage(nodes_without_tags, total_nodes)}%)"
+
+      # Total tags
+      total_tags = HTM::Models::Tag.count
+      puts "\nTotal tags in ontology: #{total_tags}"
+
+      # Tags by depth
+      if total_tags > 0
+        puts "\nTag hierarchy breakdown:"
+        HTM::Models::Tag.all.each do |tag|
+          depth = tag.name.count(':')
+          puts "  Depth #{depth}: #{tag.name.count(':', depth)} tags" if depth <= 3
+        end
+      end
+
+      # Average tags per node
+      if nodes_with_tags > 0
+        avg_tags = HTM::Models::NodeTag.count.to_f / nodes_with_tags
+        puts "\nAverage tags per node: #{avg_tags.round(2)}"
+      end
+
+      HTM::ActiveRecordConfig.disconnect!
+    end
+
+    desc "Process pending embedding jobs for nodes without embeddings"
+    task :process_embeddings => :environment do
+      require 'htm'
+
+      # Establish connection and configure HTM
+      HTM::ActiveRecordConfig.establish_connection!
+      HTM.configure  # Use default configuration
+
+      # Find nodes without embeddings
+      pending_nodes = HTM::Models::Node.where(embedding: nil)
+      total = pending_nodes.count
+
+      if total.zero?
+        puts "No pending embedding jobs"
+        HTM::ActiveRecordConfig.disconnect!
+        exit 0
+      end
+
+      puts "Processing #{total} pending embedding jobs..."
+
+      processed = 0
+      failed = 0
+
+      pending_nodes.find_each do |node|
+        begin
+          # Use the service class directly (same as job)
+          result = HTM::EmbeddingService.generate(node.content)
+          node.update!(
+            embedding: result[:storage_embedding],
+            embedding_dimension: result[:dimension]
+          )
+          processed += 1
+          print "\rProcessed: #{processed}/#{total}"
+        rescue StandardError => e
+          failed += 1
+          HTM.logger.error "Failed to process node #{node.id}: #{e.message}"
+        end
+      end
+
+      puts "\n\nCompleted:"
+      puts "  Processed: #{processed}"
+      puts "  Failed: #{failed}"
+
+      HTM::ActiveRecordConfig.disconnect!
+    end
+
+    desc "Process pending tag extraction jobs for nodes without tags"
+    task :process_tags => :environment do
+      require 'htm'
+
+      # Establish connection and configure HTM
+      HTM::ActiveRecordConfig.establish_connection!
+      HTM.configure  # Use default configuration
+
+      # Find nodes without any tags
+      nodes_without_tags = HTM::Models::Node
+        .left_joins(:node_tags)
+        .where(node_tags: { id: nil })
+
+      total = nodes_without_tags.count
+
+      if total.zero?
+        puts "No pending tag extraction jobs"
+        HTM::ActiveRecordConfig.disconnect!
+        exit 0
+      end
+
+      puts "Processing #{total} pending tag extraction jobs..."
+
+      processed = 0
+      failed = 0
+
+      nodes_without_tags.find_each do |node|
+        begin
+          # Use the service class directly (same as job)
+          existing_ontology = HTM::Models::Tag.order(created_at: :desc).limit(100).pluck(:name)
+          tag_names = HTM::TagService.extract(node.content, existing_ontology: existing_ontology)
+
+          tag_names.each do |tag_name|
+            tag = HTM::Models::Tag.find_or_create_by!(name: tag_name)
+            HTM::Models::NodeTag.find_or_create_by!(node_id: node.id, tag_id: tag.id)
+          end
+
+          processed += 1
+          print "\rProcessed: #{processed}/#{total}"
+        rescue StandardError => e
+          failed += 1
+          HTM.logger.error "Failed to process node #{node.id}: #{e.message}"
+        end
+      end
+
+      puts "\n\nCompleted:"
+      puts "  Processed: #{processed}"
+      puts "  Failed: #{failed}"
+
+      HTM::ActiveRecordConfig.disconnect!
+    end
+
+    desc "Process all pending jobs (embeddings and tags)"
+    task :process_all => [:process_embeddings, :process_tags] do
+      puts "\nAll pending jobs processed!"
+    end
+
+    desc "Reprocess embeddings for all nodes (force regeneration)"
+    task :reprocess_embeddings => :environment do
+      require 'htm'
+
+      print "This will regenerate embeddings for ALL nodes. Are you sure? (yes/no): "
+      confirmation = $stdin.gets.chomp
+
+      unless confirmation.downcase == 'yes'
+        puts "Cancelled."
+        exit 0
+      end
+
+      # Establish connection and configure HTM
+      HTM::ActiveRecordConfig.establish_connection!
+      HTM.configure  # Use default configuration
+
+      total = HTM::Models::Node.count
+
+      puts "Reprocessing embeddings for #{total} nodes..."
+
+      processed = 0
+      failed = 0
+
+      HTM::Models::Node.find_each do |node|
+        begin
+          # Use the service class directly to regenerate
+          result = HTM::EmbeddingService.generate(node.content)
+          node.update!(
+            embedding: result[:storage_embedding],
+            embedding_dimension: result[:dimension]
+          )
+          processed += 1
+          print "\rProcessed: #{processed}/#{total}"
+        rescue StandardError => e
+          failed += 1
+          HTM.logger.error "Failed to reprocess node #{node.id}: #{e.message}"
+        end
+      end
+
+      puts "\n\nCompleted:"
+      puts "  Processed: #{processed}"
+      puts "  Failed: #{failed}"
+
+      HTM::ActiveRecordConfig.disconnect!
+    end
+
+    desc "Show nodes that failed async processing"
+    task :failed => :environment do
+      require 'htm'
+
+      HTM::ActiveRecordConfig.establish_connection!
+
+      puts "Nodes with Processing Issues"
+      puts "=" * 60
+
+      # Old nodes without embeddings (created more than 1 hour ago)
+      old_without_embeddings = HTM::Models::Node
+        .where(embedding: nil)
+        .where('created_at < ?', 1.hour.ago)
+
+      if old_without_embeddings.any?
+        puts "\nNodes without embeddings (>1 hour old):"
+        old_without_embeddings.limit(10).each do |node|
+          puts "  Node #{node.id}: created #{time_ago(node.created_at)}"
+        end
+
+        if old_without_embeddings.count > 10
+          puts "  ... and #{old_without_embeddings.count - 10} more"
+        end
+      else
+        puts "\n✓ No old nodes without embeddings"
+      end
+
+      # Old nodes without tags
+      old_without_tags = HTM::Models::Node
+        .left_joins(:node_tags)
+        .where(node_tags: { id: nil })
+        .where('nodes.created_at < ?', 1.hour.ago)
+
+      if old_without_tags.any?
+        puts "\nNodes without tags (>1 hour old):"
+        old_without_tags.limit(10).each do |node|
+          puts "  Node #{node.id}: created #{time_ago(node.created_at)}"
+        end
+
+        if old_without_tags.count > 10
+          puts "  ... and #{old_without_tags.count - 10} more"
+        end
+      else
+        puts "\n✓ No old nodes without tags"
+      end
+
+      HTM::ActiveRecordConfig.disconnect!
+    end
+
+    desc "Clear all embeddings and tags (for testing/development)"
+    task :clear_all => :environment do
+      require 'htm'
+
+      print "This will clear ALL embeddings and tags. Are you sure? (yes/no): "
+      confirmation = $stdin.gets.chomp
+
+      unless confirmation.downcase == 'yes'
+        puts "Cancelled."
+        exit 0
+      end
+
+      HTM::ActiveRecordConfig.establish_connection!
+
+      puts "Clearing embeddings..."
+      HTM::Models::Node.update_all(embedding: nil, embedding_dimension: nil)
+
+      puts "Clearing tags..."
+      HTM::Models::NodeTag.delete_all
+      HTM::Models::Tag.delete_all
+
+      puts "Done! All embeddings and tags cleared."
+
+      HTM::ActiveRecordConfig.disconnect!
+    end
+
+    # Helper methods
+    def percentage(part, whole)
+      return 0 if whole.zero?
+      ((part.to_f / whole) * 100).round(1)
+    end
+
+    def time_ago(time)
+      seconds = Time.now - time
+      case seconds
+      when 0..59
+        "#{seconds.to_i} seconds ago"
+      when 60..3599
+        "#{(seconds / 60).to_i} minutes ago"
+      when 3600..86399
+        "#{(seconds / 3600).to_i} hours ago"
+      else
+        "#{(seconds / 86400).to_i} days ago"
+      end
+    end
+  end
+end
+
+# Add :environment task if not already defined (for standalone usage)
+unless Rake::Task.task_defined?(:environment)
+  task :environment do
+    # No-op for standalone usage
+    # Applications can override this to set up their environment
+  end
+end

data/mkdocs.yml

@@ -0,0 +1,190 @@
+# MkDocs Configuration for HTM Documentation
+site_name: HTM - Hierarchical Temporary Memory
+site_description: Intelligent memory management for LLM robots with two-tier architecture and RAG-based retrieval
+site_author: Dewayne VanHoozer
+site_url: https://madbomber.github.io/htm
+copyright: Copyright © 2025 Dewayne VanHoozer
+
+# Repository information
+repo_name: madbomber/htm
+repo_url: https://github.com/madbomber/htm
+edit_uri: edit/main/docs/
+
+# Configuration
+theme:
+  name: material
+  
+  # Color scheme - Default to light mode
+  palette:
+    # Palette toggle for light mode (default)
+    - scheme: default
+      primary: blue
+      accent: blue
+      toggle:
+        icon: material/brightness-7
+        name: Switch to dark mode
+    
+    # Palette toggle for dark mode
+    - scheme: slate
+      primary: blue
+      accent: blue
+      toggle:
+        icon: material/brightness-4
+        name: Switch to light mode
+
+  # Typography
+  font:
+    text: Roboto
+    code: Roboto Mono
+
+  # Theme features
+  features:
+    # Navigation
+    - navigation.instant
+    - navigation.tracking
+    - navigation.tabs
+    - navigation.tabs.sticky
+    - navigation.sections
+    - navigation.path
+    - navigation.indexes
+    - navigation.top
+
+    # Table of contents
+    - toc.follow
+
+    # Search
+    - search.suggest
+    - search.highlight
+    - search.share
+
+    # Header
+    - header.autohide
+
+    # Content
+    - content.code.copy
+    - content.code.annotate
+    - content.tabs.link
+    - content.tooltips
+    - content.action.edit
+    - content.action.view
+
+# Plugins
+plugins:
+  - search:
+      separator: '[\s\-,:!=\[\]()"`/]+|\.(?!\d)|&[lg]t;|(?!\b)(?=[A-Z][a-z])'
+
+# Extensions
+markdown_extensions:
+  # Python Markdown
+  - abbr
+  - admonition
+  - attr_list
+  - def_list
+  - footnotes
+  - md_in_html
+  - toc:
+      permalink: true
+      title: On this page
+
+  # Python Markdown Extensions
+  - pymdownx.arithmatex:
+      generic: true
+  - pymdownx.betterem:
+      smart_enable: all
+  - pymdownx.caret
+  - pymdownx.details
+  - pymdownx.emoji:
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+  - pymdownx.highlight:
+      anchor_linenums: true
+      line_spans: __span
+      pygments_lang_class: true
+  - pymdownx.inlinehilite
+  - pymdownx.keys
+  - pymdownx.magiclink:
+      repo_url_shorthand: true
+      user: madbomber
+      repo: htm
+  - pymdownx.mark
+  - pymdownx.smartsymbols
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
+  - pymdownx.tabbed:
+      alternate_style: true
+  - pymdownx.tasklist:
+      custom_checkbox: true
+  - pymdownx.tilde
+
+# Extra CSS and JavaScript
+extra_css:
+  - assets/css/custom.css
+
+extra_javascript:
+  - assets/js/mathjax.js
+  - https://polyfill.io/v3/polyfill.min.js?features=es6
+  - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js
+
+# Social media and extra configuration
+extra:
+  social:
+    - icon: fontawesome/brands/github
+      link: https://github.com/madbomber/htm
+      name: HTM on GitHub
+    - icon: fontawesome/solid/gem
+      link: https://rubygems.org/gems/htm
+      name: HTM on RubyGems
+
+# Navigation
+nav:
+  - Home: index.md
+  - Installation: installation.md
+  - Quick Start: quick-start.md
+  - Architecture:
+    - architecture/index.md
+    - Overview: architecture/overview.md
+    - Two-Tier Memory: architecture/two-tier-memory.md
+    - Hive Mind: architecture/hive-mind.md
+    - ADRs:
+      - architecture/adrs/index.md
+      - ADR-001 PostgreSQL & TimescaleDB: architecture/adrs/001-postgresql-timescaledb.md
+      - ADR-002 Two-Tier Memory: architecture/adrs/002-two-tier-memory.md
+      - ADR-003 Ollama Embeddings: architecture/adrs/003-ollama-embeddings.md
+      - ADR-004 Hive Mind: architecture/adrs/004-hive-mind.md
+      - ADR-005 RAG Retrieval: architecture/adrs/005-rag-retrieval.md
+      - ADR-006 Context Assembly: architecture/adrs/006-context-assembly.md
+      - ADR-007 Eviction Strategy: architecture/adrs/007-eviction-strategy.md
+      - ADR-008 Robot Identification: architecture/adrs/008-robot-identification.md
+      - ADR-009 Never-Forget Philosophy: architecture/adrs/009-never-forget.md
+      - ADR-010 Redis Working Memory (Rejected): architecture/adrs/010-redis-working-memory-rejected.md
+      - ADR-011 pgai Integration: architecture/adrs/011-pgai-integration.md
+  - Guides:
+    - guides/index.md
+    - Getting Started: guides/getting-started.md
+    - Adding Memories: guides/adding-memories.md
+    - Recalling Memories: guides/recalling-memories.md
+    - Working Memory: guides/working-memory.md
+    - Long-Term Memory: guides/long-term-memory.md
+    - Multi-Robot Usage: guides/multi-robot.md
+    - Search Strategies: guides/search-strategies.md
+    - Context Assembly: guides/context-assembly.md
+  - API Reference:
+    - api/index.md
+    - HTM Class: api/htm.md
+    - Working Memory: api/working-memory.md
+    - Long-Term Memory: api/long-term-memory.md
+    - Embedding Service: api/embedding-service.md
+    - Database: api/database.md
+  - Development:
+    - development/index.md
+    - Setup: development/setup.md
+    - Testing: development/testing.md
+    - Contributing: development/contributing.md
+    - Database Schema: development/schema.md
+    - Multi-Framework Support: multi_framework_support.md
+    - Database Rake Tasks: database_rake_tasks.md
+    - Using Rake Tasks: using_rake_tasks_in_your_app.md
+    - Setup Local Database: setup_local_database.md