htm 0.0.2 → 0.0.11

data/lib/tasks/htm.rake

@@ -124,6 +124,161 @@ namespace :htm do
       HTM::Database.info
     end
 
+    desc "Show record counts for all HTM tables"
+    task :stats do
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      puts "\nHTM Database Statistics"
+      puts "=" * 50
+
+      # Define tables with their models and optional extra info
+      tables = [
+        { name: 'robots', model: HTM::Models::Robot },
+        { name: 'nodes', model: HTM::Models::Node, extras: ->(m) {
+          # Node uses default_scope for active nodes, so m.count is active count
+          active = m.count
+          deleted = m.deleted.count
+          with_embedding = m.where.not(embedding: nil).count
+          "  (active: #{active}, deleted: #{deleted}, with embeddings: #{with_embedding})"
+        }},
+        { name: 'tags', model: HTM::Models::Tag },
+        { name: 'nodes_tags', model: HTM::Models::NodeTag },
+        { name: 'file_sources', model: HTM::Models::FileSource }
+      ]
+
+      total = 0
+      max_name_len = tables.map { |t| t[:name].length }.max
+
+      tables.each do |table|
+        count = table[:model].count
+        total += count
+        extras = table[:extras] ? table[:extras].call(table[:model]) : ''
+
+        printf "  %-#{max_name_len}s  %8d%s\n", table[:name], count, extras
+      end
+
+      puts "-" * 50
+      printf "  %-#{max_name_len}s  %8d\n", "Total", total
+
+      # Additional stats
+      puts "\nAdditional Statistics:"
+
+      # Nodes per robot (via robot_nodes join table)
+      robot_counts = HTM::Models::RobotNode
+        .joins(:node)
+        .where(nodes: { deleted_at: nil })
+        .group(:robot_id)
+        .count
+        .transform_keys { |id| HTM::Models::Robot.find(id).name rescue "Unknown (#{id})" }
+        .sort_by { |_, count| -count }
+        .first(5)
+
+      if robot_counts.any?
+        puts "  Top robots by node count:"
+        robot_counts.each do |name, count|
+          puts "    #{name}: #{count}"
+        end
+      end
+
+      # Tag distribution
+      top_root_tags = HTM::Models::Tag
+        .select("split_part(name, ':', 1) as root, count(*) as cnt")
+        .group("split_part(name, ':', 1)")
+        .order("cnt DESC")
+        .limit(5)
+        .map { |t| [t.root, t.cnt] }
+
+      if top_root_tags.any?
+        puts "  Top root tag categories:"
+        top_root_tags.each do |root, count|
+          puts "    #{root}: #{count}"
+        end
+      end
+
+      puts
+    end
+
+    namespace :rebuild do
+      desc "Rebuild embeddings for all nodes. Clears existing embeddings and regenerates via LLM."
+      task :embeddings do
+        require 'htm'
+        require 'ruby-progressbar'
+
+        # Ensure database connection
+        HTM::ActiveRecordConfig.establish_connection!
+
+        # Node uses default_scope for active (non-deleted) nodes
+        node_count = HTM::Models::Node.count
+        with_embeddings = HTM::Models::Node.where.not(embedding: nil).count
+        without_embeddings = node_count - with_embeddings
+
+        puts "\nHTM Embeddings Rebuild"
+        puts "=" * 50
+        puts "Current state:"
+        puts "  Total active nodes: #{node_count}"
+        puts "  With embeddings: #{with_embeddings}"
+        puts "  Without embeddings: #{without_embeddings}"
+        puts "\nThis will regenerate embeddings for ALL #{node_count} nodes."
+        puts "This operation may take a long time depending on your LLM provider."
+        print "\nType 'yes' to confirm: "
+
+        confirmation = $stdin.gets&.strip
+        unless confirmation == 'yes'
+          puts "Aborted."
+          next
+        end
+
+        puts "\nClearing existing embeddings..."
+        cleared = HTM::Models::Node.where.not(embedding: nil).update_all(embedding: nil)
+        puts "  Cleared #{cleared} embeddings"
+
+        puts "\nGenerating embeddings for #{node_count} nodes..."
+        puts "(This may take a while depending on your LLM provider)\n"
+
+        # Create progress bar with ETA
+        progressbar = ProgressBar.create(
+          total: node_count,
+          format: '%t: |%B| %c/%C (%p%%) %e',
+          title: 'Embedding',
+          output: $stdout,
+          smoothing: 0.5
+        )
+
+        # Process each active node (default_scope excludes deleted)
+        errors = 0
+        success = 0
+
+        HTM::Models::Node.find_each do |node|
+          begin
+            # Generate embedding directly (not via job since we cleared them)
+            result = HTM::EmbeddingService.generate(node.content)
+
+            node.update!(embedding: result[:storage_embedding])
+            success += 1
+          rescue StandardError => e
+            errors += 1
+            progressbar.log "  Error on node #{node.id}: #{e.message}"
+          end
+
+          progressbar.increment
+        end
+
+        progressbar.finish
+
+        # Final stats
+        final_with_embeddings = HTM::Models::Node.where.not(embedding: nil).count
+
+        puts "\nRebuild complete!"
+        puts "  Nodes processed: #{node_count}"
+        puts "  Successful: #{success}"
+        puts "  Errors: #{errors}"
+        puts "  Nodes with embeddings: #{final_with_embeddings}"
+      end
+    end
+
     namespace :schema do
       desc "Dump current schema to db/schema.sql"
       task :dump do
@@ -141,7 +296,7 @@ namespace :htm do
   end
 
   namespace :doc do
-    desc "Generate/update database documentation in dbdoc/"
+    desc "Generate/update database documentation in docs/database/ (uses .tbls.yml)"
     task :db do
       unless system("which tbls > /dev/null 2>&1")
         puts "Error: 'tbls' is not installed."
@@ -172,7 +327,7 @@ namespace :htm do
       sh "mkdocs serve"
     end
 
-    desc "Generate DB docs, build site, and serve locally"
-    task :all => [:db, :build, :serve]
+    desc "Generate DB docs, YARD API docs, build site, and serve locally"
+    task :all => [:db, :yard, :build, :serve]
   end
 end

data/lib/tasks/jobs.rake

@@ -88,10 +88,7 @@ namespace :htm do
         begin
           # Use the service class directly (same as job)
           result = HTM::EmbeddingService.generate(node.content)
-          node.update!(
-            embedding: result[:storage_embedding],
-            embedding_dimension: result[:dimension]
-          )
+          node.update!(embedding: result[:storage_embedding])
           processed += 1
           print "\rProcessed: #{processed}/#{total}"
         rescue StandardError => e
@@ -191,10 +188,7 @@ namespace :htm do
         begin
           # Use the service class directly to regenerate
           result = HTM::EmbeddingService.generate(node.content)
-          node.update!(
-            embedding: result[:storage_embedding],
-            embedding_dimension: result[:dimension]
-          )
+          node.update!(embedding: result[:storage_embedding])
           processed += 1
           print "\rProcessed: #{processed}/#{total}"
         rescue StandardError => e
@@ -274,7 +268,7 @@ namespace :htm do
       HTM::ActiveRecordConfig.establish_connection!
 
       puts "Clearing embeddings..."
-      HTM::Models::Node.update_all(embedding: nil, embedding_dimension: nil)
+      HTM::Models::Node.update_all(embedding: nil)
 
       puts "Clearing tags..."
       HTM::Models::NodeTag.delete_all

data/lib/tasks/tags.rake

@@ -10,25 +10,185 @@
 
 namespace :htm do
   namespace :tags do
-    desc "Display tags as a hierarchical tree"
-    task :tree do
+    desc "Display tags as a hierarchical tree (text format). Optional prefix filter."
+    task :tree, [:prefix] do |_t, args|
       require 'htm'
 
       # Ensure database connection
       HTM::ActiveRecordConfig.establish_connection!
 
-      count = HTM::Models::Tag.count
+      tags = args[:prefix] ? HTM::Models::Tag.with_prefix(args[:prefix]) : HTM::Models::Tag.all
+      count = tags.count
 
       if count.zero?
-        puts "No tags found in database."
+        puts args[:prefix] ? "No tags found with prefix '#{args[:prefix]}'." : "No tags found in database."
         next
       end
 
       # Display tree using Tag model method
-      puts "\nHTM Tags Tree"
+      puts "\nHTM Tags Tree#{args[:prefix] ? " (prefix: #{args[:prefix]})" : ''}"
       puts "=" * 40
-      print HTM::Models::Tag.all.tree_string
+      print tags.tree_string
       puts "\nTotal tags: #{count}"
     end
+
+    desc "Export tags as Mermaid flowchart to tags.md. Optional prefix filter."
+    task :mermaid, [:prefix] do |_t, args|
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      tags = args[:prefix] ? HTM::Models::Tag.with_prefix(args[:prefix]) : HTM::Models::Tag.all
+      count = tags.count
+
+      if count.zero?
+        puts args[:prefix] ? "No tags found with prefix '#{args[:prefix]}'." : "No tags found in database."
+        next
+      end
+
+      mermaid = tags.tree_mermaid(direction: 'TD')
+      File.write('tags.md', mermaid)
+
+      puts "Mermaid flowchart written to: tags.md"
+      puts "Tags exported: #{count}#{args[:prefix] ? " (prefix: #{args[:prefix]})" : ''}"
+    end
+
+    desc "Export tags as SVG visualization to tags.svg. Optional prefix filter."
+    task :svg, [:prefix] do |_t, args|
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      tags = args[:prefix] ? HTM::Models::Tag.with_prefix(args[:prefix]) : HTM::Models::Tag.all
+      count = tags.count
+
+      if count.zero?
+        puts args[:prefix] ? "No tags found with prefix '#{args[:prefix]}'." : "No tags found in database."
+        next
+      end
+
+      title = args[:prefix] ? "HTM Tags: #{args[:prefix]}*" : 'HTM Tag Hierarchy'
+      svg = tags.tree_svg(title: title)
+      File.write('tags.svg', svg)
+
+      puts "SVG visualization written to: tags.svg"
+      puts "Tags exported: #{count}#{args[:prefix] ? " (prefix: #{args[:prefix]})" : ''}"
+    end
+
+    desc "Rebuild all tags from node content. Clears existing tags and regenerates using LLM."
+    task :rebuild do
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      # Node uses default_scope for active (non-deleted) nodes
+      node_count = HTM::Models::Node.count
+      tag_count = HTM::Models::Tag.count
+      node_tag_count = HTM::Models::NodeTag.count
+
+      puts "\nHTM Tags Rebuild"
+      puts "=" * 40
+      puts "This will:"
+      puts "  - Delete #{node_tag_count} node-tag associations"
+      puts "  - Delete #{tag_count} tags"
+      puts "  - Regenerate tags for #{node_count} nodes using LLM"
+      puts "\nThis operation cannot be undone."
+      print "\nType 'yes' to confirm: "
+
+      confirmation = $stdin.gets&.strip
+      unless confirmation == 'yes'
+        puts "Aborted."
+        next
+      end
+
+      puts "\nClearing existing tags..."
+
+      # Clear join table first (foreign key constraint)
+      deleted_associations = HTM::Models::NodeTag.delete_all
+      puts "  Deleted #{deleted_associations} node-tag associations"
+
+      # Clear tags table
+      deleted_tags = HTM::Models::Tag.delete_all
+      puts "  Deleted #{deleted_tags} tags"
+
+      puts "\nRegenerating tags for #{node_count} nodes..."
+      puts "(This may take a while depending on your LLM provider)\n"
+
+      require 'ruby-progressbar'
+
+      # Create progress bar with ETA
+      progressbar = ProgressBar.create(
+        total: node_count,
+        format: '%t: |%B| %c/%C (%p%%) %e',
+        title: 'Processing',
+        output: $stdout,
+        smoothing: 0.5
+      )
+
+      # Process each active node (default_scope excludes deleted)
+      errors = 0
+
+      HTM::Models::Node.find_each do |node|
+        begin
+          HTM::Jobs::GenerateTagsJob.perform(node_id: node.id)
+        rescue StandardError => e
+          errors += 1
+          progressbar.log "  Error on node #{node.id}: #{e.message}"
+        end
+
+        progressbar.increment
+      end
+
+      progressbar.finish
+
+      # Final stats
+      new_tag_count = HTM::Models::Tag.count
+      new_association_count = HTM::Models::NodeTag.count
+
+      puts "\nRebuild complete!"
+      puts "  Nodes processed: #{node_count}"
+      puts "  Errors: #{errors}"
+      puts "  Tags created: #{new_tag_count}"
+      puts "  Node-tag associations: #{new_association_count}"
+    end
+
+    desc "Export tags in all formats (tags.txt, tags.md, tags.svg). Optional prefix filter."
+    task :export, [:prefix] do |_t, args|
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      tags = args[:prefix] ? HTM::Models::Tag.with_prefix(args[:prefix]) : HTM::Models::Tag.all
+      count = tags.count
+
+      if count.zero?
+        puts args[:prefix] ? "No tags found with prefix '#{args[:prefix]}'." : "No tags found in database."
+        next
+      end
+
+      prefix_note = args[:prefix] ? " (prefix: #{args[:prefix]})" : ''
+      title = args[:prefix] ? "HTM Tags: #{args[:prefix]}*" : 'HTM Tag Hierarchy'
+
+      # Export text tree
+      text_content = "HTM Tags Tree#{prefix_note}\n" + ("=" * 40) + "\n"
+      text_content += tags.tree_string
+      text_content += "\nTotal tags: #{count}\n"
+      File.write('tags.txt', text_content)
+      puts "Text tree written to: tags.txt"
+
+      # Export Mermaid
+      File.write('tags.md', tags.tree_mermaid)
+      puts "Mermaid flowchart written to: tags.md"
+
+      # Export SVG
+      File.write('tags.svg', tags.tree_svg(title: title))
+      puts "SVG visualization written to: tags.svg"
+
+      puts "\nTags exported: #{count}#{prefix_note}"
+    end
   end
 end

data/mkdocs.yml

@@ -114,6 +114,7 @@ markdown_extensions:
       repo_url_shorthand: true
       user: madbomber
       repo: htm
+      normalize_issue_symbols: true
   - pymdownx.mark
   - pymdownx.smartsymbols
   - pymdownx.snippets:
@@ -205,6 +206,40 @@ nav:
     - Long-Term Memory: api/long-term-memory.md
     - Embedding Service: api/embedding-service.md
     - Database: api/database.md
+    - YARD Reference:
+      - api/yard-reference.md
+      - HTM Module: api/yard/HTM.md
+      - Core Classes:
+        - Configuration: api/yard/HTM/Configuration.md
+        - Database: api/yard/HTM/Database.md
+        - LongTermMemory: api/yard/HTM/LongTermMemory.md
+        - WorkingMemory: api/yard/HTM/WorkingMemory.md
+      - Services:
+        - EmbeddingService: api/yard/HTM/EmbeddingService.md
+        - TagService: api/yard/HTM/TagService.md
+        - JobAdapter: api/yard/HTM/JobAdapter.md
+      - Utilities:
+        - CircuitBreaker: api/yard/HTM/CircuitBreaker.md
+        - Observability: api/yard/HTM/Observability.md
+        - Timeframe: api/yard/HTM/Timeframe.md
+        - TimeframeExtractor: api/yard/HTM/TimeframeExtractor.md
+      - Internal:
+        - ActiveRecordConfig: api/yard/HTM/ActiveRecordConfig.md
+        - Railtie: api/yard/HTM/Railtie.md
+      - Errors:
+        - Error: api/yard/HTM/Error.md
+        - AuthorizationError: api/yard/HTM/AuthorizationError.md
+        - CircuitBreakerOpenError: api/yard/HTM/CircuitBreakerOpenError.md
+        - DatabaseError: api/yard/HTM/DatabaseError.md
+        - EmbeddingError: api/yard/HTM/EmbeddingError.md
+        - NotFoundError: api/yard/HTM/NotFoundError.md
+        - QueryTimeoutError: api/yard/HTM/QueryTimeoutError.md
+        - ResourceExhaustedError: api/yard/HTM/ResourceExhaustedError.md
+        - TagError: api/yard/HTM/TagError.md
+        - ValidationError: api/yard/HTM/ValidationError.md
+      - Result Types:
+        - Timeframe Result: api/yard/HTM/Timeframe/Result.md
+        - TimeframeExtractor Result: api/yard/HTM/TimeframeExtractor/Result.md
   - Development:
     - development/index.md
     - Setup: development/setup.md
@@ -217,7 +252,7 @@ nav:
         - Robots: database/public.robots.md
         - Nodes: database/public.nodes.md
         - Tags: database/public.tags.md
-        - Working Memories: database/public.working_memories.md
+        - File Sources: database/public.file_sources.md
       - Join Tables:
         - Node Tags: database/public.node_tags.md
         - Nodes Tags (Legacy): database/public.nodes_tags.md