htm 0.0.1 → 0.0.10

data/lib/tasks/htm.rake

@@ -124,6 +124,161 @@ namespace :htm do
       HTM::Database.info
     end
 
+    desc "Show record counts for all HTM tables"
+    task :stats do
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      puts "\nHTM Database Statistics"
+      puts "=" * 50
+
+      # Define tables with their models and optional extra info
+      tables = [
+        { name: 'robots', model: HTM::Models::Robot },
+        { name: 'nodes', model: HTM::Models::Node, extras: ->(m) {
+          # Node uses default_scope for active nodes, so m.count is active count
+          active = m.count
+          deleted = m.deleted.count
+          with_embedding = m.where.not(embedding: nil).count
+          "  (active: #{active}, deleted: #{deleted}, with embeddings: #{with_embedding})"
+        }},
+        { name: 'tags', model: HTM::Models::Tag },
+        { name: 'nodes_tags', model: HTM::Models::NodeTag },
+        { name: 'file_sources', model: HTM::Models::FileSource }
+      ]
+
+      total = 0
+      max_name_len = tables.map { |t| t[:name].length }.max
+
+      tables.each do |table|
+        count = table[:model].count
+        total += count
+        extras = table[:extras] ? table[:extras].call(table[:model]) : ''
+
+        printf "  %-#{max_name_len}s  %8d%s\n", table[:name], count, extras
+      end
+
+      puts "-" * 50
+      printf "  %-#{max_name_len}s  %8d\n", "Total", total
+
+      # Additional stats
+      puts "\nAdditional Statistics:"
+
+      # Nodes per robot (via robot_nodes join table)
+      robot_counts = HTM::Models::RobotNode
+        .joins(:node)
+        .where(nodes: { deleted_at: nil })
+        .group(:robot_id)
+        .count
+        .transform_keys { |id| HTM::Models::Robot.find(id).name rescue "Unknown (#{id})" }
+        .sort_by { |_, count| -count }
+        .first(5)
+
+      if robot_counts.any?
+        puts "  Top robots by node count:"
+        robot_counts.each do |name, count|
+          puts "    #{name}: #{count}"
+        end
+      end
+
+      # Tag distribution
+      top_root_tags = HTM::Models::Tag
+        .select("split_part(name, ':', 1) as root, count(*) as cnt")
+        .group("split_part(name, ':', 1)")
+        .order("cnt DESC")
+        .limit(5)
+        .map { |t| [t.root, t.cnt] }
+
+      if top_root_tags.any?
+        puts "  Top root tag categories:"
+        top_root_tags.each do |root, count|
+          puts "    #{root}: #{count}"
+        end
+      end
+
+      puts
+    end
+
+    namespace :rebuild do
+      desc "Rebuild embeddings for all nodes. Clears existing embeddings and regenerates via LLM."
+      task :embeddings do
+        require 'htm'
+        require 'ruby-progressbar'
+
+        # Ensure database connection
+        HTM::ActiveRecordConfig.establish_connection!
+
+        # Node uses default_scope for active (non-deleted) nodes
+        node_count = HTM::Models::Node.count
+        with_embeddings = HTM::Models::Node.where.not(embedding: nil).count
+        without_embeddings = node_count - with_embeddings
+
+        puts "\nHTM Embeddings Rebuild"
+        puts "=" * 50
+        puts "Current state:"
+        puts "  Total active nodes: #{node_count}"
+        puts "  With embeddings: #{with_embeddings}"
+        puts "  Without embeddings: #{without_embeddings}"
+        puts "\nThis will regenerate embeddings for ALL #{node_count} nodes."
+        puts "This operation may take a long time depending on your LLM provider."
+        print "\nType 'yes' to confirm: "
+
+        confirmation = $stdin.gets&.strip
+        unless confirmation == 'yes'
+          puts "Aborted."
+          next
+        end
+
+        puts "\nClearing existing embeddings..."
+        cleared = HTM::Models::Node.where.not(embedding: nil).update_all(embedding: nil)
+        puts "  Cleared #{cleared} embeddings"
+
+        puts "\nGenerating embeddings for #{node_count} nodes..."
+        puts "(This may take a while depending on your LLM provider)\n"
+
+        # Create progress bar with ETA
+        progressbar = ProgressBar.create(
+          total: node_count,
+          format: '%t: |%B| %c/%C (%p%%) %e',
+          title: 'Embedding',
+          output: $stdout,
+          smoothing: 0.5
+        )
+
+        # Process each active node (default_scope excludes deleted)
+        errors = 0
+        success = 0
+
+        HTM::Models::Node.find_each do |node|
+          begin
+            # Generate embedding directly (not via job since we cleared them)
+            result = HTM::EmbeddingService.generate(node.content)
+
+            node.update!(embedding: result[:storage_embedding])
+            success += 1
+          rescue StandardError => e
+            errors += 1
+            progressbar.log "  Error on node #{node.id}: #{e.message}"
+          end
+
+          progressbar.increment
+        end
+
+        progressbar.finish
+
+        # Final stats
+        final_with_embeddings = HTM::Models::Node.where.not(embedding: nil).count
+
+        puts "\nRebuild complete!"
+        puts "  Nodes processed: #{node_count}"
+        puts "  Successful: #{success}"
+        puts "  Errors: #{errors}"
+        puts "  Nodes with embeddings: #{final_with_embeddings}"
+      end
+    end
+
     namespace :schema do
       desc "Dump current schema to db/schema.sql"
       task :dump do
@@ -138,10 +293,41 @@ namespace :htm do
       end
     end
 
-    desc "Generate/update database documentation in dbdoc/"
-    task :doc do
+  end
+
+  namespace :doc do
+    desc "Generate/update database documentation in docs/database/ (uses .tbls.yml)"
+    task :db do
+      unless system("which tbls > /dev/null 2>&1")
+        puts "Error: 'tbls' is not installed."
+        puts "Install it with: brew install tbls"
+        exit 1
+      end
       require 'htm'
       HTM::Database.generate_docs
     end
+
+    desc "Build documentation site with MkDocs"
+    task :build do
+      unless system("which mkdocs > /dev/null 2>&1")
+        puts "Error: 'mkdocs' is not installed."
+        puts "Install it with: brew install mkdocs"
+        exit 1
+      end
+      sh "mkdocs build"
+    end
+
+    desc "Serve documentation site locally with MkDocs"
+    task :serve do
+      unless system("which mkdocs > /dev/null 2>&1")
+        puts "Error: 'mkdocs' is not installed."
+        puts "Install it with: brew install mkdocs"
+        exit 1
+      end
+      sh "mkdocs serve"
+    end
+
+    desc "Generate DB docs, YARD API docs, build site, and serve locally"
+    task :all => [:db, :yard, :build, :serve]
   end
 end

data/lib/tasks/jobs.rake

@@ -42,9 +42,13 @@ namespace :htm do
       # Tags by depth
       if total_tags > 0
         puts "\nTag hierarchy breakdown:"
-        HTM::Models::Tag.all.each do |tag|
-          depth = tag.name.count(':')
-          puts "  Depth #{depth}: #{tag.name.count(':', depth)} tags" if depth <= 3
+        depth_counts = Hash.new(0)
+        HTM::Models::Tag.pluck(:name).each do |name|
+          depth = name.count(':')
+          depth_counts[depth] += 1
+        end
+        depth_counts.keys.sort.each do |depth|
+          puts "  Depth #{depth}: #{depth_counts[depth]} tags"
         end
       end
 
@@ -84,10 +88,7 @@ namespace :htm do
         begin
           # Use the service class directly (same as job)
           result = HTM::EmbeddingService.generate(node.content)
-          node.update!(
-            embedding: result[:storage_embedding],
-            embedding_dimension: result[:dimension]
-          )
+          node.update!(embedding: result[:storage_embedding])
           processed += 1
           print "\rProcessed: #{processed}/#{total}"
         rescue StandardError => e
@@ -187,10 +188,7 @@ namespace :htm do
         begin
           # Use the service class directly to regenerate
           result = HTM::EmbeddingService.generate(node.content)
-          node.update!(
-            embedding: result[:storage_embedding],
-            embedding_dimension: result[:dimension]
-          )
+          node.update!(embedding: result[:storage_embedding])
           processed += 1
           print "\rProcessed: #{processed}/#{total}"
         rescue StandardError => e
@@ -270,7 +268,7 @@ namespace :htm do
       HTM::ActiveRecordConfig.establish_connection!
 
       puts "Clearing embeddings..."
-      HTM::Models::Node.update_all(embedding: nil, embedding_dimension: nil)
+      HTM::Models::Node.update_all(embedding: nil)
 
       puts "Clearing tags..."
       HTM::Models::NodeTag.delete_all

data/lib/tasks/tags.rake

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+# HTM Tag Management Tasks
+#
+# These tasks are available to any application using the HTM gem.
+# Add to your application's Rakefile:
+#
+#   require 'htm/tasks'
+#
+
+namespace :htm do
+  namespace :tags do
+    desc "Display tags as a hierarchical tree (text format). Optional prefix filter."
+    task :tree, [:prefix] do |_t, args|
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      tags = args[:prefix] ? HTM::Models::Tag.with_prefix(args[:prefix]) : HTM::Models::Tag.all
+      count = tags.count
+
+      if count.zero?
+        puts args[:prefix] ? "No tags found with prefix '#{args[:prefix]}'." : "No tags found in database."
+        next
+      end
+
+      # Display tree using Tag model method
+      puts "\nHTM Tags Tree#{args[:prefix] ? " (prefix: #{args[:prefix]})" : ''}"
+      puts "=" * 40
+      print tags.tree_string
+      puts "\nTotal tags: #{count}"
+    end
+
+    desc "Export tags as Mermaid flowchart to tags.md. Optional prefix filter."
+    task :mermaid, [:prefix] do |_t, args|
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      tags = args[:prefix] ? HTM::Models::Tag.with_prefix(args[:prefix]) : HTM::Models::Tag.all
+      count = tags.count
+
+      if count.zero?
+        puts args[:prefix] ? "No tags found with prefix '#{args[:prefix]}'." : "No tags found in database."
+        next
+      end
+
+      mermaid = tags.tree_mermaid(direction: 'TD')
+      File.write('tags.md', mermaid)
+
+      puts "Mermaid flowchart written to: tags.md"
+      puts "Tags exported: #{count}#{args[:prefix] ? " (prefix: #{args[:prefix]})" : ''}"
+    end
+
+    desc "Export tags as SVG visualization to tags.svg. Optional prefix filter."
+    task :svg, [:prefix] do |_t, args|
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      tags = args[:prefix] ? HTM::Models::Tag.with_prefix(args[:prefix]) : HTM::Models::Tag.all
+      count = tags.count
+
+      if count.zero?
+        puts args[:prefix] ? "No tags found with prefix '#{args[:prefix]}'." : "No tags found in database."
+        next
+      end
+
+      title = args[:prefix] ? "HTM Tags: #{args[:prefix]}*" : 'HTM Tag Hierarchy'
+      svg = tags.tree_svg(title: title)
+      File.write('tags.svg', svg)
+
+      puts "SVG visualization written to: tags.svg"
+      puts "Tags exported: #{count}#{args[:prefix] ? " (prefix: #{args[:prefix]})" : ''}"
+    end
+
+    desc "Rebuild all tags from node content. Clears existing tags and regenerates using LLM."
+    task :rebuild do
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      # Node uses default_scope for active (non-deleted) nodes
+      node_count = HTM::Models::Node.count
+      tag_count = HTM::Models::Tag.count
+      node_tag_count = HTM::Models::NodeTag.count
+
+      puts "\nHTM Tags Rebuild"
+      puts "=" * 40
+      puts "This will:"
+      puts "  - Delete #{node_tag_count} node-tag associations"
+      puts "  - Delete #{tag_count} tags"
+      puts "  - Regenerate tags for #{node_count} nodes using LLM"
+      puts "\nThis operation cannot be undone."
+      print "\nType 'yes' to confirm: "
+
+      confirmation = $stdin.gets&.strip
+      unless confirmation == 'yes'
+        puts "Aborted."
+        next
+      end
+
+      puts "\nClearing existing tags..."
+
+      # Clear join table first (foreign key constraint)
+      deleted_associations = HTM::Models::NodeTag.delete_all
+      puts "  Deleted #{deleted_associations} node-tag associations"
+
+      # Clear tags table
+      deleted_tags = HTM::Models::Tag.delete_all
+      puts "  Deleted #{deleted_tags} tags"
+
+      puts "\nRegenerating tags for #{node_count} nodes..."
+      puts "(This may take a while depending on your LLM provider)\n"
+
+      require 'ruby-progressbar'
+
+      # Create progress bar with ETA
+      progressbar = ProgressBar.create(
+        total: node_count,
+        format: '%t: |%B| %c/%C (%p%%) %e',
+        title: 'Processing',
+        output: $stdout,
+        smoothing: 0.5
+      )
+
+      # Process each active node (default_scope excludes deleted)
+      errors = 0
+
+      HTM::Models::Node.find_each do |node|
+        begin
+          HTM::Jobs::GenerateTagsJob.perform(node_id: node.id)
+        rescue StandardError => e
+          errors += 1
+          progressbar.log "  Error on node #{node.id}: #{e.message}"
+        end
+
+        progressbar.increment
+      end
+
+      progressbar.finish
+
+      # Final stats
+      new_tag_count = HTM::Models::Tag.count
+      new_association_count = HTM::Models::NodeTag.count
+
+      puts "\nRebuild complete!"
+      puts "  Nodes processed: #{node_count}"
+      puts "  Errors: #{errors}"
+      puts "  Tags created: #{new_tag_count}"
+      puts "  Node-tag associations: #{new_association_count}"
+    end
+
+    desc "Export tags in all formats (tags.txt, tags.md, tags.svg). Optional prefix filter."
+    task :export, [:prefix] do |_t, args|
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      tags = args[:prefix] ? HTM::Models::Tag.with_prefix(args[:prefix]) : HTM::Models::Tag.all
+      count = tags.count
+
+      if count.zero?
+        puts args[:prefix] ? "No tags found with prefix '#{args[:prefix]}'." : "No tags found in database."
+        next
+      end
+
+      prefix_note = args[:prefix] ? " (prefix: #{args[:prefix]})" : ''
+      title = args[:prefix] ? "HTM Tags: #{args[:prefix]}*" : 'HTM Tag Hierarchy'
+
+      # Export text tree
+      text_content = "HTM Tags Tree#{prefix_note}\n" + ("=" * 40) + "\n"
+      text_content += tags.tree_string
+      text_content += "\nTotal tags: #{count}\n"
+      File.write('tags.txt', text_content)
+      puts "Text tree written to: tags.txt"
+
+      # Export Mermaid
+      File.write('tags.md', tags.tree_mermaid)
+      puts "Mermaid flowchart written to: tags.md"
+
+      # Export SVG
+      File.write('tags.svg', tags.tree_svg(title: title))
+      puts "SVG visualization written to: tags.svg"
+
+      puts "\nTags exported: #{count}#{prefix_note}"
+    end
+  end
+end

data/mkdocs.yml

@@ -13,21 +13,21 @@ edit_uri: edit/main/docs/
 # Configuration
 theme:
   name: material
-  
+
   # Color scheme - Default to light mode
   palette:
     # Palette toggle for light mode (default)
     - scheme: default
-      primary: blue
-      accent: blue
+      primary: indigo
+      accent: deep purple
       toggle:
         icon: material/brightness-7
         name: Switch to dark mode
-    
+
     # Palette toggle for dark mode
     - scheme: slate
-      primary: blue
-      accent: blue
+      primary: indigo
+      accent: deep purple
       toggle:
         icon: material/brightness-4
         name: Switch to light mode
@@ -37,6 +37,10 @@ theme:
     text: Roboto
     code: Roboto Mono
 
+  # Logo and icon
+  icon:
+    repo: fontawesome/brands/github
+
   # Theme features
   features:
     # Navigation
@@ -48,6 +52,7 @@ theme:
     - navigation.path
     - navigation.indexes
     - navigation.top
+    - navigation.footer
 
     # Table of contents
     - toc.follow
@@ -72,6 +77,7 @@ theme:
 plugins:
   - search:
       separator: '[\s\-,:!=\[\]()"`/]+|\.(?!\d)|&[lg]t;|(?!\b)(?=[A-Z][a-z])'
+  - tags
 
 # Extensions
 markdown_extensions:
@@ -82,6 +88,7 @@ markdown_extensions:
   - def_list
   - footnotes
   - md_in_html
+  - tables
   - toc:
       permalink: true
       title: On this page
@@ -92,6 +99,7 @@ markdown_extensions:
   - pymdownx.betterem:
       smart_enable: all
   - pymdownx.caret
+  - pymdownx.critic
   - pymdownx.details
   - pymdownx.emoji:
       emoji_generator: !!python/name:material.extensions.emoji.to_svg
@@ -106,8 +114,11 @@ markdown_extensions:
       repo_url_shorthand: true
       user: madbomber
       repo: htm
+      normalize_issue_symbols: true
   - pymdownx.mark
   - pymdownx.smartsymbols
+  - pymdownx.snippets:
+      check_paths: true
   - pymdownx.superfences:
       custom_fences:
         - name: mermaid
@@ -138,11 +149,28 @@ extra:
       link: https://rubygems.org/gems/htm
       name: HTM on RubyGems
 
+  analytics:
+    feedback:
+      title: Was this page helpful?
+      ratings:
+        - icon: material/emoticon-happy-outline
+          name: This page was helpful
+          data: 1
+          note: >-
+            Thanks for your feedback!
+        - icon: material/emoticon-sad-outline
+          name: This page could be improved
+          data: 0
+          note: >-
+            Thanks for your feedback! Help us improve by creating an issue.
+
 # Navigation
 nav:
   - Home: index.md
-  - Installation: installation.md
-  - Quick Start: quick-start.md
+  - Getting Started:
+      - getting-started/index.md
+      - Installation: getting-started/installation.md
+      - Quick Start: getting-started/quick-start.md
   - Architecture:
     - architecture/index.md
     - Overview: architecture/overview.md
@@ -178,13 +206,67 @@ nav:
     - Long-Term Memory: api/long-term-memory.md
     - Embedding Service: api/embedding-service.md
     - Database: api/database.md
+    - YARD Reference:
+      - api/yard-reference.md
+      - HTM Module: api/yard/HTM.md
+      - Core Classes:
+        - Configuration: api/yard/HTM/Configuration.md
+        - Database: api/yard/HTM/Database.md
+        - LongTermMemory: api/yard/HTM/LongTermMemory.md
+        - WorkingMemory: api/yard/HTM/WorkingMemory.md
+      - Services:
+        - EmbeddingService: api/yard/HTM/EmbeddingService.md
+        - TagService: api/yard/HTM/TagService.md
+        - JobAdapter: api/yard/HTM/JobAdapter.md
+      - Utilities:
+        - CircuitBreaker: api/yard/HTM/CircuitBreaker.md
+        - Observability: api/yard/HTM/Observability.md
+        - Timeframe: api/yard/HTM/Timeframe.md
+        - TimeframeExtractor: api/yard/HTM/TimeframeExtractor.md
+      - Internal:
+        - ActiveRecordConfig: api/yard/HTM/ActiveRecordConfig.md
+        - Railtie: api/yard/HTM/Railtie.md
+      - Errors:
+        - Error: api/yard/HTM/Error.md
+        - AuthorizationError: api/yard/HTM/AuthorizationError.md
+        - CircuitBreakerOpenError: api/yard/HTM/CircuitBreakerOpenError.md
+        - DatabaseError: api/yard/HTM/DatabaseError.md
+        - EmbeddingError: api/yard/HTM/EmbeddingError.md
+        - NotFoundError: api/yard/HTM/NotFoundError.md
+        - QueryTimeoutError: api/yard/HTM/QueryTimeoutError.md
+        - ResourceExhaustedError: api/yard/HTM/ResourceExhaustedError.md
+        - TagError: api/yard/HTM/TagError.md
+        - ValidationError: api/yard/HTM/ValidationError.md
+      - Result Types:
+        - Timeframe Result: api/yard/HTM/Timeframe/Result.md
+        - TimeframeExtractor Result: api/yard/HTM/TimeframeExtractor/Result.md
   - Development:
     - development/index.md
     - Setup: development/setup.md
     - Testing: development/testing.md
     - Contributing: development/contributing.md
     - Database Schema: development/schema.md
+    - Database Tables:
+      - database/README.md
+      - Core Tables:
+        - Robots: database/public.robots.md
+        - Nodes: database/public.nodes.md
+        - Tags: database/public.tags.md
+        - File Sources: database/public.file_sources.md
+      - Join Tables:
+        - Node Tags: database/public.node_tags.md
+        - Nodes Tags (Legacy): database/public.nodes_tags.md
+        - Robot Nodes: database/public.robot_nodes.md
+      - Views & Stats:
+        - Node Stats: database/public.node_stats.md
+        - Robot Activity: database/public.robot_activity.md
+        - Topic Relationships: database/public.topic_relationships.md
+        - Ontology Structure: database/public.ontology_structure.md
+        - Relationships: database/public.relationships.md
+      - System:
+        - Operations Log: database/public.operations_log.md
+        - Schema Migrations: database/public.schema_migrations.md
     - Multi-Framework Support: multi_framework_support.md
     - Database Rake Tasks: database_rake_tasks.md
     - Using Rake Tasks: using_rake_tasks_in_your_app.md
-    - Setup Local Database: setup_local_database.md
+    - Setup Local Database: setup_local_database.md