htm 0.0.2 → 0.0.10

data/lib/tasks/doc.rake

@@ -0,0 +1,300 @@
+# frozen_string_literal: true
+
+namespace :htm do
+  namespace :doc do
+    desc "Build YARD API documentation (markdown format for MkDocs)"
+    task :yard do
+      require "fileutils"
+
+      output_dir = "docs/api/yard"
+
+      puts "Building YARD documentation (markdown format)..."
+      puts "Output directory: #{output_dir}"
+      puts
+
+      # Clean previous output
+      FileUtils.rm_rf(output_dir) if Dir.exist?(output_dir)
+      FileUtils.mkdir_p(output_dir)
+
+      # Build YARD documentation in markdown format
+      # Requires yard-markdown plugin to be installed
+      options = [
+        "--plugin markdown",
+        "--output-dir #{output_dir}",
+        "--format markdown",
+        "--title 'HTM - Hierarchical Temporary Memory API'",
+        "--markup markdown",
+        "--charset utf-8",
+        "--protected",
+        "--no-private",
+        "lib/**/*.rb",
+        "-",
+        "README.md",
+        "CHANGELOG.md"
+      ]
+
+      system("yard doc #{options.join(' ')}")
+
+      if $?.success?
+        # Post-process markdown files for MkDocs compatibility
+        fix_yard_anchors_for_mkdocs(output_dir)
+
+        # Create MkDocs index page for the YARD docs
+        create_yard_index_page(output_dir)
+
+        puts
+        puts "Documentation built successfully!"
+        puts "YARD docs: #{output_dir}/"
+        puts "MkDocs index: docs/api/yard-reference.md"
+      else
+        puts
+        puts "Failed to build documentation. Make sure YARD and yard-markdown are installed:"
+        puts "  gem install yard yard-markdown"
+        exit 1
+      end
+    end
+
+    # Convert YARD anchor format to MkDocs-compatible format
+    # YARD generates: ## method_name() [](#method-i-method_name)
+    # MkDocs needs:   ## method_name() {: #method-i-method_name }
+    #
+    # Also escapes YARD annotations (@param, @return, etc.) to prevent
+    # pymdownx.magiclink from treating them as GitHub @mentions
+    def fix_yard_anchors_for_mkdocs(output_dir)
+      files_fixed = 0
+      anchors_fixed = 0
+      mentions_escaped = 0
+
+      Dir.glob(File.join(output_dir, "**/*.md")).each do |file|
+        content = File.read(file)
+        original = content.dup
+
+        # Pattern 0: Fix malformed YARD output where code fence is joined with heading
+        # "```## method_name() [](#anchor)" -> "```\n## method_name() {: #anchor }"
+        content.gsub!(%r{^(```)(\#{1,6}\s+.+?)\s*\[\]\(\#([^)]+)\)\s*$}) do
+          fence = Regexp.last_match(1)
+          heading = Regexp.last_match(2)
+          anchor_id = Regexp.last_match(3)
+          anchors_fixed += 1
+          "#{fence}\n#{heading} {: ##{anchor_id} }"
+        end
+
+        # Pattern 1: Heading with trailing anchor link
+        # "## method_name() [](#anchor-id)" -> "## method_name() {: #anchor-id }"
+        # Use %r{} to avoid # interpolation issues in regex
+        content.gsub!(%r{^(\#{1,6}\s+.+?)\s*\[\]\(\#([^)]+)\)\s*$}) do
+          heading = Regexp.last_match(1)
+          anchor_id = Regexp.last_match(2)
+          anchors_fixed += 1
+          "#{heading} {: ##{anchor_id} }"
+        end
+
+        # Pattern 2: Attribute headings with [RW]/[R]/[W] markers
+        # "## attr_name[RW] [](#attribute-i-attr_name)" -> "## attr_name [RW] {: #attribute-i-attr_name }"
+        content.gsub!(%r{^(\#{1,6}\s+\w+)\[([RW]+)\]\s*\[\]\(\#([^)]+)\)\s*$}) do
+          heading = Regexp.last_match(1)
+          rw_marker = Regexp.last_match(2)
+          anchor_id = Regexp.last_match(3)
+          anchors_fixed += 1
+          "#{heading} [#{rw_marker}] {: ##{anchor_id} }"
+        end
+
+        # Pattern 3: Escape YARD annotations to prevent GitHub @mention linking
+        # "**@param**" -> "**`@param`**" (inline code prevents magiclink processing)
+        # Common YARD tags: @param, @return, @raise, @yield, @yieldparam, @yieldreturn,
+        #                   @option, @overload, @example, @see, @note, @todo, @deprecated
+        yard_tags = %w[param return raise yield yieldparam yieldreturn option overload example see note todo deprecated abstract api author since version private]
+        yard_tags.each do |tag|
+          # Match **@tag** and replace with **`@tag`**
+          if content.gsub!(/\*\*@#{tag}\*\*/i, "**`@#{tag}`**")
+            mentions_escaped += 1
+          end
+        end
+
+        if content != original
+          File.write(file, content)
+          files_fixed += 1
+        end
+      end
+
+      if files_fixed > 0
+        puts "Fixed #{anchors_fixed} anchors in #{files_fixed} files for MkDocs compatibility"
+        puts "Escaped #{mentions_escaped} YARD annotations to prevent @mention linking" if mentions_escaped > 0
+      end
+    end
+
+    def create_yard_index_page(yard_output_dir)
+      # Generate list of documented classes/modules from YARD output
+      class_list = extract_yard_classes(yard_output_dir)
+
+      content = <<~MARKDOWN
+        # YARD API Reference
+
+        Complete API documentation generated by [YARD](https://yardoc.org/) with [yard-markdown](https://github.com/skatkov/yard-markdown).
+
+        <div class="grid cards" markdown>
+
+        -   :material-book-open-variant:{ .lg .middle } **API Documentation**
+
+            ---
+
+            Browse the complete API reference with method signatures, parameters, and examples.
+
+        -   :material-chart-bar:{ .lg .middle } **Documentation Coverage**
+
+            ---
+
+            Run `rake htm:doc:stats` to see documentation coverage statistics.
+
+        </div>
+
+        ## Classes & Modules
+
+        | Class/Module | Description |
+        |--------------|-------------|
+        #{class_list}
+
+        ## Generating Documentation
+
+        ```bash
+        # Build YARD documentation
+        rake htm:doc:yard
+
+        # Start live-reload server (HTML mode)
+        rake htm:doc:server
+
+        # Show coverage statistics
+        rake htm:doc:stats
+        ```
+      MARKDOWN
+
+      File.write("docs/api/yard-reference.md", content)
+    end
+
+    def extract_yard_classes(yard_output_dir)
+      # Class descriptions for known classes
+      descriptions = {
+        "HTM" => "Main API class for memory operations",
+        "Configuration" => "Multi-provider LLM configuration",
+        "Database" => "Database schema and connection management",
+        "EmbeddingService" => "Vector embedding generation service",
+        "LongTermMemory" => "PostgreSQL-backed permanent storage",
+        "WorkingMemory" => "Token-limited in-memory cache",
+        "TagService" => "Hierarchical tag extraction service",
+        "Observability" => "Metrics and logging instrumentation",
+        "CircuitBreaker" => "Fault tolerance for external services",
+        "Timeframe" => "Time-based query filtering",
+        "TimeframeExtractor" => "Natural language time parsing",
+        "JobAdapter" => "Background job abstraction layer"
+      }
+
+      # Dynamically discover classes from YARD markdown output
+      classes = []
+
+      # Check for markdown files in output directory
+      Dir.glob(File.join(yard_output_dir, "**/*.md")).sort.each do |file|
+        relative_path = file.sub("#{yard_output_dir}/", "")
+        basename = File.basename(file, ".md")
+
+        # Skip index files and non-class files
+        next if basename == "index" || basename == "_index"
+        next if basename.start_with?("_")
+
+        # Determine class name from path
+        class_name = relative_path.sub(".md", "").gsub("/", "::")
+
+        # Skip error classes and internal classes
+        next if class_name.end_with?("Error")
+        next if class_name.include?("Railtie")
+        next if class_name.include?("ActiveRecordConfig")
+
+        # Get description
+        simple_name = basename
+        desc = descriptions[simple_name] || "#{class_name} class"
+
+        classes << [class_name, desc, "yard/#{relative_path}"]
+      end
+
+      # If no markdown files found, fall back to checking for HTML (in case yard-markdown not installed)
+      if classes.empty?
+        htm_file = File.join(yard_output_dir, "HTM.html")
+        if File.exist?(htm_file)
+          classes << ["HTM", descriptions["HTM"] || "Main module", "yard/HTM.html"]
+        end
+
+        htm_dir = File.join(yard_output_dir, "HTM")
+        if Dir.exist?(htm_dir)
+          Dir.glob(File.join(htm_dir, "*.html")).sort.each do |file|
+            basename = File.basename(file, ".html")
+            next if basename.end_with?("Error")
+            next if basename == "Railtie"
+            next if basename == "ActiveRecordConfig"
+
+            desc = descriptions[basename] || "#{basename} class"
+            classes << ["HTM::#{basename}", desc, "yard/HTM/#{basename}.html"]
+          end
+        end
+      end
+
+      classes.map do |name, desc, path|
+        "| [#{name}](#{path}) | #{desc} |"
+      end.join("\n")
+    end
+
+    desc "Start YARD documentation server (live reload)"
+    task :server, [:port] do |_t, args|
+      port = args[:port] || 8808
+
+      puts "Starting YARD documentation server on http://localhost:#{port}"
+      puts "Press Ctrl+C to stop"
+      puts
+
+      exec("yard server --reload --port #{port}")
+    end
+
+    desc "Show documentation coverage statistics"
+    task :stats do
+      puts "YARD Documentation Coverage:"
+      puts
+      system("yard stats --list-undoc lib/**/*.rb")
+    end
+
+    desc "Fix YARD anchor links for MkDocs compatibility"
+    task :fix_anchors do
+      output_dir = "docs/api/yard"
+
+      unless Dir.exist?(output_dir)
+        puts "YARD output directory not found: #{output_dir}"
+        puts "Run 'rake htm:doc:yard' first to generate documentation"
+        exit 1
+      end
+
+      fix_yard_anchors_for_mkdocs(output_dir)
+      puts "Done! Run 'mkdocs build' to verify no anchor warnings."
+    end
+
+    desc "Clean generated documentation"
+    task :clean do
+      require "fileutils"
+
+      dirs_to_clean = ["docs/api/yard", ".yardoc"]
+      files_to_clean = ["docs/api/yard-reference.md"]
+
+      dirs_to_clean.each do |dir|
+        if Dir.exist?(dir)
+          FileUtils.rm_rf(dir)
+          puts "Removed: #{dir}"
+        end
+      end
+
+      files_to_clean.each do |file|
+        if File.exist?(file)
+          FileUtils.rm(file)
+          puts "Removed: #{file}"
+        end
+      end
+
+      puts "Documentation cleaned."
+    end
+  end
+end

data/lib/tasks/files.rake

@@ -0,0 +1,299 @@
+# frozen_string_literal: true
+
+# HTM File Loading Tasks
+#
+# These tasks are available to any application using the HTM gem.
+# Add to your application's Rakefile:
+#
+#   require 'htm/tasks'
+#
+
+namespace :htm do
+  namespace :files do
+    desc "Load a markdown file into long-term memory. Usage: rake htm:files:load[path/to/file.md]"
+    task :load, [:path] do |_t, args|
+      require 'htm'
+
+      path = args[:path]
+      unless path
+        puts "Error: File path required."
+        puts "Usage: rake 'htm:files:load[path/to/file.md]'"
+        exit 1
+      end
+
+      unless File.exist?(path)
+        puts "Error: File not found: #{path}"
+        exit 1
+      end
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      htm = HTM.new(robot_name: "FileLoader")
+      force = ENV['FORCE'] == 'true'
+
+      puts "Loading file: #{path}#{force ? ' (force)' : ''}"
+      result = htm.load_file(path, force: force)
+
+      if result[:skipped]
+        puts "Skipped: File unchanged since last sync."
+        puts "Use FORCE=true to reload anyway."
+      else
+        puts "Loaded successfully:"
+        puts "  File source ID: #{result[:file_source_id]}"
+        puts "  Chunks created: #{result[:chunks_created]}"
+        puts "  Chunks updated: #{result[:chunks_updated]}"
+        puts "  Chunks deleted: #{result[:chunks_deleted]}"
+      end
+    end
+
+    desc "Load all markdown files from a directory. Usage: rake htm:files:load_dir[path/to/dir]"
+    task :load_dir, [:path, :pattern] do |_t, args|
+      require 'htm'
+
+      path = args[:path]
+      unless path
+        puts "Error: Directory path required."
+        puts "Usage: rake 'htm:files:load_dir[path/to/dir]'"
+        puts "       rake 'htm:files:load_dir[path/to/dir,**/*.md]'"
+        exit 1
+      end
+
+      unless File.directory?(path)
+        puts "Error: Directory not found: #{path}"
+        exit 1
+      end
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      htm = HTM.new(robot_name: "FileLoader")
+      pattern = args[:pattern] || '**/*.md'
+      force = ENV['FORCE'] == 'true'
+
+      puts "Loading files from: #{path}"
+      puts "Pattern: #{pattern}#{force ? ' (force)' : ''}"
+      puts
+
+      results = htm.load_directory(path, pattern: pattern, force: force)
+
+      total_created = 0
+      total_updated = 0
+      total_deleted = 0
+      skipped = 0
+
+      results.each do |result|
+        if result[:skipped]
+          skipped += 1
+        else
+          total_created += result[:chunks_created]
+          total_updated += result[:chunks_updated]
+          total_deleted += result[:chunks_deleted]
+          puts "  #{result[:file_path]}: #{result[:chunks_created]} created, #{result[:chunks_updated]} updated, #{result[:chunks_deleted]} deleted"
+        end
+      end
+
+      puts
+      puts "Summary:"
+      puts "  Files processed: #{results.size}"
+      puts "  Files skipped (unchanged): #{skipped}"
+      puts "  Total chunks created: #{total_created}"
+      puts "  Total chunks updated: #{total_updated}"
+      puts "  Total chunks deleted: #{total_deleted}"
+    end
+
+    desc "List all loaded file sources"
+    task :list do
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      sources = HTM::Models::FileSource.order(:file_path)
+      count = sources.count
+
+      if count.zero?
+        puts "No files loaded."
+        next
+      end
+
+      puts "Loaded files (#{count}):"
+      puts "-" * 80
+
+      sources.each do |source|
+        chunks = source.chunks.count
+        sync_status = ""
+        if File.exist?(source.file_path)
+          current_mtime = File.mtime(source.file_path)
+          sync_status = source.needs_sync?(current_mtime) ? " [needs sync]" : ""
+        else
+          sync_status = " [missing]"
+        end
+        puts "  #{source.file_path}"
+        puts "    ID: #{source.id} | Chunks: #{chunks} | Last synced: #{source.last_synced_at&.strftime('%Y-%m-%d %H:%M')}#{sync_status}"
+      end
+    end
+
+    desc "Show details for a loaded file. Usage: rake htm:files:info[path/to/file.md]"
+    task :info, [:path] do |_t, args|
+      require 'htm'
+
+      path = args[:path]
+      unless path
+        puts "Error: File path required."
+        puts "Usage: rake 'htm:files:info[path/to/file.md]'"
+        exit 1
+      end
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      # Try to find by exact path or expanded path
+      source = HTM::Models::FileSource.find_by(file_path: path) ||
+               HTM::Models::FileSource.find_by(file_path: File.expand_path(path))
+
+      unless source
+        puts "Error: File not loaded: #{path}"
+        exit 1
+      end
+
+      puts "File: #{source.file_path}"
+      puts "-" * 60
+      puts "  ID: #{source.id}"
+      puts "  File size: #{source.file_size} bytes"
+      puts "  Last synced: #{source.last_synced_at}"
+
+      if File.exist?(source.file_path)
+        current_mtime = File.mtime(source.file_path)
+        puts "  Needs sync: #{source.needs_sync?(current_mtime) ? 'Yes' : 'No'}"
+      else
+        puts "  Needs sync: File missing!"
+      end
+
+      puts "  Created: #{source.created_at}"
+      puts
+
+      if source.frontmatter.any?
+        puts "Frontmatter:"
+        source.frontmatter.each do |key, value|
+          puts "  #{key}: #{value}"
+        end
+        puts
+      end
+
+      chunks = source.chunks
+      puts "Chunks (#{chunks.count}):"
+      chunks.each_with_index do |chunk, idx|
+        preview = chunk.content[0..60].gsub("\n", " ")
+        preview += "..." if chunk.content.length > 60
+        puts "  [#{idx}] #{preview}"
+      end
+    end
+
+    desc "Unload a file from memory. Usage: rake htm:files:unload[path/to/file.md]"
+    task :unload, [:path] do |_t, args|
+      require 'htm'
+
+      path = args[:path]
+      unless path
+        puts "Error: File path required."
+        puts "Usage: rake 'htm:files:unload[path/to/file.md]'"
+        exit 1
+      end
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      htm = HTM.new(robot_name: "FileLoader")
+      result = htm.unload_file(path)
+
+      if result
+        puts "Unloaded: #{path}"
+      else
+        puts "File not found: #{path}"
+      end
+    end
+
+    desc "Sync all loaded files (reload changed files)"
+    task :sync do
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      htm = HTM.new(robot_name: "FileLoader")
+      sources = HTM::Models::FileSource.all
+
+      if sources.count.zero?
+        puts "No files loaded."
+        next
+      end
+
+      puts "Syncing #{sources.count} files..."
+      puts
+
+      synced = 0
+      skipped = 0
+      missing = 0
+
+      sources.each do |source|
+        unless File.exist?(source.file_path)
+          puts "  [missing] #{source.file_path}"
+          missing += 1
+          next
+        end
+
+        current_mtime = File.mtime(source.file_path)
+        unless source.needs_sync?(current_mtime)
+          skipped += 1
+          next
+        end
+
+        result = htm.load_file(source.file_path)
+        puts "  [synced] #{source.file_path}: #{result[:chunks_created]} created, #{result[:chunks_updated]} updated, #{result[:chunks_deleted]} deleted"
+        synced += 1
+      end
+
+      puts
+      puts "Summary:"
+      puts "  Synced: #{synced}"
+      puts "  Skipped (unchanged): #{skipped}"
+      puts "  Missing files: #{missing}"
+    end
+
+    desc "Show file loading statistics"
+    task :stats do
+      require 'htm'
+
+      # Ensure database connection
+      HTM::ActiveRecordConfig.establish_connection!
+
+      total_sources = HTM::Models::FileSource.count
+      total_chunks = HTM::Models::Node.where.not(source_id: nil).count
+
+      # Count files needing sync (checking actual file mtime)
+      needs_sync = 0
+      missing = 0
+      HTM::Models::FileSource.find_each do |source|
+        if File.exist?(source.file_path)
+          current_mtime = File.mtime(source.file_path)
+          needs_sync += 1 if source.needs_sync?(current_mtime)
+        else
+          missing += 1
+        end
+      end
+
+      puts "File Loading Statistics"
+      puts "=" * 40
+      puts "  Total files loaded: #{total_sources}"
+      puts "  Total chunks: #{total_chunks}"
+      puts "  Files needing sync: #{needs_sync}"
+      puts "  Missing files: #{missing}" if missing > 0
+
+      if total_sources > 0
+        avg_chunks = (total_chunks.to_f / total_sources).round(1)
+        puts "  Average chunks per file: #{avg_chunks}"
+      end
+    end
+  end
+end