htm 0.0.11 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c6e41109b86fd5c51c8f96cec8a11ea9455035da54d72ae461b3b2f0a7ab7a8d
-  data.tar.gz: 1fe15917933aefa827a12c6e92406711ab95449097cb78bdab4c0f6510f595c1
+  metadata.gz: 5658951570d47a6988a3e6c5078ae90c83a8bbbaebd4bc18b8a70152d0648ab1
+  data.tar.gz: e4216f5bbe7cbbcfb1b78058f16fb5c190a8902f4778874c858acf7401f0da14
 SHA512:
-  metadata.gz: c02e326797a6986e1a5b987bec948f09c0ffc7cb45045898e3fc6faf5c57c9bf3ca4d399324e88528f994c7e3e1989bb71f462aae8adbb270b7c99654aa135d3
-  data.tar.gz: 4ee90ea84f36ab2d85c72cae4f5a1382e9a174d354f2d976d73b8915cb5dc8df68f6ae799ec082076293ce9b386500b94b4fe12d291903c0804351f5fd027151
+  metadata.gz: 3d206a81c9120d7c2ffb8a8afb41056ee93a9b614a7abfabe1e14196e60570daaab048e37cc74601bd04afed0af5290616ff925a7610dc37e8816cb4d797956a
+  data.tar.gz: d2977a3e77e2d7a0b6cd886d3edadbfe8c04b3af55e4280d8b70393de41ad019a76fe12e19c1bf141d3f6b604ac4a221caca5ea914c64985521a4ccbe2fbfce9

data/.dictate.toml

@@ -0,0 +1,46 @@
+# Dictator Configuration for HTM
+# Decree-based structural enforcement
+# https://github.com/seuros/dictator
+
+[decree.supreme]
+# Universal structural rules (applies to ALL files)
+trailing_whitespace   = "deny"
+tabs_vs_spaces        = "spaces"
+tab_width             = 2
+final_newline         = "require"
+line_endings          = "lf"
+blank_line_whitespace = "deny"
+
+[decree.ruby]
+# Ruby-specific structural enforcement
+max_line_length         = 120
+max_lines               = 300
+ignore_comments         = true
+ignore_blank_lines      = true
+method_visibility_order = ["public", "protected", "private"]
+comment_spacing         = true
+
+[decree.ruby.linter]
+# External linter - Dictator adds: -A --format json
+command = "rubocop"
+
+[decree.frontmatter]
+# YAML frontmatter ordering for .md files
+order    = ["title", "slug", "pubDate", "description", "tags", "draft"]
+required = ["title"]
+
+# Disable unused language decrees
+[decree.typescript]
+enabled = false
+
+[decree.golang]
+enabled = false
+
+[decree.rust]
+enabled = false
+
+[decree.python]
+enabled = false
+
+[decree.kjr]
+enabled = false

data/.envrc

@@ -2,6 +2,8 @@
 
 export RR=`pwd`
 
+export HTM_EXTRACT_PROPOSITIONS=true
+
 # Database connection - Localhost PostgreSQL
 export HTM_DBHOST=localhost
 export HTM_DBPORT=5432

data/CHANGELOG.md

@@ -7,6 +7,56 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.0.14] - 2025-12-05
+
+### Added
+- **OpenTelemetry metrics** - Optional observability with zero overhead when disabled
+  - New `HTM::Telemetry` module with null object pattern
+  - Metrics: `htm.jobs` (counter), `htm.embedding.latency`, `htm.tag.latency`, `htm.search.latency` (histograms), `htm.cache.operations` (counter)
+  - Instrumented: `GenerateEmbeddingJob`, `GenerateTagsJob`, search methods, `QueryCache`
+  - Enable via `HTM_TELEMETRY_ENABLED=true` or `config.telemetry_enabled = true`
+  - Works with 50+ OTLP-compatible backends (Jaeger, Prometheus, Datadog, etc.)
+  - Comprehensive documentation in `docs/telemetry.md`
+  - 18 new telemetry tests
+- **Telemetry demo with Grafana visualization** - `examples/telemetry/`
+  - Live dashboard showing HTM metrics (job counts, latencies, cache hit rates)
+  - Uses Homebrew-installed Prometheus and Grafana (no Docker required)
+  - Auto-configures Prometheus scrape target
+  - Pre-built Grafana dashboard JSON for easy import
+  - Interactive shutdown prompt for service management
+- **`htm:db:create` rake task** - Create database if it doesn't exist (respects `RAILS_ENV`)
+
+### Changed
+- **All `htm:db:*` tasks now respect `RAILS_ENV`** - Following Rails conventions
+  - Database selection based on environment: `htm_development`, `htm_test`, `htm_production`
+  - `rake test` automatically sets `RAILS_ENV=test`
+  - Example: `RAILS_ENV=test rake htm:db:setup` operates on `htm_test`
+- **`config/database.yml` refactored** - Extracts base name from `HTM_DBURL` and appends environment suffix
+  - `HTM_DBURL=postgresql://...htm_development` + `RAILS_ENV=test` → connects to `htm_test`
+- **`HTM::Database.default_config` now respects `RAILS_ENV`** - Uses `ActiveRecordConfig.load_database_config`
+- **Renamed `htm:db:test` to `htm:db:verify`** - Avoids naming collision with test database namespace
+  - `htm:db:verify` verifies database connection
+  - `RAILS_ENV=test rake htm:db:*` operates on test database
+
+## [0.0.13] - 2025-12-04
+
+### Changed
+- **MarkdownChunker now uses Baran gem** - Replaced custom ParagraphChunker with Baran's `MarkdownSplitter`
+  - Respects markdown structure (headers, code blocks, horizontal rules)
+  - Configurable `chunk_size` and `chunk_overlap` settings
+  - Returns cursor positions for each chunk
+- **Fuzzy tag search with trigram matching** - Tags now searchable with fuzzy matching via pg_trgm
+- **LongTermMemory modularization** - Refactored into separate concerns for better maintainability
+  - Search optimizations for vector, fulltext, and hybrid strategies
+
+### Fixed
+- **Configurable limits** - The following are now configurable (previously hard-coded):
+  - `max_embedding_dimension` (default: 2000)
+  - `max_tag_depth` (default: 4)
+  - Circuit breaker settings: `failure_threshold`, `reset_timeout`, `half_open_max_calls`
+  - Relevance scoring weights: `semantic_weight`, `tag_weight`, `recency_weight`, `access_weight`
+  - `relevance_recency_half_life_hours` (default: 168 = 1 week)
+
 ## [0.0.11] - 2025-12-02
 
 ### Added
@@ -260,7 +310,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Markdown file loader** - Load markdown files into long-term memory
   - `FileSource` model to track loaded files with metadata and sync status
   - `MarkdownLoader` with YAML frontmatter extraction
-  - `ParagraphChunker` for splitting content into semantic chunks
+  - `MarkdownChunker` for splitting content into semantic chunks (uses Baran gem)
   - DELTA_TIME tolerance (5 seconds) for reliable file change detection
 - **New HTM API methods** for file operations:
   - `htm.load_file(path, force: false)` - Load single markdown file
@@ -347,7 +397,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [0.0.1] - 2025-10-25
 
 ### Added
-- Initial release of HTM (Hierarchical Temporary Memory)
+- Initial release of HTM (Hierarchical Temporal Memory)
 - Two-tier memory system:
   - Working memory: Token-limited, in-memory active context
   - Long-term memory: Durable PostgreSQL/TimescaleDB storage

data/README.md

@@ -68,6 +68,12 @@
     - Source file tracking with re-sync support
     - YAML frontmatter extraction as metadata
 
+- **Telemetry (OpenTelemetry)**
+    - Optional metrics collection via OpenTelemetry
+    - Zero overhead when disabled (null object pattern)
+    - Works with 50+ backends (Jaeger, Prometheus, Datadog, etc.)
+    - Tracks job latency, search performance, and cache effectiveness
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -373,6 +379,68 @@ rake htm:jobs:clear_all
 
 See `rake -T htm:jobs` for complete list of job management tasks.
 
+## Telemetry (OpenTelemetry)
+
+HTM includes optional OpenTelemetry-based metrics for production observability. Telemetry is **disabled by default** with zero overhead when off.
+
+### Enabling Telemetry
+
+```ruby
+HTM.configure do |config|
+  config.telemetry_enabled = true
+end
+
+# Or via environment variable
+# HTM_TELEMETRY_ENABLED=true
+```
+
+### Configuring the Destination
+
+HTM emits metrics via standard OpenTelemetry protocols. Configure your destination using environment variables:
+
+```bash
+# Export to any OTLP-compatible backend
+export OTEL_METRICS_EXPORTER="otlp"
+export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
+```
+
+### Available Metrics
+
+| Metric | Type | Attributes | Description |
+|--------|------|------------|-------------|
+| `htm.jobs` | Counter | `job`, `status` | Job execution counts (embedding, tags) |
+| `htm.embedding.latency` | Histogram | `provider`, `status` | Embedding generation time (ms) |
+| `htm.tag.latency` | Histogram | `provider`, `status` | Tag extraction time (ms) |
+| `htm.search.latency` | Histogram | `strategy` | Search operation time (ms) |
+| `htm.cache.operations` | Counter | `operation` | Cache hits/misses |
+
+### Compatible Backends
+
+HTM works with any OTLP-compatible observability platform:
+
+**Open Source:** Jaeger, Prometheus, Grafana Tempo/Mimir, SigNoz, Uptrace
+
+**Commercial:** Datadog, New Relic, Honeycomb, Splunk, Dynatrace, AWS X-Ray, Google Cloud Trace, Azure Monitor
+
+### Optional Dependencies
+
+Users who want telemetry should add these gems:
+
+```ruby
+gem 'opentelemetry-sdk'
+gem 'opentelemetry-metrics-sdk'
+gem 'opentelemetry-exporter-otlp'  # For OTLP export
+```
+
+### Design
+
+HTM uses a **null object pattern** for telemetry. When disabled or when the SDK is not installed:
+- All metric operations are no-ops
+- Zero runtime overhead
+- No errors or exceptions
+
+See [docs/telemetry.md](docs/telemetry.md) for detailed configuration and usage examples.
+
 ## Configuration
 
 HTM uses dependency injection for LLM access, allowing you to configure embedding generation, tag extraction, logging, and token counting.
@@ -1160,6 +1228,17 @@ export HTM_LOG_LEVEL="INFO"   # Default
 
 This is used by the default logger when `HTM.configure` is called without a custom logger.
 
+#### HTM_TELEMETRY_ENABLED
+
+Enable OpenTelemetry metrics collection:
+
+```bash
+export HTM_TELEMETRY_ENABLED="true"   # Enable telemetry
+export HTM_TELEMETRY_ENABLED="false"  # Disable (default)
+```
+
+When enabled, HTM emits metrics to configured OpenTelemetry collectors. See [Telemetry](#telemetry-opentelemetry) for details.
+
 ### Quick Setup Examples
 
 #### Local Development (PostgreSQL)

data/Rakefile

@@ -10,6 +10,18 @@ Rake::TestTask.new(:test) do |t|
   t.verbose = true
 end
 
+# Ensure test task runs with RAILS_ENV=test
+task :test do
+  ENV['RAILS_ENV'] = 'test'
+end
+
+# Prepend environment setup before test runs
+Rake::Task[:test].enhance [:set_test_env]
+
+task :set_test_env do
+  ENV['RAILS_ENV'] = 'test'
+end
+
 task default: :test
 
 # Load HTM database tasks from lib/tasks/htm.rake
@@ -20,8 +32,8 @@ require_relative "lib/htm/tasks"
 desc "Run database setup (deprecated: use htm:db:setup)"
 task :db_setup => "htm:db:setup"
 
-desc "Test database connection (deprecated: use htm:db:test)"
-task :db_test => "htm:db:test"
+desc "Verify database connection (deprecated: use htm:db:verify)"
+task :db_test => "htm:db:verify"
 
 desc "Run example"
 task :example do

data/bin/htm_mcp.rb

@@ -395,6 +395,98 @@ class ListTagsTool < FastMcp::Tool
   end
 end
 
+# Tool: Search tags with fuzzy matching
+class SearchTagsTool < FastMcp::Tool
+  description "Search for tags using fuzzy matching (typo-tolerant). Use this when you're unsure of exact tag names."
+
+  arguments do
+    required(:query).filled(:string).description("Search query - can contain typos (e.g., 'postgrsql' finds 'database:postgresql')")
+    optional(:limit).filled(:integer).description("Maximum number of results (default: 20)")
+    optional(:min_similarity).filled(:float).description("Minimum similarity threshold 0.0-1.0 (default: 0.3, lower = more fuzzy)")
+  end
+
+  def call(query:, limit: 20, min_similarity: 0.3)
+    MCP_STDERR_LOG.info "SearchTagsTool called: query=#{query.inspect}, limit=#{limit}, min_similarity=#{min_similarity}"
+
+    htm = MCPSession.htm_instance
+    ltm = htm.instance_variable_get(:@long_term_memory)
+
+    results = ltm.search_tags(query, limit: limit, min_similarity: min_similarity)
+
+    # Enrich with node counts
+    tags = results.map do |result|
+      tag = HTM::Models::Tag.find_by(name: result[:name])
+      {
+        name: result[:name],
+        similarity: result[:similarity].round(3),
+        node_count: tag&.nodes&.count || 0
+      }
+    end
+
+    MCP_STDERR_LOG.info "SearchTagsTool complete: found #{tags.length} tags"
+
+    {
+      success: true,
+      query: query,
+      min_similarity: min_similarity,
+      count: tags.length,
+      tags: tags
+    }.to_json
+  end
+end
+
+# Tool: Find nodes by topic with fuzzy option
+class FindByTopicTool < FastMcp::Tool
+  description "Find memory nodes by topic/tag with optional fuzzy matching for typo tolerance"
+
+  arguments do
+    required(:topic).filled(:string).description("Topic or tag to search for (e.g., 'database:postgresql' or 'postgrsql' with fuzzy)")
+    optional(:fuzzy).filled(:bool).description("Enable fuzzy matching for typo tolerance (default: false)")
+    optional(:exact).filled(:bool).description("Require exact tag match (default: false, uses prefix matching)")
+    optional(:limit).filled(:integer).description("Maximum number of results (default: 20)")
+    optional(:min_similarity).filled(:float).description("Minimum similarity for fuzzy mode (default: 0.3)")
+  end
+
+  def call(topic:, fuzzy: false, exact: false, limit: 20, min_similarity: 0.3)
+    MCP_STDERR_LOG.info "FindByTopicTool called: topic=#{topic.inspect}, fuzzy=#{fuzzy}, exact=#{exact}"
+
+    htm = MCPSession.htm_instance
+    ltm = htm.instance_variable_get(:@long_term_memory)
+
+    nodes = ltm.nodes_by_topic(
+      topic,
+      fuzzy: fuzzy,
+      exact: exact,
+      min_similarity: min_similarity,
+      limit: limit
+    )
+
+    # Enrich with tags
+    results = nodes.map do |node_attrs|
+      node = HTM::Models::Node.includes(:tags).find_by(id: node_attrs['id'])
+      next unless node
+
+      {
+        id: node.id,
+        content: node.content[0..200],
+        tags: node.tags.map(&:name),
+        created_at: node.created_at.iso8601
+      }
+    end.compact
+
+    MCP_STDERR_LOG.info "FindByTopicTool complete: found #{results.length} nodes"
+
+    {
+      success: true,
+      topic: topic,
+      fuzzy: fuzzy,
+      exact: exact,
+      count: results.length,
+      results: results
+    }.to_json
+  end
+end
+
 # Tool: Get memory statistics
 class StatsTool < FastMcp::Tool
   description "Get statistics about HTM memory usage"
@@ -516,6 +608,8 @@ server.register_tool(RecallTool)
 server.register_tool(ForgetTool)
 server.register_tool(RestoreTool)
 server.register_tool(ListTagsTool)
+server.register_tool(SearchTagsTool)   # Fuzzy tag search with typo tolerance
+server.register_tool(FindByTopicTool)  # Find nodes by topic with fuzzy option
 server.register_tool(StatsTool)
 
 # Register resources

data/config/database.yml

@@ -2,36 +2,46 @@
 # Uses ERB to read from environment variables
 #
 # Priority:
-# 1. HTM_DBURL - Full connection URL (preferred)
+# 1. HTM_DBURL - Full connection URL (preferred for development/production)
 # 2. Individual HTM_DB* variables - Host, name, user, password, port
 # 3. Defaults for development/test
 #
 # Example HTM_DBURL format:
 #   postgresql://user:password@host:port/database?sslmode=require
+#
+# Test database:
+#   Tests always use htm_test database (Rails convention).
+#   Set RAILS_ENV=test or RACK_ENV=test to use the test configuration.
 
 <%
   require 'uri'
-  
+
+  # Determine current environment
+  current_env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+
   # Parse connection from HTM_DBURL or use individual variables
   if ENV['HTM_DBURL']
     uri = URI.parse(ENV['HTM_DBURL'])
     params = URI.decode_www_form(uri.query || '').to_h
-    
+    base_database = uri.path[1..-1]
+
+    # Extract base name (remove _development, _test, _production suffixes if present)
+    base_name = base_database.sub(/_(development|test|production)$/, '')
+
     db_config = {
       'host' => uri.host,
       'port' => uri.port || 5432,
-      'database' => uri.path[1..-1],
+      'base_name' => base_name,
       'username' => uri.user,
       'password' => uri.password,
       'sslmode' => params['sslmode'] || 'prefer'
     }
   else
-    env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
     db_config = {
       'host' => ENV.fetch('HTM_DBHOST', 'localhost'),
       'port' => ENV.fetch('HTM_DBPORT', 5432).to_i,
-      'database' => ENV.fetch('HTM_DBNAME', "htm_#{env}"),
-      'username' => ENV.fetch('HTM_DBUSER', 'postgres'),
+      'base_name' => ENV.fetch('HTM_DBNAME', 'htm').sub(/_(development|test|production)$/, ''),
+      'username' => ENV.fetch('HTM_DBUSER', ENV['USER']),
       'password' => ENV.fetch('HTM_DBPASS', ''),
       'sslmode' => ENV.fetch('HTM_SSLMODE', 'prefer')
     }
@@ -53,15 +63,12 @@ default: &default
 
 development:
   <<: *default
-  database: <%= db_config['database'] %>
+  database: <%= db_config['base_name'] %>_development
 
 test:
   <<: *default
-  database: <%= db_config['database'] %>_test
+  database: <%= db_config['base_name'] %>_test
 
 production:
   <<: *default
-  database: <%= db_config['database'] %>
-  <% unless ENV['HTM_DBURL'] %>
-  # WARNING: Production should use HTM_DBURL with SSL
-  <% end %>
+  database: <%= db_config['base_name'] %>_production

data/db/migrate/00010_add_soft_delete_to_associations.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class AddSoftDeleteToAssociations < ActiveRecord::Migration[7.0]
+  def change
+    # Add deleted_at to robot_nodes for soft delete support
+    unless column_exists?(:robot_nodes, :deleted_at)
+      add_column :robot_nodes, :deleted_at, :datetime, null: true
+    end
+    unless index_exists?(:robot_nodes, :deleted_at)
+      add_index :robot_nodes, :deleted_at
+    end
+
+    # Add deleted_at to node_tags for soft delete support
+    unless column_exists?(:node_tags, :deleted_at)
+      add_column :node_tags, :deleted_at, :datetime, null: true
+    end
+    unless index_exists?(:node_tags, :deleted_at)
+      add_index :node_tags, :deleted_at
+    end
+
+    # Add deleted_at to tags for soft delete support
+    unless column_exists?(:tags, :deleted_at)
+      add_column :tags, :deleted_at, :datetime, null: true
+    end
+    unless index_exists?(:tags, :deleted_at)
+      add_index :tags, :deleted_at
+    end
+  end
+end

data/db/migrate/00011_add_performance_indexes.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class AddPerformanceIndexes < ActiveRecord::Migration[7.1]
+  def change
+    # Partial index for soft-delete filter (used in almost every query)
+    # This complements idx_nodes_not_deleted_created_at for queries that
+    # don't need created_at ordering but still filter by deleted_at IS NULL
+    add_index :nodes, :id,
+              name: 'idx_nodes_active',
+              where: 'deleted_at IS NULL',
+              comment: 'Partial index for active (non-deleted) node queries'
+
+    # Composite index for embedding-based searches on active nodes
+    # Helps vector_search and hybrid_search which filter by deleted_at IS NULL
+    # and embedding IS NOT NULL
+    execute <<-SQL
+      CREATE INDEX idx_nodes_active_with_embedding ON nodes (id)
+        WHERE deleted_at IS NULL AND embedding IS NOT NULL
+    SQL
+  end
+end

data/db/migrate/00012_add_tags_trigram_index.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class AddTagsTrigramIndex < ActiveRecord::Migration[7.0]
+  def up
+    # Add GIN trigram index on tags.name for fuzzy search
+    # Enables queries like: WHERE name % 'postgrsql' (typo-tolerant)
+    # Also speeds up LIKE '%pattern%' queries
+    execute <<~SQL
+      CREATE INDEX idx_tags_name_trgm ON tags USING gin(name gin_trgm_ops);
+    SQL
+  end
+
+  def down
+    execute <<~SQL
+      DROP INDEX IF EXISTS idx_tags_name_trgm;
+    SQL
+  end
+end

data/db/migrate/00013_enable_lz4_compression.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class EnableLz4Compression < ActiveRecord::Migration[7.0]
+  def up
+    # Switch TOAST compression from pglz to lz4 for better read performance
+    # LZ4 decompression is ~32% faster than pglz with only marginally lower compression ratio
+    # See: https://www.depesz.com/2025/11/29/using-json-json-vs-jsonb-pglz-vs-lz4-key-optimization-parsing-speed/
+
+    # nodes.metadata - JSONB column for flexible key-value storage
+    execute <<~SQL
+      ALTER TABLE nodes ALTER COLUMN metadata SET COMPRESSION lz4;
+    SQL
+
+    # nodes.content - TEXT column containing memory content
+    execute <<~SQL
+      ALTER TABLE nodes ALTER COLUMN content SET COMPRESSION lz4;
+    SQL
+
+    # file_sources.frontmatter - JSONB column for parsed YAML frontmatter
+    execute <<~SQL
+      ALTER TABLE file_sources ALTER COLUMN frontmatter SET COMPRESSION lz4;
+    SQL
+
+    # Note: Existing rows retain their original compression until rewritten.
+    # To recompress existing data, run: VACUUM FULL nodes; VACUUM FULL file_sources;
+    # This is optional and can be done during a maintenance window.
+  end
+
+  def down
+    # Revert to default pglz compression
+    execute <<~SQL
+      ALTER TABLE nodes ALTER COLUMN metadata SET COMPRESSION pglz;
+    SQL
+
+    execute <<~SQL
+      ALTER TABLE nodes ALTER COLUMN content SET COMPRESSION pglz;
+    SQL
+
+    execute <<~SQL
+      ALTER TABLE file_sources ALTER COLUMN frontmatter SET COMPRESSION pglz;
+    SQL
+  end
+end