htm 0.0.10 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b552d9b1e6a197c35d1ff2c64d4c8eb94a36a0e67d05efe1c10d305cf8f23d78
-  data.tar.gz: 58fa199e1a3af9fb9c3a9e8fc0fe9ee779209a44baecbccdfdf6f570f5e7c692
+  metadata.gz: 5658951570d47a6988a3e6c5078ae90c83a8bbbaebd4bc18b8a70152d0648ab1
+  data.tar.gz: e4216f5bbe7cbbcfb1b78058f16fb5c190a8902f4778874c858acf7401f0da14
 SHA512:
-  metadata.gz: 9326f318f1677ea5cad5e5f6cdda4125421a13bde13d1c7bef0a2b2e8096873a7762f11d008c8d0e64365f90272aad58699b39a3af40f0c188f7e961eef6d2df
-  data.tar.gz: 6f25fa0089349b0872db11427747e525ab46a9528d1ada22be183be260bad40d48ef692f78d922ba0803b817f2f661fa143721bf180882f34c575f4650e65448
+  metadata.gz: 3d206a81c9120d7c2ffb8a8afb41056ee93a9b614a7abfabe1e14196e60570daaab048e37cc74601bd04afed0af5290616ff925a7610dc37e8816cb4d797956a
+  data.tar.gz: d2977a3e77e2d7a0b6cd886d3edadbfe8c04b3af55e4280d8b70393de41ad019a76fe12e19c1bf141d3f6b604ac4a221caca5ea914c64985521a4ccbe2fbfce9

data/.dictate.toml

@@ -0,0 +1,46 @@
+# Dictator Configuration for HTM
+# Decree-based structural enforcement
+# https://github.com/seuros/dictator
+
+[decree.supreme]
+# Universal structural rules (applies to ALL files)
+trailing_whitespace   = "deny"
+tabs_vs_spaces        = "spaces"
+tab_width             = 2
+final_newline         = "require"
+line_endings          = "lf"
+blank_line_whitespace = "deny"
+
+[decree.ruby]
+# Ruby-specific structural enforcement
+max_line_length         = 120
+max_lines               = 300
+ignore_comments         = true
+ignore_blank_lines      = true
+method_visibility_order = ["public", "protected", "private"]
+comment_spacing         = true
+
+[decree.ruby.linter]
+# External linter - Dictator adds: -A --format json
+command = "rubocop"
+
+[decree.frontmatter]
+# YAML frontmatter ordering for .md files
+order    = ["title", "slug", "pubDate", "description", "tags", "draft"]
+required = ["title"]
+
+# Disable unused language decrees
+[decree.typescript]
+enabled = false
+
+[decree.golang]
+enabled = false
+
+[decree.rust]
+enabled = false
+
+[decree.python]
+enabled = false
+
+[decree.kjr]
+enabled = false

data/.envrc

@@ -2,6 +2,8 @@
 
 export RR=`pwd`
 
+export HTM_EXTRACT_PROPOSITIONS=true
+
 # Database connection - Localhost PostgreSQL
 export HTM_DBHOST=localhost
 export HTM_DBPORT=5432

data/CHANGELOG.md

@@ -7,6 +7,88 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.0.14] - 2025-12-05
+
+### Added
+- **OpenTelemetry metrics** - Optional observability with zero overhead when disabled
+  - New `HTM::Telemetry` module with null object pattern
+  - Metrics: `htm.jobs` (counter), `htm.embedding.latency`, `htm.tag.latency`, `htm.search.latency` (histograms), `htm.cache.operations` (counter)
+  - Instrumented: `GenerateEmbeddingJob`, `GenerateTagsJob`, search methods, `QueryCache`
+  - Enable via `HTM_TELEMETRY_ENABLED=true` or `config.telemetry_enabled = true`
+  - Works with 50+ OTLP-compatible backends (Jaeger, Prometheus, Datadog, etc.)
+  - Comprehensive documentation in `docs/telemetry.md`
+  - 18 new telemetry tests
+- **Telemetry demo with Grafana visualization** - `examples/telemetry/`
+  - Live dashboard showing HTM metrics (job counts, latencies, cache hit rates)
+  - Uses Homebrew-installed Prometheus and Grafana (no Docker required)
+  - Auto-configures Prometheus scrape target
+  - Pre-built Grafana dashboard JSON for easy import
+  - Interactive shutdown prompt for service management
+- **`htm:db:create` rake task** - Create database if it doesn't exist (respects `RAILS_ENV`)
+
+### Changed
+- **All `htm:db:*` tasks now respect `RAILS_ENV`** - Following Rails conventions
+  - Database selection based on environment: `htm_development`, `htm_test`, `htm_production`
+  - `rake test` automatically sets `RAILS_ENV=test`
+  - Example: `RAILS_ENV=test rake htm:db:setup` operates on `htm_test`
+- **`config/database.yml` refactored** - Extracts base name from `HTM_DBURL` and appends environment suffix
+  - `HTM_DBURL=postgresql://...htm_development` + `RAILS_ENV=test` → connects to `htm_test`
+- **`HTM::Database.default_config` now respects `RAILS_ENV`** - Uses `ActiveRecordConfig.load_database_config`
+- **Renamed `htm:db:test` to `htm:db:verify`** - Avoids naming collision with test database namespace
+  - `htm:db:verify` verifies database connection
+  - `RAILS_ENV=test rake htm:db:*` operates on test database
+
+## [0.0.13] - 2025-12-04
+
+### Changed
+- **MarkdownChunker now uses Baran gem** - Replaced custom ParagraphChunker with Baran's `MarkdownSplitter`
+  - Respects markdown structure (headers, code blocks, horizontal rules)
+  - Configurable `chunk_size` and `chunk_overlap` settings
+  - Returns cursor positions for each chunk
+- **Fuzzy tag search with trigram matching** - Tags now searchable with fuzzy matching via pg_trgm
+- **LongTermMemory modularization** - Refactored into separate concerns for better maintainability
+  - Search optimizations for vector, fulltext, and hybrid strategies
+
+### Fixed
+- **Configurable limits** - The following are now configurable (previously hard-coded):
+  - `max_embedding_dimension` (default: 2000)
+  - `max_tag_depth` (default: 4)
+  - Circuit breaker settings: `failure_threshold`, `reset_timeout`, `half_open_max_calls`
+  - Relevance scoring weights: `semantic_weight`, `tag_weight`, `recency_weight`, `access_weight`
+  - `relevance_recency_half_life_hours` (default: 168 = 1 week)
+
+## [0.0.11] - 2025-12-02
+
+### Added
+- **MCP Server and Client** - Model Context Protocol integration for AI assistants
+  - `bin/htm_mcp.rb` - FastMCP-based server exposing HTM tools:
+    - `SetRobotTool` - Set robot identity for the session (per-client isolation)
+    - `GetRobotTool` - Get current robot information
+    - `GetWorkingMemoryTool` - Retrieve working memory for session restore
+    - `RememberTool` - Store information with tags and metadata
+    - `RecallTool` - Search memories with vector/fulltext/hybrid strategies
+    - `ForgetTool` / `RestoreTool` - Soft delete and restore memories
+    - `ListTagsTool` - List tags with optional prefix filtering
+    - `StatsTool` - Memory usage statistics
+  - Resources: `htm://statistics`, `htm://tags/hierarchy`, `htm://memories/recent`
+  - STDERR logging to avoid corrupting MCP JSON-RPC protocol
+  - Session-based robot identity via `MCPSession` module
+  - `examples/mcp_client.rb` - Interactive chat client using ruby_llm-mcp:
+    - Prompts for robot name on startup (or uses `HTM_ROBOT_NAME` env var)
+    - Session restore: offers to restore working memory from previous session
+    - Interactive chat loop with Ollama LLM (gpt-oss model)
+    - Tool call logging for visibility
+    - Slash commands: `/tools`, `/resources`, `/stats`, `/tags`, `/clear`, `/help`, `/exit`
+- **Session restore feature** - MCP client can restore previous session context
+  - `GetWorkingMemoryTool` returns all nodes in working memory for a robot
+  - Client prompts user to restore previous session on startup
+  - Working memory injected into chat context for continuity
+
+### Fixed
+- **GetWorkingMemoryTool** now uses `joins(:node)` to exclude soft-deleted nodes
+- **StatsTool** fixed scope error (`Node.active` → `Node.count` with default_scope)
+- **MCP tool response parsing** - Extract `.text` from `RubyLLM::MCP::Content` objects
+
 ## [0.0.10] - 2025-12-02
 
 ### Added
@@ -228,7 +310,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Markdown file loader** - Load markdown files into long-term memory
   - `FileSource` model to track loaded files with metadata and sync status
   - `MarkdownLoader` with YAML frontmatter extraction
-  - `ParagraphChunker` for splitting content into semantic chunks
+  - `MarkdownChunker` for splitting content into semantic chunks (uses Baran gem)
   - DELTA_TIME tolerance (5 seconds) for reliable file change detection
 - **New HTM API methods** for file operations:
   - `htm.load_file(path, force: false)` - Load single markdown file
@@ -315,7 +397,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [0.0.1] - 2025-10-25
 
 ### Added
-- Initial release of HTM (Hierarchical Temporary Memory)
+- Initial release of HTM (Hierarchical Temporal Memory)
 - Two-tier memory system:
   - Working memory: Token-limited, in-memory active context
   - Long-term memory: Durable PostgreSQL/TimescaleDB storage
@@ -436,7 +518,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Working memory size is user-configurable
 - See ADRs for detailed architectural decisions and rationale
 
-[Unreleased]: https://github.com/madbomber/htm/compare/v0.0.10...HEAD
+[Unreleased]: https://github.com/madbomber/htm/compare/v0.0.12...HEAD
+[0.0.12]: https://github.com/madbomber/htm/compare/v0.0.10...v0.0.12
 [0.0.10]: https://github.com/madbomber/htm/compare/v0.0.9...v0.0.10
 [0.0.9]: https://github.com/madbomber/htm/compare/v0.0.8...v0.0.9
 [0.0.8]: https://github.com/madbomber/htm/compare/v0.0.7...v0.0.8

data/README.md

@@ -68,6 +68,12 @@
     - Source file tracking with re-sync support
     - YAML frontmatter extraction as metadata
 
+- **Telemetry (OpenTelemetry)**
+    - Optional metrics collection via OpenTelemetry
+    - Zero overhead when disabled (null object pattern)
+    - Works with 50+ backends (Jaeger, Prometheus, Datadog, etc.)
+    - Tracks job latency, search performance, and cache effectiveness
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -373,6 +379,68 @@ rake htm:jobs:clear_all
 
 See `rake -T htm:jobs` for complete list of job management tasks.
 
+## Telemetry (OpenTelemetry)
+
+HTM includes optional OpenTelemetry-based metrics for production observability. Telemetry is **disabled by default** with zero overhead when off.
+
+### Enabling Telemetry
+
+```ruby
+HTM.configure do |config|
+  config.telemetry_enabled = true
+end
+
+# Or via environment variable
+# HTM_TELEMETRY_ENABLED=true
+```
+
+### Configuring the Destination
+
+HTM emits metrics via standard OpenTelemetry protocols. Configure your destination using environment variables:
+
+```bash
+# Export to any OTLP-compatible backend
+export OTEL_METRICS_EXPORTER="otlp"
+export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
+```
+
+### Available Metrics
+
+| Metric | Type | Attributes | Description |
+|--------|------|------------|-------------|
+| `htm.jobs` | Counter | `job`, `status` | Job execution counts (embedding, tags) |
+| `htm.embedding.latency` | Histogram | `provider`, `status` | Embedding generation time (ms) |
+| `htm.tag.latency` | Histogram | `provider`, `status` | Tag extraction time (ms) |
+| `htm.search.latency` | Histogram | `strategy` | Search operation time (ms) |
+| `htm.cache.operations` | Counter | `operation` | Cache hits/misses |
+
+### Compatible Backends
+
+HTM works with any OTLP-compatible observability platform:
+
+**Open Source:** Jaeger, Prometheus, Grafana Tempo/Mimir, SigNoz, Uptrace
+
+**Commercial:** Datadog, New Relic, Honeycomb, Splunk, Dynatrace, AWS X-Ray, Google Cloud Trace, Azure Monitor
+
+### Optional Dependencies
+
+Users who want telemetry should add these gems:
+
+```ruby
+gem 'opentelemetry-sdk'
+gem 'opentelemetry-metrics-sdk'
+gem 'opentelemetry-exporter-otlp'  # For OTLP export
+```
+
+### Design
+
+HTM uses a **null object pattern** for telemetry. When disabled or when the SDK is not installed:
+- All metric operations are no-ops
+- Zero runtime overhead
+- No errors or exceptions
+
+See [docs/telemetry.md](docs/telemetry.md) for detailed configuration and usage examples.
+
 ## Configuration
 
 HTM uses dependency injection for LLM access, allowing you to configure embedding generation, tag extraction, logging, and token counting.
@@ -1160,6 +1228,17 @@ export HTM_LOG_LEVEL="INFO"   # Default
 
 This is used by the default logger when `HTM.configure` is called without a custom logger.
 
+#### HTM_TELEMETRY_ENABLED
+
+Enable OpenTelemetry metrics collection:
+
+```bash
+export HTM_TELEMETRY_ENABLED="true"   # Enable telemetry
+export HTM_TELEMETRY_ENABLED="false"  # Disable (default)
+```
+
+When enabled, HTM emits metrics to configured OpenTelemetry collectors. See [Telemetry](#telemetry-opentelemetry) for details.
+
 ### Quick Setup Examples
 
 #### Local Development (PostgreSQL)
@@ -1400,13 +1479,13 @@ This separation allows you to provide any LLM implementation while HTM handles r
 ## Roadmap
 
 - [x] Phase 1: Foundation (basic two-tier memory)
-- [ ] Phase 2: RAG retrieval (semantic search)
-- [ ] Phase 3: Relationships & tags
-- [ ] Phase 4: Working memory management
-- [ ] Phase 5: Hive mind features
-- [ ] Phase 6: Operations & observability
-- [ ] Phase 7: Advanced features
-- [ ] Phase 8: Production-ready gem
+- [x] Phase 2: RAG retrieval (semantic search)
+- [x] Phase 3: Relationships & tags
+- [x] Phase 4: Working memory management
+- [x] Phase 5: Hive mind features
+- [x] Phase 6: Operations & observability
+- [x] Phase 7: Advanced features
+- [x] Phase 8: Production-ready gem
 
 
 ## Contributing

data/Rakefile

@@ -10,6 +10,18 @@ Rake::TestTask.new(:test) do |t|
   t.verbose = true
 end
 
+# Ensure test task runs with RAILS_ENV=test
+task :test do
+  ENV['RAILS_ENV'] = 'test'
+end
+
+# Prepend environment setup before test runs
+Rake::Task[:test].enhance [:set_test_env]
+
+task :set_test_env do
+  ENV['RAILS_ENV'] = 'test'
+end
+
 task default: :test
 
 # Load HTM database tasks from lib/tasks/htm.rake
@@ -20,8 +32,8 @@ require_relative "lib/htm/tasks"
 desc "Run database setup (deprecated: use htm:db:setup)"
 task :db_setup => "htm:db:setup"
 
-desc "Test database connection (deprecated: use htm:db:test)"
-task :db_test => "htm:db:test"
+desc "Verify database connection (deprecated: use htm:db:verify)"
+task :db_test => "htm:db:verify"
 
 desc "Run example"
 task :example do