htm 0.0.14 → 0.0.17

data/bin/htm_mcp

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# HTM MCP Server with CLI management commands
+#
+# Usage:
+#   htm_mcp              # Start MCP server (default)
+#   htm_mcp server       # Start MCP server (explicit)
+#   htm_mcp setup        # Initialize database
+#   htm_mcp init         # Initialize database (alias for setup)
+#   htm_mcp verify       # Verify database connection
+#   htm_mcp stats        # Show memory statistics
+#   htm_mcp version      # Show HTM version
+#   htm_mcp help         # Show help and environment variables
+
+# Setup bundler to load gems from Gemfile
+# This allows running the script directly without `bundle exec`
+require 'bundler/setup'
+
+require_relative '../lib/htm'
+require_relative '../lib/htm/mcp/cli'
+
+# Handle CLI commands (exits if command handled)
+# Returns false if server should start
+unless HTM::MCP::CLI.run(ARGV) == false
+  exit 0
+end
+
+# Start MCP server
+require_relative '../lib/htm/mcp/server'
+HTM::MCP::Server.start

data/config/database.yml

@@ -9,15 +9,18 @@
 # Example HTM_DBURL format:
 #   postgresql://user:password@host:port/database?sslmode=require
 #
+# Environment detection priority:
+#   HTM_ENV > RAILS_ENV > RACK_ENV > 'development'
+#
 # Test database:
-#   Tests always use htm_test database (Rails convention).
-#   Set RAILS_ENV=test or RACK_ENV=test to use the test configuration.
+#   Tests always use htm_test database.
+#   Set HTM_ENV=test (or RAILS_ENV=test) to use the test configuration.
 
 <%
   require 'uri'
 
-  # Determine current environment
-  current_env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+  # Determine current environment (HTM_ENV takes priority for non-Rails users)
+  current_env = ENV['HTM_ENV'] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
 
   # Parse connection from HTM_DBURL or use individual variables
   if ENV['HTM_DBURL']

data/db/migrate/00003_create_file_sources.rb

@@ -16,5 +16,10 @@ class CreateFileSources < ActiveRecord::Migration[7.1]
     add_index :file_sources, :file_path, unique: true, name: 'idx_file_sources_path_unique'
     add_index :file_sources, :file_hash, name: 'idx_file_sources_hash'
     add_index :file_sources, :last_synced_at, name: 'idx_file_sources_last_synced'
+
+    # LZ4 compression for better read performance on JSONB column
+    execute <<~SQL
+      ALTER TABLE file_sources ALTER COLUMN frontmatter SET COMPRESSION lz4;
+    SQL
   end
 end

data/db/migrate/00004_create_nodes.rb

@@ -59,6 +59,23 @@ class CreateNodes < ActiveRecord::Migration[7.1]
         CHECK (embedding_dimension IS NULL OR (embedding_dimension > 0 AND embedding_dimension <= 2000))
     SQL
 
+    # Partial index for active (non-deleted) node queries
+    add_index :nodes, :id,
+              name: 'idx_nodes_active',
+              where: 'deleted_at IS NULL'
+
+    # Composite index for embedding-based searches on active nodes
+    execute <<-SQL
+      CREATE INDEX idx_nodes_active_with_embedding ON nodes (id)
+        WHERE deleted_at IS NULL AND embedding IS NOT NULL
+    SQL
+
+    # LZ4 compression for better read performance
+    execute <<-SQL
+      ALTER TABLE nodes ALTER COLUMN metadata SET COMPRESSION lz4;
+      ALTER TABLE nodes ALTER COLUMN content SET COMPRESSION lz4;
+    SQL
+
     # Foreign key to file_sources table
     add_foreign_key :nodes, :file_sources, column: :source_id, on_delete: :nullify
   end

data/db/migrate/00005_create_tags.rb

@@ -5,9 +5,16 @@ class CreateTags < ActiveRecord::Migration[7.1]
     create_table :tags, comment: 'Unique tag names for categorization' do |t|
       t.text :name, null: false, comment: 'Hierarchical tag in format: root:level1:level2 (e.g., database:postgresql:timescaledb)'
       t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this tag was created'
+      t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
     end
 
     add_index :tags, :name, unique: true, name: 'idx_tags_name_unique'
     add_index :tags, :name, using: :btree, opclass: :text_pattern_ops, name: 'idx_tags_name_pattern'
+    add_index :tags, :deleted_at, name: 'idx_tags_deleted_at'
+
+    # GIN trigram index for fuzzy search (typo-tolerant queries)
+    execute <<~SQL
+      CREATE INDEX idx_tags_name_trgm ON tags USING gin(name gin_trgm_ops);
+    SQL
   end
 end

data/db/migrate/00006_create_node_tags.rb

@@ -6,11 +6,13 @@ class CreateNodeTags < ActiveRecord::Migration[7.1]
       t.bigint :node_id, null: false, comment: 'ID of the node being tagged'
       t.bigint :tag_id, null: false, comment: 'ID of the tag being applied'
       t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this association was created'
+      t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
     end
 
     add_index :node_tags, [:node_id, :tag_id], unique: true, name: 'idx_node_tags_unique'
     add_index :node_tags, :node_id, name: 'idx_node_tags_node_id'
     add_index :node_tags, :tag_id, name: 'idx_node_tags_tag_id'
+    add_index :node_tags, :deleted_at, name: 'idx_node_tags_deleted_at'
 
     add_foreign_key :node_tags, :nodes, column: :node_id, on_delete: :cascade
     add_foreign_key :node_tags, :tags, column: :tag_id, on_delete: :cascade

data/db/migrate/00007_create_robot_nodes.rb

@@ -11,14 +11,21 @@ class CreateRobotNodes < ActiveRecord::Migration[7.1]
                     comment: 'When this robot last tried to remember this content'
       t.integer :remember_count, default: 1, null: false,
                 comment: 'Number of times this robot has tried to remember this content'
+      t.boolean :working_memory, default: false, null: false,
+                comment: 'True if this node is currently in the robot working memory'
       t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }
       t.timestamptz :updated_at, default: -> { 'CURRENT_TIMESTAMP' }
+      t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
     end
 
     add_index :robot_nodes, [:robot_id, :node_id], unique: true, name: 'idx_robot_nodes_unique'
     add_index :robot_nodes, :robot_id, name: 'idx_robot_nodes_robot_id'
     add_index :robot_nodes, :node_id, name: 'idx_robot_nodes_node_id'
     add_index :robot_nodes, :last_remembered_at, name: 'idx_robot_nodes_last_remembered_at'
+    add_index :robot_nodes, :deleted_at, name: 'idx_robot_nodes_deleted_at'
+    add_index :robot_nodes, [:robot_id, :working_memory],
+              where: 'working_memory = true',
+              name: 'idx_robot_nodes_working_memory'
 
     add_foreign_key :robot_nodes, :robots, column: :robot_id, on_delete: :cascade
     add_foreign_key :robot_nodes, :nodes, column: :node_id, on_delete: :cascade

data/db/schema.sql

@@ -108,7 +108,7 @@ CREATE TABLE public.node_tags (
     node_id bigint NOT NULL,
     tag_id bigint NOT NULL,
     created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
-    deleted_at timestamp(6) without time zone
+    deleted_at timestamp with time zone
 );
 
 --
@@ -135,6 +135,12 @@ COMMENT ON COLUMN public.node_tags.tag_id IS 'ID of the tag being applied';
 
 COMMENT ON COLUMN public.node_tags.created_at IS 'When this association was created';
 
+--
+-- Name: COLUMN node_tags.deleted_at; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.node_tags.deleted_at IS 'Soft delete timestamp';
+
 --
 -- Name: node_tags_id_seq; Type: SEQUENCE; Schema: public; Owner: -
 --
@@ -288,10 +294,10 @@ CREATE TABLE public.robot_nodes (
     first_remembered_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
     last_remembered_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
     remember_count integer DEFAULT 1 NOT NULL,
+    working_memory boolean DEFAULT false NOT NULL,
     created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
     updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
-    working_memory boolean DEFAULT false NOT NULL,
-    deleted_at timestamp(6) without time zone
+    deleted_at timestamp with time zone
 );
 
 --
@@ -336,6 +342,12 @@ COMMENT ON COLUMN public.robot_nodes.remember_count IS 'Number of times this rob
 
 COMMENT ON COLUMN public.robot_nodes.working_memory IS 'True if this node is currently in the robot working memory';
 
+--
+-- Name: COLUMN robot_nodes.deleted_at; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.robot_nodes.deleted_at IS 'Soft delete timestamp';
+
 --
 -- Name: robot_nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
 --
@@ -421,7 +433,7 @@ CREATE TABLE public.tags (
     id bigint NOT NULL,
     name text NOT NULL,
     created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
-    deleted_at timestamp(6) without time zone
+    deleted_at timestamp with time zone
 );
 
 --
@@ -442,6 +454,12 @@ COMMENT ON COLUMN public.tags.name IS 'Hierarchical tag in format: root:level1:l
 
 COMMENT ON COLUMN public.tags.created_at IS 'When this tag was created';
 
+--
+-- Name: COLUMN tags.deleted_at; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.tags.deleted_at IS 'Soft delete timestamp';
+
 --
 -- Name: tags_id_seq; Type: SEQUENCE; Schema: public; Owner: -
 --
@@ -562,6 +580,12 @@ CREATE INDEX idx_file_sources_last_synced ON public.file_sources USING btree (la
 
 CREATE UNIQUE INDEX idx_file_sources_path_unique ON public.file_sources USING btree (file_path);
 
+--
+-- Name: idx_node_tags_deleted_at; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_node_tags_deleted_at ON public.node_tags USING btree (deleted_at);
+
 --
 -- Name: idx_node_tags_node_id; Type: INDEX; Schema: public; Owner: -
 --
@@ -592,12 +616,6 @@ CREATE INDEX idx_nodes_access_count ON public.nodes USING btree (access_count);
 
 CREATE INDEX idx_nodes_active ON public.nodes USING btree (id) WHERE (deleted_at IS NULL);
 
---
--- Name: INDEX idx_nodes_active; Type: COMMENT; Schema: public; Owner: -
---
-
-COMMENT ON INDEX public.idx_nodes_active IS 'Partial index for active (non-deleted) node queries';
-
 --
 -- Name: idx_nodes_active_with_embedding; Type: INDEX; Schema: public; Owner: -
 --
@@ -676,6 +694,12 @@ CREATE INDEX idx_nodes_source_id ON public.nodes USING btree (source_id);
 
 CREATE INDEX idx_nodes_updated_at ON public.nodes USING btree (updated_at);
 
+--
+-- Name: idx_robot_nodes_deleted_at; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_robot_nodes_deleted_at ON public.robot_nodes USING btree (deleted_at);
+
 --
 -- Name: idx_robot_nodes_last_remembered_at; Type: INDEX; Schema: public; Owner: -
 --
@@ -706,6 +730,12 @@ CREATE UNIQUE INDEX idx_robot_nodes_unique ON public.robot_nodes USING btree (ro
 
 CREATE INDEX idx_robot_nodes_working_memory ON public.robot_nodes USING btree (robot_id, working_memory) WHERE (working_memory = true);
 
+--
+-- Name: idx_tags_deleted_at; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_tags_deleted_at ON public.tags USING btree (deleted_at);
+
 --
 -- Name: idx_tags_name_pattern; Type: INDEX; Schema: public; Owner: -
 --
@@ -724,24 +754,6 @@ CREATE INDEX idx_tags_name_trgm ON public.tags USING gin (name public.gin_trgm_o
 
 CREATE UNIQUE INDEX idx_tags_name_unique ON public.tags USING btree (name);
 
---
--- Name: index_node_tags_on_deleted_at; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX index_node_tags_on_deleted_at ON public.node_tags USING btree (deleted_at);
-
---
--- Name: index_robot_nodes_on_deleted_at; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX index_robot_nodes_on_deleted_at ON public.robot_nodes USING btree (deleted_at);
-
---
--- Name: index_tags_on_deleted_at; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX index_tags_on_deleted_at ON public.tags USING btree (deleted_at);
-
 --
 -- Name: nodes fk_rails_920ad16d08; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -781,4 +793,4 @@ ALTER TABLE ONLY public.robot_nodes
 -- PostgreSQL database dump complete
 --
 
-\unrestrict bjkKIo8iBhvKUCAsIDDWdDnwALL0oxUsmZVbvgEad6ZmhcLmalGL4Ih7bfe9nlh
+\unrestrict 4WlUqnJzNHaNhcr67XLIIhAvRPidZODUPGkM34l27SvmC0zu6dIsQdJ8dtu589Z

data/docs/api/yard/HTM/Configuration.md

@@ -81,6 +81,21 @@ Returns the value of attribute bedrock_region.
 ## bedrock_secret_key[RW] {: #attribute-i-bedrock_secret_key }
 Returns the value of attribute bedrock_secret_key.
 
+## chunk_overlap[RW] {: #attribute-i-chunk_overlap }
+Character overlap between chunks (default: 64)
+
+## chunk_size[RW] {: #attribute-i-chunk_size }
+Chunking configuration (for file loading)
+
+## circuit_breaker_failure_threshold[RW] {: #attribute-i-circuit_breaker_failure_threshold }
+Circuit breaker configuration
+
+## circuit_breaker_half_open_max_calls[RW] {: #attribute-i-circuit_breaker_half_open_max_calls }
+Successes to close (default: 3)
+
+## circuit_breaker_reset_timeout[RW] {: #attribute-i-circuit_breaker_reset_timeout }
+Seconds before half-open (default: 60)
+
 ## connection_timeout[RW] {: #attribute-i-connection_timeout }
 Returns the value of attribute connection_timeout.
 
@@ -102,6 +117,9 @@ Returns the value of attribute embedding_provider.
 ## embedding_timeout[RW] {: #attribute-i-embedding_timeout }
 Returns the value of attribute embedding_timeout.
 
+## extract_propositions[RW] {: #attribute-i-extract_propositions }
+Returns the value of attribute extract_propositions.
+
 ## gemini_api_key[RW] {: #attribute-i-gemini_api_key }
 Returns the value of attribute gemini_api_key.
 
@@ -114,6 +132,12 @@ Returns the value of attribute job_backend.
 ## logger[RW] {: #attribute-i-logger }
 Returns the value of attribute logger.
 
+## max_embedding_dimension[RW] {: #attribute-i-max_embedding_dimension }
+Limit configuration
+
+## max_tag_depth[RW] {: #attribute-i-max_tag_depth }
+Max tag hierarchy depth (default: 4)
+
 ## ollama_url[RW] {: #attribute-i-ollama_url }
 Returns the value of attribute ollama_url.
 
@@ -129,6 +153,33 @@ Provider-specific API keys and endpoints
 ## openrouter_api_key[RW] {: #attribute-i-openrouter_api_key }
 Returns the value of attribute openrouter_api_key.
 
+## proposition_extractor[RW] {: #attribute-i-proposition_extractor }
+Returns the value of attribute proposition_extractor.
+
+## proposition_model[RW] {: #attribute-i-proposition_model }
+Returns the value of attribute proposition_model.
+
+## proposition_provider[RW] {: #attribute-i-proposition_provider }
+Returns the value of attribute proposition_provider.
+
+## proposition_timeout[RW] {: #attribute-i-proposition_timeout }
+Returns the value of attribute proposition_timeout.
+
+## relevance_access_weight[RW] {: #attribute-i-relevance_access_weight }
+Access frequency weight (default: 0.1)
+
+## relevance_recency_half_life_hours[RW] {: #attribute-i-relevance_recency_half_life_hours }
+Decay half-life in hours (default: 168 = 1 week)
+
+## relevance_recency_weight[RW] {: #attribute-i-relevance_recency_weight }
+Temporal freshness weight (default: 0.1)
+
+## relevance_semantic_weight[RW] {: #attribute-i-relevance_semantic_weight }
+Relevance scoring weights (must sum to 1.0)
+
+## relevance_tag_weight[RW] {: #attribute-i-relevance_tag_weight }
+Tag overlap weight (default: 0.3)
+
 ## tag_extractor[RW] {: #attribute-i-tag_extractor }
 Returns the value of attribute tag_extractor.
 
@@ -141,6 +192,9 @@ Returns the value of attribute tag_provider.
 ## tag_timeout[RW] {: #attribute-i-tag_timeout }
 Returns the value of attribute tag_timeout.
 
+## telemetry_enabled[RW] {: #attribute-i-telemetry_enabled }
+Enable OpenTelemetry metrics (default: false)
+
 ## token_counter[RW] {: #attribute-i-token_counter }
 Returns the value of attribute token_counter.
 

data/docs/api/yard/HTM/Database.md

@@ -8,20 +8,23 @@ initialization
 
 # Class Methods
 ## default_config() {: #method-c-default_config }
-Get default database configuration
-**`@return`** [Hash, nil] Connection configuration hash
+Get default database configuration (respects RAILS_ENV)
+
+Uses ActiveRecordConfig which reads from config/database.yml and respects
+RAILS_ENV for environment-specific database selection.
+**`@return`** [Hash, nil] Connection configuration hash with PG-style keys
 
 ## drop(db_url nil) {: #method-c-drop }
-Drop all HTM tables
-**`@param`** [String] Database connection URL (uses ENV['HTM_DBURL'] if not provided)
+Drop all HTM tables (respects RAILS_ENV)
+**`@param`** [String] Database connection URL (uses default_config if not provided)
 
 **`@return`** [void] 
 
 ## dump_schema(db_url nil) {: #method-c-dump_schema }
-Dump current database schema to db/schema.sql
+Dump current database schema to db/schema.sql (respects RAILS_ENV)
 
 Uses pg_dump to create a clean SQL schema file without data
-**`@param`** [String] Database connection URL (uses ENV['HTM_DBURL'] if not provided)
+**`@param`** [String] Database connection URL (uses default_config if not provided)
 
 **`@return`** [void] 
 
@@ -39,16 +42,16 @@ comprehensive database documentation including:
 **`@return`** [void] 
 
 ## info(db_url nil) {: #method-c-info }
-Show database info
-**`@param`** [String] Database connection URL (uses ENV['HTM_DBURL'] if not provided)
+Show database info (respects RAILS_ENV)
+**`@param`** [String] Database connection URL (uses default_config if not provided)
 
 **`@return`** [void] 
 
 ## load_schema(db_url nil) {: #method-c-load_schema }
-Load schema from db/schema.sql
+Load schema from db/schema.sql (respects RAILS_ENV)
 
 Uses psql to load the schema file
-**`@param`** [String] Database connection URL (uses ENV['HTM_DBURL'] if not provided)
+**`@param`** [String] Database connection URL (uses default_config if not provided)
 
 **`@return`** [void] 
 

data/docs/api/yard/HTM/EmbeddingService.md

@@ -39,8 +39,12 @@ Generate embedding with validation and processing
   storage_dimension: Integer         # Padded dimension (2000)
 }
 
+## max_dimension() {: #method-c-max_dimension }
+Maximum embedding dimension (configurable, default 2000)
+**`@return`** [Integer] Max dimensions for pgvector HNSW index
+
 ## pad_embedding(embedding ) {: #method-c-pad_embedding }
-Pad embedding to MAX_DIMENSION with zeros
+Pad embedding to max_dimension with zeros
 **`@param`** [Array<Float>] Original embedding
 
 **`@return`** [Array<Float>] Padded embedding