htm 0.0.1

data/Rakefile

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.verbose = true
+end
+
+task default: :test
+
+# Load HTM database tasks from lib/tasks/htm.rake
+# This uses the same loader that external applications use
+require_relative "lib/htm/tasks"
+
+# Legacy tasks for backwards compatibility
+desc "Run database setup (deprecated: use htm:db:setup)"
+task :db_setup => "htm:db:setup"
+
+desc "Test database connection (deprecated: use htm:db:test)"
+task :db_test => "htm:db:test"
+
+desc "Run example"
+task :example do
+  ruby "examples/basic_usage.rb"
+end
+
+desc "Show gem stats"
+task :stats do
+  puts "\nHTM Gem Statistics:"
+  puts "=" * 60
+
+  # Count lines of code
+  lib_files = Dir.glob("lib/**/*.rb")
+  lib_lines = lib_files.sum { |f| File.readlines(f).size }
+
+  test_files = Dir.glob("test/**/*.rb")
+  test_lines = test_files.sum { |f| File.readlines(f).size }
+
+  puts "Library:"
+  puts "  Files: #{lib_files.size}"
+  puts "  Lines: #{lib_lines}"
+  puts "\nTests:"
+  puts "  Files: #{test_files.size}"
+  puts "  Lines: #{test_lines}"
+  puts "\nTotal lines: #{lib_lines + test_lines}"
+  puts "=" * 60
+end

data/SETUP.md

@@ -0,0 +1,268 @@
+# HTM Setup Guide
+
+## Prerequisites
+
+1. **Ruby** (version 3.0 or higher)
+2. **TimescaleDB Cloud Account** (already set up)
+3. **Database Environment Variables** (already configured)
+4. **Ollama** (for embeddings via RubyLLM)
+
+## Ollama Setup
+
+HTM uses RubyLLM with the Ollama provider for generating embeddings. You need to install and run Ollama locally.
+
+### 1. Install Ollama
+
+**macOS:**
+```bash
+curl https://ollama.ai/install.sh | sh
+```
+
+**Or download from:** https://ollama.ai/download
+
+### 2. Start Ollama Service
+
+```bash
+# Ollama typically starts automatically after installation
+# Verify it's running:
+curl http://localhost:11434/api/version
+```
+
+### 3. Pull the gpt-oss Model
+
+```bash
+# Pull the default model used by HTM
+ollama pull gpt-oss
+
+# Verify the model is available
+ollama list
+```
+
+### 4. Test Embedding Generation
+
+```bash
+# Test that embeddings work
+ollama run gpt-oss "Hello, world!"
+```
+
+### Optional: Custom Ollama URL
+
+If Ollama is running on a different host/port, set the environment variable:
+
+```bash
+export OLLAMA_URL="http://custom-host:11434"
+```
+
+## Database Setup
+
+### 1. Load Database Credentials
+
+The HTM project uses environment variables to manage database credentials. These are defined in `~/.bashrc__tiger`.
+
+```bash
+# Load the Tiger database environment variables
+source ~/.bashrc__tiger
+```
+
+To make these variables available automatically in new shell sessions, ensure `~/.bashrc__tiger` is sourced in your `~/.bashrc` or `~/.bash_profile`.
+
+### 2. Verify Connection
+
+Test the database connection:
+
+```bash
+cd /path/to/HTM
+ruby test_connection.rb
+```
+
+You should see:
+```
+✓ Connected successfully!
+✓ TimescaleDB Extension: Version 2.22.1
+✓ pgvector Extension: Version 0.8.1
+✓ pg_trgm Extension: Version 1.6
+```
+
+### 3. Enable Extensions (One-time)
+
+Enable required PostgreSQL extensions (already done, but can be re-run safely):
+
+```bash
+ruby enable_extensions.rb
+```
+
+## Environment Variables Reference
+
+After sourcing `~/.bashrc__tiger`, these variables are available:
+
+| Variable | Description | Example Value |
+|----------|-------------|---------------|
+| `HTM_SERVICE_NAME` | Service identifier | `db-67977` |
+| `HTM_DBNAME` | Database name | `tsdb` |
+| `HTM_DBUSER` | Database user | `tsdbadmin` |
+| `HTM_DBPASS` | Database password | `***` |
+| `HTM_DBURL` | Full connection URL (preferred) | `postgres://...` |
+| `HTM_DBPORT` | Database port | `37807` |
+
+## Development Workflow
+
+### Quick Start
+
+```bash
+# 1. Source environment variables (if not in .bashrc)
+source ~/.bashrc__tiger
+
+# 2. Install dependencies (when gem is created)
+bundle install
+
+# 3. Initialize database schema (when ready)
+ruby -r ./lib/htm -e "HTMDatabase.setup"
+
+# 4. Test HTM functionality (when implemented)
+ruby examples/basic_usage.rb
+```
+
+### Testing
+
+HTM uses Minitest for testing:
+
+```bash
+# Run all tests
+rake test
+
+# Or run directly with Ruby
+ruby test/htm_test.rb
+
+# Run specific test file
+ruby test/embedding_service_test.rb
+
+# Run integration tests (requires database)
+ruby test/integration_test.rb
+```
+
+## Project Structure
+
+```
+HTM/
+├── lib/
+│   ├── htm.rb                    # Main HTM class
+│   ├── htm/
+│   │   ├── database.rb           # Database setup and schema
+│   │   ├── long_term_memory.rb   # PostgreSQL-backed storage
+│   │   ├── working_memory.rb     # In-memory active context
+│   │   ├── embedding_service.rb  # RubyLLM embedding generation (Ollama/gpt-oss)
+│   │   └── version.rb            # Version constant
+├── sql/
+│   └── schema.sql                # Database schema
+├── test/
+│   ├── test_helper.rb            # Minitest configuration
+│   ├── htm_test.rb               # Basic HTM tests
+│   ├── embedding_service_test.rb # Embedding tests (RubyLLM/Ollama)
+│   └── integration_test.rb       # Full integration tests
+├── examples/
+│   └── basic_usage.rb            # Basic usage example
+├── test_connection.rb            # Verify database connection
+├── enable_extensions.rb          # Enable PostgreSQL extensions
+├── SETUP.md                      # This file
+├── README.md                     # Project overview
+├── htm_teamwork.md               # Planning and design doc
+├── Gemfile
+├── htm.gemspec
+└── Rakefile                      # Rake tasks
+```
+
+## Next Steps
+
+1. **Phase 1**: Create basic gem structure
+2. **Phase 2**: Implement database schema
+3. **Phase 3**: Implement LongTermMemory class
+4. **Phase 4**: Implement WorkingMemory class
+5. **Phase 5**: Implement HTM main class
+6. **Phase 6**: Add tests
+7. **Phase 7**: Create examples
+
+See `htm_teamwork.md` for detailed roadmap.
+
+## Troubleshooting
+
+### Ollama Issues
+
+If you encounter embedding errors:
+
+```bash
+# Verify Ollama is running
+curl http://localhost:11434/api/version
+
+# Check if gpt-oss model is available
+ollama list | grep gpt-oss
+
+# Test embedding generation
+ollama run gpt-oss "Test embedding"
+
+# View Ollama logs
+ollama logs
+
+# Restart Ollama service
+# On macOS, Ollama runs as a background service
+# Check Activity Monitor or restart from the menu bar
+```
+
+**Common Ollama Errors:**
+
+- **"connection refused"**: Ollama service is not running. Start Ollama from Applications or via CLI.
+- **"model not found"**: Run `ollama pull gpt-oss` to download the model.
+- **Custom URL not working**: Ensure `OLLAMA_URL` environment variable is set correctly.
+
+### Database Connection Issues
+
+If you get connection errors:
+
+```bash
+# Verify environment variables are set
+echo $HTM_DBURL
+
+# Test connection manually
+psql $HTM_DBURL
+
+# Check if ~/.bashrc__tiger is sourced
+grep "bashrc__tiger" ~/.bashrc
+```
+
+### Extension Issues
+
+If extensions aren't available:
+
+```bash
+# Re-run extension setup
+ruby enable_extensions.rb
+
+# Check extension status manually
+psql $HTM_DBURL -c "SELECT extname, extversion FROM pg_extension ORDER BY extname"
+```
+
+### SSL Issues
+
+The TimescaleDB Cloud instance requires SSL. If you see SSL errors:
+
+```bash
+# Ensure sslmode is set in connection URL
+echo $HTM_DBURL | grep sslmode
+# Should show: sslmode=require
+```
+
+## Resources
+
+- **Ollama**: https://ollama.ai/
+- **RubyLLM**: https://github.com/madbomber/ruby_llm
+- **TimescaleDB Docs**: https://docs.timescale.com/
+- **pgvector Docs**: https://github.com/pgvector/pgvector
+- **Planning Document**: `htm_teamwork.md`
+- **PostgreSQL Docs**: https://www.postgresql.org/docs/
+
+## Support
+
+For issues or questions:
+1. Check `htm_teamwork.md` for design decisions
+2. Review examples in `examples/` directory
+3. Run tests with `rake test` (Minitest framework)
+4. Check Ollama status for embedding issues

data/config/database.yml

@@ -0,0 +1,67 @@
+# HTM Database Configuration
+# Uses ERB to read from environment variables
+#
+# Priority:
+# 1. HTM_DBURL - Full connection URL (preferred)
+# 2. Individual HTM_DB* variables - Host, name, user, password, port
+# 3. Defaults for development/test
+#
+# Example HTM_DBURL format:
+#   postgresql://user:password@host:port/database?sslmode=require
+
+<%
+  require 'uri'
+  
+  # Parse connection from HTM_DBURL or use individual variables
+  if ENV['HTM_DBURL']
+    uri = URI.parse(ENV['HTM_DBURL'])
+    params = URI.decode_www_form(uri.query || '').to_h
+    
+    db_config = {
+      'host' => uri.host,
+      'port' => uri.port || 5432,
+      'database' => uri.path[1..-1],
+      'username' => uri.user,
+      'password' => uri.password,
+      'sslmode' => params['sslmode'] || 'prefer'
+    }
+  else
+    env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+    db_config = {
+      'host' => ENV.fetch('HTM_DBHOST', 'localhost'),
+      'port' => ENV.fetch('HTM_DBPORT', 5432).to_i,
+      'database' => ENV.fetch('HTM_DBNAME', "htm_#{env}"),
+      'username' => ENV.fetch('HTM_DBUSER', 'postgres'),
+      'password' => ENV.fetch('HTM_DBPASS', ''),
+      'sslmode' => ENV.fetch('HTM_SSLMODE', 'prefer')
+    }
+  end
+%>
+
+default: &default
+  adapter: postgresql
+  encoding: unicode
+  pool: <%= ENV.fetch("HTM_DB_POOL_SIZE", "10").to_i %>
+  timeout: 5000
+  prepared_statements: false
+  advisory_locks: false
+  host: <%= db_config['host'] %>
+  port: <%= db_config['port'] %>
+  username: <%= db_config['username'] %>
+  password: <%= db_config['password'] %>
+  sslmode: <%= db_config['sslmode'] %>
+
+development:
+  <<: *default
+  database: <%= db_config['database'] %>
+
+test:
+  <<: *default
+  database: <%= db_config['database'] %>_test
+
+production:
+  <<: *default
+  database: <%= db_config['database'] %>
+  <% unless ENV['HTM_DBURL'] %>
+  # WARNING: Production should use HTM_DBURL with SSL
+  <% end %>

data/db/migrate/20250101000001_enable_extensions.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class EnableExtensions < ActiveRecord::Migration[7.1]
+  def up
+    # Note: On TimescaleDB Cloud, the extension is named 'vector' not 'pgvector'
+    enable_extension 'vector'
+    enable_extension 'pg_trgm'
+  end
+
+  def down
+    disable_extension 'pg_trgm'
+    disable_extension 'vector'
+  end
+end

data/db/migrate/20250101000002_create_robots.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class CreateRobots < ActiveRecord::Migration[7.1]
+  def change
+    unless table_exists?(:robots)
+      create_table :robots, comment: 'Registry of all LLM robots using the HTM system' do |t|
+        t.text :name, comment: 'Human-readable name for the robot'
+        t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When the robot was first registered'
+        t.timestamptz :last_active, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'Last time the robot accessed the system'
+        t.jsonb :metadata, comment: 'Robot-specific configuration and metadata'
+      end
+    end
+  end
+end

data/db/migrate/20250101000003_create_nodes.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class CreateNodes < ActiveRecord::Migration[7.1]
+  def change
+    unless table_exists?(:nodes)
+      create_table :nodes, comment: 'Core memory storage for conversation messages and context' do |t|
+        t.text :content, null: false, comment: 'The conversation message/utterance content'
+        t.text :source, default: '', comment: 'From where the content came (empty string if unknown)'
+        t.integer :access_count, default: 0, null: false, comment: 'Number of times this node has been accessed/retrieved'
+        t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this memory was created'
+        t.timestamptz :updated_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this memory was last modified'
+        t.timestamptz :last_accessed, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this memory was last accessed'
+        t.integer :token_count, comment: 'Number of tokens in the content (for context budget management)'
+        t.boolean :in_working_memory, default: false, comment: 'Whether this memory is currently in working memory'
+        t.bigint :robot_id, null: false, comment: 'ID of the robot that owns this memory'
+        t.vector :embedding, limit: 2000, comment: 'Vector embedding (max 2000 dimensions) for semantic search'
+        t.integer :embedding_dimension, comment: 'Actual number of dimensions used in the embedding vector (max 2000)'
+      end
+
+      # Basic indexes for common queries
+      add_index :nodes, :created_at, name: 'idx_nodes_created_at'
+      add_index :nodes, :updated_at, name: 'idx_nodes_updated_at'
+      add_index :nodes, :last_accessed, name: 'idx_nodes_last_accessed'
+      add_index :nodes, :access_count, name: 'idx_nodes_access_count'
+      add_index :nodes, :robot_id, name: 'idx_nodes_robot_id'
+      add_index :nodes, :source, name: 'idx_nodes_source'
+      add_index :nodes, :in_working_memory, name: 'idx_nodes_in_working_memory'
+
+      # Add check constraint for embedding dimensions
+      # Only validates when embedding_dimension is provided (allows NULL for nodes without embeddings)
+      execute <<-SQL
+        ALTER TABLE nodes ADD CONSTRAINT check_embedding_dimension
+          CHECK (embedding_dimension IS NULL OR (embedding_dimension > 0 AND embedding_dimension <= 2000))
+      SQL
+    end
+
+    # Foreign key to robots table (outside table_exists check so it gets added even if table already exists)
+    unless foreign_key_exists?(:nodes, :robots, column: :robot_id)
+      add_foreign_key :nodes, :robots, column: :robot_id, primary_key: :id, on_delete: :cascade
+    end
+  end
+end

data/db/migrate/20250101000005_create_tags.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class CreateTags < ActiveRecord::Migration[7.1]
+  def change
+    # Create tags table with unique tag names
+    unless table_exists?(:tags)
+      create_table :tags, comment: 'Unique tag names for categorization' do |t|
+        t.text :name, null: false, comment: 'Hierarchical tag in format: root:level1:level2 (e.g., database:postgresql:timescaledb)'
+        t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this tag was created'
+      end
+
+      add_index :tags, :name, unique: true, name: 'idx_tags_name_unique'
+      add_index :tags, :name, using: :btree, opclass: :text_pattern_ops, name: 'idx_tags_name_pattern'
+    end
+
+    # Create join table for many-to-many relationship
+    unless table_exists?(:node_tags)
+      create_table :node_tags, comment: 'Join table connecting nodes to tags (many-to-many)' do |t|
+        t.bigint :node_id, null: false, comment: 'ID of the node being tagged'
+        t.bigint :tag_id, null: false, comment: 'ID of the tag being applied'
+        t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this association was created'
+      end
+
+      add_index :node_tags, [:node_id, :tag_id], unique: true, name: 'idx_node_tags_unique'
+      add_index :node_tags, :node_id, name: 'idx_node_tags_node_id'
+      add_index :node_tags, :tag_id, name: 'idx_node_tags_tag_id'
+    end
+
+    # Add foreign keys (outside table_exists check so they get added even if table already exists)
+    unless foreign_key_exists?(:node_tags, :nodes, column: :node_id)
+      add_foreign_key :node_tags, :nodes, column: :node_id, primary_key: :id, on_delete: :cascade
+    end
+
+    unless foreign_key_exists?(:node_tags, :tags, column: :tag_id)
+      add_foreign_key :node_tags, :tags, column: :tag_id, primary_key: :id, on_delete: :cascade
+    end
+  end
+end

data/db/migrate/20250101000007_add_node_vector_indexes.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+class AddNodeVectorIndexes < ActiveRecord::Migration[7.1]
+  def up
+    # Vector similarity search index (HNSW for better performance)
+    execute <<-SQL
+      CREATE INDEX IF NOT EXISTS idx_nodes_embedding ON nodes
+        USING hnsw (embedding vector_cosine_ops)
+        WITH (m = 16, ef_construction = 64)
+    SQL
+
+    # Full-text search on conversation content
+    execute <<-SQL
+      CREATE INDEX IF NOT EXISTS idx_nodes_content_gin ON nodes
+        USING gin(to_tsvector('english', content))
+    SQL
+
+    # Trigram indexes for fuzzy matching on conversation content
+    execute <<-SQL
+      CREATE INDEX IF NOT EXISTS idx_nodes_content_trgm ON nodes
+        USING gin(content gin_trgm_ops)
+    SQL
+  end
+
+  def down
+    remove_index :nodes, name: 'idx_nodes_embedding'
+    remove_index :nodes, name: 'idx_nodes_content_gin'
+    remove_index :nodes, name: 'idx_nodes_content_trgm'
+  end
+end