htm 0.0.20 → 0.0.30

data/db/migrate/00007_create_robot_nodes.rb

@@ -1,33 +1,47 @@
 # frozen_string_literal: true
 
-class CreateRobotNodes < ActiveRecord::Migration[7.1]
-  def change
-    create_table :robot_nodes, comment: 'Join table connecting robots to nodes (many-to-many)' do |t|
-      t.bigint :robot_id, null: false, comment: 'ID of the robot that remembered this node'
-      t.bigint :node_id, null: false, comment: 'ID of the node being remembered'
-      t.timestamptz :first_remembered_at, default: -> { 'CURRENT_TIMESTAMP' },
-                    comment: 'When this robot first remembered this content'
-      t.timestamptz :last_remembered_at, default: -> { 'CURRENT_TIMESTAMP' },
-                    comment: 'When this robot last tried to remember this content'
-      t.integer :remember_count, default: 1, null: false,
-                comment: 'Number of times this robot has tried to remember this content'
-      t.boolean :working_memory, default: false, null: false,
-                comment: 'True if this node is currently in the robot working memory'
-      t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }
-      t.timestamptz :updated_at, default: -> { 'CURRENT_TIMESTAMP' }
-      t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
+require_relative '../../lib/htm/migration'
+
+class CreateRobotNodes < HTM::Migration
+  def up
+    create_table(:robot_nodes) do
+      primary_key :id
+      Bignum :robot_id, null: false
+      Bignum :node_id, null: false
+      DateTime :first_remembered_at, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :last_remembered_at, default: Sequel::CURRENT_TIMESTAMP
+      Integer :remember_count, default: 1, null: false
+      TrueClass :working_memory, default: false, null: false
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :updated_at, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :deleted_at
+    end
+
+    add_index :robot_nodes, [:robot_id, :node_id], unique: true, name: :idx_robot_nodes_unique
+    add_index :robot_nodes, :robot_id, name: :idx_robot_nodes_robot_id
+    add_index :robot_nodes, :node_id, name: :idx_robot_nodes_node_id
+    add_index :robot_nodes, :last_remembered_at, name: :idx_robot_nodes_last_remembered_at
+    add_index :robot_nodes, :deleted_at, name: :idx_robot_nodes_deleted_at
+
+    # Partial index for working memory queries
+    run "CREATE INDEX idx_robot_nodes_working_memory ON robot_nodes (robot_id, working_memory) WHERE working_memory = true"
+
+    alter_table(:robot_nodes) do
+      add_foreign_key [:robot_id], :robots, on_delete: :cascade
+      add_foreign_key [:node_id], :nodes, on_delete: :cascade
     end
 
-    add_index :robot_nodes, [:robot_id, :node_id], unique: true, name: 'idx_robot_nodes_unique'
-    add_index :robot_nodes, :robot_id, name: 'idx_robot_nodes_robot_id'
-    add_index :robot_nodes, :node_id, name: 'idx_robot_nodes_node_id'
-    add_index :robot_nodes, :last_remembered_at, name: 'idx_robot_nodes_last_remembered_at'
-    add_index :robot_nodes, :deleted_at, name: 'idx_robot_nodes_deleted_at'
-    add_index :robot_nodes, [:robot_id, :working_memory],
-              where: 'working_memory = true',
-              name: 'idx_robot_nodes_working_memory'
+    run "COMMENT ON TABLE robot_nodes IS 'Join table connecting robots to nodes (many-to-many)'"
+    run "COMMENT ON COLUMN robot_nodes.robot_id IS 'ID of the robot that remembered this node'"
+    run "COMMENT ON COLUMN robot_nodes.node_id IS 'ID of the node being remembered'"
+    run "COMMENT ON COLUMN robot_nodes.first_remembered_at IS 'When this robot first remembered this content'"
+    run "COMMENT ON COLUMN robot_nodes.last_remembered_at IS 'When this robot last tried to remember this content'"
+    run "COMMENT ON COLUMN robot_nodes.remember_count IS 'Number of times this robot has tried to remember this content'"
+    run "COMMENT ON COLUMN robot_nodes.working_memory IS 'True if this node is currently in the robot working memory'"
+    run "COMMENT ON COLUMN robot_nodes.deleted_at IS 'Soft delete timestamp'"
+  end
 
-    add_foreign_key :robot_nodes, :robots, column: :robot_id, on_delete: :cascade
-    add_foreign_key :robot_nodes, :nodes, column: :node_id, on_delete: :cascade
+  def down
+    drop_table(:robot_nodes)
   end
 end

data/db/schema.sql

@@ -3,6 +3,22 @@
 -- DO NOT EDIT THIS FILE MANUALLY
 -- Run 'rake htm:db:schema:dump' to regenerate
 
+--
+-- Name: paradedb; Type: SCHEMA; Schema: -; Owner: -
+--
+
+CREATE SCHEMA paradedb;
+
+--
+-- Name: pg_search; Type: EXTENSION; Schema: -; Owner: -
+--
+
+CREATE EXTENSION IF NOT EXISTS pg_search WITH SCHEMA paradedb;
+
+--
+-- Name: EXTENSION pg_search; Type: COMMENT; Schema: -; Owner: -
+--
+
 --
 -- Name: pg_trgm; Type: EXTENSION; Schema: -; Owner: -
 --
@@ -793,4 +809,4 @@ ALTER TABLE ONLY public.robot_nodes
 -- PostgreSQL database dump complete
 --
 
-\unrestrict 4WlUqnJzNHaNhcr67XLIIhAvRPidZODUPGkM34l27SvmC0zu6dIsQdJ8dtu589Z
+\unrestrict 1ItB7RQU4jC5IvOL40FU9j9sS6bzk9jcKeDUYSOd78ym0sA7pq0FXYSOEoWsPh7

data/db/seeds.rb

@@ -6,14 +6,14 @@
 # and creates memory nodes with embeddings and tags.
 #
 # Configuration is read from environment variables:
-#   HTM_EMBEDDING_PROVIDER - Embedding provider (default: ollama)
-#   HTM_EMBEDDING_MODEL - Embedding model (default: nomic-embed-text)
-#   HTM_EMBEDDING_DIMENSIONS - Embedding dimensions (default: 768)
-#   HTM_TAG_PROVIDER - Tag extraction provider (default: ollama)
-#   HTM_TAG_MODEL - Tag extraction model (default: gemma3)
-#   OLLAMA_URL - Ollama server URL (default: http://localhost:11434)
-#   HTM_EMBEDDING_TIMEOUT - Embedding generation timeout in seconds (default: 120)
-#   HTM_TAG_TIMEOUT - Tag generation timeout in seconds (default: 180)
+#   HTM_EMBEDDING__PROVIDER - Embedding provider (default: ollama)
+#   HTM_EMBEDDING__MODEL - Embedding model (default: nomic-embed-text)
+#   HTM_EMBEDDING__DIMENSIONS - Embedding dimensions (default: 768)
+#   HTM_TAG__PROVIDER - Tag extraction provider (default: ollama)
+#   HTM_TAG__MODEL - Tag extraction model (default: gemma3)
+#   HTM_PROVIDERS__OLLAMA__URL - Ollama server URL (default: http://localhost:11434)
+#   HTM_EMBEDDING__TIMEOUT - Embedding generation timeout in seconds (default: 120)
+#   HTM_TAG__TIMEOUT - Tag generation timeout in seconds (default: 180)
 #   HTM_CONNECTION_TIMEOUT - LLM connection timeout in seconds (default: 30)
 #   HTM_DATABASE__URL - Database connection URL
 #
@@ -30,13 +30,13 @@ puts "=" * 80
 puts
 
 # Configure HTM using environment variables or defaults
-embedding_provider = (ENV['HTM_EMBEDDING_PROVIDER'] || 'ollama').to_sym
-embedding_model = ENV['HTM_EMBEDDING_MODEL'] || 'nomic-embed-text'
-embedding_dimensions = (ENV['HTM_EMBEDDING_DIMENSIONS'] || '768').to_i
-tag_provider = (ENV['HTM_TAG_PROVIDER'] || 'ollama').to_sym
-tag_model = ENV['HTM_TAG_MODEL'] || 'gemma3'
-embedding_timeout = (ENV['HTM_EMBEDDING_TIMEOUT'] || '120').to_i
-tag_timeout = (ENV['HTM_TAG_TIMEOUT'] || '180').to_i
+embedding_provider = (ENV['HTM_EMBEDDING__PROVIDER'] || 'ollama').to_sym
+embedding_model = ENV['HTM_EMBEDDING__MODEL'] || 'nomic-embed-text'
+embedding_dimensions = (ENV['HTM_EMBEDDING__DIMENSIONS'] || '768').to_i
+tag_provider = (ENV['HTM_TAG__PROVIDER'] || 'ollama').to_sym
+tag_model = ENV['HTM_TAG__MODEL'] || 'gemma3'
+embedding_timeout = (ENV['HTM_EMBEDDING__TIMEOUT'] || '120').to_i
+tag_timeout = (ENV['HTM_TAG__TIMEOUT'] || '180').to_i
 connection_timeout = (ENV['HTM_CONNECTION_TIMEOUT'] || '60').to_i
 
 puts "Configuration:"
@@ -49,15 +49,15 @@ puts "  Timeouts: embedding=#{embedding_timeout}s, tag=#{tag_timeout}s, connecti
 puts
 
 HTM.configure do |c|
-  c.embedding_provider = embedding_provider
-  c.embedding_model = embedding_model
-  c.embedding_dimensions = embedding_dimensions
-  c.tag_provider = tag_provider
-  c.tag_model = tag_model
-  c.embedding_timeout = embedding_timeout
-  c.tag_timeout = tag_timeout
+  c.embedding.provider = embedding_provider
+  c.embedding.model = embedding_model
+  c.embedding.dimensions = embedding_dimensions
+  c.tag.provider = tag_provider
+  c.tag.model = tag_model
+  c.embedding.timeout = embedding_timeout
+  c.tag.timeout = tag_timeout
   c.connection_timeout = connection_timeout
-  c.ollama_url = ENV['OLLAMA_URL'] if ENV['OLLAMA_URL']
+  c.providers.ollama.url = ENV['HTM_PROVIDERS__OLLAMA__URL'] if ENV['HTM_PROVIDERS__OLLAMA__URL']
   c.reset_to_defaults  # Apply default implementations with configured settings
 end
 
@@ -72,32 +72,32 @@ puts "Creating sample conversation..."
 
 htm.remember(
   "What is TimescaleDB good for?",
-  source: "user"
+  metadata: { source: "user" }
 )
 
 htm.remember(
   "PostgreSQL with TimescaleDB provides efficient time-series data storage and querying capabilities.",
-  source: "assistant"
+  metadata: { source: "assistant" }
 )
 
 htm.remember(
   "How much training data do ML models need?",
-  source: "user"
+  metadata: { source: "user" }
 )
 
 htm.remember(
   "Machine learning models require large amounts of training data to achieve good performance.",
-  source: "assistant"
+  metadata: { source: "assistant" }
 )
 
 htm.remember(
   "Tell me about Ruby on Rails",
-  source: "user"
+  metadata: { source: "user" }
 )
 
 htm.remember(
   "Ruby on Rails is a web framework for building database-backed applications.",
-  source: "assistant"
+  metadata: { source: "assistant" }
 )
 
 puts "✓ Created 6 conversation messages (3 exchanges)"
@@ -135,7 +135,7 @@ if Dir.exist?(seed_data_dir)
           # Save previous section if we have one
           if current_section && current_paragraph.any?
             paragraph_text = current_paragraph.join(' ')
-            htm.remember(paragraph_text, source: filename)
+            htm.remember(paragraph_text, metadata: { source: filename })
             count += 1
             print "." if count % 10 == 0
           end
@@ -152,7 +152,7 @@ if Dir.exist?(seed_data_dir)
       # Don't forget the last section
       if current_section && current_paragraph.any?
         paragraph_text = current_paragraph.join(' ')
-        htm.remember(paragraph_text, source: filename)
+        htm.remember(paragraph_text, metadata: { source: filename })
         count += 1
       end
 
@@ -187,7 +187,7 @@ puts
 puts "Checking completion status..."
 
 # Check completion status
-nodes_with_embeddings = HTM::Models::Node.where.not(embedding: nil).count
+nodes_with_embeddings = HTM::Models::Node.exclude(embedding: nil).count
 puts "  - Nodes with embeddings: #{nodes_with_embeddings}/#{total_records}"
 
 total_tags = HTM::Models::NodeTag.count
@@ -203,6 +203,6 @@ if nodes_with_embeddings == total_records && total_tags > 0
 else
   puts "⚠ Some background jobs may still be running."
   puts "  Run this query to check progress:"
-  puts "  HTM::Models::Node.where.not(embedding: nil).count"
+  puts "  HTM::Models::Node.exclude(embedding: nil).count"
   puts "=" * 80
 end

data/docs/database/naming-convention.md

@@ -0,0 +1,244 @@
+# Database Naming Convention
+
+HTM enforces a strict database naming convention to prevent accidental data corruption or loss from operating on the wrong database.
+
+## The Convention
+
+Database names **must** follow this exact format:
+
+```
+{service_name}_{environment}
+```
+
+Where:
+- `service_name` is the value of `config.service.name` (default: `htm`)
+- `environment` is the value of `HTM_ENV` (or `RAILS_ENV` / `RACK_ENV` fallback)
+
+## Valid Examples
+
+| Service Name | Environment | Expected Database Name |
+|--------------|-------------|------------------------|
+| `htm` | `development` | `htm_development` |
+| `htm` | `test` | `htm_test` |
+| `htm` | `production` | `htm_production` |
+| `payroll` | `development` | `payroll_development` |
+| `payroll` | `production` | `payroll_production` |
+
+## Why This Matters
+
+Without strict enforcement, dangerous misconfigurations can go undetected:
+
+### Scenario 1: Environment Mismatch
+
+```bash
+# Developer thinks they're in test, but connected to production
+HTM_ENV=test
+HTM_DATABASE__URL="postgresql://user@host/htm_production"
+
+rake htm:db:drop  # DISASTER: Drops production database!
+```
+
+With the naming convention enforced, this command fails immediately:
+
+```
+Error: Database name does not follow naming convention!
+
+  Database names must be: {service_name}_{environment}
+
+  Service name: htm
+  Environment:  test
+  Expected:     htm_test
+  Actual:       htm_production
+```
+
+### Scenario 2: Service Mismatch
+
+```bash
+# HTM configured to use another application's database
+HTM_ENV=production
+# service.name = "htm" (default)
+HTM_DATABASE__URL="postgresql://user@host/payroll_production"
+
+rake htm:db:setup  # DISASTER: Corrupts payroll application's data!
+```
+
+With enforcement, this fails:
+
+```
+Error: Database name does not follow naming convention!
+
+  Service name: htm
+  Environment:  production
+  Expected:     htm_production
+  Actual:       payroll_production
+```
+
+## How Enforcement Works
+
+### Validation Points
+
+The naming convention is validated at these points:
+
+1. **All rake tasks** that depend on `htm:db:validate` (setup, migrate, drop, etc.)
+2. **Programmatic access** via `HTM.config.validate_database_name!`
+
+### No Bypass Option
+
+There is no way to skip this validation. If your database name doesn't match the convention, you must either:
+
+1. Rename your database to match the convention
+2. Change `HTM_ENV` to match the database suffix
+3. Change `config.service.name` to match the database prefix
+
+## Configuration
+
+### Setting the Service Name
+
+The service name defaults to `htm`. To use a different name:
+
+**Via environment variable:**
+```bash
+export HTM_SERVICE__NAME="myapp"
+```
+
+**Via configuration file (`config/htm.yml`):**
+```yaml
+service:
+  name: myapp
+```
+
+**Via Ruby configuration:**
+```ruby
+HTM.configure do |config|
+  config.service.name = "myapp"
+end
+```
+
+### Setting the Environment
+
+Environment is determined by (in priority order):
+
+1. `HTM_ENV`
+2. `RAILS_ENV`
+3. `RACK_ENV`
+4. Default: `development`
+
+**Valid environments:**
+- `development`
+- `test`
+- `production`
+
+These correspond to the top-level keys in `config/defaults.yml`.
+
+## Validation Methods
+
+### Check if Database Name is Valid
+
+```ruby
+config = HTM.config
+
+# Boolean check
+if config.valid_database_name?
+  puts "Database name is correct"
+else
+  puts "Expected: #{config.expected_database_name}"
+  puts "Actual: #{config.actual_database_name}"
+end
+```
+
+### Raise Error on Invalid Name
+
+```ruby
+config = HTM.config
+
+# Raises HTM::ConfigurationError if invalid
+config.validate_database_name!
+```
+
+### Get Expected and Actual Names
+
+```ruby
+config = HTM.config
+
+config.expected_database_name  # => "htm_test"
+config.actual_database_name    # => Extracted from URL or config
+```
+
+## Rake Task Validation
+
+All database-related rake tasks run validation automatically:
+
+```bash
+# These all validate the naming convention first:
+rake htm:db:setup
+rake htm:db:migrate
+rake htm:db:drop
+rake htm:db:reset
+rake htm:db:create
+```
+
+To validate manually without performing any operation:
+
+```bash
+rake htm:db:validate
+```
+
+## Migration Guide
+
+If you have existing databases that don't follow the convention:
+
+### Option 1: Rename the Database
+
+```bash
+# PostgreSQL
+psql -c "ALTER DATABASE old_name RENAME TO htm_development;"
+```
+
+### Option 2: Export and Import
+
+```bash
+# Export from old database
+pg_dump old_database > backup.sql
+
+# Create new database with correct name
+createdb htm_development
+
+# Import to new database
+psql htm_development < backup.sql
+```
+
+### Option 3: Change Your Service Name
+
+If your database is named `myapp_production`, set your service name to match:
+
+```bash
+export HTM_SERVICE__NAME="myapp"
+```
+
+## Error Messages
+
+When validation fails, you'll see a clear error message:
+
+```
+Error: Database name 'wrong_db' does not match expected 'htm_test'.
+Database names must follow the convention: {service_name}_{environment}
+  Service name: htm
+  Environment:  test
+  Expected:     htm_test
+  Actual:       wrong_db
+
+Either:
+  - Set HTM_DATABASE__URL to point to 'htm_test'
+  - Set HTM_DATABASE__NAME=htm_test
+  - Change HTM_ENV to match the database suffix
+```
+
+## Summary
+
+The strict database naming convention:
+
+- **Prevents** accidental operations on wrong environments
+- **Prevents** cross-application database corruption
+- **Requires** exact match of `{service_name}_{environment}`
+- **Has no bypass** - you must fix the configuration
+- **Validates automatically** on all database operations

data/docs/database_rake_tasks.md

@@ -219,6 +219,37 @@ $ rake htm:db:reset
 # Runs drop (with confirmation) then setup
 ```
 
+#### `rake htm:db:purge_all`
+Permanently removes all soft-deleted records from all tables.
+
+**What it does:**
+- Removes soft-deleted nodes, node_tags, and robot_nodes
+- Removes orphaned join table entries (pointing to non-existent nodes)
+- Removes orphaned propositions (where source_node_id no longer exists)
+- Removes orphaned robots (with no associated memory nodes)
+- Deletes in correct order for referential integrity
+
+**Safety:** Prompts for confirmation before deletion
+
+```bash
+$ rake htm:db:purge_all
+
+HTM Purge All Soft-Deleted Records
+============================================================
+
+Records to permanently delete:
+--------------------------------------------------------------
+Soft-deleted nodes:              23
+Soft-deleted node_tags:          45
+Orphaned propositions:           5
+Orphaned robots (no nodes):      2
+--------------------------------------------------------------
+Total records to delete:         75
+
+Proceed with permanent deletion? (yes/no): yes
+✓ Purge complete!
+```
+
 ---
 
 ## Environment Variables