htm 0.0.31 → 0.0.32

data/docs/robots/two-tier-memory.md

@@ -189,20 +189,13 @@ end
 
 ```ruby
 # User having back-and-forth coding conversation
-context = htm.create_context(strategy: :recent, max_tokens: 8000)
+context = htm.working_memory.assemble_context(strategy: :recent, max_tokens: 8000)
 # Recent messages prioritized for coherent conversation flow
 ```
 
-#### 2. Important (`:important`)
+#### 2. Frequent (`:frequent`)
 
-Sort by importance score, highest first.
-
-```ruby
-def assemble_context(strategy: :important, max_tokens: nil)
-  nodes = @nodes.sort_by { |k, v| -v[:importance] }.map(&:last)
-  build_context(nodes, max_tokens || @max_tokens)
-end
-```
+Sort by access count, most accessed first.
 
 **Best For:**
 
@@ -215,8 +208,8 @@ end
 
 ```ruby
 # LLM helping with architectural review
-context = htm.create_context(strategy: :important)
-# Critical decisions and facts prioritized over recent chat
+context = htm.working_memory.assemble_context(strategy: :frequent)
+# Most frequently accessed decisions and facts prioritized
 ```
 
 #### 3. Balanced (`:balanced`) - **Recommended Default**
@@ -252,7 +245,7 @@ end
 
 ```ruby
 # Code helper assisting with debugging and design
-context = htm.create_context(strategy: :balanced)
+context = htm.working_memory.assemble_context(strategy: :balanced)
 # Recent debugging context + important architectural decisions
 ```
 
@@ -287,7 +280,7 @@ Long-term memory provides unlimited, durable storage for all memories with advan
 |--------|---------|
 | **Purpose** | Permanent knowledge base |
 | **Capacity** | Effectively unlimited |
-| **Storage** | PostgreSQL 16+ with TimescaleDB |
+| **Storage** | PostgreSQL 16+ with pgvector and pg_trgm |
 | **Access Pattern** | RAG-based retrieval (semantic + temporal) |
 | **Retention** | Permanent (explicit deletion only) |
 | **Lifetime** | Forever (survives process restarts) |
@@ -298,16 +291,16 @@ Long-term memory provides unlimited, durable storage for all memories with advan
 ```sql
 CREATE TABLE nodes (
   id BIGSERIAL PRIMARY KEY,
-  key TEXT UNIQUE NOT NULL,
-  value TEXT NOT NULL,
+  content TEXT NOT NULL,
+  content_hash TEXT UNIQUE,
   type TEXT,
-  importance REAL DEFAULT 1.0,
   token_count INTEGER,
-  in_working_memory BOOLEAN DEFAULT FALSE,
-  robot_id TEXT NOT NULL REFERENCES robots(id),
-  embedding vector(1536),
+  metadata JSONB DEFAULT '{}',
+  source_id BIGINT REFERENCES file_sources(id),
+  embedding vector(3072),
+  deleted_at TIMESTAMP,
   created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-  ...
+  updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
 );
 
 -- HNSW index for vector similarity
@@ -316,58 +309,44 @@ CREATE INDEX idx_nodes_embedding ON nodes
   WITH (m = 16, ef_construction = 64);
 
 -- Full-text search
-CREATE INDEX idx_nodes_value_gin ON nodes
-  USING gin(to_tsvector('english', value));
+CREATE INDEX idx_nodes_content_gin ON nodes
+  USING gin(to_tsvector('english', content));
 ```
 
-### Why PostgreSQL + TimescaleDB?
+### Why PostgreSQL?
 
 **PostgreSQL provides:**
 
 - ACID guarantees for data integrity
 - Rich ecosystem and tooling
 - pgvector for vector similarity search
-- Full-text search with GIN indexes
+- Full-text search with GIN indexes and pg_trgm for fuzzy matching
+- Soft delete support via `deleted_at` column
+- JSONB metadata with GIN index for flexible filtering
 - Mature, production-proven
 
-**TimescaleDB adds:**
-
-- Hypertable partitioning by time
-- Automatic compression (70-90% reduction)
-- Time-range query optimization
-- Retention policies for data lifecycle
-
 !!! info "Related ADR"
-    See [ADR-001: Use PostgreSQL with TimescaleDB for Storage](../architecture/adrs/001-postgresql-timescaledb.md) for complete rationale.
+    See [ADR-001: Use PostgreSQL for Storage](../architecture/adrs/001-postgresql-timescaledb.md) for complete rationale.
 
 ### Long-Term Memory Operations
 
 #### Add Node
 
 ```ruby
-def add(key:, value:, embedding:, importance:, token_count:, robot_id:, type: nil)
-  result = @db.exec_params(<<~SQL, [key, value, type, importance, token_count, robot_id, embedding])
-    INSERT INTO nodes (key, value, type, importance, token_count, robot_id, embedding, in_working_memory)
-    VALUES ($1, $2, $3, $4, $5, $6, $7, TRUE)
-    RETURNING id
-  SQL
-
-  result[0]['id'].to_i
-end
+# High-level API (recommended)
+node_id = htm.remember("PostgreSQL is great for vector search",
+                       type: :fact,
+                       tags: ["database:postgresql"],
+                       metadata: { source: "docs" })
 ```
 
 **Time Complexity:** O(log n) with B-tree and HNSW indexes
 
-#### Retrieve by Key
+#### Retrieve by ID
 
 ```ruby
-def retrieve(key)
-  result = @db.exec_params(<<~SQL, [key])
-    SELECT * FROM nodes WHERE key = $1
-  SQL
-
-  result.first
-end
+# Direct model lookup
+node = HTM::Models::Node.first(id: node_id)
 ```
 
 **Time Complexity:** O(1) with unique index on key
@@ -447,7 +426,7 @@ Pure semantic similarity using cosine distance between embeddings.
 ```ruby
 # Query: "database optimization"
 # Finds: "PostgreSQL index tuning", "query performance", "EXPLAIN ANALYZE"
-memories = htm.recall(timeframe: "last month", topic: "database optimization", strategy: :vector)
+memories = htm.recall("database optimization", timeframe: "last month", strategy: :vector)
 ```
 
 ##### 2. Full-Text Search (`:fulltext`)
@@ -466,7 +445,7 @@ Keyword-based matching using PostgreSQL GIN indexes.
 ```ruby
 # Query: "TimescaleDB compression"
 # Finds exact matches for "TimescaleDB" and "compression"
-memories = htm.recall(timeframe: "last week", topic: "TimescaleDB compression", strategy: :fulltext)
+memories = htm.recall("TimescaleDB compression", timeframe: "last week", strategy: :fulltext)
 ```
 
 ##### 3. Hybrid Search (`:hybrid`) - **Recommended**
@@ -483,7 +462,7 @@ Combines vector and full-text with RRF scoring.
 
 ```ruby
 # Combines semantic similarity AND keyword matching
-memories = htm.recall(timeframe: "last month", topic: "PostgreSQL performance", strategy: :hybrid)
+memories = htm.recall("PostgreSQL performance", timeframe: "last month", strategy: :hybrid)
 ```
 
 ### Performance Characteristics
@@ -499,11 +478,11 @@ memories = htm.recall(timeframe: "last month", topic: "PostgreSQL performance",
 
 **Storage Efficiency:**
 
-With TimescaleDB compression (after 30 days):
+With PostgreSQL TOAST compression:
 
-- Text node: ~1KB → ~200 bytes (80% reduction)
-- Node with embedding: ~7KB → ~1-2KB (70-85% reduction)
-- 100,000 nodes: ~700MB → ~100-200MB
+- Text node: ~1KB (minimal overhead)
+- Node with embedding: ~7KB (1536-dim float32 vector)
+- 100,000 nodes: ~700MB typical storage
 
 ## Memory Flow: Add → Evict → Recall
 
@@ -524,7 +503,7 @@ htm.memory_stats[:working_memory]
 # }
 
 # Add large memory
-htm.add_node("large_doc", large_documentation, importance: 7.0)
+htm.remember(large_documentation)
 
 # HTM automatically:
 # 1. Generates embedding (Ollama: ~50ms)
@@ -545,8 +524,8 @@ htm.memory_stats[:working_memory]
 # }
 
 # Evicted nodes still in long-term memory!
-evicted_node = htm.retrieve("old_debug_log")  # Still works
-# => { "key" => "old_debug_log", "in_working_memory" => false, ... }
+# Retrieve via recall with the original content
+evicted_results = htm.recall("old_debug_log", strategy: :fulltext)  # Still works
 ```
 
 ## Context Assembly Strategies in Detail
@@ -556,8 +535,8 @@ evicted_node = htm.retrieve("old_debug_log")  # Still works
 | Strategy | Sort Key | Best Use Case | Typical Output |
 |----------|----------|---------------|----------------|
 | **Recent** | Access order (newest first) | Conversations, debugging | Recent 5-10 messages |
-| **Important** | Importance score (highest first) | Planning, decisions | Top 10 most important facts |
-| **Balanced** | `importance / (1 + hours)` | General assistant | Mix of recent + important |
+| **Frequent** | Access count (highest first) | Planning, decisions | Top 10 most accessed facts |
+| **Balanced** | `importance / (1 + hours)` | General assistant | Mix of recent + frequent |
 
 ### Code Examples
 
@@ -565,7 +544,7 @@ evicted_node = htm.retrieve("old_debug_log")  # Still works
 
 ```ruby
 # Assemble context from most recent memories
-context = htm.create_context(strategy: :recent, max_tokens: 8000)
+context = htm.working_memory.assemble_context(strategy: :recent, max_tokens: 8000)
 
 # Typical output (recent conversation):
 # """
@@ -577,17 +556,17 @@ context = htm.create_context(strategy: :recent, max_tokens: 8000)
 # """
 ```
 
-#### Important Strategy
+#### Frequent Strategy
 
 ```ruby
-# Assemble context from most important memories
-context = htm.create_context(strategy: :important, max_tokens: 4000)
+# Assemble context from most frequently accessed memories
+context = htm.working_memory.assemble_context(strategy: :frequent, max_tokens: 4000)
 
 # Typical output (critical facts):
 # """
-# Decision: Use PostgreSQL with TimescaleDB for storage (importance: 10.0)
-# User preference: Always use debug_me over puts (importance: 9.0)
-# Architecture: Two-tier memory system (importance: 9.0)
+# Decision: Use PostgreSQL for storage (accessed 10 times)
+# User preference: Always use debug_me over puts (accessed 8 times)
+# Architecture: Two-tier memory system (accessed 7 times)
 # ...
 # """
 ```
@@ -596,13 +575,13 @@ context = htm.create_context(strategy: :important, max_tokens: 4000)
 
 ```ruby
 # Assemble context with time decay
-context = htm.create_context(strategy: :balanced)
+context = htm.working_memory.assemble_context(strategy: :balanced)
 
 # Typical output (hybrid):
 # """
-# Recent debugging: ValueError in embedding service (importance: 7.0, 10 min ago) [score: 42.0]
-# Critical decision: PostgreSQL chosen (importance: 10.0, 3 days ago) [score: 0.14]
-# Current task: Implementing RAG search (importance: 6.0, 1 hour ago) [score: 3.0]
+# Recent debugging: ValueError in embedding service (10 min ago)
+# Current task: Implementing RAG search (1 hour ago)
+# Critical decision: PostgreSQL chosen (accessed 10 times, 3 days ago)
 # ...
 # """
 ```
@@ -624,55 +603,54 @@ htm = HTM.new(working_memory_size: 128_000)  # Default
 #### 2. Adjust Importance Scores
 
 ```ruby
-# High importance for critical information
-htm.add_node("user_preference", "User prefers Vim keybindings", importance: 9.0)
+# Store critical information
+htm.remember("User prefers Vim keybindings", metadata: { priority: "high" })
 
-# Low importance for transient information
-htm.add_node("debug_log", "Temporary debug output", importance: 1.0)
+# Store transient information
+htm.remember("Temporary debug output", metadata: { category: "debug" })
 
-# Medium importance for general context
-htm.add_node("discussion", "Discussed API design patterns", importance: 5.0)
+# Store general context
+htm.remember("Discussed API design patterns")
 ```
 
 #### 3. Use Appropriate Context Strategy
 
 ```ruby
 # For chat: recent strategy
-chat_context = htm.create_context(strategy: :recent, max_tokens: 8000)
+chat_context = htm.working_memory.assemble_context(strategy: :recent, max_tokens: 8000)
 
-# For planning: important strategy
-planning_context = htm.create_context(strategy: :important, max_tokens: 4000)
+# For planning: frequent strategy
+planning_context = htm.working_memory.assemble_context(strategy: :frequent, max_tokens: 4000)
 
 # For general: balanced strategy (default)
-general_context = htm.create_context(strategy: :balanced)
+general_context = htm.working_memory.assemble_context(strategy: :balanced)
 ```
 
 ### Long-Term Memory Optimization
 
-#### 1. Leverage TimescaleDB Compression
+#### 1. Leverage PostgreSQL Partitioning
 
 ```sql
--- Enable compression after 30 days
-SELECT add_compression_policy('nodes', INTERVAL '30 days');
+-- Create a partial index for active (non-deleted) nodes
+CREATE INDEX CONCURRENTLY idx_nodes_active
+  ON nodes (created_at DESC)
+  WHERE deleted_at IS NULL;
 
--- Compress by robot_id and type for better ratio
-ALTER TABLE nodes SET (
-  timescaledb.compress,
-  timescaledb.compress_segmentby = 'robot_id,type'
-);
+-- Periodically VACUUM to reclaim space from soft-deleted rows
+VACUUM ANALYZE nodes;
 ```
 
 #### 2. Use Appropriate Search Strategy
 
 ```ruby
 # For exact matches: full-text
-exact_matches = htm.recall(timeframe: "last week", topic: "PostgreSQL", strategy: :fulltext)
+exact_matches = htm.recall("PostgreSQL", timeframe: "last week", strategy: :fulltext)
 
 # For semantic similarity: vector
-similar_concepts = htm.recall(timeframe: "last month", topic: "database performance", strategy: :vector)
+similar_concepts = htm.recall("database performance", timeframe: "last month", strategy: :vector)
 
 # For best results: hybrid (default)
-best_results = htm.recall(timeframe: "last month", topic: "PostgreSQL performance", strategy: :hybrid)
+best_results = htm.recall("PostgreSQL performance", timeframe: "last month", strategy: :hybrid)
 ```
 
 #### 3. Index Tuning

data/docs/setup_local_database.md

@@ -36,24 +36,7 @@ psql --version
 brew install pgvector
 ```
 
-### 2.2 Install TimescaleDB (Time-Series Database)
-
-```bash
-# Add TimescaleDB tap
-brew tap timescale/tap
-
-# Install TimescaleDB
-brew install timescaledb
-
-# Configure PostgreSQL for TimescaleDB
-# This updates your postgresql.conf with TimescaleDB settings
-timescaledb-tune --quiet --yes
-
-# Restart PostgreSQL to load TimescaleDB
-brew services restart postgresql@17
-```
-
-### 2.3 pg_trgm (Trigram Matching)
+### 2.2 pg_trgm (Trigram Matching)
 
 This extension is included with PostgreSQL, no installation needed.
 
@@ -96,9 +79,6 @@ createdb htm_development
 # Enable pgvector
 psql -d htm_development -c "CREATE EXTENSION IF NOT EXISTS vector;"
 
-# Enable TimescaleDB
-psql -d htm_development -c "CREATE EXTENSION IF NOT EXISTS timescaledb;"
-
 # Enable pg_trgm (trigram matching)
 psql -d htm_development -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;"
 ```
@@ -112,20 +92,15 @@ be rake htm:db:setup
 This will:
 1. Verify extensions are available
 2. Create HTM schema (tables, indexes, triggers)
-3. Set up TimescaleDB hypertables
-4. Run any pending migrations
+3. Run any pending migrations
 
 Expected output:
 
 ```
-✓ TimescaleDB version: X.X.X
 ✓ pgvector version: X.X.X
 ✓ pg_trgm version: X.X.X
 Creating HTM schema...
 ✓ Schema created
-✓ Created hypertable for operations_log
-✓ Created hypertable for nodes
-✓ Enabled compression for nodes older than 30 days
 ✓ HTM database schema created successfully
 ```
 
@@ -176,17 +151,13 @@ brew install pgvector
 psql -d htm_development -c "CREATE EXTENSION IF NOT EXISTS vector;"
 ```
 
-### Error: "TimescaleDB extension not found"
+### Error: "pg_trgm extension not found"
 
-**Problem:** TimescaleDB not installed or not enabled.
+**Problem:** pg_trgm not enabled (it ships with PostgreSQL but needs to be enabled).
 
 **Solution:**
 ```bash
-brew tap timescale/tap
-brew install timescaledb
-timescaledb-tune --quiet --yes
-brew services restart postgresql@17
-psql -d htm_development -c "CREATE EXTENSION IF NOT EXISTS timescaledb;"
+psql -d htm_development -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;"
 ```
 
 ### Error: "Connection refused" to Ollama
@@ -212,27 +183,12 @@ direnv allow
 echo $HTM_DATABASE__URL  # Verify it's set
 ```
 
-## Switching Back to TimescaleDB Cloud
+## Switching to a Remote Database
 
-To switch back to TimescaleDB Cloud (production), edit `.envrc`:
+To switch to a remote PostgreSQL database (e.g. production), update your environment:
 
 ```bash
-# Comment out localhost config
-# export HTM_DATABASE__HOST=localhost
-# export HTM_DATABASE__PORT=5432
-# export HTM_DATABASE__NAME=htm_development
-# export HTM_DATABASE__USER=${USER}
-# export HTM_DATABASE__PASSWORD=
-# export HTM_DATABASE__URL="postgresql://${HTM_DATABASE__USER}@${HTM_DATABASE__HOST}:${HTM_DATABASE__PORT}/${HTM_DATABASE__NAME}?sslmode=prefer"
-
-# Uncomment TimescaleDB Cloud config
-export HTM_SERVICE_NAME=$TIGER_SERVICE_NAME
-export HTM_DATABASE__URL=$TIGER_DBURL
-export HTM_DATABASE__NAME=$TIGER_DBNAME
-export HTM_DATABASE__USER=$TIGER_DBUSER
-export HTM_DATABASE__PASSWORD=$TIGER_DBPASS
-export HTM_DATABASE__HOST=$TIGER_DBHOST
-export HTM_DATABASE__PORT=$TIGER_DBPORT
+export HTM_DATABASE__URL="postgresql://user:password@remote-host:5432/htm_production"
 ```
 
 Then reload:
@@ -266,14 +222,12 @@ PostgreSQL Version:
 Extensions:
   pg_trgm (X.X.X)
   plpgsql (X.X.X)
-  timescaledb (X.X.X)
   vector (X.X.X)
 
 HTM Tables:
   nodes: X rows
   tags: X rows
   robots: X rows
-  operations_log: X rows
   schema_migrations: X rows
 
 Database Size: XX MB

data/docs/using_rake_tasks_in_your_app.md

@@ -66,10 +66,10 @@ Create a `.envrc` file in your application's root:
 export HTM_DATABASE__URL="postgresql://user:password@host:port/dbname?sslmode=require"
 
 # Or use individual parameters
-export HTM_DATABASE__HOST="your-host.tsdb.cloud.timescale.com"
-export HTM_DATABASE__PORT="37807"
-export HTM_DATABASE__NAME="tsdb"
-export HTM_DATABASE__USER="tsdbadmin"
+export HTM_DATABASE__HOST="your-db-host.example.com"
+export HTM_DATABASE__PORT="5432"
+export HTM_DATABASE__NAME="htm_production"
+export HTM_DATABASE__USER="postgres"
 export HTM_DATABASE__PASSWORD="your_password"
 
 # Embedding configuration
@@ -277,7 +277,7 @@ jobs:
 
     services:
       postgres:
-        image: timescale/timescaledb-ha:pg17
+        image: pgvector/pgvector:pg17
         env:
           POSTGRES_PASSWORD: postgres
         options: >-
@@ -323,7 +323,7 @@ services:
     command: bash -c "rake htm:db:setup && rake app:start"
 
   db:
-    image: timescale/timescaledb-ha:pg17
+    image: pgvector/pgvector:pg17
     environment:
       - POSTGRES_PASSWORD=postgres
 ```

data/examples/01_basic_usage.rb

@@ -18,6 +18,7 @@ ExamplesHelper.section "HTM Basic Usage Example"
 ExamplesHelper.print_environment
 
 # Verify database is available
+ExamplesHelper.reset_if_requested!
 ExamplesHelper.require_database!
 
 begin

data/examples/03_custom_llm_configuration.rb

@@ -19,6 +19,7 @@ ExamplesHelper.section "HTM Custom LLM Configuration Example"
 ExamplesHelper.print_environment
 
 # Verify database is available
+ExamplesHelper.reset_if_requested!
 ExamplesHelper.require_database!
 
 # Example 1: Use Default Configuration (RubyLLM with Ollama)

data/examples/04_file_loader_usage.rb

@@ -24,6 +24,7 @@ ExamplesHelper.section "HTM File Loader Example"
 ExamplesHelper.print_environment
 
 # Verify database is available
+ExamplesHelper.reset_if_requested!
 ExamplesHelper.require_database!
 
 begin

data/examples/05_timeframe_demo.rb

@@ -13,6 +13,7 @@
 
 require_relative 'examples_helper'
 
+ExamplesHelper.reset_if_requested!
 ExamplesHelper.require_database!
 
 puts <<~HEADER

data/examples/06_example_app/app.rb

@@ -20,6 +20,7 @@ class ExampleApp
     ExamplesHelper.print_environment
 
     # Verify database is available
+    ExamplesHelper.reset_if_requested!
     ExamplesHelper.require_database!
 
     puts "Database configured: #{HTM.config.actual_database_name}"

data/examples/07_cli_app/htm_cli.rb

@@ -379,6 +379,7 @@ class HTMCli
 end
 
 # Check database configuration
+ExamplesHelper.reset_if_requested!
 ExamplesHelper.require_database!
 
 # Check Ollama connection (optional but recommended)

data/examples/09_mcp_client.rb

@@ -58,6 +58,7 @@ class HTMMcpClient
   private
 
   def validate_environment
+    ExamplesHelper.reset_if_requested!
     ExamplesHelper.require_database!
 
     unless File.exist?(MCP_SERVER_PATH)

data/examples/10_telemetry/demo.rb

@@ -109,6 +109,7 @@ class TelemetryDemo
     puts "  [OK] Environment: #{HTM.config.environment}"
     puts "  [OK] Database: #{HTM.config.actual_database_name}"
 
+    ExamplesHelper.reset_if_requested!
     ExamplesHelper.require_database!
     puts
   end

data/examples/11_robot_groups/multi_process.rb

@@ -123,6 +123,7 @@ def run_demo
   BANNER
 
   ExamplesHelper.print_environment
+  ExamplesHelper.reset_if_requested!
   ExamplesHelper.require_database!
 
   group_name = "demo-#{Time.now.to_i}"

data/examples/11_robot_groups/same_process.rb

@@ -34,6 +34,7 @@ require 'json'
 
 ExamplesHelper.section "HTM Robot Group Demo - Shared Working Memory & Failover"
 ExamplesHelper.print_environment
+ExamplesHelper.reset_if_requested!
 ExamplesHelper.require_database!
 
 begin

data/examples/12_rails_app/.envrc

@@ -0,0 +1,12 @@
+# htm/examples/12_rails_app/.envrc
+
+source_up
+
+export RR=`pwd`
+
+# ChatBot settings
+export RAILS_ENV=development
+
+# HTM Settings
+unset HTM_DATABASE__URL
+export HTM_DATABASE__NAME=htm_$HTM_ENV

data/examples/12_rails_app/Gemfile

@@ -2,18 +2,23 @@
 
 source 'https://rubygems.org'
 
-# Rails - full-stack setup
-gem 'rails', '~> 7.1'
+# Rails 8.1 - full-stack setup
+gem 'rails', '~> 8.1'
 gem 'puma', '~> 6.0'
 
 # HTM (use local development version)
 gem 'htm', path: '../..'
 
+# RubyLLM for chat persistence
+gem 'ruby_llm', '>= 1.0'
+
 # PostgreSQL (required for HTM)
 gem 'pg', '~> 1.5'
 
-# Ruby 4.0+ bundled gems (no longer in stdlib)
+# Ruby 3.2+ bundled gems (no longer in stdlib)
 gem 'ostruct'
+gem 'logger'
+gem 'benchmark'
 
 # Frontend - using CDN for Tailwind and Hotwire (no build step)
 gem 'propshaft'                    # Asset pipeline for CSS/images