htm 0.0.31 → 0.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.irbrc +2 -3
- data/.rubocop.yml +184 -0
- data/CHANGELOG.md +46 -0
- data/README.md +2 -0
- data/Rakefile +93 -12
- data/db/migrate/00008_create_node_relationships.rb +54 -0
- data/db/migrate/00009_fix_node_relationships_column_types.rb +17 -0
- data/db/schema.sql +124 -1
- data/docs/api/database.md +35 -57
- data/docs/api/embedding-service.md +1 -1
- data/docs/api/index.md +26 -15
- data/docs/api/working-memory.md +8 -8
- data/docs/architecture/index.md +5 -7
- data/docs/architecture/overview.md +5 -8
- data/docs/assets/images/htm-architecture-overview.svg +1 -1
- data/docs/assets/images/htm-context-assembly-flow.svg +2 -2
- data/docs/assets/images/htm-layered-architecture.svg +3 -3
- data/docs/assets/images/two-tier-memory-architecture.svg +1 -1
- data/docs/database/README.md +1 -0
- data/docs/database_rake_tasks.md +20 -28
- data/docs/development/contributing.md +5 -5
- data/docs/development/index.md +4 -7
- data/docs/development/schema.md +71 -1
- data/docs/development/setup.md +40 -82
- data/docs/development/testing.md +1 -1
- data/docs/examples/file-loading.md +4 -4
- data/docs/examples/mcp-client.md +1 -1
- data/docs/getting-started/quick-start.md +4 -4
- data/docs/guides/adding-memories.md +14 -1
- data/docs/guides/configuration.md +5 -5
- data/docs/guides/context-assembly.md +4 -4
- data/docs/guides/file-loading.md +12 -12
- data/docs/guides/getting-started.md +2 -2
- data/docs/guides/long-term-memory.md +7 -27
- data/docs/guides/propositions.md +20 -19
- data/docs/guides/recalling-memories.md +5 -5
- data/docs/guides/tags.md +18 -13
- data/docs/multi_framework_support.md +1 -1
- data/docs/robots/hive-mind.md +1 -1
- data/docs/robots/multi-robot.md +2 -2
- data/docs/robots/robot-groups.md +1 -1
- data/docs/robots/two-tier-memory.md +72 -94
- data/docs/setup_local_database.md +8 -54
- data/docs/using_rake_tasks_in_your_app.md +6 -6
- data/examples/01_basic_usage.rb +1 -0
- data/examples/03_custom_llm_configuration.rb +1 -0
- data/examples/04_file_loader_usage.rb +1 -0
- data/examples/05_timeframe_demo.rb +1 -0
- data/examples/06_example_app/app.rb +1 -0
- data/examples/07_cli_app/htm_cli.rb +1 -0
- data/examples/09_mcp_client.rb +1 -0
- data/examples/10_telemetry/demo.rb +1 -0
- data/examples/11_robot_groups/multi_process.rb +1 -0
- data/examples/11_robot_groups/same_process.rb +1 -0
- data/examples/12_rails_app/.envrc +12 -0
- data/examples/12_rails_app/Gemfile +8 -3
- data/examples/12_rails_app/Gemfile.lock +94 -89
- data/examples/12_rails_app/README.md +70 -19
- data/examples/12_rails_app/app/controllers/application_controller.rb +6 -0
- data/examples/12_rails_app/app/controllers/chats_controller.rb +305 -0
- data/examples/12_rails_app/app/controllers/dashboard_controller.rb +3 -0
- data/examples/12_rails_app/app/controllers/files_controller.rb +17 -2
- data/examples/12_rails_app/app/controllers/home_controller.rb +8 -0
- data/examples/12_rails_app/app/controllers/memories_controller.rb +9 -4
- data/examples/12_rails_app/app/controllers/messages_controller.rb +214 -0
- data/examples/12_rails_app/app/controllers/robots_controller.rb +11 -1
- data/examples/12_rails_app/app/controllers/tags_controller.rb +14 -1
- data/examples/12_rails_app/app/javascript/application.js +1 -1
- data/examples/12_rails_app/app/models/application_record.rb +5 -0
- data/examples/12_rails_app/app/models/chat.rb +36 -0
- data/examples/12_rails_app/app/models/message.rb +5 -0
- data/examples/12_rails_app/app/models/model.rb +5 -0
- data/examples/12_rails_app/app/models/tool_call.rb +5 -0
- data/examples/12_rails_app/app/views/chats/index.html.erb +61 -0
- data/examples/12_rails_app/app/views/chats/show.html.erb +213 -0
- data/examples/12_rails_app/app/views/dashboard/index.html.erb +3 -0
- data/examples/12_rails_app/app/views/files/index.html.erb +10 -5
- data/examples/12_rails_app/app/views/files/new.html.erb +4 -2
- data/examples/12_rails_app/app/views/files/show.html.erb +19 -3
- data/examples/12_rails_app/app/views/home/index.html.erb +45 -0
- data/examples/12_rails_app/app/views/layouts/application.html.erb +20 -18
- data/examples/12_rails_app/app/views/memories/_memory_card.html.erb +1 -1
- data/examples/12_rails_app/app/views/memories/deleted.html.erb +3 -1
- data/examples/12_rails_app/app/views/memories/edit.html.erb +2 -0
- data/examples/12_rails_app/app/views/memories/index.html.erb +2 -0
- data/examples/12_rails_app/app/views/memories/new.html.erb +2 -0
- data/examples/12_rails_app/app/views/memories/show.html.erb +4 -2
- data/examples/12_rails_app/app/views/messages/_message.html.erb +20 -0
- data/examples/12_rails_app/app/views/robots/index.html.erb +2 -0
- data/examples/12_rails_app/app/views/robots/new.html.erb +2 -0
- data/examples/12_rails_app/app/views/robots/show.html.erb +2 -0
- data/examples/12_rails_app/app/views/search/index.html.erb +59 -8
- data/examples/12_rails_app/app/views/shared/_navbar.html.erb +75 -29
- data/examples/12_rails_app/app/views/tags/index.html.erb +2 -0
- data/examples/12_rails_app/app/views/tags/show.html.erb +3 -1
- data/examples/12_rails_app/config/application.rb +1 -1
- data/examples/12_rails_app/config/database.yml +9 -5
- data/examples/12_rails_app/config/importmap.rb +1 -1
- data/examples/12_rails_app/config/initializers/htm.rb +9 -2
- data/examples/12_rails_app/config/initializers/ruby_llm.rb +33 -0
- data/examples/12_rails_app/config/routes.rb +39 -23
- data/examples/12_rails_app/db/migrate/20250124000001_create_ruby_llm_tables.rb +34 -0
- data/examples/12_rails_app/db/migrate/20250124000002_create_models_table.rb +28 -0
- data/examples/12_rails_app/db/schema.rb +67 -0
- data/examples/examples_helper.rb +25 -0
- data/lib/htm/circuit_breaker.rb +5 -6
- data/lib/htm/config/builder.rb +12 -12
- data/lib/htm/config/database.rb +21 -27
- data/lib/htm/config/validator.rb +12 -18
- data/lib/htm/config.rb +76 -65
- data/lib/htm/database.rb +193 -199
- data/lib/htm/embedding_service.rb +4 -9
- data/lib/htm/integrations/sinatra.rb +7 -7
- data/lib/htm/job_adapter.rb +14 -21
- data/lib/htm/jobs/generate_embedding_job.rb +28 -44
- data/lib/htm/jobs/generate_propositions_job.rb +29 -55
- data/lib/htm/jobs/generate_relationships_job.rb +137 -0
- data/lib/htm/jobs/generate_tags_job.rb +45 -67
- data/lib/htm/loaders/markdown_loader.rb +65 -112
- data/lib/htm/long_term_memory/fulltext_search.rb +1 -1
- data/lib/htm/long_term_memory/hybrid_search.rb +300 -128
- data/lib/htm/long_term_memory/node_operations.rb +2 -2
- data/lib/htm/long_term_memory/relevance_scorer.rb +100 -68
- data/lib/htm/long_term_memory/tag_operations.rb +87 -120
- data/lib/htm/long_term_memory/vector_search.rb +1 -1
- data/lib/htm/long_term_memory.rb +2 -1
- data/lib/htm/mcp/cli.rb +59 -58
- data/lib/htm/mcp/server.rb +5 -6
- data/lib/htm/mcp/tools.rb +30 -36
- data/lib/htm/migration.rb +10 -10
- data/lib/htm/models/node.rb +2 -3
- data/lib/htm/models/node_relationship.rb +72 -0
- data/lib/htm/models/node_tag.rb +2 -2
- data/lib/htm/models/robot_node.rb +2 -2
- data/lib/htm/models/tag.rb +41 -28
- data/lib/htm/observability.rb +45 -51
- data/lib/htm/proposition_service.rb +3 -7
- data/lib/htm/query_cache.rb +13 -15
- data/lib/htm/railtie.rb +1 -2
- data/lib/htm/robot_group.rb +9 -9
- data/lib/htm/sequel_config.rb +1 -0
- data/lib/htm/sql_builder.rb +1 -1
- data/lib/htm/tag_service.rb +2 -6
- data/lib/htm/timeframe.rb +4 -5
- data/lib/htm/timeframe_extractor.rb +42 -83
- data/lib/htm/version.rb +1 -1
- data/lib/htm/workflows/remember_workflow.rb +112 -115
- data/lib/htm/working_memory.rb +21 -26
- data/lib/htm.rb +103 -116
- data/lib/tasks/db.rake +0 -2
- data/lib/tasks/doc.rake +14 -13
- data/lib/tasks/files.rake +5 -12
- data/lib/tasks/htm.rake +70 -71
- data/lib/tasks/jobs.rake +41 -47
- data/lib/tasks/tags.rake +3 -8
- metadata +25 -100
data/docs/api/database.md
CHANGED
|
@@ -4,15 +4,13 @@ Database schema setup and configuration utilities for HTM.
|
|
|
4
4
|
|
|
5
5
|
## Overview
|
|
6
6
|
|
|
7
|
-
`HTM::Database` provides class methods for setting up the HTM database schema, managing PostgreSQL connections, and
|
|
7
|
+
`HTM::Database` provides class methods for setting up the HTM database schema, managing PostgreSQL connections, and running migrations.
|
|
8
8
|
|
|
9
9
|
**Key Features:**
|
|
10
10
|
|
|
11
11
|
- Schema creation and migration
|
|
12
|
-
-
|
|
13
|
-
- Extension verification (TimescaleDB, pgvector, pg_trgm)
|
|
12
|
+
- Extension verification (pgvector, pg_trgm)
|
|
14
13
|
- Connection configuration parsing
|
|
15
|
-
- Automatic compression policies
|
|
16
14
|
|
|
17
15
|
## Class Definition
|
|
18
16
|
|
|
@@ -28,7 +26,7 @@ end
|
|
|
28
26
|
|
|
29
27
|
### `setup(db_url = nil)` {: #setup }
|
|
30
28
|
|
|
31
|
-
Set up the HTM database schema and
|
|
29
|
+
Set up the HTM database schema and run migrations.
|
|
32
30
|
|
|
33
31
|
```ruby
|
|
34
32
|
HTM::Database.setup(db_url = nil)
|
|
@@ -52,10 +50,9 @@ HTM::Database.setup(db_url = nil)
|
|
|
52
50
|
#### Side Effects
|
|
53
51
|
|
|
54
52
|
- Connects to PostgreSQL database
|
|
55
|
-
- Verifies required extensions (
|
|
53
|
+
- Verifies required extensions (pgvector, pg_trgm)
|
|
56
54
|
- Creates schema (tables, indexes, views)
|
|
57
|
-
-
|
|
58
|
-
- Sets up compression policies
|
|
55
|
+
- Runs pending migrations
|
|
59
56
|
- Prints status messages to stdout
|
|
60
57
|
|
|
61
58
|
#### Examples
|
|
@@ -67,22 +64,18 @@ HTM::Database.setup
|
|
|
67
64
|
# Use specific database URL
|
|
68
65
|
HTM::Database.setup('postgresql://user:pass@host:5432/dbname')
|
|
69
66
|
|
|
70
|
-
# Use
|
|
71
|
-
url = 'postgresql://
|
|
67
|
+
# Use a remote PostgreSQL database
|
|
68
|
+
url = 'postgresql://user:pass@db.example.com:5432/htm_production?sslmode=require'
|
|
72
69
|
HTM::Database.setup(url)
|
|
73
70
|
```
|
|
74
71
|
|
|
75
72
|
#### Output
|
|
76
73
|
|
|
77
74
|
```
|
|
78
|
-
✓
|
|
79
|
-
✓ pgvector version: 0.5.1
|
|
75
|
+
✓ pgvector version: 0.8.1+
|
|
80
76
|
✓ pg_trgm version: 1.6
|
|
81
77
|
Creating HTM schema...
|
|
82
78
|
✓ Schema created
|
|
83
|
-
✓ Created hypertable for operations_log
|
|
84
|
-
✓ Created hypertable for nodes
|
|
85
|
-
✓ Enabled compression for nodes older than 30 days
|
|
86
79
|
✓ HTM database schema created successfully
|
|
87
80
|
```
|
|
88
81
|
|
|
@@ -140,14 +133,14 @@ url = 'postgresql://user:pass@host:5432/db?sslmode=require'
|
|
|
140
133
|
config = HTM::Database.parse_connection_url(url)
|
|
141
134
|
# => { ..., sslmode: "require" }
|
|
142
135
|
|
|
143
|
-
#
|
|
144
|
-
url = 'postgresql://
|
|
136
|
+
# Remote PostgreSQL URL
|
|
137
|
+
url = 'postgresql://myuser:secret@db.example.com:5432/htm_production?sslmode=require'
|
|
145
138
|
config = HTM::Database.parse_connection_url(url)
|
|
146
139
|
# => {
|
|
147
|
-
# host: "
|
|
148
|
-
# port:
|
|
149
|
-
# dbname: "
|
|
150
|
-
# user: "
|
|
140
|
+
# host: "db.example.com",
|
|
141
|
+
# port: 5432,
|
|
142
|
+
# dbname: "htm_production",
|
|
143
|
+
# user: "myuser",
|
|
151
144
|
# password: "secret",
|
|
152
145
|
# sslmode: "require"
|
|
153
146
|
# }
|
|
@@ -176,8 +169,8 @@ HTM::Database.parse_connection_params()
|
|
|
176
169
|
|
|
177
170
|
| Variable | Description | Default |
|
|
178
171
|
|----------|-------------|---------|
|
|
179
|
-
| `HTM_DATABASE__HOST` | Database hostname | `'
|
|
180
|
-
| `HTM_DATABASE__PORT` | Database port | `
|
|
172
|
+
| `HTM_DATABASE__HOST` | Database hostname | `'localhost'` |
|
|
173
|
+
| `HTM_DATABASE__PORT` | Database port | `5432` |
|
|
181
174
|
| `HTM_DATABASE__NAME` | Database name | *required* |
|
|
182
175
|
| `HTM_DATABASE__USER` | Database user | *required* |
|
|
183
176
|
| `HTM_DATABASE__PASSWORD` | Database password | *required* |
|
|
@@ -186,18 +179,17 @@ HTM::Database.parse_connection_params()
|
|
|
186
179
|
|
|
187
180
|
```ruby
|
|
188
181
|
# Set environment variables
|
|
189
|
-
ENV['HTM_DATABASE__NAME'] = '
|
|
190
|
-
ENV['HTM_DATABASE__USER'] = '
|
|
182
|
+
ENV['HTM_DATABASE__NAME'] = 'htm_development'
|
|
183
|
+
ENV['HTM_DATABASE__USER'] = 'postgres'
|
|
191
184
|
ENV['HTM_DATABASE__PASSWORD'] = 'secret'
|
|
192
185
|
|
|
193
186
|
config = HTM::Database.parse_connection_params()
|
|
194
187
|
# => {
|
|
195
|
-
# host: "
|
|
196
|
-
# port:
|
|
197
|
-
# dbname: "
|
|
198
|
-
# user: "
|
|
199
|
-
# password: "secret"
|
|
200
|
-
# sslmode: "require"
|
|
188
|
+
# host: "localhost",
|
|
189
|
+
# port: 5432,
|
|
190
|
+
# dbname: "htm_development",
|
|
191
|
+
# user: "postgres",
|
|
192
|
+
# password: "secret"
|
|
201
193
|
# }
|
|
202
194
|
|
|
203
195
|
# Custom host and port
|
|
@@ -293,13 +285,9 @@ For detailed database schema documentation, see:
|
|
|
293
285
|
### 1. Verify Extensions
|
|
294
286
|
|
|
295
287
|
```ruby
|
|
296
|
-
# Check TimescaleDB
|
|
297
|
-
timescale = conn.exec("SELECT extversion FROM pg_extension WHERE extname='timescaledb'").first
|
|
298
|
-
# => {"extversion"=>"2.13.0"}
|
|
299
|
-
|
|
300
288
|
# Check pgvector
|
|
301
289
|
pgvector = conn.exec("SELECT extversion FROM pg_extension WHERE extname='vector'").first
|
|
302
|
-
# => {"extversion"=>"0.
|
|
290
|
+
# => {"extversion"=>"0.8.1"}
|
|
303
291
|
|
|
304
292
|
# Check pg_trgm
|
|
305
293
|
pg_trgm = conn.exec("SELECT extversion FROM pg_extension WHERE extname='pg_trgm'").first
|
|
@@ -335,33 +323,22 @@ conn.exec("SELECT add_compression_policy('nodes', INTERVAL '30 days', if_not_exi
|
|
|
335
323
|
|
|
336
324
|
## Environment Configuration
|
|
337
325
|
|
|
338
|
-
###
|
|
339
|
-
|
|
340
|
-
Using URL (recommended):
|
|
326
|
+
### Local PostgreSQL (macOS)
|
|
341
327
|
|
|
342
328
|
```bash
|
|
343
|
-
|
|
344
|
-
export HTM_DATABASE__URL='postgresql://tsdbadmin:PASSWORD@SERVICE.tsdb.cloud.timescale.com:37807/tsdb?sslmode=require'
|
|
329
|
+
export HTM_DATABASE__URL='postgresql://localhost/htm_development'
|
|
345
330
|
```
|
|
346
331
|
|
|
347
|
-
|
|
332
|
+
### Local PostgreSQL (with auth)
|
|
348
333
|
|
|
349
334
|
```bash
|
|
350
|
-
|
|
351
|
-
export HTM_DATABASE__HOST='xxx.tsdb.cloud.timescale.com'
|
|
352
|
-
export HTM_DATABASE__PORT=37807
|
|
353
|
-
export HTM_DATABASE__NAME='tsdb'
|
|
354
|
-
export HTM_DATABASE__USER='tsdbadmin'
|
|
355
|
-
export HTM_DATABASE__PASSWORD='your_password'
|
|
335
|
+
export HTM_DATABASE__URL='postgresql://user:pass@localhost:5432/htm_development'
|
|
356
336
|
```
|
|
357
337
|
|
|
358
|
-
###
|
|
338
|
+
### Remote PostgreSQL (with SSL)
|
|
359
339
|
|
|
360
340
|
```bash
|
|
361
|
-
export HTM_DATABASE__URL='postgresql://
|
|
362
|
-
|
|
363
|
-
# Or with auth
|
|
364
|
-
export HTM_DATABASE__URL='postgresql://user:pass@localhost:5432/htm_dev'
|
|
341
|
+
export HTM_DATABASE__URL='postgresql://user:pass@db.example.com:5432/htm_production?sslmode=require'
|
|
365
342
|
```
|
|
366
343
|
|
|
367
344
|
### Docker PostgreSQL
|
|
@@ -461,7 +438,6 @@ htm = HTM.new(db_config: config)
|
|
|
461
438
|
### Extensions Not Available
|
|
462
439
|
|
|
463
440
|
```
|
|
464
|
-
⚠ Warning: TimescaleDB extension not found
|
|
465
441
|
⚠ Warning: pgvector extension not found
|
|
466
442
|
```
|
|
467
443
|
|
|
@@ -469,12 +445,14 @@ htm = HTM.new(db_config: config)
|
|
|
469
445
|
|
|
470
446
|
```bash
|
|
471
447
|
# Ubuntu/Debian
|
|
472
|
-
sudo apt install postgresql-
|
|
448
|
+
sudo apt install postgresql-17-pgvector
|
|
473
449
|
|
|
474
450
|
# macOS with Homebrew
|
|
475
|
-
brew install
|
|
451
|
+
brew install pgvector
|
|
476
452
|
|
|
477
|
-
#
|
|
453
|
+
# Then enable in your database:
|
|
454
|
+
psql $HTM_DATABASE__URL -c "CREATE EXTENSION IF NOT EXISTS vector;"
|
|
455
|
+
psql $HTM_DATABASE__URL -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;"
|
|
478
456
|
```
|
|
479
457
|
|
|
480
458
|
### Connection Refused
|
|
@@ -476,8 +476,8 @@ Search queries also generate embeddings:
|
|
|
476
476
|
```ruby
|
|
477
477
|
# User searches for "database performance"
|
|
478
478
|
results = htm.recall(
|
|
479
|
+
"database performance",
|
|
479
480
|
timeframe: "last week",
|
|
480
|
-
topic: "database performance",
|
|
481
481
|
strategy: :vector
|
|
482
482
|
)
|
|
483
483
|
|
data/docs/api/index.md
CHANGED
|
@@ -33,11 +33,11 @@ HTM is a two-tier intelligent memory management system for LLM-based robots:
|
|
|
33
33
|
<rect x="300" y="20" width="200" height="180" class="class-box" rx="5"/>
|
|
34
34
|
<text x="400" y="45" text-anchor="middle" class="class-title">HTM</text>
|
|
35
35
|
<line x1="310" y1="55" x2="490" y2="55" stroke="#4a9eff" stroke-width="1"/>
|
|
36
|
-
<text x="310" y="75" class="class-method">+
|
|
36
|
+
<text x="310" y="75" class="class-method">+ remember()</text>
|
|
37
37
|
<text x="310" y="90" class="class-method">+ recall()</text>
|
|
38
|
-
<text x="310" y="105" class="class-method">+
|
|
38
|
+
<text x="310" y="105" class="class-method">+ restore()</text>
|
|
39
39
|
<text x="310" y="120" class="class-method">+ forget()</text>
|
|
40
|
-
<text x="310" y="135" class="class-method">+
|
|
40
|
+
<text x="310" y="135" class="class-method">+ working_memory()</text>
|
|
41
41
|
<text x="310" y="150" class="class-method">+ memory_stats()</text>
|
|
42
42
|
<text x="310" y="165" class="class-method">+ which_robot_said()</text>
|
|
43
43
|
<text x="310" y="180" class="class-method">+ conversation_timeline()</text>
|
|
@@ -102,12 +102,23 @@ HTM is a two-tier intelligent memory management system for LLM-based robots:
|
|
|
102
102
|
|
|
103
103
|
| Class | Purpose | Key Methods |
|
|
104
104
|
|-------|---------|-------------|
|
|
105
|
-
| [HTM](htm.md) | Main interface for memory management | `
|
|
105
|
+
| [HTM](htm.md) | Main interface for memory management | `remember`, `recall`, `forget`, `restore`, `working_memory` |
|
|
106
106
|
| [WorkingMemory](working-memory.md) | Token-limited active context | `add`, `evict_to_make_space`, `assemble_context` |
|
|
107
107
|
| [LongTermMemory](long-term-memory.md) | Persistent PostgreSQL storage | `add`, `search`, `search_fulltext`, `search_hybrid` |
|
|
108
108
|
| [EmbeddingService](embedding-service.md) | Vector embedding generation | `embed`, `count_tokens` |
|
|
109
109
|
| [Database](database.md) | Schema setup and configuration | `setup`, `default_config` |
|
|
110
110
|
|
|
111
|
+
### Sequel Models
|
|
112
|
+
|
|
113
|
+
| Class | Table | Purpose |
|
|
114
|
+
|-------|-------|---------|
|
|
115
|
+
| `HTM::Models::Node` | nodes | Core memory storage |
|
|
116
|
+
| `HTM::Models::Tag` | tags | Hierarchical tag names |
|
|
117
|
+
| `HTM::Models::NodeTag` | node_tags | Node ↔ tag join |
|
|
118
|
+
| `HTM::Models::NodeRelationship` | node_relationships | Weighted directed edges between nodes; Jaccard-similarity scores from tag co-occurrence |
|
|
119
|
+
| `HTM::Models::Robot` | robots | Robot registry |
|
|
120
|
+
| `HTM::Models::FileSource` | file_sources | Source file tracking |
|
|
121
|
+
|
|
111
122
|
### Common Usage Patterns
|
|
112
123
|
|
|
113
124
|
#### Basic Memory Operations
|
|
@@ -117,14 +128,14 @@ HTM is a two-tier intelligent memory management system for LLM-based robots:
|
|
|
117
128
|
htm = HTM.new(robot_name: "Assistant")
|
|
118
129
|
|
|
119
130
|
# Add memories
|
|
120
|
-
htm.
|
|
121
|
-
type: :fact,
|
|
131
|
+
htm.remember("PostgreSQL is our database",
|
|
132
|
+
type: :fact, tags: ["database"])
|
|
122
133
|
|
|
123
134
|
# Recall memories
|
|
124
|
-
memories = htm.recall(timeframe: "last week"
|
|
135
|
+
memories = htm.recall("PostgreSQL", timeframe: "last week")
|
|
125
136
|
|
|
126
137
|
# Create LLM context
|
|
127
|
-
context = htm.
|
|
138
|
+
context = htm.working_memory.assemble_context(strategy: :balanced)
|
|
128
139
|
```
|
|
129
140
|
|
|
130
141
|
#### Multi-Robot Collaboration
|
|
@@ -143,16 +154,16 @@ timeline = htm.conversation_timeline("deployment", limit: 20)
|
|
|
143
154
|
```ruby
|
|
144
155
|
# Vector similarity search
|
|
145
156
|
memories = htm.recall(
|
|
157
|
+
"API design decisions",
|
|
146
158
|
timeframe: "last 30 days",
|
|
147
|
-
topic: "API design decisions",
|
|
148
159
|
strategy: :vector,
|
|
149
160
|
limit: 10
|
|
150
161
|
)
|
|
151
162
|
|
|
152
163
|
# Hybrid search (fulltext + vector)
|
|
153
164
|
memories = htm.recall(
|
|
165
|
+
"security vulnerabilities",
|
|
154
166
|
timeframe: "this month",
|
|
155
|
-
topic: "security vulnerabilities",
|
|
156
167
|
strategy: :hybrid,
|
|
157
168
|
limit: 20
|
|
158
169
|
)
|
|
@@ -194,13 +205,13 @@ When adding nodes, you can specify a type:
|
|
|
194
205
|
|
|
195
206
|
## Context Assembly Strategies
|
|
196
207
|
|
|
197
|
-
When creating context with `
|
|
208
|
+
When creating context with `working_memory.assemble_context`:
|
|
198
209
|
|
|
199
210
|
| Strategy | Behavior |
|
|
200
211
|
|----------|----------|
|
|
201
212
|
| `:recent` | Most recently accessed first |
|
|
202
|
-
| `:
|
|
203
|
-
| `:balanced` | Weighted by
|
|
213
|
+
| `:frequent` | Most frequently accessed first |
|
|
214
|
+
| `:balanced` | Weighted by access frequency and recency |
|
|
204
215
|
|
|
205
216
|
## API Documentation
|
|
206
217
|
|
|
@@ -243,8 +254,8 @@ HTM is **not thread-safe** by default. Each instance maintains its own working m
|
|
|
243
254
|
For large memory stores (>100K nodes):
|
|
244
255
|
|
|
245
256
|
- Use hybrid search with appropriate `prefilter_limit`
|
|
246
|
-
-
|
|
247
|
-
-
|
|
257
|
+
- Purge soft-deleted nodes periodically with `htm.purge_deleted`
|
|
258
|
+
- Add partial indexes for frequently filtered columns
|
|
248
259
|
|
|
249
260
|
## Next Steps
|
|
250
261
|
|
data/docs/api/working-memory.md
CHANGED
|
@@ -297,7 +297,7 @@ assemble_context(strategy:, max_tokens: nil)
|
|
|
297
297
|
|
|
298
298
|
| Parameter | Type | Default | Description |
|
|
299
299
|
|-----------|------|---------|-------------|
|
|
300
|
-
| `strategy` | Symbol | *required* | Assembly strategy (`:recent`, `:
|
|
300
|
+
| `strategy` | Symbol | *required* | Assembly strategy (`:recent`, `:frequent`, `:balanced`) |
|
|
301
301
|
| `max_tokens` | Integer, nil | `@max_tokens` | Optional token limit |
|
|
302
302
|
|
|
303
303
|
#### Assembly Strategies
|
|
@@ -309,10 +309,10 @@ assemble_context(strategy:, max_tokens: nil)
|
|
|
309
309
|
# Good for: Conversational continuity
|
|
310
310
|
```
|
|
311
311
|
|
|
312
|
-
**`:
|
|
312
|
+
**`:frequent`** - Most frequently accessed first
|
|
313
313
|
|
|
314
314
|
```ruby
|
|
315
|
-
# Ordered by
|
|
315
|
+
# Ordered by access count (highest first)
|
|
316
316
|
# Good for: Critical information prioritization
|
|
317
317
|
```
|
|
318
318
|
|
|
@@ -336,8 +336,8 @@ context = wm.assemble_context(strategy: :balanced)
|
|
|
336
336
|
# Recent conversations only
|
|
337
337
|
context = wm.assemble_context(strategy: :recent)
|
|
338
338
|
|
|
339
|
-
# Most
|
|
340
|
-
context = wm.assemble_context(strategy: :
|
|
339
|
+
# Most frequently accessed information
|
|
340
|
+
context = wm.assemble_context(strategy: :frequent)
|
|
341
341
|
|
|
342
342
|
# Limited tokens
|
|
343
343
|
context = wm.assemble_context(
|
|
@@ -524,8 +524,8 @@ class ContextManager
|
|
|
524
524
|
end
|
|
525
525
|
|
|
526
526
|
def for_analysis
|
|
527
|
-
#
|
|
528
|
-
@wm.assemble_context(strategy: :
|
|
527
|
+
# Frequently accessed context for deep analysis
|
|
528
|
+
@wm.assemble_context(strategy: :frequent, max_tokens: 32000)
|
|
529
529
|
end
|
|
530
530
|
|
|
531
531
|
def for_general_task
|
|
@@ -652,7 +652,7 @@ end
|
|
|
652
652
|
context = wm.assemble_context(strategy: :recent)
|
|
653
653
|
|
|
654
654
|
# For analysis/reasoning
|
|
655
|
-
context = wm.assemble_context(strategy: :
|
|
655
|
+
context = wm.assemble_context(strategy: :frequent)
|
|
656
656
|
|
|
657
657
|
# For general use
|
|
658
658
|
context = wm.assemble_context(strategy: :balanced)
|
data/docs/architecture/index.md
CHANGED
|
@@ -37,7 +37,7 @@ HTM provides intelligent memory management through five core components that wor
|
|
|
37
37
|
<rect x="300" y="380" width="200" height="120" fill="rgba(244, 67, 54, 0.2)" stroke="#F44336" stroke-width="2" rx="5"/>
|
|
38
38
|
<text x="400" y="415" text-anchor="middle" fill="#E0E0E0" font-size="14" font-weight="bold">Database</text>
|
|
39
39
|
<text x="400" y="435" text-anchor="middle" fill="#B0B0B0" font-size="11">PostgreSQL 16+</text>
|
|
40
|
-
<text x="400" y="455" text-anchor="middle" fill="#B0B0B0" font-size="11">
|
|
40
|
+
<text x="400" y="455" text-anchor="middle" fill="#B0B0B0" font-size="11">Sequel ORM</text>
|
|
41
41
|
<text x="400" y="475" text-anchor="middle" fill="#B0B0B0" font-size="11">pgvector + pg_trgm</text>
|
|
42
42
|
|
|
43
43
|
<!-- Connections -->
|
|
@@ -102,7 +102,7 @@ Durable PostgreSQL storage for permanent knowledge retention. All memories are s
|
|
|
102
102
|
**Characteristics:**
|
|
103
103
|
|
|
104
104
|
- **Capacity:** Effectively unlimited
|
|
105
|
-
- **Storage:** PostgreSQL with
|
|
105
|
+
- **Storage:** PostgreSQL with pgvector and pg_trgm extensions
|
|
106
106
|
- **Retention:** Permanent (explicit deletion only)
|
|
107
107
|
- **Access Pattern:** RAG-based retrieval (semantic + temporal)
|
|
108
108
|
- **Lifetime:** Forever
|
|
@@ -126,11 +126,10 @@ Generates vector embeddings for semantic search and manages token counting for m
|
|
|
126
126
|
|
|
127
127
|
### Database
|
|
128
128
|
|
|
129
|
-
PostgreSQL 16+ with extensions for
|
|
129
|
+
PostgreSQL 16+ with extensions for vector similarity search, full-text search, and fuzzy matching.
|
|
130
130
|
|
|
131
131
|
**Key Extensions:**
|
|
132
132
|
|
|
133
|
-
- **TimescaleDB**: Hypertable partitioning, compression policies, time-range optimization
|
|
134
133
|
- **pgvector**: Vector similarity search with HNSW indexing
|
|
135
134
|
- **pg_trgm**: Trigram-based fuzzy text matching
|
|
136
135
|
|
|
@@ -282,8 +281,7 @@ Explore detailed architecture documentation:
|
|
|
282
281
|
| Layer | Technology | Purpose |
|
|
283
282
|
|-------|-----------|---------|
|
|
284
283
|
| **Language** | Ruby 3.2+ | Core implementation |
|
|
285
|
-
| **Database** | PostgreSQL 16+ | Relational storage |
|
|
286
|
-
| **Time-Series** | TimescaleDB | Hypertable partitioning, compression |
|
|
284
|
+
| **Database** | PostgreSQL 16+ | Relational storage with soft delete |
|
|
287
285
|
| **Vector Search** | pgvector | Semantic similarity (HNSW) |
|
|
288
286
|
| **Full-Text** | pg_trgm | Fuzzy text matching |
|
|
289
287
|
| **Embeddings** | RubyLLM (multi-provider) | Vector generation |
|
|
@@ -334,7 +332,7 @@ Explore detailed architecture documentation:
|
|
|
334
332
|
|
|
335
333
|
## Related Documentation
|
|
336
334
|
|
|
337
|
-
- [Installation Guide](../getting-started/installation.md) - Setup PostgreSQL
|
|
335
|
+
- [Installation Guide](../getting-started/installation.md) - Setup PostgreSQL and dependencies
|
|
338
336
|
- [Quick Start](../getting-started/quick-start.md) - Get started with HTM in 5 minutes
|
|
339
337
|
- [API Reference](../api/htm.md) - Complete API documentation
|
|
340
338
|
- [Architecture Decision Records](adrs/index.md) - Detailed decision history
|
|
@@ -79,9 +79,9 @@ HTM implements a layered architecture with clear separation of concerns between
|
|
|
79
79
|
#### Data Layer
|
|
80
80
|
|
|
81
81
|
- **PostgreSQL**: Relational storage with ACID guarantees
|
|
82
|
-
- **TimescaleDB**: Time-series optimization and compression
|
|
83
82
|
- **pgvector**: Vector similarity search with HNSW
|
|
84
83
|
- **pg_trgm**: Fuzzy text matching for search
|
|
84
|
+
- **Sequel**: Ruby ORM for database access
|
|
85
85
|
|
|
86
86
|
## Component Diagrams
|
|
87
87
|
|
|
@@ -238,7 +238,6 @@ Audit trail of all memory operations for debugging and replay.
|
|
|
238
238
|
|-----------|---------|---------|------------|
|
|
239
239
|
| **Ruby** | 3.2+ | Implementation language | Readable, expressive, mature ecosystem |
|
|
240
240
|
| **PostgreSQL** | 16+ | Relational database | ACID guarantees, rich extensions, production-proven |
|
|
241
|
-
| **TimescaleDB** | 2.13+ | Time-series extension | Hypertable partitioning, automatic compression |
|
|
242
241
|
| **pgvector** | 0.5+ | Vector similarity | HNSW indexing, PostgreSQL-native, fast approximate search |
|
|
243
242
|
| **pg_trgm** | - | Fuzzy text search | Built-in PostgreSQL extension for trigram matching |
|
|
244
243
|
|
|
@@ -284,8 +283,8 @@ Based on typical production workloads with 10,000 nodes in long-term memory (cli
|
|
|
284
283
|
| `recall()` (vector) | 80ms | 140ms | 230ms | Client-side query embedding + vector search |
|
|
285
284
|
| `recall()` (fulltext) | 30ms | 60ms | 100ms | GIN index search (no embedding needed) |
|
|
286
285
|
| `recall()` (hybrid) | 110ms | 190ms | 330ms | Client-side embedding + hybrid search |
|
|
287
|
-
| `
|
|
288
|
-
| `
|
|
286
|
+
| `HTM::Models::Node.first()` | 5ms | 10ms | 20ms | Simple primary key lookup |
|
|
287
|
+
| `working_memory.assemble_context()` | 8ms | 15ms | 25ms | In-memory sort + join |
|
|
289
288
|
| `forget()` | 10ms | 20ms | 40ms | DELETE with cascades |
|
|
290
289
|
|
|
291
290
|
!!! tip "Performance Optimization"
|
|
@@ -308,15 +307,13 @@ Based on typical production workloads with 10,000 nodes in long-term memory (cli
|
|
|
308
307
|
| Component | Size Estimate | Compression |
|
|
309
308
|
|-----------|--------------|-------------|
|
|
310
309
|
| Node (text only) | ~1KB average | None |
|
|
311
|
-
| Node (with embedding) | ~7KB (1536 dims × 4 bytes) |
|
|
310
|
+
| Node (with embedding) | ~7KB (1536 dims × 4 bytes) | TOAST (auto) |
|
|
312
311
|
| Indexes | ~2x data size | Minimal |
|
|
313
|
-
| Operations log | ~200 bytes/op | TimescaleDB compression |
|
|
314
312
|
|
|
315
313
|
**Example:** 100,000 nodes with embeddings:
|
|
316
314
|
|
|
317
315
|
- Raw data: ~700 MB
|
|
318
316
|
- With indexes: ~2.1 GB
|
|
319
|
-
- With compression (after 30 days): ~300 MB
|
|
320
317
|
|
|
321
318
|
## Scalability Considerations
|
|
322
319
|
|
|
@@ -354,7 +351,7 @@ Based on typical production workloads with 10,000 nodes in long-term memory (cli
|
|
|
354
351
|
- Add PostgreSQL read replicas
|
|
355
352
|
- Route `recall()` and `retrieve()` to replicas
|
|
356
353
|
- Primary handles writes only
|
|
357
|
-
-
|
|
354
|
+
- PostgreSQL native replication support
|
|
358
355
|
|
|
359
356
|
!!! warning "Consistency Considerations"
|
|
360
357
|
Read replicas may lag primary by seconds. For strong consistency requirements, query primary database.
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
<!-- HTM API -->
|
|
14
14
|
<rect x="200" y="190" width="500" height="80" fill="rgba(33, 150, 243, 0.3)" stroke="#2196F3" stroke-width="3" rx="8"/>
|
|
15
15
|
<text x="450" y="225" text-anchor="middle" fill="#FFFFFF" font-size="16" font-weight="bold">HTM API</text>
|
|
16
|
-
<text x="450" y="250" text-anchor="middle" fill="#B0B0B0" font-size="13">
|
|
16
|
+
<text x="450" y="250" text-anchor="middle" fill="#B0B0B0" font-size="13">remember(), recall(), forget(), restore(), etc.</text>
|
|
17
17
|
|
|
18
18
|
<!-- Arrows down to both memories -->
|
|
19
19
|
<path d="M 320 270 L 320 330" stroke="#2196F3" stroke-width="3" fill="none" marker-end="url(#arrow1)"/>
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
<!-- User -->
|
|
6
6
|
<rect x="50" y="80" width="200" height="60" fill="rgba(76, 175, 80, 0.3)" stroke="#4CAF50" stroke-width="3" rx="8"/>
|
|
7
|
-
<text x="150" y="110" text-anchor="middle" fill="#FFFFFF" font-size="13" font-weight="bold">User:
|
|
7
|
+
<text x="150" y="110" text-anchor="middle" fill="#FFFFFF" font-size="13" font-weight="bold">User: assemble_context</text>
|
|
8
8
|
<text x="150" y="128" text-anchor="middle" fill="#B0B0B0" font-size="11">(with strategy)</text>
|
|
9
9
|
|
|
10
10
|
<!-- HTM -->
|
|
@@ -81,7 +81,7 @@
|
|
|
81
81
|
<text x="695" y="165" text-anchor="middle" fill="#673AB7" font-size="10">:recent</text>
|
|
82
82
|
|
|
83
83
|
<path d="M 670 210 L 720 210 L 720 250" stroke="#F44336" stroke-width="2" fill="none" marker-end="url(#arrowR)"/>
|
|
84
|
-
<text x="695" y="215" text-anchor="middle" fill="#F44336" font-size="10">:
|
|
84
|
+
<text x="695" y="215" text-anchor="middle" fill="#F44336" font-size="10">:frequent</text>
|
|
85
85
|
|
|
86
86
|
<path d="M 670 245 L 720 245 L 720 330" stroke="#4CAF50" stroke-width="2" fill="none" marker-end="url(#arrowG)"/>
|
|
87
87
|
<text x="695" y="260" text-anchor="middle" fill="#4CAF50" font-size="10">:balanced</text>
|
|
@@ -5,11 +5,11 @@
|
|
|
5
5
|
<!-- Layer 1: API Layer -->
|
|
6
6
|
<rect x="100" y="60" width="600" height="100" fill="rgba(76, 175, 80, 0.2)" stroke="#4CAF50" stroke-width="2" rx="5"/>
|
|
7
7
|
<text x="400" y="85" text-anchor="middle" fill="#E0E0E0" font-size="16" font-weight="bold">API Layer</text>
|
|
8
|
-
<text x="200" y="110" text-anchor="middle" fill="#B0B0B0" font-size="12">
|
|
8
|
+
<text x="200" y="110" text-anchor="middle" fill="#B0B0B0" font-size="12">remember()</text>
|
|
9
9
|
<text x="320" y="110" text-anchor="middle" fill="#B0B0B0" font-size="12">recall()</text>
|
|
10
|
-
<text x="440" y="110" text-anchor="middle" fill="#B0B0B0" font-size="12">
|
|
10
|
+
<text x="440" y="110" text-anchor="middle" fill="#B0B0B0" font-size="12">restore()</text>
|
|
11
11
|
<text x="560" y="110" text-anchor="middle" fill="#B0B0B0" font-size="12">forget()</text>
|
|
12
|
-
<text x="400" y="135" text-anchor="middle" fill="#B0B0B0" font-size="12">
|
|
12
|
+
<text x="400" y="135" text-anchor="middle" fill="#B0B0B0" font-size="12">assemble_context() • memory_stats() • which_robot_said()</text>
|
|
13
13
|
|
|
14
14
|
<!-- Layer 2: Coordination Layer -->
|
|
15
15
|
<rect x="100" y="180" width="600" height="100" fill="rgba(33, 150, 243, 0.2)" stroke="#2196F3" stroke-width="2" rx="5"/>
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
<rect x="450" y="80" width="300" height="180" fill="rgba(156, 39, 176, 0.2)" stroke="#9C27B0" stroke-width="3" rx="5"/>
|
|
17
17
|
<text x="600" y="110" text-anchor="middle" fill="#E0E0E0" font-size="16" font-weight="bold">Long-Term Memory (Cold)</text>
|
|
18
18
|
<text x="480" y="140" fill="#B0B0B0" font-size="12">Capacity: Unlimited</text>
|
|
19
|
-
<text x="480" y="160" fill="#B0B0B0" font-size="12">Storage: PostgreSQL +
|
|
19
|
+
<text x="480" y="160" fill="#B0B0B0" font-size="12">Storage: PostgreSQL + pgvector</text>
|
|
20
20
|
<text x="480" y="180" fill="#B0B0B0" font-size="12">Speed: O(log n) with indexes</text>
|
|
21
21
|
<text x="480" y="200" fill="#B0B0B0" font-size="12">Lifetime: Permanent</text>
|
|
22
22
|
<text x="480" y="220" fill="#B0B0B0" font-size="12">Retrieval: RAG (semantic + temporal)</text>
|
data/docs/database/README.md
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
| Name | Columns | Comment | Type |
|
|
6
6
|
| ---- | ------- | ------- | ---- |
|
|
7
7
|
| [public.file_sources](public.file_sources.md) | 9 | Source file metadata for loaded documents | BASE TABLE |
|
|
8
|
+
| [public.node_relationships](public.node_relationships.md) | 8 | Weighted directed edges between related nodes (Jaccard similarity) | BASE TABLE |
|
|
8
9
|
| [public.node_tags](public.node_tags.md) | 5 | Join table connecting nodes to tags (many-to-many) | BASE TABLE |
|
|
9
10
|
| [public.nodes](public.nodes.md) | 14 | Core memory storage for conversation messages and context | BASE TABLE |
|
|
10
11
|
| [public.robot_nodes](public.robot_nodes.md) | 10 | Join table connecting robots to nodes (many-to-many) | BASE TABLE |
|