htm 0.0.11 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.dictate.toml +46 -0
- data/.envrc +2 -0
- data/CHANGELOG.md +85 -2
- data/README.md +348 -79
- data/Rakefile +14 -2
- data/bin/htm_mcp.rb +94 -0
- data/config/database.yml +20 -13
- data/db/migrate/00003_create_file_sources.rb +5 -0
- data/db/migrate/00004_create_nodes.rb +17 -0
- data/db/migrate/00005_create_tags.rb +7 -0
- data/db/migrate/00006_create_node_tags.rb +2 -0
- data/db/migrate/00007_create_robot_nodes.rb +7 -0
- data/db/schema.sql +69 -100
- data/docs/api/index.md +1 -1
- data/docs/api/yard/HTM/Configuration.md +54 -0
- data/docs/api/yard/HTM/Database.md +13 -10
- data/docs/api/yard/HTM/EmbeddingService.md +5 -1
- data/docs/api/yard/HTM/LongTermMemory.md +18 -277
- data/docs/api/yard/HTM/PropositionError.md +18 -0
- data/docs/api/yard/HTM/PropositionService.md +66 -0
- data/docs/api/yard/HTM/QueryCache.md +88 -0
- data/docs/api/yard/HTM/RobotGroup.md +481 -0
- data/docs/api/yard/HTM/SqlBuilder.md +108 -0
- data/docs/api/yard/HTM/TagService.md +4 -0
- data/docs/api/yard/HTM/Telemetry/NullInstrument.md +13 -0
- data/docs/api/yard/HTM/Telemetry/NullMeter.md +15 -0
- data/docs/api/yard/HTM/Telemetry.md +109 -0
- data/docs/api/yard/HTM/WorkingMemoryChannel.md +176 -0
- data/docs/api/yard/HTM.md +8 -22
- data/docs/api/yard/index.csv +102 -25
- data/docs/api/yard-reference.md +8 -0
- data/docs/architecture/index.md +1 -1
- data/docs/assets/images/multi-provider-failover.svg +51 -0
- data/docs/assets/images/robot-group-architecture.svg +65 -0
- data/docs/database/README.md +3 -3
- data/docs/database/public.file_sources.svg +29 -21
- data/docs/database/public.node_tags.md +2 -0
- data/docs/database/public.node_tags.svg +53 -41
- data/docs/database/public.nodes.md +2 -0
- data/docs/database/public.nodes.svg +52 -40
- data/docs/database/public.robot_nodes.md +2 -0
- data/docs/database/public.robot_nodes.svg +30 -22
- data/docs/database/public.robots.svg +16 -12
- data/docs/database/public.tags.md +3 -0
- data/docs/database/public.tags.svg +41 -33
- data/docs/database/schema.json +66 -0
- data/docs/database/schema.svg +60 -48
- data/docs/development/index.md +14 -1
- data/docs/development/rake-tasks.md +1068 -0
- data/docs/getting-started/index.md +1 -1
- data/docs/getting-started/quick-start.md +144 -155
- data/docs/guides/adding-memories.md +2 -3
- data/docs/guides/context-assembly.md +185 -184
- data/docs/guides/getting-started.md +154 -148
- data/docs/guides/index.md +8 -1
- data/docs/guides/long-term-memory.md +60 -92
- data/docs/guides/mcp-server.md +617 -0
- data/docs/guides/multi-robot.md +249 -345
- data/docs/guides/recalling-memories.md +153 -163
- data/docs/guides/robot-groups.md +604 -0
- data/docs/guides/search-strategies.md +61 -58
- data/docs/guides/working-memory.md +103 -136
- data/docs/images/telemetry-architecture.svg +153 -0
- data/docs/index.md +30 -26
- data/docs/telemetry.md +391 -0
- data/examples/README.md +46 -1
- data/examples/cli_app/README.md +1 -1
- data/examples/cli_app/htm_cli.rb +1 -1
- data/examples/robot_groups/robot_worker.rb +1 -2
- data/examples/robot_groups/same_process.rb +1 -4
- data/examples/sinatra_app/app.rb +1 -1
- data/examples/telemetry/README.md +147 -0
- data/examples/telemetry/SETUP_README.md +169 -0
- data/examples/telemetry/demo.rb +498 -0
- data/examples/telemetry/grafana/dashboards/htm-metrics.json +457 -0
- data/lib/htm/configuration.rb +261 -70
- data/lib/htm/database.rb +46 -22
- data/lib/htm/embedding_service.rb +24 -14
- data/lib/htm/errors.rb +15 -1
- data/lib/htm/jobs/generate_embedding_job.rb +19 -0
- data/lib/htm/jobs/generate_propositions_job.rb +103 -0
- data/lib/htm/jobs/generate_tags_job.rb +24 -0
- data/lib/htm/loaders/markdown_chunker.rb +79 -0
- data/lib/htm/loaders/markdown_loader.rb +41 -15
- data/lib/htm/long_term_memory/fulltext_search.rb +138 -0
- data/lib/htm/long_term_memory/hybrid_search.rb +324 -0
- data/lib/htm/long_term_memory/node_operations.rb +209 -0
- data/lib/htm/long_term_memory/relevance_scorer.rb +355 -0
- data/lib/htm/long_term_memory/robot_operations.rb +34 -0
- data/lib/htm/long_term_memory/tag_operations.rb +428 -0
- data/lib/htm/long_term_memory/vector_search.rb +109 -0
- data/lib/htm/long_term_memory.rb +51 -1153
- data/lib/htm/models/node.rb +35 -2
- data/lib/htm/models/node_tag.rb +31 -0
- data/lib/htm/models/robot_node.rb +31 -0
- data/lib/htm/models/tag.rb +44 -0
- data/lib/htm/proposition_service.rb +169 -0
- data/lib/htm/query_cache.rb +214 -0
- data/lib/htm/robot_group.rb +721 -0
- data/lib/htm/sql_builder.rb +178 -0
- data/lib/htm/tag_service.rb +16 -6
- data/lib/htm/tasks.rb +8 -2
- data/lib/htm/telemetry.rb +224 -0
- data/lib/htm/version.rb +1 -1
- data/lib/htm/working_memory_channel.rb +250 -0
- data/lib/htm.rb +66 -3
- data/lib/tasks/doc.rake +1 -1
- data/lib/tasks/htm.rake +259 -13
- data/mkdocs.yml +98 -96
- metadata +55 -20
- data/.aigcm_msg +0 -1
- data/.claude/settings.local.json +0 -95
- data/CLAUDE.md +0 -603
- data/db/migrate/00009_add_working_memory_to_robot_nodes.rb +0 -12
- data/examples/cli_app/temp.log +0 -93
- data/examples/robot_groups/lib/robot_group.rb +0 -419
- data/examples/robot_groups/lib/working_memory_channel.rb +0 -140
- data/lib/htm/loaders/paragraph_chunker.rb +0 -112
- data/notes/ARCHITECTURE_REVIEW.md +0 -1167
- data/notes/IMPLEMENTATION_SUMMARY.md +0 -606
- data/notes/MULTI_FRAMEWORK_IMPLEMENTATION.md +0 -451
- data/notes/next_steps.md +0 -100
- data/notes/plan.md +0 -627
- data/notes/tag_ontology_enhancement_ideas.md +0 -222
- data/notes/timescaledb_removal_summary.md +0 -200
data/Rakefile
CHANGED
|
@@ -10,6 +10,18 @@ Rake::TestTask.new(:test) do |t|
|
|
|
10
10
|
t.verbose = true
|
|
11
11
|
end
|
|
12
12
|
|
|
13
|
+
# Ensure test task runs with RAILS_ENV=test
|
|
14
|
+
task :test do
|
|
15
|
+
ENV['RAILS_ENV'] = 'test'
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Prepend environment setup before test runs
|
|
19
|
+
Rake::Task[:test].enhance [:set_test_env]
|
|
20
|
+
|
|
21
|
+
task :set_test_env do
|
|
22
|
+
ENV['RAILS_ENV'] = 'test'
|
|
23
|
+
end
|
|
24
|
+
|
|
13
25
|
task default: :test
|
|
14
26
|
|
|
15
27
|
# Load HTM database tasks from lib/tasks/htm.rake
|
|
@@ -20,8 +32,8 @@ require_relative "lib/htm/tasks"
|
|
|
20
32
|
desc "Run database setup (deprecated: use htm:db:setup)"
|
|
21
33
|
task :db_setup => "htm:db:setup"
|
|
22
34
|
|
|
23
|
-
desc "
|
|
24
|
-
task :db_test => "htm:db:
|
|
35
|
+
desc "Verify database connection (deprecated: use htm:db:verify)"
|
|
36
|
+
task :db_test => "htm:db:verify"
|
|
25
37
|
|
|
26
38
|
desc "Run example"
|
|
27
39
|
task :example do
|
data/bin/htm_mcp.rb
CHANGED
|
@@ -395,6 +395,98 @@ class ListTagsTool < FastMcp::Tool
|
|
|
395
395
|
end
|
|
396
396
|
end
|
|
397
397
|
|
|
398
|
+
# Tool: Search tags with fuzzy matching
|
|
399
|
+
class SearchTagsTool < FastMcp::Tool
|
|
400
|
+
description "Search for tags using fuzzy matching (typo-tolerant). Use this when you're unsure of exact tag names."
|
|
401
|
+
|
|
402
|
+
arguments do
|
|
403
|
+
required(:query).filled(:string).description("Search query - can contain typos (e.g., 'postgrsql' finds 'database:postgresql')")
|
|
404
|
+
optional(:limit).filled(:integer).description("Maximum number of results (default: 20)")
|
|
405
|
+
optional(:min_similarity).filled(:float).description("Minimum similarity threshold 0.0-1.0 (default: 0.3, lower = more fuzzy)")
|
|
406
|
+
end
|
|
407
|
+
|
|
408
|
+
def call(query:, limit: 20, min_similarity: 0.3)
|
|
409
|
+
MCP_STDERR_LOG.info "SearchTagsTool called: query=#{query.inspect}, limit=#{limit}, min_similarity=#{min_similarity}"
|
|
410
|
+
|
|
411
|
+
htm = MCPSession.htm_instance
|
|
412
|
+
ltm = htm.instance_variable_get(:@long_term_memory)
|
|
413
|
+
|
|
414
|
+
results = ltm.search_tags(query, limit: limit, min_similarity: min_similarity)
|
|
415
|
+
|
|
416
|
+
# Enrich with node counts
|
|
417
|
+
tags = results.map do |result|
|
|
418
|
+
tag = HTM::Models::Tag.find_by(name: result[:name])
|
|
419
|
+
{
|
|
420
|
+
name: result[:name],
|
|
421
|
+
similarity: result[:similarity].round(3),
|
|
422
|
+
node_count: tag&.nodes&.count || 0
|
|
423
|
+
}
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
MCP_STDERR_LOG.info "SearchTagsTool complete: found #{tags.length} tags"
|
|
427
|
+
|
|
428
|
+
{
|
|
429
|
+
success: true,
|
|
430
|
+
query: query,
|
|
431
|
+
min_similarity: min_similarity,
|
|
432
|
+
count: tags.length,
|
|
433
|
+
tags: tags
|
|
434
|
+
}.to_json
|
|
435
|
+
end
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
# Tool: Find nodes by topic with fuzzy option
|
|
439
|
+
class FindByTopicTool < FastMcp::Tool
|
|
440
|
+
description "Find memory nodes by topic/tag with optional fuzzy matching for typo tolerance"
|
|
441
|
+
|
|
442
|
+
arguments do
|
|
443
|
+
required(:topic).filled(:string).description("Topic or tag to search for (e.g., 'database:postgresql' or 'postgrsql' with fuzzy)")
|
|
444
|
+
optional(:fuzzy).filled(:bool).description("Enable fuzzy matching for typo tolerance (default: false)")
|
|
445
|
+
optional(:exact).filled(:bool).description("Require exact tag match (default: false, uses prefix matching)")
|
|
446
|
+
optional(:limit).filled(:integer).description("Maximum number of results (default: 20)")
|
|
447
|
+
optional(:min_similarity).filled(:float).description("Minimum similarity for fuzzy mode (default: 0.3)")
|
|
448
|
+
end
|
|
449
|
+
|
|
450
|
+
def call(topic:, fuzzy: false, exact: false, limit: 20, min_similarity: 0.3)
|
|
451
|
+
MCP_STDERR_LOG.info "FindByTopicTool called: topic=#{topic.inspect}, fuzzy=#{fuzzy}, exact=#{exact}"
|
|
452
|
+
|
|
453
|
+
htm = MCPSession.htm_instance
|
|
454
|
+
ltm = htm.instance_variable_get(:@long_term_memory)
|
|
455
|
+
|
|
456
|
+
nodes = ltm.nodes_by_topic(
|
|
457
|
+
topic,
|
|
458
|
+
fuzzy: fuzzy,
|
|
459
|
+
exact: exact,
|
|
460
|
+
min_similarity: min_similarity,
|
|
461
|
+
limit: limit
|
|
462
|
+
)
|
|
463
|
+
|
|
464
|
+
# Enrich with tags
|
|
465
|
+
results = nodes.map do |node_attrs|
|
|
466
|
+
node = HTM::Models::Node.includes(:tags).find_by(id: node_attrs['id'])
|
|
467
|
+
next unless node
|
|
468
|
+
|
|
469
|
+
{
|
|
470
|
+
id: node.id,
|
|
471
|
+
content: node.content[0..200],
|
|
472
|
+
tags: node.tags.map(&:name),
|
|
473
|
+
created_at: node.created_at.iso8601
|
|
474
|
+
}
|
|
475
|
+
end.compact
|
|
476
|
+
|
|
477
|
+
MCP_STDERR_LOG.info "FindByTopicTool complete: found #{results.length} nodes"
|
|
478
|
+
|
|
479
|
+
{
|
|
480
|
+
success: true,
|
|
481
|
+
topic: topic,
|
|
482
|
+
fuzzy: fuzzy,
|
|
483
|
+
exact: exact,
|
|
484
|
+
count: results.length,
|
|
485
|
+
results: results
|
|
486
|
+
}.to_json
|
|
487
|
+
end
|
|
488
|
+
end
|
|
489
|
+
|
|
398
490
|
# Tool: Get memory statistics
|
|
399
491
|
class StatsTool < FastMcp::Tool
|
|
400
492
|
description "Get statistics about HTM memory usage"
|
|
@@ -516,6 +608,8 @@ server.register_tool(RecallTool)
|
|
|
516
608
|
server.register_tool(ForgetTool)
|
|
517
609
|
server.register_tool(RestoreTool)
|
|
518
610
|
server.register_tool(ListTagsTool)
|
|
611
|
+
server.register_tool(SearchTagsTool) # Fuzzy tag search with typo tolerance
|
|
612
|
+
server.register_tool(FindByTopicTool) # Find nodes by topic with fuzzy option
|
|
519
613
|
server.register_tool(StatsTool)
|
|
520
614
|
|
|
521
615
|
# Register resources
|
data/config/database.yml
CHANGED
|
@@ -2,36 +2,46 @@
|
|
|
2
2
|
# Uses ERB to read from environment variables
|
|
3
3
|
#
|
|
4
4
|
# Priority:
|
|
5
|
-
# 1. HTM_DBURL - Full connection URL (preferred)
|
|
5
|
+
# 1. HTM_DBURL - Full connection URL (preferred for development/production)
|
|
6
6
|
# 2. Individual HTM_DB* variables - Host, name, user, password, port
|
|
7
7
|
# 3. Defaults for development/test
|
|
8
8
|
#
|
|
9
9
|
# Example HTM_DBURL format:
|
|
10
10
|
# postgresql://user:password@host:port/database?sslmode=require
|
|
11
|
+
#
|
|
12
|
+
# Test database:
|
|
13
|
+
# Tests always use htm_test database (Rails convention).
|
|
14
|
+
# Set RAILS_ENV=test or RACK_ENV=test to use the test configuration.
|
|
11
15
|
|
|
12
16
|
<%
|
|
13
17
|
require 'uri'
|
|
14
|
-
|
|
18
|
+
|
|
19
|
+
# Determine current environment
|
|
20
|
+
current_env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
|
|
21
|
+
|
|
15
22
|
# Parse connection from HTM_DBURL or use individual variables
|
|
16
23
|
if ENV['HTM_DBURL']
|
|
17
24
|
uri = URI.parse(ENV['HTM_DBURL'])
|
|
18
25
|
params = URI.decode_www_form(uri.query || '').to_h
|
|
19
|
-
|
|
26
|
+
base_database = uri.path[1..-1]
|
|
27
|
+
|
|
28
|
+
# Extract base name (remove _development, _test, _production suffixes if present)
|
|
29
|
+
base_name = base_database.sub(/_(development|test|production)$/, '')
|
|
30
|
+
|
|
20
31
|
db_config = {
|
|
21
32
|
'host' => uri.host,
|
|
22
33
|
'port' => uri.port || 5432,
|
|
23
|
-
'
|
|
34
|
+
'base_name' => base_name,
|
|
24
35
|
'username' => uri.user,
|
|
25
36
|
'password' => uri.password,
|
|
26
37
|
'sslmode' => params['sslmode'] || 'prefer'
|
|
27
38
|
}
|
|
28
39
|
else
|
|
29
|
-
env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
|
|
30
40
|
db_config = {
|
|
31
41
|
'host' => ENV.fetch('HTM_DBHOST', 'localhost'),
|
|
32
42
|
'port' => ENV.fetch('HTM_DBPORT', 5432).to_i,
|
|
33
|
-
'
|
|
34
|
-
'username' => ENV.fetch('HTM_DBUSER', '
|
|
43
|
+
'base_name' => ENV.fetch('HTM_DBNAME', 'htm').sub(/_(development|test|production)$/, ''),
|
|
44
|
+
'username' => ENV.fetch('HTM_DBUSER', ENV['USER']),
|
|
35
45
|
'password' => ENV.fetch('HTM_DBPASS', ''),
|
|
36
46
|
'sslmode' => ENV.fetch('HTM_SSLMODE', 'prefer')
|
|
37
47
|
}
|
|
@@ -53,15 +63,12 @@ default: &default
|
|
|
53
63
|
|
|
54
64
|
development:
|
|
55
65
|
<<: *default
|
|
56
|
-
database: <%= db_config['
|
|
66
|
+
database: <%= db_config['base_name'] %>_development
|
|
57
67
|
|
|
58
68
|
test:
|
|
59
69
|
<<: *default
|
|
60
|
-
database: <%= db_config['
|
|
70
|
+
database: <%= db_config['base_name'] %>_test
|
|
61
71
|
|
|
62
72
|
production:
|
|
63
73
|
<<: *default
|
|
64
|
-
database: <%= db_config['
|
|
65
|
-
<% unless ENV['HTM_DBURL'] %>
|
|
66
|
-
# WARNING: Production should use HTM_DBURL with SSL
|
|
67
|
-
<% end %>
|
|
74
|
+
database: <%= db_config['base_name'] %>_production
|
|
@@ -16,5 +16,10 @@ class CreateFileSources < ActiveRecord::Migration[7.1]
|
|
|
16
16
|
add_index :file_sources, :file_path, unique: true, name: 'idx_file_sources_path_unique'
|
|
17
17
|
add_index :file_sources, :file_hash, name: 'idx_file_sources_hash'
|
|
18
18
|
add_index :file_sources, :last_synced_at, name: 'idx_file_sources_last_synced'
|
|
19
|
+
|
|
20
|
+
# LZ4 compression for better read performance on JSONB column
|
|
21
|
+
execute <<~SQL
|
|
22
|
+
ALTER TABLE file_sources ALTER COLUMN frontmatter SET COMPRESSION lz4;
|
|
23
|
+
SQL
|
|
19
24
|
end
|
|
20
25
|
end
|
|
@@ -59,6 +59,23 @@ class CreateNodes < ActiveRecord::Migration[7.1]
|
|
|
59
59
|
CHECK (embedding_dimension IS NULL OR (embedding_dimension > 0 AND embedding_dimension <= 2000))
|
|
60
60
|
SQL
|
|
61
61
|
|
|
62
|
+
# Partial index for active (non-deleted) node queries
|
|
63
|
+
add_index :nodes, :id,
|
|
64
|
+
name: 'idx_nodes_active',
|
|
65
|
+
where: 'deleted_at IS NULL'
|
|
66
|
+
|
|
67
|
+
# Composite index for embedding-based searches on active nodes
|
|
68
|
+
execute <<-SQL
|
|
69
|
+
CREATE INDEX idx_nodes_active_with_embedding ON nodes (id)
|
|
70
|
+
WHERE deleted_at IS NULL AND embedding IS NOT NULL
|
|
71
|
+
SQL
|
|
72
|
+
|
|
73
|
+
# LZ4 compression for better read performance
|
|
74
|
+
execute <<-SQL
|
|
75
|
+
ALTER TABLE nodes ALTER COLUMN metadata SET COMPRESSION lz4;
|
|
76
|
+
ALTER TABLE nodes ALTER COLUMN content SET COMPRESSION lz4;
|
|
77
|
+
SQL
|
|
78
|
+
|
|
62
79
|
# Foreign key to file_sources table
|
|
63
80
|
add_foreign_key :nodes, :file_sources, column: :source_id, on_delete: :nullify
|
|
64
81
|
end
|
|
@@ -5,9 +5,16 @@ class CreateTags < ActiveRecord::Migration[7.1]
|
|
|
5
5
|
create_table :tags, comment: 'Unique tag names for categorization' do |t|
|
|
6
6
|
t.text :name, null: false, comment: 'Hierarchical tag in format: root:level1:level2 (e.g., database:postgresql:timescaledb)'
|
|
7
7
|
t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this tag was created'
|
|
8
|
+
t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
|
|
8
9
|
end
|
|
9
10
|
|
|
10
11
|
add_index :tags, :name, unique: true, name: 'idx_tags_name_unique'
|
|
11
12
|
add_index :tags, :name, using: :btree, opclass: :text_pattern_ops, name: 'idx_tags_name_pattern'
|
|
13
|
+
add_index :tags, :deleted_at, name: 'idx_tags_deleted_at'
|
|
14
|
+
|
|
15
|
+
# GIN trigram index for fuzzy search (typo-tolerant queries)
|
|
16
|
+
execute <<~SQL
|
|
17
|
+
CREATE INDEX idx_tags_name_trgm ON tags USING gin(name gin_trgm_ops);
|
|
18
|
+
SQL
|
|
12
19
|
end
|
|
13
20
|
end
|
|
@@ -6,11 +6,13 @@ class CreateNodeTags < ActiveRecord::Migration[7.1]
|
|
|
6
6
|
t.bigint :node_id, null: false, comment: 'ID of the node being tagged'
|
|
7
7
|
t.bigint :tag_id, null: false, comment: 'ID of the tag being applied'
|
|
8
8
|
t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this association was created'
|
|
9
|
+
t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
|
|
9
10
|
end
|
|
10
11
|
|
|
11
12
|
add_index :node_tags, [:node_id, :tag_id], unique: true, name: 'idx_node_tags_unique'
|
|
12
13
|
add_index :node_tags, :node_id, name: 'idx_node_tags_node_id'
|
|
13
14
|
add_index :node_tags, :tag_id, name: 'idx_node_tags_tag_id'
|
|
15
|
+
add_index :node_tags, :deleted_at, name: 'idx_node_tags_deleted_at'
|
|
14
16
|
|
|
15
17
|
add_foreign_key :node_tags, :nodes, column: :node_id, on_delete: :cascade
|
|
16
18
|
add_foreign_key :node_tags, :tags, column: :tag_id, on_delete: :cascade
|
|
@@ -11,14 +11,21 @@ class CreateRobotNodes < ActiveRecord::Migration[7.1]
|
|
|
11
11
|
comment: 'When this robot last tried to remember this content'
|
|
12
12
|
t.integer :remember_count, default: 1, null: false,
|
|
13
13
|
comment: 'Number of times this robot has tried to remember this content'
|
|
14
|
+
t.boolean :working_memory, default: false, null: false,
|
|
15
|
+
comment: 'True if this node is currently in the robot working memory'
|
|
14
16
|
t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }
|
|
15
17
|
t.timestamptz :updated_at, default: -> { 'CURRENT_TIMESTAMP' }
|
|
18
|
+
t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
|
|
16
19
|
end
|
|
17
20
|
|
|
18
21
|
add_index :robot_nodes, [:robot_id, :node_id], unique: true, name: 'idx_robot_nodes_unique'
|
|
19
22
|
add_index :robot_nodes, :robot_id, name: 'idx_robot_nodes_robot_id'
|
|
20
23
|
add_index :robot_nodes, :node_id, name: 'idx_robot_nodes_node_id'
|
|
21
24
|
add_index :robot_nodes, :last_remembered_at, name: 'idx_robot_nodes_last_remembered_at'
|
|
25
|
+
add_index :robot_nodes, :deleted_at, name: 'idx_robot_nodes_deleted_at'
|
|
26
|
+
add_index :robot_nodes, [:robot_id, :working_memory],
|
|
27
|
+
where: 'working_memory = true',
|
|
28
|
+
name: 'idx_robot_nodes_working_memory'
|
|
22
29
|
|
|
23
30
|
add_foreign_key :robot_nodes, :robots, column: :robot_id, on_delete: :cascade
|
|
24
31
|
add_foreign_key :robot_nodes, :nodes, column: :node_id, on_delete: :cascade
|
data/db/schema.sql
CHANGED
|
@@ -38,6 +38,7 @@ CREATE TABLE public.file_sources (
|
|
|
38
38
|
created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
39
39
|
updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP
|
|
40
40
|
);
|
|
41
|
+
ALTER TABLE ONLY public.file_sources ALTER COLUMN frontmatter SET COMPRESSION lz4;
|
|
41
42
|
|
|
42
43
|
--
|
|
43
44
|
-- Name: TABLE file_sources; Type: COMMENT; Schema: public; Owner: -
|
|
@@ -106,7 +107,8 @@ CREATE TABLE public.node_tags (
|
|
|
106
107
|
id bigint NOT NULL,
|
|
107
108
|
node_id bigint NOT NULL,
|
|
108
109
|
tag_id bigint NOT NULL,
|
|
109
|
-
created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP
|
|
110
|
+
created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
111
|
+
deleted_at timestamp with time zone
|
|
110
112
|
);
|
|
111
113
|
|
|
112
114
|
--
|
|
@@ -133,6 +135,12 @@ COMMENT ON COLUMN public.node_tags.tag_id IS 'ID of the tag being applied';
|
|
|
133
135
|
|
|
134
136
|
COMMENT ON COLUMN public.node_tags.created_at IS 'When this association was created';
|
|
135
137
|
|
|
138
|
+
--
|
|
139
|
+
-- Name: COLUMN node_tags.deleted_at; Type: COMMENT; Schema: public; Owner: -
|
|
140
|
+
--
|
|
141
|
+
|
|
142
|
+
COMMENT ON COLUMN public.node_tags.deleted_at IS 'Soft delete timestamp';
|
|
143
|
+
|
|
136
144
|
--
|
|
137
145
|
-- Name: node_tags_id_seq; Type: SEQUENCE; Schema: public; Owner: -
|
|
138
146
|
--
|
|
@@ -171,6 +179,8 @@ CREATE TABLE public.nodes (
|
|
|
171
179
|
metadata jsonb DEFAULT '{}'::jsonb NOT NULL,
|
|
172
180
|
CONSTRAINT check_embedding_dimension CHECK (((embedding_dimension IS NULL) OR ((embedding_dimension > 0) AND (embedding_dimension <= 2000))))
|
|
173
181
|
);
|
|
182
|
+
ALTER TABLE ONLY public.nodes ALTER COLUMN content SET COMPRESSION lz4;
|
|
183
|
+
ALTER TABLE ONLY public.nodes ALTER COLUMN metadata SET COMPRESSION lz4;
|
|
174
184
|
|
|
175
185
|
--
|
|
176
186
|
-- Name: TABLE nodes; Type: COMMENT; Schema: public; Owner: -
|
|
@@ -284,8 +294,10 @@ CREATE TABLE public.robot_nodes (
|
|
|
284
294
|
first_remembered_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
285
295
|
last_remembered_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
286
296
|
remember_count integer DEFAULT 1 NOT NULL,
|
|
297
|
+
working_memory boolean DEFAULT false NOT NULL,
|
|
287
298
|
created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
288
|
-
updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP
|
|
299
|
+
updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
300
|
+
deleted_at timestamp with time zone
|
|
289
301
|
);
|
|
290
302
|
|
|
291
303
|
--
|
|
@@ -324,6 +336,18 @@ COMMENT ON COLUMN public.robot_nodes.last_remembered_at IS 'When this robot last
|
|
|
324
336
|
|
|
325
337
|
COMMENT ON COLUMN public.robot_nodes.remember_count IS 'Number of times this robot has tried to remember this content';
|
|
326
338
|
|
|
339
|
+
--
|
|
340
|
+
-- Name: COLUMN robot_nodes.working_memory; Type: COMMENT; Schema: public; Owner: -
|
|
341
|
+
--
|
|
342
|
+
|
|
343
|
+
COMMENT ON COLUMN public.robot_nodes.working_memory IS 'True if this node is currently in the robot working memory';
|
|
344
|
+
|
|
345
|
+
--
|
|
346
|
+
-- Name: COLUMN robot_nodes.deleted_at; Type: COMMENT; Schema: public; Owner: -
|
|
347
|
+
--
|
|
348
|
+
|
|
349
|
+
COMMENT ON COLUMN public.robot_nodes.deleted_at IS 'Soft delete timestamp';
|
|
350
|
+
|
|
327
351
|
--
|
|
328
352
|
-- Name: robot_nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
|
|
329
353
|
--
|
|
@@ -408,7 +432,8 @@ CREATE TABLE public.schema_migrations (
|
|
|
408
432
|
CREATE TABLE public.tags (
|
|
409
433
|
id bigint NOT NULL,
|
|
410
434
|
name text NOT NULL,
|
|
411
|
-
created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP
|
|
435
|
+
created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
436
|
+
deleted_at timestamp with time zone
|
|
412
437
|
);
|
|
413
438
|
|
|
414
439
|
--
|
|
@@ -430,69 +455,16 @@ COMMENT ON COLUMN public.tags.name IS 'Hierarchical tag in format: root:level1:l
|
|
|
430
455
|
COMMENT ON COLUMN public.tags.created_at IS 'When this tag was created';
|
|
431
456
|
|
|
432
457
|
--
|
|
433
|
-
-- Name:
|
|
458
|
+
-- Name: COLUMN tags.deleted_at; Type: COMMENT; Schema: public; Owner: -
|
|
434
459
|
--
|
|
435
460
|
|
|
436
|
-
|
|
437
|
-
START WITH 1
|
|
438
|
-
INCREMENT BY 1
|
|
439
|
-
NO MINVALUE
|
|
440
|
-
NO MAXVALUE
|
|
441
|
-
CACHE 1;
|
|
461
|
+
COMMENT ON COLUMN public.tags.deleted_at IS 'Soft delete timestamp';
|
|
442
462
|
|
|
443
463
|
--
|
|
444
|
-
-- Name: tags_id_seq; Type: SEQUENCE
|
|
445
|
-
--
|
|
446
|
-
|
|
447
|
-
ALTER SEQUENCE public.tags_id_seq OWNED BY public.tags.id;
|
|
448
|
-
|
|
449
|
-
--
|
|
450
|
-
-- Name: working_memories; Type: TABLE; Schema: public; Owner: -
|
|
451
|
-
--
|
|
452
|
-
|
|
453
|
-
CREATE TABLE public.working_memories (
|
|
454
|
-
id bigint NOT NULL,
|
|
455
|
-
robot_id bigint NOT NULL,
|
|
456
|
-
node_id bigint NOT NULL,
|
|
457
|
-
added_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
458
|
-
token_count integer
|
|
459
|
-
);
|
|
460
|
-
|
|
461
|
-
--
|
|
462
|
-
-- Name: TABLE working_memories; Type: COMMENT; Schema: public; Owner: -
|
|
463
|
-
--
|
|
464
|
-
|
|
465
|
-
COMMENT ON TABLE public.working_memories IS 'Per-robot working memory state (optional persistence)';
|
|
466
|
-
|
|
467
|
-
--
|
|
468
|
-
-- Name: COLUMN working_memories.robot_id; Type: COMMENT; Schema: public; Owner: -
|
|
469
|
-
--
|
|
470
|
-
|
|
471
|
-
COMMENT ON COLUMN public.working_memories.robot_id IS 'Robot whose working memory this belongs to';
|
|
472
|
-
|
|
473
|
-
--
|
|
474
|
-
-- Name: COLUMN working_memories.node_id; Type: COMMENT; Schema: public; Owner: -
|
|
475
|
-
--
|
|
476
|
-
|
|
477
|
-
COMMENT ON COLUMN public.working_memories.node_id IS 'Node currently in working memory';
|
|
478
|
-
|
|
479
|
-
--
|
|
480
|
-
-- Name: COLUMN working_memories.added_at; Type: COMMENT; Schema: public; Owner: -
|
|
481
|
-
--
|
|
482
|
-
|
|
483
|
-
COMMENT ON COLUMN public.working_memories.added_at IS 'When node was added to working memory';
|
|
484
|
-
|
|
485
|
-
--
|
|
486
|
-
-- Name: COLUMN working_memories.token_count; Type: COMMENT; Schema: public; Owner: -
|
|
487
|
-
--
|
|
488
|
-
|
|
489
|
-
COMMENT ON COLUMN public.working_memories.token_count IS 'Cached token count for budget tracking';
|
|
490
|
-
|
|
491
|
-
--
|
|
492
|
-
-- Name: working_memories_id_seq; Type: SEQUENCE; Schema: public; Owner: -
|
|
464
|
+
-- Name: tags_id_seq; Type: SEQUENCE; Schema: public; Owner: -
|
|
493
465
|
--
|
|
494
466
|
|
|
495
|
-
CREATE SEQUENCE public.
|
|
467
|
+
CREATE SEQUENCE public.tags_id_seq
|
|
496
468
|
START WITH 1
|
|
497
469
|
INCREMENT BY 1
|
|
498
470
|
NO MINVALUE
|
|
@@ -500,10 +472,10 @@ CREATE SEQUENCE public.working_memories_id_seq
|
|
|
500
472
|
CACHE 1;
|
|
501
473
|
|
|
502
474
|
--
|
|
503
|
-
-- Name:
|
|
475
|
+
-- Name: tags_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
|
|
504
476
|
--
|
|
505
477
|
|
|
506
|
-
ALTER SEQUENCE public.
|
|
478
|
+
ALTER SEQUENCE public.tags_id_seq OWNED BY public.tags.id;
|
|
507
479
|
|
|
508
480
|
--
|
|
509
481
|
-- Name: file_sources id; Type: DEFAULT; Schema: public; Owner: -
|
|
@@ -541,12 +513,6 @@ ALTER TABLE ONLY public.robots ALTER COLUMN id SET DEFAULT nextval('public.robot
|
|
|
541
513
|
|
|
542
514
|
ALTER TABLE ONLY public.tags ALTER COLUMN id SET DEFAULT nextval('public.tags_id_seq'::regclass);
|
|
543
515
|
|
|
544
|
-
--
|
|
545
|
-
-- Name: working_memories id; Type: DEFAULT; Schema: public; Owner: -
|
|
546
|
-
--
|
|
547
|
-
|
|
548
|
-
ALTER TABLE ONLY public.working_memories ALTER COLUMN id SET DEFAULT nextval('public.working_memories_id_seq'::regclass);
|
|
549
|
-
|
|
550
516
|
--
|
|
551
517
|
-- Name: file_sources file_sources_pkey; Type: CONSTRAINT; Schema: public; Owner: -
|
|
552
518
|
--
|
|
@@ -596,13 +562,6 @@ ALTER TABLE ONLY public.schema_migrations
|
|
|
596
562
|
ALTER TABLE ONLY public.tags
|
|
597
563
|
ADD CONSTRAINT tags_pkey PRIMARY KEY (id);
|
|
598
564
|
|
|
599
|
-
--
|
|
600
|
-
-- Name: working_memories working_memories_pkey; Type: CONSTRAINT; Schema: public; Owner: -
|
|
601
|
-
--
|
|
602
|
-
|
|
603
|
-
ALTER TABLE ONLY public.working_memories
|
|
604
|
-
ADD CONSTRAINT working_memories_pkey PRIMARY KEY (id);
|
|
605
|
-
|
|
606
565
|
--
|
|
607
566
|
-- Name: idx_file_sources_hash; Type: INDEX; Schema: public; Owner: -
|
|
608
567
|
--
|
|
@@ -621,6 +580,12 @@ CREATE INDEX idx_file_sources_last_synced ON public.file_sources USING btree (la
|
|
|
621
580
|
|
|
622
581
|
CREATE UNIQUE INDEX idx_file_sources_path_unique ON public.file_sources USING btree (file_path);
|
|
623
582
|
|
|
583
|
+
--
|
|
584
|
+
-- Name: idx_node_tags_deleted_at; Type: INDEX; Schema: public; Owner: -
|
|
585
|
+
--
|
|
586
|
+
|
|
587
|
+
CREATE INDEX idx_node_tags_deleted_at ON public.node_tags USING btree (deleted_at);
|
|
588
|
+
|
|
624
589
|
--
|
|
625
590
|
-- Name: idx_node_tags_node_id; Type: INDEX; Schema: public; Owner: -
|
|
626
591
|
--
|
|
@@ -645,6 +610,18 @@ CREATE UNIQUE INDEX idx_node_tags_unique ON public.node_tags USING btree (node_i
|
|
|
645
610
|
|
|
646
611
|
CREATE INDEX idx_nodes_access_count ON public.nodes USING btree (access_count);
|
|
647
612
|
|
|
613
|
+
--
|
|
614
|
+
-- Name: idx_nodes_active; Type: INDEX; Schema: public; Owner: -
|
|
615
|
+
--
|
|
616
|
+
|
|
617
|
+
CREATE INDEX idx_nodes_active ON public.nodes USING btree (id) WHERE (deleted_at IS NULL);
|
|
618
|
+
|
|
619
|
+
--
|
|
620
|
+
-- Name: idx_nodes_active_with_embedding; Type: INDEX; Schema: public; Owner: -
|
|
621
|
+
--
|
|
622
|
+
|
|
623
|
+
CREATE INDEX idx_nodes_active_with_embedding ON public.nodes USING btree (id) WHERE ((deleted_at IS NULL) AND (embedding IS NOT NULL));
|
|
624
|
+
|
|
648
625
|
--
|
|
649
626
|
-- Name: idx_nodes_content_gin; Type: INDEX; Schema: public; Owner: -
|
|
650
627
|
--
|
|
@@ -717,6 +694,12 @@ CREATE INDEX idx_nodes_source_id ON public.nodes USING btree (source_id);
|
|
|
717
694
|
|
|
718
695
|
CREATE INDEX idx_nodes_updated_at ON public.nodes USING btree (updated_at);
|
|
719
696
|
|
|
697
|
+
--
|
|
698
|
+
-- Name: idx_robot_nodes_deleted_at; Type: INDEX; Schema: public; Owner: -
|
|
699
|
+
--
|
|
700
|
+
|
|
701
|
+
CREATE INDEX idx_robot_nodes_deleted_at ON public.robot_nodes USING btree (deleted_at);
|
|
702
|
+
|
|
720
703
|
--
|
|
721
704
|
-- Name: idx_robot_nodes_last_remembered_at; Type: INDEX; Schema: public; Owner: -
|
|
722
705
|
--
|
|
@@ -742,48 +725,34 @@ CREATE INDEX idx_robot_nodes_robot_id ON public.robot_nodes USING btree (robot_i
|
|
|
742
725
|
CREATE UNIQUE INDEX idx_robot_nodes_unique ON public.robot_nodes USING btree (robot_id, node_id);
|
|
743
726
|
|
|
744
727
|
--
|
|
745
|
-
-- Name:
|
|
728
|
+
-- Name: idx_robot_nodes_working_memory; Type: INDEX; Schema: public; Owner: -
|
|
746
729
|
--
|
|
747
730
|
|
|
748
|
-
CREATE INDEX
|
|
731
|
+
CREATE INDEX idx_robot_nodes_working_memory ON public.robot_nodes USING btree (robot_id, working_memory) WHERE (working_memory = true);
|
|
749
732
|
|
|
750
733
|
--
|
|
751
|
-
-- Name:
|
|
734
|
+
-- Name: idx_tags_deleted_at; Type: INDEX; Schema: public; Owner: -
|
|
752
735
|
--
|
|
753
736
|
|
|
754
|
-
CREATE
|
|
737
|
+
CREATE INDEX idx_tags_deleted_at ON public.tags USING btree (deleted_at);
|
|
755
738
|
|
|
756
739
|
--
|
|
757
|
-
-- Name:
|
|
758
|
-
--
|
|
759
|
-
|
|
760
|
-
CREATE INDEX idx_working_memories_node_id ON public.working_memories USING btree (node_id);
|
|
761
|
-
|
|
762
|
-
--
|
|
763
|
-
-- Name: idx_working_memories_robot_id; Type: INDEX; Schema: public; Owner: -
|
|
764
|
-
--
|
|
765
|
-
|
|
766
|
-
CREATE INDEX idx_working_memories_robot_id ON public.working_memories USING btree (robot_id);
|
|
767
|
-
|
|
768
|
-
--
|
|
769
|
-
-- Name: idx_working_memories_unique; Type: INDEX; Schema: public; Owner: -
|
|
740
|
+
-- Name: idx_tags_name_pattern; Type: INDEX; Schema: public; Owner: -
|
|
770
741
|
--
|
|
771
742
|
|
|
772
|
-
CREATE
|
|
743
|
+
CREATE INDEX idx_tags_name_pattern ON public.tags USING btree (name text_pattern_ops);
|
|
773
744
|
|
|
774
745
|
--
|
|
775
|
-
-- Name:
|
|
746
|
+
-- Name: idx_tags_name_trgm; Type: INDEX; Schema: public; Owner: -
|
|
776
747
|
--
|
|
777
748
|
|
|
778
|
-
|
|
779
|
-
ADD CONSTRAINT fk_rails_2c1d8b383c FOREIGN KEY (node_id) REFERENCES public.nodes(id) ON DELETE CASCADE;
|
|
749
|
+
CREATE INDEX idx_tags_name_trgm ON public.tags USING gin (name public.gin_trgm_ops);
|
|
780
750
|
|
|
781
751
|
--
|
|
782
|
-
-- Name:
|
|
752
|
+
-- Name: idx_tags_name_unique; Type: INDEX; Schema: public; Owner: -
|
|
783
753
|
--
|
|
784
754
|
|
|
785
|
-
|
|
786
|
-
ADD CONSTRAINT fk_rails_4b7c3eb07b FOREIGN KEY (robot_id) REFERENCES public.robots(id) ON DELETE CASCADE;
|
|
755
|
+
CREATE UNIQUE INDEX idx_tags_name_unique ON public.tags USING btree (name);
|
|
787
756
|
|
|
788
757
|
--
|
|
789
758
|
-- Name: nodes fk_rails_920ad16d08; Type: FK CONSTRAINT; Schema: public; Owner: -
|
|
@@ -824,4 +793,4 @@ ALTER TABLE ONLY public.robot_nodes
|
|
|
824
793
|
-- PostgreSQL database dump complete
|
|
825
794
|
--
|
|
826
795
|
|
|
827
|
-
\unrestrict
|
|
796
|
+
\unrestrict 4WlUqnJzNHaNhcr67XLIIhAvRPidZODUPGkM34l27SvmC0zu6dIsQdJ8dtu589Z
|