htm 0.0.11 → 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. checksums.yaml +4 -4
  2. data/.dictate.toml +46 -0
  3. data/.envrc +2 -0
  4. data/CHANGELOG.md +85 -2
  5. data/README.md +348 -79
  6. data/Rakefile +14 -2
  7. data/bin/htm_mcp.rb +94 -0
  8. data/config/database.yml +20 -13
  9. data/db/migrate/00003_create_file_sources.rb +5 -0
  10. data/db/migrate/00004_create_nodes.rb +17 -0
  11. data/db/migrate/00005_create_tags.rb +7 -0
  12. data/db/migrate/00006_create_node_tags.rb +2 -0
  13. data/db/migrate/00007_create_robot_nodes.rb +7 -0
  14. data/db/schema.sql +69 -100
  15. data/docs/api/index.md +1 -1
  16. data/docs/api/yard/HTM/Configuration.md +54 -0
  17. data/docs/api/yard/HTM/Database.md +13 -10
  18. data/docs/api/yard/HTM/EmbeddingService.md +5 -1
  19. data/docs/api/yard/HTM/LongTermMemory.md +18 -277
  20. data/docs/api/yard/HTM/PropositionError.md +18 -0
  21. data/docs/api/yard/HTM/PropositionService.md +66 -0
  22. data/docs/api/yard/HTM/QueryCache.md +88 -0
  23. data/docs/api/yard/HTM/RobotGroup.md +481 -0
  24. data/docs/api/yard/HTM/SqlBuilder.md +108 -0
  25. data/docs/api/yard/HTM/TagService.md +4 -0
  26. data/docs/api/yard/HTM/Telemetry/NullInstrument.md +13 -0
  27. data/docs/api/yard/HTM/Telemetry/NullMeter.md +15 -0
  28. data/docs/api/yard/HTM/Telemetry.md +109 -0
  29. data/docs/api/yard/HTM/WorkingMemoryChannel.md +176 -0
  30. data/docs/api/yard/HTM.md +8 -22
  31. data/docs/api/yard/index.csv +102 -25
  32. data/docs/api/yard-reference.md +8 -0
  33. data/docs/architecture/index.md +1 -1
  34. data/docs/assets/images/multi-provider-failover.svg +51 -0
  35. data/docs/assets/images/robot-group-architecture.svg +65 -0
  36. data/docs/database/README.md +3 -3
  37. data/docs/database/public.file_sources.svg +29 -21
  38. data/docs/database/public.node_tags.md +2 -0
  39. data/docs/database/public.node_tags.svg +53 -41
  40. data/docs/database/public.nodes.md +2 -0
  41. data/docs/database/public.nodes.svg +52 -40
  42. data/docs/database/public.robot_nodes.md +2 -0
  43. data/docs/database/public.robot_nodes.svg +30 -22
  44. data/docs/database/public.robots.svg +16 -12
  45. data/docs/database/public.tags.md +3 -0
  46. data/docs/database/public.tags.svg +41 -33
  47. data/docs/database/schema.json +66 -0
  48. data/docs/database/schema.svg +60 -48
  49. data/docs/development/index.md +14 -1
  50. data/docs/development/rake-tasks.md +1068 -0
  51. data/docs/getting-started/index.md +1 -1
  52. data/docs/getting-started/quick-start.md +144 -155
  53. data/docs/guides/adding-memories.md +2 -3
  54. data/docs/guides/context-assembly.md +185 -184
  55. data/docs/guides/getting-started.md +154 -148
  56. data/docs/guides/index.md +8 -1
  57. data/docs/guides/long-term-memory.md +60 -92
  58. data/docs/guides/mcp-server.md +617 -0
  59. data/docs/guides/multi-robot.md +249 -345
  60. data/docs/guides/recalling-memories.md +153 -163
  61. data/docs/guides/robot-groups.md +604 -0
  62. data/docs/guides/search-strategies.md +61 -58
  63. data/docs/guides/working-memory.md +103 -136
  64. data/docs/images/telemetry-architecture.svg +153 -0
  65. data/docs/index.md +30 -26
  66. data/docs/telemetry.md +391 -0
  67. data/examples/README.md +46 -1
  68. data/examples/cli_app/README.md +1 -1
  69. data/examples/cli_app/htm_cli.rb +1 -1
  70. data/examples/robot_groups/robot_worker.rb +1 -2
  71. data/examples/robot_groups/same_process.rb +1 -4
  72. data/examples/sinatra_app/app.rb +1 -1
  73. data/examples/telemetry/README.md +147 -0
  74. data/examples/telemetry/SETUP_README.md +169 -0
  75. data/examples/telemetry/demo.rb +498 -0
  76. data/examples/telemetry/grafana/dashboards/htm-metrics.json +457 -0
  77. data/lib/htm/configuration.rb +261 -70
  78. data/lib/htm/database.rb +46 -22
  79. data/lib/htm/embedding_service.rb +24 -14
  80. data/lib/htm/errors.rb +15 -1
  81. data/lib/htm/jobs/generate_embedding_job.rb +19 -0
  82. data/lib/htm/jobs/generate_propositions_job.rb +103 -0
  83. data/lib/htm/jobs/generate_tags_job.rb +24 -0
  84. data/lib/htm/loaders/markdown_chunker.rb +79 -0
  85. data/lib/htm/loaders/markdown_loader.rb +41 -15
  86. data/lib/htm/long_term_memory/fulltext_search.rb +138 -0
  87. data/lib/htm/long_term_memory/hybrid_search.rb +324 -0
  88. data/lib/htm/long_term_memory/node_operations.rb +209 -0
  89. data/lib/htm/long_term_memory/relevance_scorer.rb +355 -0
  90. data/lib/htm/long_term_memory/robot_operations.rb +34 -0
  91. data/lib/htm/long_term_memory/tag_operations.rb +428 -0
  92. data/lib/htm/long_term_memory/vector_search.rb +109 -0
  93. data/lib/htm/long_term_memory.rb +51 -1153
  94. data/lib/htm/models/node.rb +35 -2
  95. data/lib/htm/models/node_tag.rb +31 -0
  96. data/lib/htm/models/robot_node.rb +31 -0
  97. data/lib/htm/models/tag.rb +44 -0
  98. data/lib/htm/proposition_service.rb +169 -0
  99. data/lib/htm/query_cache.rb +214 -0
  100. data/lib/htm/robot_group.rb +721 -0
  101. data/lib/htm/sql_builder.rb +178 -0
  102. data/lib/htm/tag_service.rb +16 -6
  103. data/lib/htm/tasks.rb +8 -2
  104. data/lib/htm/telemetry.rb +224 -0
  105. data/lib/htm/version.rb +1 -1
  106. data/lib/htm/working_memory_channel.rb +250 -0
  107. data/lib/htm.rb +66 -3
  108. data/lib/tasks/doc.rake +1 -1
  109. data/lib/tasks/htm.rake +259 -13
  110. data/mkdocs.yml +98 -96
  111. metadata +55 -20
  112. data/.aigcm_msg +0 -1
  113. data/.claude/settings.local.json +0 -95
  114. data/CLAUDE.md +0 -603
  115. data/db/migrate/00009_add_working_memory_to_robot_nodes.rb +0 -12
  116. data/examples/cli_app/temp.log +0 -93
  117. data/examples/robot_groups/lib/robot_group.rb +0 -419
  118. data/examples/robot_groups/lib/working_memory_channel.rb +0 -140
  119. data/lib/htm/loaders/paragraph_chunker.rb +0 -112
  120. data/notes/ARCHITECTURE_REVIEW.md +0 -1167
  121. data/notes/IMPLEMENTATION_SUMMARY.md +0 -606
  122. data/notes/MULTI_FRAMEWORK_IMPLEMENTATION.md +0 -451
  123. data/notes/next_steps.md +0 -100
  124. data/notes/plan.md +0 -627
  125. data/notes/tag_ontology_enhancement_ideas.md +0 -222
  126. data/notes/timescaledb_removal_summary.md +0 -200
data/Rakefile CHANGED
@@ -10,6 +10,18 @@ Rake::TestTask.new(:test) do |t|
10
10
  t.verbose = true
11
11
  end
12
12
 
13
+ # Ensure test task runs with RAILS_ENV=test
14
+ task :test do
15
+ ENV['RAILS_ENV'] = 'test'
16
+ end
17
+
18
+ # Prepend environment setup before test runs
19
+ Rake::Task[:test].enhance [:set_test_env]
20
+
21
+ task :set_test_env do
22
+ ENV['RAILS_ENV'] = 'test'
23
+ end
24
+
13
25
  task default: :test
14
26
 
15
27
  # Load HTM database tasks from lib/tasks/htm.rake
@@ -20,8 +32,8 @@ require_relative "lib/htm/tasks"
20
32
  desc "Run database setup (deprecated: use htm:db:setup)"
21
33
  task :db_setup => "htm:db:setup"
22
34
 
23
- desc "Test database connection (deprecated: use htm:db:test)"
24
- task :db_test => "htm:db:test"
35
+ desc "Verify database connection (deprecated: use htm:db:verify)"
36
+ task :db_test => "htm:db:verify"
25
37
 
26
38
  desc "Run example"
27
39
  task :example do
data/bin/htm_mcp.rb CHANGED
@@ -395,6 +395,98 @@ class ListTagsTool < FastMcp::Tool
395
395
  end
396
396
  end
397
397
 
398
+ # Tool: Search tags with fuzzy matching
399
+ class SearchTagsTool < FastMcp::Tool
400
+ description "Search for tags using fuzzy matching (typo-tolerant). Use this when you're unsure of exact tag names."
401
+
402
+ arguments do
403
+ required(:query).filled(:string).description("Search query - can contain typos (e.g., 'postgrsql' finds 'database:postgresql')")
404
+ optional(:limit).filled(:integer).description("Maximum number of results (default: 20)")
405
+ optional(:min_similarity).filled(:float).description("Minimum similarity threshold 0.0-1.0 (default: 0.3, lower = more fuzzy)")
406
+ end
407
+
408
+ def call(query:, limit: 20, min_similarity: 0.3)
409
+ MCP_STDERR_LOG.info "SearchTagsTool called: query=#{query.inspect}, limit=#{limit}, min_similarity=#{min_similarity}"
410
+
411
+ htm = MCPSession.htm_instance
412
+ ltm = htm.instance_variable_get(:@long_term_memory)
413
+
414
+ results = ltm.search_tags(query, limit: limit, min_similarity: min_similarity)
415
+
416
+ # Enrich with node counts
417
+ tags = results.map do |result|
418
+ tag = HTM::Models::Tag.find_by(name: result[:name])
419
+ {
420
+ name: result[:name],
421
+ similarity: result[:similarity].round(3),
422
+ node_count: tag&.nodes&.count || 0
423
+ }
424
+ end
425
+
426
+ MCP_STDERR_LOG.info "SearchTagsTool complete: found #{tags.length} tags"
427
+
428
+ {
429
+ success: true,
430
+ query: query,
431
+ min_similarity: min_similarity,
432
+ count: tags.length,
433
+ tags: tags
434
+ }.to_json
435
+ end
436
+ end
437
+
438
+ # Tool: Find nodes by topic with fuzzy option
439
+ class FindByTopicTool < FastMcp::Tool
440
+ description "Find memory nodes by topic/tag with optional fuzzy matching for typo tolerance"
441
+
442
+ arguments do
443
+ required(:topic).filled(:string).description("Topic or tag to search for (e.g., 'database:postgresql' or 'postgrsql' with fuzzy)")
444
+ optional(:fuzzy).filled(:bool).description("Enable fuzzy matching for typo tolerance (default: false)")
445
+ optional(:exact).filled(:bool).description("Require exact tag match (default: false, uses prefix matching)")
446
+ optional(:limit).filled(:integer).description("Maximum number of results (default: 20)")
447
+ optional(:min_similarity).filled(:float).description("Minimum similarity for fuzzy mode (default: 0.3)")
448
+ end
449
+
450
+ def call(topic:, fuzzy: false, exact: false, limit: 20, min_similarity: 0.3)
451
+ MCP_STDERR_LOG.info "FindByTopicTool called: topic=#{topic.inspect}, fuzzy=#{fuzzy}, exact=#{exact}"
452
+
453
+ htm = MCPSession.htm_instance
454
+ ltm = htm.instance_variable_get(:@long_term_memory)
455
+
456
+ nodes = ltm.nodes_by_topic(
457
+ topic,
458
+ fuzzy: fuzzy,
459
+ exact: exact,
460
+ min_similarity: min_similarity,
461
+ limit: limit
462
+ )
463
+
464
+ # Enrich with tags
465
+ results = nodes.map do |node_attrs|
466
+ node = HTM::Models::Node.includes(:tags).find_by(id: node_attrs['id'])
467
+ next unless node
468
+
469
+ {
470
+ id: node.id,
471
+ content: node.content[0..200],
472
+ tags: node.tags.map(&:name),
473
+ created_at: node.created_at.iso8601
474
+ }
475
+ end.compact
476
+
477
+ MCP_STDERR_LOG.info "FindByTopicTool complete: found #{results.length} nodes"
478
+
479
+ {
480
+ success: true,
481
+ topic: topic,
482
+ fuzzy: fuzzy,
483
+ exact: exact,
484
+ count: results.length,
485
+ results: results
486
+ }.to_json
487
+ end
488
+ end
489
+
398
490
  # Tool: Get memory statistics
399
491
  class StatsTool < FastMcp::Tool
400
492
  description "Get statistics about HTM memory usage"
@@ -516,6 +608,8 @@ server.register_tool(RecallTool)
516
608
  server.register_tool(ForgetTool)
517
609
  server.register_tool(RestoreTool)
518
610
  server.register_tool(ListTagsTool)
611
+ server.register_tool(SearchTagsTool) # Fuzzy tag search with typo tolerance
612
+ server.register_tool(FindByTopicTool) # Find nodes by topic with fuzzy option
519
613
  server.register_tool(StatsTool)
520
614
 
521
615
  # Register resources
data/config/database.yml CHANGED
@@ -2,36 +2,46 @@
2
2
  # Uses ERB to read from environment variables
3
3
  #
4
4
  # Priority:
5
- # 1. HTM_DBURL - Full connection URL (preferred)
5
+ # 1. HTM_DBURL - Full connection URL (preferred for development/production)
6
6
  # 2. Individual HTM_DB* variables - Host, name, user, password, port
7
7
  # 3. Defaults for development/test
8
8
  #
9
9
  # Example HTM_DBURL format:
10
10
  # postgresql://user:password@host:port/database?sslmode=require
11
+ #
12
+ # Test database:
13
+ # Tests always use htm_test database (Rails convention).
14
+ # Set RAILS_ENV=test or RACK_ENV=test to use the test configuration.
11
15
 
12
16
  <%
13
17
  require 'uri'
14
-
18
+
19
+ # Determine current environment
20
+ current_env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
21
+
15
22
  # Parse connection from HTM_DBURL or use individual variables
16
23
  if ENV['HTM_DBURL']
17
24
  uri = URI.parse(ENV['HTM_DBURL'])
18
25
  params = URI.decode_www_form(uri.query || '').to_h
19
-
26
+ base_database = uri.path[1..-1]
27
+
28
+ # Extract base name (remove _development, _test, _production suffixes if present)
29
+ base_name = base_database.sub(/_(development|test|production)$/, '')
30
+
20
31
  db_config = {
21
32
  'host' => uri.host,
22
33
  'port' => uri.port || 5432,
23
- 'database' => uri.path[1..-1],
34
+ 'base_name' => base_name,
24
35
  'username' => uri.user,
25
36
  'password' => uri.password,
26
37
  'sslmode' => params['sslmode'] || 'prefer'
27
38
  }
28
39
  else
29
- env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
30
40
  db_config = {
31
41
  'host' => ENV.fetch('HTM_DBHOST', 'localhost'),
32
42
  'port' => ENV.fetch('HTM_DBPORT', 5432).to_i,
33
- 'database' => ENV.fetch('HTM_DBNAME', "htm_#{env}"),
34
- 'username' => ENV.fetch('HTM_DBUSER', 'postgres'),
43
+ 'base_name' => ENV.fetch('HTM_DBNAME', 'htm').sub(/_(development|test|production)$/, ''),
44
+ 'username' => ENV.fetch('HTM_DBUSER', ENV['USER']),
35
45
  'password' => ENV.fetch('HTM_DBPASS', ''),
36
46
  'sslmode' => ENV.fetch('HTM_SSLMODE', 'prefer')
37
47
  }
@@ -53,15 +63,12 @@ default: &default
53
63
 
54
64
  development:
55
65
  <<: *default
56
- database: <%= db_config['database'] %>
66
+ database: <%= db_config['base_name'] %>_development
57
67
 
58
68
  test:
59
69
  <<: *default
60
- database: <%= db_config['database'] %>_test
70
+ database: <%= db_config['base_name'] %>_test
61
71
 
62
72
  production:
63
73
  <<: *default
64
- database: <%= db_config['database'] %>
65
- <% unless ENV['HTM_DBURL'] %>
66
- # WARNING: Production should use HTM_DBURL with SSL
67
- <% end %>
74
+ database: <%= db_config['base_name'] %>_production
@@ -16,5 +16,10 @@ class CreateFileSources < ActiveRecord::Migration[7.1]
16
16
  add_index :file_sources, :file_path, unique: true, name: 'idx_file_sources_path_unique'
17
17
  add_index :file_sources, :file_hash, name: 'idx_file_sources_hash'
18
18
  add_index :file_sources, :last_synced_at, name: 'idx_file_sources_last_synced'
19
+
20
+ # LZ4 compression for better read performance on JSONB column
21
+ execute <<~SQL
22
+ ALTER TABLE file_sources ALTER COLUMN frontmatter SET COMPRESSION lz4;
23
+ SQL
19
24
  end
20
25
  end
@@ -59,6 +59,23 @@ class CreateNodes < ActiveRecord::Migration[7.1]
59
59
  CHECK (embedding_dimension IS NULL OR (embedding_dimension > 0 AND embedding_dimension <= 2000))
60
60
  SQL
61
61
 
62
+ # Partial index for active (non-deleted) node queries
63
+ add_index :nodes, :id,
64
+ name: 'idx_nodes_active',
65
+ where: 'deleted_at IS NULL'
66
+
67
+ # Composite index for embedding-based searches on active nodes
68
+ execute <<-SQL
69
+ CREATE INDEX idx_nodes_active_with_embedding ON nodes (id)
70
+ WHERE deleted_at IS NULL AND embedding IS NOT NULL
71
+ SQL
72
+
73
+ # LZ4 compression for better read performance
74
+ execute <<-SQL
75
+ ALTER TABLE nodes ALTER COLUMN metadata SET COMPRESSION lz4;
76
+ ALTER TABLE nodes ALTER COLUMN content SET COMPRESSION lz4;
77
+ SQL
78
+
62
79
  # Foreign key to file_sources table
63
80
  add_foreign_key :nodes, :file_sources, column: :source_id, on_delete: :nullify
64
81
  end
@@ -5,9 +5,16 @@ class CreateTags < ActiveRecord::Migration[7.1]
5
5
  create_table :tags, comment: 'Unique tag names for categorization' do |t|
6
6
  t.text :name, null: false, comment: 'Hierarchical tag in format: root:level1:level2 (e.g., database:postgresql:timescaledb)'
7
7
  t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this tag was created'
8
+ t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
8
9
  end
9
10
 
10
11
  add_index :tags, :name, unique: true, name: 'idx_tags_name_unique'
11
12
  add_index :tags, :name, using: :btree, opclass: :text_pattern_ops, name: 'idx_tags_name_pattern'
13
+ add_index :tags, :deleted_at, name: 'idx_tags_deleted_at'
14
+
15
+ # GIN trigram index for fuzzy search (typo-tolerant queries)
16
+ execute <<~SQL
17
+ CREATE INDEX idx_tags_name_trgm ON tags USING gin(name gin_trgm_ops);
18
+ SQL
12
19
  end
13
20
  end
@@ -6,11 +6,13 @@ class CreateNodeTags < ActiveRecord::Migration[7.1]
6
6
  t.bigint :node_id, null: false, comment: 'ID of the node being tagged'
7
7
  t.bigint :tag_id, null: false, comment: 'ID of the tag being applied'
8
8
  t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this association was created'
9
+ t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
9
10
  end
10
11
 
11
12
  add_index :node_tags, [:node_id, :tag_id], unique: true, name: 'idx_node_tags_unique'
12
13
  add_index :node_tags, :node_id, name: 'idx_node_tags_node_id'
13
14
  add_index :node_tags, :tag_id, name: 'idx_node_tags_tag_id'
15
+ add_index :node_tags, :deleted_at, name: 'idx_node_tags_deleted_at'
14
16
 
15
17
  add_foreign_key :node_tags, :nodes, column: :node_id, on_delete: :cascade
16
18
  add_foreign_key :node_tags, :tags, column: :tag_id, on_delete: :cascade
@@ -11,14 +11,21 @@ class CreateRobotNodes < ActiveRecord::Migration[7.1]
11
11
  comment: 'When this robot last tried to remember this content'
12
12
  t.integer :remember_count, default: 1, null: false,
13
13
  comment: 'Number of times this robot has tried to remember this content'
14
+ t.boolean :working_memory, default: false, null: false,
15
+ comment: 'True if this node is currently in the robot working memory'
14
16
  t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }
15
17
  t.timestamptz :updated_at, default: -> { 'CURRENT_TIMESTAMP' }
18
+ t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
16
19
  end
17
20
 
18
21
  add_index :robot_nodes, [:robot_id, :node_id], unique: true, name: 'idx_robot_nodes_unique'
19
22
  add_index :robot_nodes, :robot_id, name: 'idx_robot_nodes_robot_id'
20
23
  add_index :robot_nodes, :node_id, name: 'idx_robot_nodes_node_id'
21
24
  add_index :robot_nodes, :last_remembered_at, name: 'idx_robot_nodes_last_remembered_at'
25
+ add_index :robot_nodes, :deleted_at, name: 'idx_robot_nodes_deleted_at'
26
+ add_index :robot_nodes, [:robot_id, :working_memory],
27
+ where: 'working_memory = true',
28
+ name: 'idx_robot_nodes_working_memory'
22
29
 
23
30
  add_foreign_key :robot_nodes, :robots, column: :robot_id, on_delete: :cascade
24
31
  add_foreign_key :robot_nodes, :nodes, column: :node_id, on_delete: :cascade
data/db/schema.sql CHANGED
@@ -38,6 +38,7 @@ CREATE TABLE public.file_sources (
38
38
  created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
39
39
  updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP
40
40
  );
41
+ ALTER TABLE ONLY public.file_sources ALTER COLUMN frontmatter SET COMPRESSION lz4;
41
42
 
42
43
  --
43
44
  -- Name: TABLE file_sources; Type: COMMENT; Schema: public; Owner: -
@@ -106,7 +107,8 @@ CREATE TABLE public.node_tags (
106
107
  id bigint NOT NULL,
107
108
  node_id bigint NOT NULL,
108
109
  tag_id bigint NOT NULL,
109
- created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP
110
+ created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
111
+ deleted_at timestamp with time zone
110
112
  );
111
113
 
112
114
  --
@@ -133,6 +135,12 @@ COMMENT ON COLUMN public.node_tags.tag_id IS 'ID of the tag being applied';
133
135
 
134
136
  COMMENT ON COLUMN public.node_tags.created_at IS 'When this association was created';
135
137
 
138
+ --
139
+ -- Name: COLUMN node_tags.deleted_at; Type: COMMENT; Schema: public; Owner: -
140
+ --
141
+
142
+ COMMENT ON COLUMN public.node_tags.deleted_at IS 'Soft delete timestamp';
143
+
136
144
  --
137
145
  -- Name: node_tags_id_seq; Type: SEQUENCE; Schema: public; Owner: -
138
146
  --
@@ -171,6 +179,8 @@ CREATE TABLE public.nodes (
171
179
  metadata jsonb DEFAULT '{}'::jsonb NOT NULL,
172
180
  CONSTRAINT check_embedding_dimension CHECK (((embedding_dimension IS NULL) OR ((embedding_dimension > 0) AND (embedding_dimension <= 2000))))
173
181
  );
182
+ ALTER TABLE ONLY public.nodes ALTER COLUMN content SET COMPRESSION lz4;
183
+ ALTER TABLE ONLY public.nodes ALTER COLUMN metadata SET COMPRESSION lz4;
174
184
 
175
185
  --
176
186
  -- Name: TABLE nodes; Type: COMMENT; Schema: public; Owner: -
@@ -284,8 +294,10 @@ CREATE TABLE public.robot_nodes (
284
294
  first_remembered_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
285
295
  last_remembered_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
286
296
  remember_count integer DEFAULT 1 NOT NULL,
297
+ working_memory boolean DEFAULT false NOT NULL,
287
298
  created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
288
- updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP
299
+ updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
300
+ deleted_at timestamp with time zone
289
301
  );
290
302
 
291
303
  --
@@ -324,6 +336,18 @@ COMMENT ON COLUMN public.robot_nodes.last_remembered_at IS 'When this robot last
324
336
 
325
337
  COMMENT ON COLUMN public.robot_nodes.remember_count IS 'Number of times this robot has tried to remember this content';
326
338
 
339
+ --
340
+ -- Name: COLUMN robot_nodes.working_memory; Type: COMMENT; Schema: public; Owner: -
341
+ --
342
+
343
+ COMMENT ON COLUMN public.robot_nodes.working_memory IS 'True if this node is currently in the robot working memory';
344
+
345
+ --
346
+ -- Name: COLUMN robot_nodes.deleted_at; Type: COMMENT; Schema: public; Owner: -
347
+ --
348
+
349
+ COMMENT ON COLUMN public.robot_nodes.deleted_at IS 'Soft delete timestamp';
350
+
327
351
  --
328
352
  -- Name: robot_nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
329
353
  --
@@ -408,7 +432,8 @@ CREATE TABLE public.schema_migrations (
408
432
  CREATE TABLE public.tags (
409
433
  id bigint NOT NULL,
410
434
  name text NOT NULL,
411
- created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP
435
+ created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
436
+ deleted_at timestamp with time zone
412
437
  );
413
438
 
414
439
  --
@@ -430,69 +455,16 @@ COMMENT ON COLUMN public.tags.name IS 'Hierarchical tag in format: root:level1:l
430
455
  COMMENT ON COLUMN public.tags.created_at IS 'When this tag was created';
431
456
 
432
457
  --
433
- -- Name: tags_id_seq; Type: SEQUENCE; Schema: public; Owner: -
458
+ -- Name: COLUMN tags.deleted_at; Type: COMMENT; Schema: public; Owner: -
434
459
  --
435
460
 
436
- CREATE SEQUENCE public.tags_id_seq
437
- START WITH 1
438
- INCREMENT BY 1
439
- NO MINVALUE
440
- NO MAXVALUE
441
- CACHE 1;
461
+ COMMENT ON COLUMN public.tags.deleted_at IS 'Soft delete timestamp';
442
462
 
443
463
  --
444
- -- Name: tags_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
445
- --
446
-
447
- ALTER SEQUENCE public.tags_id_seq OWNED BY public.tags.id;
448
-
449
- --
450
- -- Name: working_memories; Type: TABLE; Schema: public; Owner: -
451
- --
452
-
453
- CREATE TABLE public.working_memories (
454
- id bigint NOT NULL,
455
- robot_id bigint NOT NULL,
456
- node_id bigint NOT NULL,
457
- added_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
458
- token_count integer
459
- );
460
-
461
- --
462
- -- Name: TABLE working_memories; Type: COMMENT; Schema: public; Owner: -
463
- --
464
-
465
- COMMENT ON TABLE public.working_memories IS 'Per-robot working memory state (optional persistence)';
466
-
467
- --
468
- -- Name: COLUMN working_memories.robot_id; Type: COMMENT; Schema: public; Owner: -
469
- --
470
-
471
- COMMENT ON COLUMN public.working_memories.robot_id IS 'Robot whose working memory this belongs to';
472
-
473
- --
474
- -- Name: COLUMN working_memories.node_id; Type: COMMENT; Schema: public; Owner: -
475
- --
476
-
477
- COMMENT ON COLUMN public.working_memories.node_id IS 'Node currently in working memory';
478
-
479
- --
480
- -- Name: COLUMN working_memories.added_at; Type: COMMENT; Schema: public; Owner: -
481
- --
482
-
483
- COMMENT ON COLUMN public.working_memories.added_at IS 'When node was added to working memory';
484
-
485
- --
486
- -- Name: COLUMN working_memories.token_count; Type: COMMENT; Schema: public; Owner: -
487
- --
488
-
489
- COMMENT ON COLUMN public.working_memories.token_count IS 'Cached token count for budget tracking';
490
-
491
- --
492
- -- Name: working_memories_id_seq; Type: SEQUENCE; Schema: public; Owner: -
464
+ -- Name: tags_id_seq; Type: SEQUENCE; Schema: public; Owner: -
493
465
  --
494
466
 
495
- CREATE SEQUENCE public.working_memories_id_seq
467
+ CREATE SEQUENCE public.tags_id_seq
496
468
  START WITH 1
497
469
  INCREMENT BY 1
498
470
  NO MINVALUE
@@ -500,10 +472,10 @@ CREATE SEQUENCE public.working_memories_id_seq
500
472
  CACHE 1;
501
473
 
502
474
  --
503
- -- Name: working_memories_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
475
+ -- Name: tags_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
504
476
  --
505
477
 
506
- ALTER SEQUENCE public.working_memories_id_seq OWNED BY public.working_memories.id;
478
+ ALTER SEQUENCE public.tags_id_seq OWNED BY public.tags.id;
507
479
 
508
480
  --
509
481
  -- Name: file_sources id; Type: DEFAULT; Schema: public; Owner: -
@@ -541,12 +513,6 @@ ALTER TABLE ONLY public.robots ALTER COLUMN id SET DEFAULT nextval('public.robot
541
513
 
542
514
  ALTER TABLE ONLY public.tags ALTER COLUMN id SET DEFAULT nextval('public.tags_id_seq'::regclass);
543
515
 
544
- --
545
- -- Name: working_memories id; Type: DEFAULT; Schema: public; Owner: -
546
- --
547
-
548
- ALTER TABLE ONLY public.working_memories ALTER COLUMN id SET DEFAULT nextval('public.working_memories_id_seq'::regclass);
549
-
550
516
  --
551
517
  -- Name: file_sources file_sources_pkey; Type: CONSTRAINT; Schema: public; Owner: -
552
518
  --
@@ -596,13 +562,6 @@ ALTER TABLE ONLY public.schema_migrations
596
562
  ALTER TABLE ONLY public.tags
597
563
  ADD CONSTRAINT tags_pkey PRIMARY KEY (id);
598
564
 
599
- --
600
- -- Name: working_memories working_memories_pkey; Type: CONSTRAINT; Schema: public; Owner: -
601
- --
602
-
603
- ALTER TABLE ONLY public.working_memories
604
- ADD CONSTRAINT working_memories_pkey PRIMARY KEY (id);
605
-
606
565
  --
607
566
  -- Name: idx_file_sources_hash; Type: INDEX; Schema: public; Owner: -
608
567
  --
@@ -621,6 +580,12 @@ CREATE INDEX idx_file_sources_last_synced ON public.file_sources USING btree (la
621
580
 
622
581
  CREATE UNIQUE INDEX idx_file_sources_path_unique ON public.file_sources USING btree (file_path);
623
582
 
583
+ --
584
+ -- Name: idx_node_tags_deleted_at; Type: INDEX; Schema: public; Owner: -
585
+ --
586
+
587
+ CREATE INDEX idx_node_tags_deleted_at ON public.node_tags USING btree (deleted_at);
588
+
624
589
  --
625
590
  -- Name: idx_node_tags_node_id; Type: INDEX; Schema: public; Owner: -
626
591
  --
@@ -645,6 +610,18 @@ CREATE UNIQUE INDEX idx_node_tags_unique ON public.node_tags USING btree (node_i
645
610
 
646
611
  CREATE INDEX idx_nodes_access_count ON public.nodes USING btree (access_count);
647
612
 
613
+ --
614
+ -- Name: idx_nodes_active; Type: INDEX; Schema: public; Owner: -
615
+ --
616
+
617
+ CREATE INDEX idx_nodes_active ON public.nodes USING btree (id) WHERE (deleted_at IS NULL);
618
+
619
+ --
620
+ -- Name: idx_nodes_active_with_embedding; Type: INDEX; Schema: public; Owner: -
621
+ --
622
+
623
+ CREATE INDEX idx_nodes_active_with_embedding ON public.nodes USING btree (id) WHERE ((deleted_at IS NULL) AND (embedding IS NOT NULL));
624
+
648
625
  --
649
626
  -- Name: idx_nodes_content_gin; Type: INDEX; Schema: public; Owner: -
650
627
  --
@@ -717,6 +694,12 @@ CREATE INDEX idx_nodes_source_id ON public.nodes USING btree (source_id);
717
694
 
718
695
  CREATE INDEX idx_nodes_updated_at ON public.nodes USING btree (updated_at);
719
696
 
697
+ --
698
+ -- Name: idx_robot_nodes_deleted_at; Type: INDEX; Schema: public; Owner: -
699
+ --
700
+
701
+ CREATE INDEX idx_robot_nodes_deleted_at ON public.robot_nodes USING btree (deleted_at);
702
+
720
703
  --
721
704
  -- Name: idx_robot_nodes_last_remembered_at; Type: INDEX; Schema: public; Owner: -
722
705
  --
@@ -742,48 +725,34 @@ CREATE INDEX idx_robot_nodes_robot_id ON public.robot_nodes USING btree (robot_i
742
725
  CREATE UNIQUE INDEX idx_robot_nodes_unique ON public.robot_nodes USING btree (robot_id, node_id);
743
726
 
744
727
  --
745
- -- Name: idx_tags_name_pattern; Type: INDEX; Schema: public; Owner: -
728
+ -- Name: idx_robot_nodes_working_memory; Type: INDEX; Schema: public; Owner: -
746
729
  --
747
730
 
748
- CREATE INDEX idx_tags_name_pattern ON public.tags USING btree (name text_pattern_ops);
731
+ CREATE INDEX idx_robot_nodes_working_memory ON public.robot_nodes USING btree (robot_id, working_memory) WHERE (working_memory = true);
749
732
 
750
733
  --
751
- -- Name: idx_tags_name_unique; Type: INDEX; Schema: public; Owner: -
734
+ -- Name: idx_tags_deleted_at; Type: INDEX; Schema: public; Owner: -
752
735
  --
753
736
 
754
- CREATE UNIQUE INDEX idx_tags_name_unique ON public.tags USING btree (name);
737
+ CREATE INDEX idx_tags_deleted_at ON public.tags USING btree (deleted_at);
755
738
 
756
739
  --
757
- -- Name: idx_working_memories_node_id; Type: INDEX; Schema: public; Owner: -
758
- --
759
-
760
- CREATE INDEX idx_working_memories_node_id ON public.working_memories USING btree (node_id);
761
-
762
- --
763
- -- Name: idx_working_memories_robot_id; Type: INDEX; Schema: public; Owner: -
764
- --
765
-
766
- CREATE INDEX idx_working_memories_robot_id ON public.working_memories USING btree (robot_id);
767
-
768
- --
769
- -- Name: idx_working_memories_unique; Type: INDEX; Schema: public; Owner: -
740
+ -- Name: idx_tags_name_pattern; Type: INDEX; Schema: public; Owner: -
770
741
  --
771
742
 
772
- CREATE UNIQUE INDEX idx_working_memories_unique ON public.working_memories USING btree (robot_id, node_id);
743
+ CREATE INDEX idx_tags_name_pattern ON public.tags USING btree (name text_pattern_ops);
773
744
 
774
745
  --
775
- -- Name: working_memories fk_rails_2c1d8b383c; Type: FK CONSTRAINT; Schema: public; Owner: -
746
+ -- Name: idx_tags_name_trgm; Type: INDEX; Schema: public; Owner: -
776
747
  --
777
748
 
778
- ALTER TABLE ONLY public.working_memories
779
- ADD CONSTRAINT fk_rails_2c1d8b383c FOREIGN KEY (node_id) REFERENCES public.nodes(id) ON DELETE CASCADE;
749
+ CREATE INDEX idx_tags_name_trgm ON public.tags USING gin (name public.gin_trgm_ops);
780
750
 
781
751
  --
782
- -- Name: working_memories fk_rails_4b7c3eb07b; Type: FK CONSTRAINT; Schema: public; Owner: -
752
+ -- Name: idx_tags_name_unique; Type: INDEX; Schema: public; Owner: -
783
753
  --
784
754
 
785
- ALTER TABLE ONLY public.working_memories
786
- ADD CONSTRAINT fk_rails_4b7c3eb07b FOREIGN KEY (robot_id) REFERENCES public.robots(id) ON DELETE CASCADE;
755
+ CREATE UNIQUE INDEX idx_tags_name_unique ON public.tags USING btree (name);
787
756
 
788
757
  --
789
758
  -- Name: nodes fk_rails_920ad16d08; Type: FK CONSTRAINT; Schema: public; Owner: -
@@ -824,4 +793,4 @@ ALTER TABLE ONLY public.robot_nodes
824
793
  -- PostgreSQL database dump complete
825
794
  --
826
795
 
827
- \unrestrict DUrF24Zrve4qSBwlDrJ4qAzzZhvhX5s2S57oHYVJ0ZPbaDC4ItMZ29Pv9oI3Q9d
796
+ \unrestrict 4WlUqnJzNHaNhcr67XLIIhAvRPidZODUPGkM34l27SvmC0zu6dIsQdJ8dtu589Z
data/docs/api/index.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # API Reference
2
2
 
3
- Complete API documentation for HTM (Hierarchical Temporary Memory).
3
+ Complete API documentation for HTM (Hierarchical Temporal Memory).
4
4
 
5
5
  ## Overview
6
6