htm 0.0.30 → 0.0.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. checksums.yaml +4 -4
  2. data/.irbrc +2 -3
  3. data/.rubocop.yml +184 -0
  4. data/CHANGELOG.md +46 -0
  5. data/README.md +2 -0
  6. data/Rakefile +93 -12
  7. data/db/migrate/00008_create_node_relationships.rb +54 -0
  8. data/db/migrate/00009_fix_node_relationships_column_types.rb +17 -0
  9. data/db/schema.sql +124 -1
  10. data/docs/api/database.md +35 -57
  11. data/docs/api/embedding-service.md +1 -1
  12. data/docs/api/index.md +26 -15
  13. data/docs/api/working-memory.md +8 -8
  14. data/docs/architecture/index.md +5 -7
  15. data/docs/architecture/overview.md +5 -8
  16. data/docs/assets/images/htm-architecture-overview.svg +1 -1
  17. data/docs/assets/images/htm-context-assembly-flow.svg +2 -2
  18. data/docs/assets/images/htm-layered-architecture.svg +3 -3
  19. data/docs/assets/images/two-tier-memory-architecture.svg +1 -1
  20. data/docs/database/README.md +1 -0
  21. data/docs/database_rake_tasks.md +20 -28
  22. data/docs/development/contributing.md +5 -5
  23. data/docs/development/index.md +4 -7
  24. data/docs/development/schema.md +71 -1
  25. data/docs/development/setup.md +40 -82
  26. data/docs/development/testing.md +1 -1
  27. data/docs/examples/file-loading.md +4 -4
  28. data/docs/examples/mcp-client.md +1 -1
  29. data/docs/getting-started/quick-start.md +4 -4
  30. data/docs/guides/adding-memories.md +14 -1
  31. data/docs/guides/configuration.md +5 -5
  32. data/docs/guides/context-assembly.md +4 -4
  33. data/docs/guides/file-loading.md +12 -12
  34. data/docs/guides/getting-started.md +2 -2
  35. data/docs/guides/long-term-memory.md +7 -27
  36. data/docs/guides/propositions.md +20 -19
  37. data/docs/guides/recalling-memories.md +5 -5
  38. data/docs/guides/tags.md +18 -13
  39. data/docs/multi_framework_support.md +1 -1
  40. data/docs/robots/hive-mind.md +1 -1
  41. data/docs/robots/multi-robot.md +2 -2
  42. data/docs/robots/robot-groups.md +1 -1
  43. data/docs/robots/two-tier-memory.md +72 -94
  44. data/docs/setup_local_database.md +8 -54
  45. data/docs/using_rake_tasks_in_your_app.md +6 -6
  46. data/examples/01_basic_usage.rb +1 -0
  47. data/examples/03_custom_llm_configuration.rb +1 -0
  48. data/examples/04_file_loader_usage.rb +1 -0
  49. data/examples/05_timeframe_demo.rb +1 -0
  50. data/examples/06_example_app/app.rb +1 -0
  51. data/examples/07_cli_app/htm_cli.rb +1 -0
  52. data/examples/09_mcp_client.rb +1 -0
  53. data/examples/10_telemetry/demo.rb +1 -0
  54. data/examples/11_robot_groups/multi_process.rb +1 -0
  55. data/examples/11_robot_groups/same_process.rb +1 -0
  56. data/examples/12_rails_app/.envrc +12 -0
  57. data/examples/12_rails_app/Gemfile +8 -3
  58. data/examples/12_rails_app/Gemfile.lock +94 -89
  59. data/examples/12_rails_app/README.md +70 -19
  60. data/examples/12_rails_app/app/controllers/application_controller.rb +6 -0
  61. data/examples/12_rails_app/app/controllers/chats_controller.rb +305 -0
  62. data/examples/12_rails_app/app/controllers/dashboard_controller.rb +3 -0
  63. data/examples/12_rails_app/app/controllers/files_controller.rb +17 -2
  64. data/examples/12_rails_app/app/controllers/home_controller.rb +8 -0
  65. data/examples/12_rails_app/app/controllers/memories_controller.rb +9 -4
  66. data/examples/12_rails_app/app/controllers/messages_controller.rb +214 -0
  67. data/examples/12_rails_app/app/controllers/robots_controller.rb +11 -1
  68. data/examples/12_rails_app/app/controllers/tags_controller.rb +14 -1
  69. data/examples/12_rails_app/app/javascript/application.js +1 -1
  70. data/examples/12_rails_app/app/models/application_record.rb +5 -0
  71. data/examples/12_rails_app/app/models/chat.rb +36 -0
  72. data/examples/12_rails_app/app/models/message.rb +5 -0
  73. data/examples/12_rails_app/app/models/model.rb +5 -0
  74. data/examples/12_rails_app/app/models/tool_call.rb +5 -0
  75. data/examples/12_rails_app/app/views/chats/index.html.erb +61 -0
  76. data/examples/12_rails_app/app/views/chats/show.html.erb +213 -0
  77. data/examples/12_rails_app/app/views/dashboard/index.html.erb +3 -0
  78. data/examples/12_rails_app/app/views/files/index.html.erb +10 -5
  79. data/examples/12_rails_app/app/views/files/new.html.erb +4 -2
  80. data/examples/12_rails_app/app/views/files/show.html.erb +19 -3
  81. data/examples/12_rails_app/app/views/home/index.html.erb +45 -0
  82. data/examples/12_rails_app/app/views/layouts/application.html.erb +20 -18
  83. data/examples/12_rails_app/app/views/memories/_memory_card.html.erb +1 -1
  84. data/examples/12_rails_app/app/views/memories/deleted.html.erb +3 -1
  85. data/examples/12_rails_app/app/views/memories/edit.html.erb +2 -0
  86. data/examples/12_rails_app/app/views/memories/index.html.erb +2 -0
  87. data/examples/12_rails_app/app/views/memories/new.html.erb +2 -0
  88. data/examples/12_rails_app/app/views/memories/show.html.erb +4 -2
  89. data/examples/12_rails_app/app/views/messages/_message.html.erb +20 -0
  90. data/examples/12_rails_app/app/views/robots/index.html.erb +2 -0
  91. data/examples/12_rails_app/app/views/robots/new.html.erb +2 -0
  92. data/examples/12_rails_app/app/views/robots/show.html.erb +2 -0
  93. data/examples/12_rails_app/app/views/search/index.html.erb +59 -8
  94. data/examples/12_rails_app/app/views/shared/_navbar.html.erb +75 -29
  95. data/examples/12_rails_app/app/views/tags/index.html.erb +2 -0
  96. data/examples/12_rails_app/app/views/tags/show.html.erb +3 -1
  97. data/examples/12_rails_app/config/application.rb +1 -1
  98. data/examples/12_rails_app/config/database.yml +9 -5
  99. data/examples/12_rails_app/config/importmap.rb +1 -1
  100. data/examples/12_rails_app/config/initializers/htm.rb +9 -2
  101. data/examples/12_rails_app/config/initializers/ruby_llm.rb +33 -0
  102. data/examples/12_rails_app/config/routes.rb +39 -23
  103. data/examples/12_rails_app/db/migrate/20250124000001_create_ruby_llm_tables.rb +34 -0
  104. data/examples/12_rails_app/db/migrate/20250124000002_create_models_table.rb +28 -0
  105. data/examples/12_rails_app/db/schema.rb +67 -0
  106. data/examples/examples_helper.rb +25 -0
  107. data/lib/htm/circuit_breaker.rb +5 -6
  108. data/lib/htm/config/builder.rb +12 -12
  109. data/lib/htm/config/database.rb +21 -27
  110. data/lib/htm/config/defaults.yml +25 -13
  111. data/lib/htm/config/validator.rb +12 -18
  112. data/lib/htm/config.rb +93 -173
  113. data/lib/htm/database.rb +193 -199
  114. data/lib/htm/embedding_service.rb +4 -9
  115. data/lib/htm/integrations/sinatra.rb +7 -7
  116. data/lib/htm/job_adapter.rb +14 -21
  117. data/lib/htm/jobs/generate_embedding_job.rb +28 -44
  118. data/lib/htm/jobs/generate_propositions_job.rb +29 -55
  119. data/lib/htm/jobs/generate_relationships_job.rb +137 -0
  120. data/lib/htm/jobs/generate_tags_job.rb +45 -67
  121. data/lib/htm/loaders/markdown_loader.rb +65 -112
  122. data/lib/htm/long_term_memory/fulltext_search.rb +1 -1
  123. data/lib/htm/long_term_memory/hybrid_search.rb +300 -128
  124. data/lib/htm/long_term_memory/node_operations.rb +2 -2
  125. data/lib/htm/long_term_memory/relevance_scorer.rb +100 -68
  126. data/lib/htm/long_term_memory/tag_operations.rb +87 -120
  127. data/lib/htm/long_term_memory/vector_search.rb +1 -1
  128. data/lib/htm/long_term_memory.rb +2 -1
  129. data/lib/htm/mcp/cli.rb +59 -58
  130. data/lib/htm/mcp/server.rb +5 -6
  131. data/lib/htm/mcp/tools.rb +30 -36
  132. data/lib/htm/migration.rb +10 -10
  133. data/lib/htm/models/node.rb +2 -3
  134. data/lib/htm/models/node_relationship.rb +72 -0
  135. data/lib/htm/models/node_tag.rb +2 -2
  136. data/lib/htm/models/robot_node.rb +2 -2
  137. data/lib/htm/models/tag.rb +41 -28
  138. data/lib/htm/observability.rb +45 -51
  139. data/lib/htm/proposition_service.rb +3 -7
  140. data/lib/htm/query_cache.rb +13 -15
  141. data/lib/htm/railtie.rb +1 -2
  142. data/lib/htm/robot_group.rb +9 -9
  143. data/lib/htm/sequel_config.rb +1 -0
  144. data/lib/htm/sql_builder.rb +1 -1
  145. data/lib/htm/tag_service.rb +2 -6
  146. data/lib/htm/timeframe.rb +4 -5
  147. data/lib/htm/timeframe_extractor.rb +42 -83
  148. data/lib/htm/version.rb +1 -1
  149. data/lib/htm/workflows/remember_workflow.rb +112 -115
  150. data/lib/htm/working_memory.rb +21 -26
  151. data/lib/htm.rb +103 -116
  152. data/lib/tasks/db.rake +0 -2
  153. data/lib/tasks/doc.rake +14 -13
  154. data/lib/tasks/files.rake +5 -12
  155. data/lib/tasks/htm.rake +70 -71
  156. data/lib/tasks/jobs.rake +41 -47
  157. data/lib/tasks/tags.rake +3 -8
  158. metadata +28 -106
  159. data/lib/htm/config/section.rb +0 -74
  160. data/lib/htm/loaders/defaults_loader.rb +0 -166
  161. data/lib/htm/loaders/xdg_config_loader.rb +0 -116
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5765fbc2b92d89be3f2bae2f53b4353a76343efd3d092456138bea73ac7803d9
4
- data.tar.gz: e594cc16f789745267ce527dcc182fbd533b7484c36e09887f702007388803a3
3
+ metadata.gz: f9b4fa074840f3acee69bbdf85e5149872d40b5466ad2032793c839af025049c
4
+ data.tar.gz: e153891af2ce3a7649e8103733ccabb9d832d1ef28b435bd8cd5af0afd43d404
5
5
  SHA512:
6
- metadata.gz: 78b4b7e226b9911b429e7e0d8735dd0d50d1e56abf7e9e5f494ab50321e33e574605d6a3991d4f0ce18a95ae6dd8ee7928004e3d455730aa388ee2515ffc53c3
7
- data.tar.gz: c19a0c15d79342d08e406724724cf64afc68c75ea26f1bf332b4a9b661f3ebe9d50e3f9dbf41db780c0542d4cd7a77ac0bb0da9bf45b5ea7d171def21b4ffc4a
6
+ metadata.gz: 38398e6f1856e259f5c7e1b18e0b833fc9cdc08acf5f664d68b96f8bc409b9e7ce9c536f85c331e63f1e8cee28141be0e5c33775fa057bc4f5083102519b3e4b
7
+ data.tar.gz: 4dd7f77aa05cc5d6e4a96f0c8b389729a9e96a831082df715ca6fc9b44f41e856df56c7b9bbdd4c16cdfc614f03aa9e0de9ab9cf96494b90578bf823308c8e2f
data/.irbrc CHANGED
@@ -12,8 +12,8 @@ puts "=" * 60
12
12
 
13
13
  # Database Connection
14
14
  begin
15
- HTM::ActiveRecordConfig.establish_connection! unless HTM::ActiveRecordConfig.connected?
16
- db_name = HTM::Database.default_config[:dbname] rescue 'unknown'
15
+ HTM::SequelConfig.establish_connection! unless HTM::SequelConfig.connected?
16
+ db_name = HTM::SequelConfig.db.opts[:database] rescue 'unknown'
17
17
  puts "✓ Database connected: #{db_name}"
18
18
  rescue => e
19
19
  puts "✗ Database connection failed: #{e.message}"
@@ -27,7 +27,6 @@ HTM.configure do |config|
27
27
  config.embedding.dimensions = 768
28
28
  config.tag.provider = :ollama
29
29
  config.tag.model = 'gemma3:latest'
30
- config.reset_to_defaults
31
30
  end
32
31
  puts "✓ HTM configured (inline jobs, Ollama provider)"
33
32
 
data/.rubocop.yml ADDED
@@ -0,0 +1,184 @@
1
+ AllCops:
2
+ NewCops: enable
3
+ SuggestExtensions: false
4
+ TargetRubyVersion: 4.0
5
+ Exclude:
6
+ - 'examples/**/*'
7
+ - 'vendor/**/*'
8
+ - 'db/**/*'
9
+ - 'scripts/**/*'
10
+
11
+ # ── Style: disabled cops ───────────────────────────────────────────────────
12
+ Style/StringLiterals:
13
+ Enabled: false
14
+
15
+ Style/StringLiteralsInInterpolation:
16
+ Enabled: false
17
+
18
+ Style/Documentation:
19
+ Enabled: false
20
+
21
+ # Ruby 4.0 freezes string literals by default
22
+ Style/FrozenStringLiteralComment:
23
+ Enabled: false
24
+
25
+ Style/IfUnlessModifier:
26
+ Enabled: false
27
+
28
+ Style/RescueModifier:
29
+ Enabled: false
30
+
31
+ Style/TrivialAccessors:
32
+ Enabled: false
33
+
34
+ Style/MultilineTernaryOperator:
35
+ Enabled: false
36
+
37
+ Style/SafeNavigation:
38
+ Enabled: false
39
+
40
+ Style/EmptyClassDefinition:
41
+ Enabled: false
42
+
43
+ Style/ClassAndModuleChildren:
44
+ Enabled: false
45
+
46
+ Style/RescueStandardError:
47
+ Enabled: false
48
+
49
+ Style/OneClassPerFile:
50
+ Enabled: false
51
+
52
+ Style/FormatString:
53
+ Enabled: false
54
+
55
+ Style/StringConcatenation:
56
+ Enabled: false
57
+
58
+ # ── Layout ─────────────────────────────────────────────────────────────────
59
+ Layout/LineLength:
60
+ Max: 140
61
+ Exclude:
62
+ - 'lib/tasks/**/*'
63
+ - 'test/**/*'
64
+
65
+ Layout/ExtraSpacing:
66
+ Enabled: false
67
+
68
+ Layout/HashAlignment:
69
+ Enabled: false
70
+
71
+ Layout/FirstHashElementIndentation:
72
+ Enabled: false
73
+
74
+ Layout/EmptyLineAfterGuardClause:
75
+ Enabled: false
76
+
77
+ # ── Naming ─────────────────────────────────────────────────────────────────
78
+ Naming/MethodParameterName:
79
+ Enabled: false
80
+
81
+ Naming/VariableNumber:
82
+ Exclude:
83
+ - 'test/**/*'
84
+
85
+ Naming/RescuedExceptionsVariableName:
86
+ Enabled: false
87
+
88
+ Naming/AccessorMethodName:
89
+ Enabled: false
90
+
91
+ Naming/PredicatePrefix:
92
+ Enabled: false
93
+
94
+ Naming/PredicateMethod:
95
+ AllowedMethods:
96
+ - soft_delete!
97
+ - restore!
98
+ - forget
99
+ - restore
100
+ - run
101
+ - update_existing_chunk
102
+ - verify_extensions!
103
+ - validate_database!
104
+ Exclude:
105
+ - 'test/**/*'
106
+
107
+ # ── Lint ───────────────────────────────────────────────────────────────────
108
+ Lint/EmptyBlock:
109
+ AllowedMethods:
110
+ - arguments
111
+ Exclude:
112
+ - 'test/**/*'
113
+ - 'lib/htm/mcp/**/*'
114
+
115
+ Lint/UnusedMethodArgument:
116
+ Enabled: false
117
+
118
+ Lint/ConstantDefinitionInBlock:
119
+ Exclude:
120
+ - 'Rakefile'
121
+ - 'test/**/*'
122
+
123
+ # ── Gemspec ────────────────────────────────────────────────────────────────
124
+ Gemspec/DevelopmentDependencies:
125
+ EnforcedStyle: Gemfile
126
+
127
+ Gemspec/RequiredRubyVersion:
128
+ Enabled: false
129
+
130
+ Gemspec/OrderedDependencies:
131
+ Enabled: false
132
+
133
+ # ── Metrics ────────────────────────────────────────────────────────────────
134
+ Metrics/MethodLength:
135
+ Max: 35
136
+ CountAsOne:
137
+ - heredoc
138
+ - array
139
+ - hash
140
+ Exclude:
141
+ - 'test/**/*'
142
+ - 'lib/tasks/**/*'
143
+
144
+ Metrics/AbcSize:
145
+ Max: 40
146
+ Exclude:
147
+ - 'test/**/*'
148
+
149
+ Metrics/ClassLength:
150
+ Max: 600
151
+ Exclude:
152
+ - 'test/**/*'
153
+
154
+ Metrics/ModuleLength:
155
+ Max: 450
156
+ Exclude:
157
+ - 'test/**/*'
158
+ - 'lib/tasks/**/*'
159
+
160
+ Metrics/CyclomaticComplexity:
161
+ Max: 20
162
+ Exclude:
163
+ - 'test/**/*'
164
+
165
+ Metrics/PerceivedComplexity:
166
+ Max: 20
167
+ Exclude:
168
+ - 'test/**/*'
169
+
170
+ Metrics/ParameterLists:
171
+ Enabled: false
172
+
173
+ Metrics/BlockLength:
174
+ Exclude:
175
+ - 'Rakefile'
176
+ - '*.gemspec'
177
+ - 'test/**/*'
178
+ - 'lib/tasks/**/*'
179
+ - 'lib/htm/models/**/*'
180
+ - 'lib/htm/long_term_memory/**/*'
181
+
182
+ Style/FormatStringToken:
183
+ Exclude:
184
+ - 'lib/tasks/**/*'
data/CHANGELOG.md CHANGED
@@ -8,6 +8,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
8
8
  ## [Unreleased]
9
9
 
10
10
  ### Added
11
+ - **`node_relationships` table** - Weighted directed edges between related nodes
12
+ - Stores Jaccard-similarity scores computed from tag co-occurrence
13
+ - Both directions (A→B and B→A) stored explicitly for O(1) index lookups during CTE traversal
14
+ - Unique constraint on `(source_id, target_id, rel_type)` — re-runs refresh stale weights via upsert
15
+ - DB CHECK constraints enforce weight in [0.0, 1.0] and prevent self-loops
16
+ - FK constraints (ON DELETE CASCADE) from both `source_id` and `target_id` to nodes
17
+ - **`HTM::Models::NodeRelationship`** - Sequel model for the new table
18
+ - Constants: `REL_TYPES` (`related_to`, `supports`, `contradicts`, `derived_from`) and `ORIGINS` (`tag_cooccurrence`, `tag_hierarchy`, `explicit`)
19
+ - Dataset scopes: `neighbors_of`, `above_weight`, `by_origin`, `by_rel_type`, `between_nodes`
20
+ - Full validation: presence, allowed values, weight range, self-loop rejection, uniqueness
21
+ - **`HTM::Jobs::GenerateRelationshipsJob`** - Background job that computes and upserts Jaccard edges
22
+ - Chained automatically after `GenerateTagsJob` completes
23
+ - Single SQL query computes Jaccard similarity for all tag-sharing nodes at once
24
+ - Skips edges below `MIN_WEIGHT_THRESHOLD` (0.1); caps at `MAX_EDGES_PER_NODE` (50) per node
25
+ - Idempotent: `INSERT ... ON CONFLICT DO UPDATE` refreshes stale weights on re-run
26
+ - **Migrations** `00008_create_node_relationships` and `00009_fix_node_relationships_column_types`
27
+
28
+
11
29
  - **`sslmode` database configuration support** - SSL mode now extracted from URL and included when building URL
12
30
  - `parse_database_url` extracts `sslmode` from URL query string (e.g., `?sslmode=require`)
13
31
  - `build_database_url` includes `sslmode` as query parameter when set
@@ -61,6 +79,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
61
79
  - Increased specificity requirements for self-contained facts
62
80
 
63
81
  ### Fixed
82
+ - **`token_counter` never set on config load** - `setup_defaults` on-load hook assigned to `@setup_defaults` (typo) instead of `@token_counter`, causing `HTM::ValidationError: token_counter must be callable` in any test that triggered a fresh config load via `HTM.configure`
83
+ - **`reset_htm_configuration` lost mock services** - After `HTM.reset_configuration!` the helper only set `job.backend = :inline`, leaving the real Ollama embedding generator active; subsequent tests accumulated circuit breaker failures and left nodes without embeddings. Now calls `configure_htm_with_mocks` to fully restore test defaults
84
+ - **Integration test circuit breaker leak** - `IntegrationTest#setup` now explicitly resets `EmbeddingService` and `TagService` circuit breakers before each test to prevent state bleed from other test classes
85
+ - **Duplicate rescue branches** - Merged redundant `rescue HTM::CircuitBreakerOpenError` + service-specific error rescues (both just `raise`) into single `rescue A, B` in `EmbeddingService`, `TagService`, and `PropositionService`
86
+ - **Duplicate `when "cosine"` branch in Node distance operator** - Removed branch whose body was identical to the `else` fallback (`"<=>"`), eliminating the `Lint/DuplicateBranch` offense
87
+ - **Identical conditional branches in `Tag.format_tree_branch`** - `branch = is_last ? '+-- ' : '+-- '` both sides were identical; simplified to a direct string literal
88
+ - **`EmptyElse` in `Observability#process_memory_mb`** - Removed redundant `else nil` branch from the platform detection conditional
89
+
90
+ ### Refactored
91
+ - **Flog complexity reduced across 15+ files** - All methods now score below the 50.0 failure threshold (previously 9 methods failed):
92
+ - `HTM::Models::Tag::generate_tree_svg` (93.6 → passing): extracted `svg_title_element`, `svg_node_elements`, `svg_node_coords`, `svg_edge_path`, `svg_node_rect`, `svg_node_label`
93
+ - `HybridSearch#search_hybrid_rrf` (83.2 → passing): extracted `extract_rrf_tag_info`, `build_rrf_literals`, `build_rrf_filter_sql`, `post_process_rrf_results`
94
+ - `HybridSearch#merge_with_rrf` (65.7 → passing): extracted `merge_vector_rrf_entry`, `merge_fulltext_rrf_entry`, `merge_tag_rrf_entry`
95
+ - `RelevanceScorer#fetch_nodes_by_tags` (62.6 → passing): extracted `build_tag_base_query`, `apply_match_all_constraint`
96
+ - `MarkdownLoader#sync_chunks` (61.2 → passing): extracted `update_existing_chunk`, `soft_delete_removed_chunks`
97
+ - `HybridSearch#build_hybrid_cte_sql` (55.9 → passing): extracted per-CTE builder methods
98
+ - `Config::Database#build_database_url` (55.6 → passing): extracted `database_auth_segment`
99
+ - `MCP::GetWorkingMemoryTool#call` (56.0 → passing): extracted `format_working_memory_entry`
100
+ - `MarkdownLoader#load_file` (50.1 → passing): extracted `validate_file_path!`, `read_file_content`, `prepend_frontmatter_to_chunk`
101
+ - `Config#apply_provider_config` (77.4 → passing): replaced `case` statement with dynamic dispatch (`send(:"apply_#{provider}_provider_config", config)`)
102
+ - `RememberWorkflow#finalize_step` (50.7 → passing): extracted `finalize_node`, `evict_working_memory_if_needed`
103
+ - `HTM#remember`: extracted `validate_remember_content!`, `validate_remember_tags!`, `enqueue_background_jobs`, `handle_existing_node_tags`, `store_in_working_memory`
104
+ - **RuboCop: 128 offenses resolved** - Zero offenses across 89 files:
105
+ - Replaced all `[[x, min].max, max].min` patterns with `x.clamp(min, max)` in `tag_operations.rb`, `fulltext_search.rb`, `vector_search.rb`, `hybrid_search.rb`
106
+ - Broke all `end.sort_by {}` / `end.map {}` multiline block chains into two statements across `database.rb`, `hybrid_search.rb`, `relevance_scorer.rb`, `mcp/cli.rb`, `working_memory.rb`
107
+ - Updated `.rubocop.yml`: added `AllowedMethods` for bang methods (`soft_delete!`, `restore!`) and MCP DSL (`arguments` block), excluded `lib/tasks/**/*` from `BlockLength`, `MethodLength`, `FormatStringToken`, and `LineLength` cops
108
+ - Removed development dependencies from `htm.gemspec` in favour of `Gemfile` group blocks (enforced by `Gemspec/DevelopmentDependencies`)
109
+
64
110
  - **Test database isolation** - Two-layer protection prevents tests from polluting development/production
65
111
  - `Rakefile`: `set_test_env` task now ALWAYS overrides `HTM_DATABASE__URL` to test database
66
112
  - Uses `#{service_name}_test` pattern (e.g., `htm_test`) based on `HTM_SERVICE__NAME` env var
data/README.md CHANGED
@@ -150,8 +150,10 @@ For MCP integration, database setup, and configuration options, see the [full do
150
150
  | **Hive Mind** | Shared memory across all robots |
151
151
  | **Vector Search** | Semantic retrieval with pgvector |
152
152
  | **Hybrid Search** | Combines vector + full-text matching |
153
+ | **Graph Traversal** | CTE-based relationship traversal via Jaccard-weighted edges |
153
154
  | **Temporal Queries** | "last week", "yesterday", date ranges |
154
155
  | **Auto-Tagging** | LLM extracts hierarchical tags automatically |
156
+ | **Auto-Relationships** | Tag co-occurrence edges computed after tagging |
155
157
  | **Robot Groups** | High-availability with failover |
156
158
  | **Rails Integration** | Auto-configures via Railtie, uses ActiveJob |
157
159
  | **Telemetry** | Optional OpenTelemetry metrics |
data/Rakefile CHANGED
@@ -3,11 +3,14 @@
3
3
  require "bundler/gem_tasks"
4
4
  require "rake/testtask"
5
5
 
6
+ RUBOCOP_ENV = { "RUBOCOP_CACHE_ROOT" => "tmp/rubocop_cache" }.freeze
7
+
6
8
  Rake::TestTask.new(:test) do |t|
7
9
  t.libs << "test"
8
10
  t.libs << "lib"
9
11
  t.test_files = FileList["test/**/*_test.rb"]
10
12
  t.verbose = true
13
+ t.ruby_opts << "-rtest_helper"
11
14
  end
12
15
 
13
16
  # Prepend environment setup before test runs
@@ -23,7 +26,7 @@ task :set_test_env do
23
26
 
24
27
  # ALWAYS use the test database - never allow tests to run against other databases
25
28
  # This prevents accidental pollution of development/production data
26
- test_db_url = "postgresql://#{ENV['USER']}@localhost:5432/#{test_db_name}"
29
+ test_db_url = "postgresql://#{ENV.fetch('USER', nil)}@localhost:5432/#{test_db_name}"
27
30
 
28
31
  if ENV['HTM_DATABASE__URL'] && !ENV['HTM_DATABASE__URL'].include?('_test')
29
32
  warn "WARNING: HTM_DATABASE__URL was set to '#{ENV['HTM_DATABASE__URL']}'"
@@ -52,7 +55,7 @@ task :set_examples_env do
52
55
  examples_db_name = "#{service_name}_examples"
53
56
 
54
57
  # ALWAYS use the examples database
55
- examples_db_url = "postgresql://#{ENV['USER']}@localhost:5432/#{examples_db_name}"
58
+ examples_db_url = "postgresql://#{ENV.fetch('USER', nil)}@localhost:5432/#{examples_db_name}"
56
59
 
57
60
  if ENV['HTM_DATABASE__URL'] && !ENV['HTM_DATABASE__URL'].include?('_examples')
58
61
  warn "WARNING: HTM_DATABASE__URL was set to '#{ENV['HTM_DATABASE__URL']}'"
@@ -80,7 +83,7 @@ namespace :examples do
80
83
  end
81
84
 
82
85
  desc "Run all standalone examples"
83
- task :all => :set_examples_env do
86
+ task all: :set_examples_env do
84
87
  examples = %w[
85
88
  examples/01_basic_usage.rb
86
89
  examples/03_custom_llm_configuration.rb
@@ -88,12 +91,11 @@ namespace :examples do
88
91
  examples/05_timeframe_demo.rb
89
92
  ]
90
93
  examples.each do |example|
91
- if File.exist?(example)
92
- puts "\n#{'=' * 60}"
93
- puts "Running: #{example}"
94
- puts "#{'=' * 60}"
95
- ruby example
96
- end
94
+ next unless File.exist?(example)
95
+ puts "\n#{'=' * 60}"
96
+ puts "Running: #{example}"
97
+ puts('=' * 60)
98
+ ruby example
97
99
  end
98
100
  end
99
101
 
@@ -102,8 +104,8 @@ namespace :examples do
102
104
  require_relative 'lib/htm'
103
105
  puts "Examples Environment Status"
104
106
  puts "=" * 40
105
- puts "HTM_ENV: #{ENV['HTM_ENV']}"
106
- puts "Database URL: #{ENV['HTM_DATABASE__URL']}"
107
+ puts "HTM_ENV: #{ENV.fetch('HTM_ENV', nil)}"
108
+ puts "Database URL: #{ENV.fetch('HTM_DATABASE__URL', nil)}"
107
109
  puts "Expected database: #{HTM.config.expected_database_name}"
108
110
  if HTM.config.database_configured?
109
111
  puts "Database configured: Yes"
@@ -129,13 +131,92 @@ namespace :examples do
129
131
  end
130
132
 
131
133
  desc "Run example (alias for examples:basic)"
132
- task :example => 'examples:basic'
134
+ task example: 'examples:basic'
133
135
 
134
136
  desc "Run timeframe demo"
135
137
  task :timeframe_demo do
136
138
  ruby "examples/05_timeframe_demo.rb"
137
139
  end
138
140
 
141
+ desc "Check code style with RuboCop"
142
+ task :rubocop do
143
+ sh RUBOCOP_ENV, "bundle exec rubocop"
144
+ end
145
+
146
+ desc "Auto-correct RuboCop offenses"
147
+ task :rubocop_fix do
148
+ sh RUBOCOP_ENV, "bundle exec rubocop -a"
149
+ end
150
+
151
+ desc "Check code complexity with Flog (warn >=20, fail >=50)"
152
+ task :flog_check do
153
+ require 'flog'
154
+
155
+ method_warn = 20.0
156
+ method_fail = 50.0
157
+
158
+ flogger = Flog.new(all: true)
159
+ flogger.flog(*Dir.glob('lib/**/*.rb').reject { |f| f.start_with?('lib/tasks/') })
160
+
161
+ warnings = []
162
+ failures = []
163
+
164
+ flogger.each_by_score do |method, score|
165
+ next if method.end_with?('#none')
166
+
167
+ if score > method_fail
168
+ failures << "#{format('%.1f', score)}: #{method}"
169
+ elsif score > method_warn
170
+ warnings << "#{format('%.1f', score)}: #{method}"
171
+ end
172
+ end
173
+
174
+ unless warnings.empty?
175
+ puts "\nFlog warnings (#{method_warn}–#{method_fail}) — target for future refactoring:"
176
+ warnings.each { |v| puts " #{v}" }
177
+ end
178
+
179
+ if failures.empty?
180
+ puts "\nFlog: no methods exceed the failure threshold (>=#{method_fail})"
181
+ else
182
+ puts "\nFlog failures (>=#{method_fail}) — must be refactored:"
183
+ failures.each { |v| puts " #{v}" }
184
+ abort "\nFlog quality gate failed: #{failures.size} method(s) exceed #{method_fail}"
185
+ end
186
+ end
187
+
188
+ desc "Run all quality checks: tests (with coverage), RuboCop, and Flog"
189
+ task :quality do
190
+ results = {}
191
+
192
+ puts "\n#{'=' * 60}"
193
+ puts "Quality Gate: Tests + Coverage"
194
+ puts '=' * 60
195
+ results[:tests] = system("bundle exec rake test") ? :pass : :fail
196
+
197
+ puts "\n#{'=' * 60}"
198
+ puts "Quality Gate: RuboCop"
199
+ puts '=' * 60
200
+ results[:rubocop] = system(RUBOCOP_ENV, "bundle exec rubocop") ? :pass : :fail
201
+
202
+ puts "\n#{'=' * 60}"
203
+ puts "Quality Gate: Flog Complexity"
204
+ puts '=' * 60
205
+ results[:flog] = system("bundle exec rake flog_check") ? :pass : :fail
206
+
207
+ puts "\n#{'=' * 60}"
208
+ puts "Quality Summary"
209
+ puts '=' * 60
210
+ results.each do |gate, status|
211
+ icon = status == :pass ? 'PASS' : 'FAIL'
212
+ puts " [#{icon}] #{gate}"
213
+ end
214
+ puts '=' * 60
215
+
216
+ abort "\nQuality gate failed" if results.values.any?(:fail)
217
+ puts "\nAll quality gates passed."
218
+ end
219
+
139
220
  desc "Show gem stats"
140
221
  task :stats do
141
222
  puts "\nHTM Gem Statistics:"
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../../lib/htm/migration'
4
+
5
+ class CreateNodeRelationships < HTM::Migration
6
+ def up
7
+ create_table(:node_relationships) do
8
+ primary_key :id
9
+ Bignum :source_id, null: false
10
+ Bignum :target_id, null: false
11
+ String :rel_type, null: false, default: 'related_to'
12
+ String :origin, null: false, default: 'tag_cooccurrence'
13
+ Float :weight, null: false, default: 1.0
14
+ DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
15
+ DateTime :updated_at, default: Sequel::CURRENT_TIMESTAMP
16
+ end
17
+
18
+ add_index :node_relationships, [:source_id, :target_id, :rel_type],
19
+ unique: true, name: :idx_node_relationships_unique
20
+ add_index :node_relationships, :source_id, name: :idx_node_relationships_source_id
21
+ add_index :node_relationships, :target_id, name: :idx_node_relationships_target_id
22
+ add_index :node_relationships, :weight, name: :idx_node_relationships_weight
23
+ add_index :node_relationships, :origin, name: :idx_node_relationships_origin
24
+
25
+ alter_table(:node_relationships) do
26
+ add_foreign_key [:source_id], :nodes, name: :fk_node_relationships_source, on_delete: :cascade
27
+ add_foreign_key [:target_id], :nodes, name: :fk_node_relationships_target, on_delete: :cascade
28
+ end
29
+
30
+ run <<~SQL
31
+ ALTER TABLE node_relationships
32
+ ADD CONSTRAINT chk_node_relationships_weight
33
+ CHECK (weight >= 0.0 AND weight <= 1.0)
34
+ SQL
35
+
36
+ run <<~SQL
37
+ ALTER TABLE node_relationships
38
+ ADD CONSTRAINT chk_node_relationships_no_self_loop
39
+ CHECK (source_id <> target_id)
40
+ SQL
41
+
42
+ run "COMMENT ON TABLE node_relationships IS 'Weighted directed edges between nodes for graph traversal'"
43
+ run "COMMENT ON COLUMN node_relationships.source_id IS 'Starting node of the relationship'"
44
+ run "COMMENT ON COLUMN node_relationships.target_id IS 'Ending node of the relationship'"
45
+ run "COMMENT ON COLUMN node_relationships.rel_type IS 'Semantic label: related_to, supports, contradicts, derived_from'"
46
+ run "COMMENT ON COLUMN node_relationships.origin IS 'How created: tag_cooccurrence, tag_hierarchy, explicit'"
47
+ run "COMMENT ON COLUMN node_relationships.weight IS 'Relationship strength 0.0-1.0 (Jaccard similarity for tag_cooccurrence)'"
48
+ run "COMMENT ON COLUMN node_relationships.updated_at IS 'When weight was last recalculated'"
49
+ end
50
+
51
+ def down
52
+ drop_table(:node_relationships)
53
+ end
54
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../../lib/htm/migration'
4
+
5
+ class FixNodeRelationshipsColumnTypes < HTM::Migration
6
+ def up
7
+ run "ALTER TABLE node_relationships ALTER COLUMN id TYPE bigint"
8
+ run "ALTER TABLE node_relationships ALTER COLUMN created_at TYPE timestamp with time zone"
9
+ run "ALTER TABLE node_relationships ALTER COLUMN updated_at TYPE timestamp with time zone"
10
+ end
11
+
12
+ def down
13
+ run "ALTER TABLE node_relationships ALTER COLUMN id TYPE integer"
14
+ run "ALTER TABLE node_relationships ALTER COLUMN created_at TYPE timestamp without time zone"
15
+ run "ALTER TABLE node_relationships ALTER COLUMN updated_at TYPE timestamp without time zone"
16
+ end
17
+ end
data/db/schema.sql CHANGED
@@ -115,6 +115,78 @@ CREATE SEQUENCE public.file_sources_id_seq
115
115
 
116
116
  ALTER SEQUENCE public.file_sources_id_seq OWNED BY public.file_sources.id;
117
117
 
118
+ --
119
+ -- Name: node_relationships; Type: TABLE; Schema: public; Owner: -
120
+ --
121
+
122
+ CREATE TABLE public.node_relationships (
123
+ id bigint NOT NULL,
124
+ source_id bigint NOT NULL,
125
+ target_id bigint NOT NULL,
126
+ rel_type text DEFAULT 'related_to'::text NOT NULL,
127
+ origin text DEFAULT 'tag_cooccurrence'::text NOT NULL,
128
+ weight double precision DEFAULT 1.0 NOT NULL,
129
+ created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
130
+ updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
131
+ CONSTRAINT chk_node_relationships_no_self_loop CHECK ((source_id <> target_id)),
132
+ CONSTRAINT chk_node_relationships_weight CHECK (((weight >= (0.0)::double precision) AND (weight <= (1.0)::double precision)))
133
+ );
134
+
135
+ --
136
+ -- Name: TABLE node_relationships; Type: COMMENT; Schema: public; Owner: -
137
+ --
138
+
139
+ COMMENT ON TABLE public.node_relationships IS 'Weighted directed edges between nodes for graph traversal';
140
+
141
+ --
142
+ -- Name: COLUMN node_relationships.source_id; Type: COMMENT; Schema: public; Owner: -
143
+ --
144
+
145
+ COMMENT ON COLUMN public.node_relationships.source_id IS 'Starting node of the relationship';
146
+
147
+ --
148
+ -- Name: COLUMN node_relationships.target_id; Type: COMMENT; Schema: public; Owner: -
149
+ --
150
+
151
+ COMMENT ON COLUMN public.node_relationships.target_id IS 'Ending node of the relationship';
152
+
153
+ --
154
+ -- Name: COLUMN node_relationships.rel_type; Type: COMMENT; Schema: public; Owner: -
155
+ --
156
+
157
+ COMMENT ON COLUMN public.node_relationships.rel_type IS 'Semantic label: related_to, supports, contradicts, derived_from';
158
+
159
+ --
160
+ -- Name: COLUMN node_relationships.origin; Type: COMMENT; Schema: public; Owner: -
161
+ --
162
+
163
+ COMMENT ON COLUMN public.node_relationships.origin IS 'How created: tag_cooccurrence, tag_hierarchy, explicit';
164
+
165
+ --
166
+ -- Name: COLUMN node_relationships.weight; Type: COMMENT; Schema: public; Owner: -
167
+ --
168
+
169
+ COMMENT ON COLUMN public.node_relationships.weight IS 'Relationship strength 0.0-1.0 (Jaccard similarity for tag_cooccurrence)';
170
+
171
+ --
172
+ -- Name: COLUMN node_relationships.updated_at; Type: COMMENT; Schema: public; Owner: -
173
+ --
174
+
175
+ COMMENT ON COLUMN public.node_relationships.updated_at IS 'When weight was last recalculated';
176
+
177
+ --
178
+ -- Name: node_relationships_id_seq; Type: SEQUENCE; Schema: public; Owner: -
179
+ --
180
+
181
+ ALTER TABLE public.node_relationships ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
182
+ SEQUENCE NAME public.node_relationships_id_seq
183
+ START WITH 1
184
+ INCREMENT BY 1
185
+ NO MINVALUE
186
+ NO MAXVALUE
187
+ CACHE 1
188
+ );
189
+
118
190
  --
119
191
  -- Name: node_tags; Type: TABLE; Schema: public; Owner: -
120
192
  --
@@ -536,6 +608,13 @@ ALTER TABLE ONLY public.tags ALTER COLUMN id SET DEFAULT nextval('public.tags_id
536
608
  ALTER TABLE ONLY public.file_sources
537
609
  ADD CONSTRAINT file_sources_pkey PRIMARY KEY (id);
538
610
 
611
+ --
612
+ -- Name: node_relationships node_relationships_pkey; Type: CONSTRAINT; Schema: public; Owner: -
613
+ --
614
+
615
+ ALTER TABLE ONLY public.node_relationships
616
+ ADD CONSTRAINT node_relationships_pkey PRIMARY KEY (id);
617
+
539
618
  --
540
619
  -- Name: node_tags node_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: -
541
620
  --
@@ -596,6 +675,36 @@ CREATE INDEX idx_file_sources_last_synced ON public.file_sources USING btree (la
596
675
 
597
676
  CREATE UNIQUE INDEX idx_file_sources_path_unique ON public.file_sources USING btree (file_path);
598
677
 
678
+ --
679
+ -- Name: idx_node_relationships_origin; Type: INDEX; Schema: public; Owner: -
680
+ --
681
+
682
+ CREATE INDEX idx_node_relationships_origin ON public.node_relationships USING btree (origin);
683
+
684
+ --
685
+ -- Name: idx_node_relationships_source_id; Type: INDEX; Schema: public; Owner: -
686
+ --
687
+
688
+ CREATE INDEX idx_node_relationships_source_id ON public.node_relationships USING btree (source_id);
689
+
690
+ --
691
+ -- Name: idx_node_relationships_target_id; Type: INDEX; Schema: public; Owner: -
692
+ --
693
+
694
+ CREATE INDEX idx_node_relationships_target_id ON public.node_relationships USING btree (target_id);
695
+
696
+ --
697
+ -- Name: idx_node_relationships_unique; Type: INDEX; Schema: public; Owner: -
698
+ --
699
+
700
+ CREATE UNIQUE INDEX idx_node_relationships_unique ON public.node_relationships USING btree (source_id, target_id, rel_type);
701
+
702
+ --
703
+ -- Name: idx_node_relationships_weight; Type: INDEX; Schema: public; Owner: -
704
+ --
705
+
706
+ CREATE INDEX idx_node_relationships_weight ON public.node_relationships USING btree (weight);
707
+
599
708
  --
600
709
  -- Name: idx_node_tags_deleted_at; Type: INDEX; Schema: public; Owner: -
601
710
  --
@@ -770,6 +879,20 @@ CREATE INDEX idx_tags_name_trgm ON public.tags USING gin (name public.gin_trgm_o
770
879
 
771
880
  CREATE UNIQUE INDEX idx_tags_name_unique ON public.tags USING btree (name);
772
881
 
882
+ --
883
+ -- Name: node_relationships fk_node_relationships_source; Type: FK CONSTRAINT; Schema: public; Owner: -
884
+ --
885
+
886
+ ALTER TABLE ONLY public.node_relationships
887
+ ADD CONSTRAINT fk_node_relationships_source FOREIGN KEY (source_id) REFERENCES public.nodes(id) ON DELETE CASCADE;
888
+
889
+ --
890
+ -- Name: node_relationships fk_node_relationships_target; Type: FK CONSTRAINT; Schema: public; Owner: -
891
+ --
892
+
893
+ ALTER TABLE ONLY public.node_relationships
894
+ ADD CONSTRAINT fk_node_relationships_target FOREIGN KEY (target_id) REFERENCES public.nodes(id) ON DELETE CASCADE;
895
+
773
896
  --
774
897
  -- Name: nodes fk_rails_920ad16d08; Type: FK CONSTRAINT; Schema: public; Owner: -
775
898
  --
@@ -809,4 +932,4 @@ ALTER TABLE ONLY public.robot_nodes
809
932
  -- PostgreSQL database dump complete
810
933
  --
811
934
 
812
- \unrestrict 1ItB7RQU4jC5IvOL40FU9j9sS6bzk9jcKeDUYSOd78ym0sA7pq0FXYSOEoWsPh7
935
+ \unrestrict w7upGmgsSOptbQDRWSmlGQx3H1ForwMj2X115QW1LhH1csS9lg06winM4mrfbpS