htm 0.0.1 → 0.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. checksums.yaml +4 -4
  2. data/.aigcm_msg +1 -0
  3. data/.architecture/reviews/comprehensive-codebase-review.md +577 -0
  4. data/.claude/settings.local.json +92 -0
  5. data/.envrc +1 -0
  6. data/.irbrc +283 -80
  7. data/.tbls.yml +31 -0
  8. data/CHANGELOG.md +314 -16
  9. data/CLAUDE.md +603 -0
  10. data/README.md +76 -5
  11. data/Rakefile +5 -0
  12. data/SETUP.md +132 -101
  13. data/db/migrate/{20250101000001_enable_extensions.rb → 00001_enable_extensions.rb} +0 -1
  14. data/db/migrate/00002_create_robots.rb +11 -0
  15. data/db/migrate/00003_create_file_sources.rb +20 -0
  16. data/db/migrate/00004_create_nodes.rb +65 -0
  17. data/db/migrate/00005_create_tags.rb +13 -0
  18. data/db/migrate/00006_create_node_tags.rb +18 -0
  19. data/db/migrate/00007_create_robot_nodes.rb +26 -0
  20. data/db/migrate/00009_add_working_memory_to_robot_nodes.rb +12 -0
  21. data/db/schema.sql +390 -36
  22. data/docs/api/database.md +19 -232
  23. data/docs/api/embedding-service.md +1 -7
  24. data/docs/api/htm.md +305 -364
  25. data/docs/api/index.md +1 -7
  26. data/docs/api/long-term-memory.md +342 -590
  27. data/docs/api/yard/HTM/ActiveRecordConfig.md +23 -0
  28. data/docs/api/yard/HTM/AuthorizationError.md +11 -0
  29. data/docs/api/yard/HTM/CircuitBreaker.md +92 -0
  30. data/docs/api/yard/HTM/CircuitBreakerOpenError.md +34 -0
  31. data/docs/api/yard/HTM/Configuration.md +175 -0
  32. data/docs/api/yard/HTM/Database.md +99 -0
  33. data/docs/api/yard/HTM/DatabaseError.md +14 -0
  34. data/docs/api/yard/HTM/EmbeddingError.md +18 -0
  35. data/docs/api/yard/HTM/EmbeddingService.md +58 -0
  36. data/docs/api/yard/HTM/Error.md +11 -0
  37. data/docs/api/yard/HTM/JobAdapter.md +39 -0
  38. data/docs/api/yard/HTM/LongTermMemory.md +342 -0
  39. data/docs/api/yard/HTM/NotFoundError.md +17 -0
  40. data/docs/api/yard/HTM/Observability.md +107 -0
  41. data/docs/api/yard/HTM/QueryTimeoutError.md +19 -0
  42. data/docs/api/yard/HTM/Railtie.md +27 -0
  43. data/docs/api/yard/HTM/ResourceExhaustedError.md +13 -0
  44. data/docs/api/yard/HTM/TagError.md +18 -0
  45. data/docs/api/yard/HTM/TagService.md +67 -0
  46. data/docs/api/yard/HTM/Timeframe/Result.md +24 -0
  47. data/docs/api/yard/HTM/Timeframe.md +40 -0
  48. data/docs/api/yard/HTM/TimeframeExtractor/Result.md +24 -0
  49. data/docs/api/yard/HTM/TimeframeExtractor.md +45 -0
  50. data/docs/api/yard/HTM/ValidationError.md +20 -0
  51. data/docs/api/yard/HTM/WorkingMemory.md +131 -0
  52. data/docs/api/yard/HTM.md +80 -0
  53. data/docs/api/yard/index.csv +179 -0
  54. data/docs/api/yard-reference.md +51 -0
  55. data/docs/architecture/adrs/001-postgresql-timescaledb.md +1 -1
  56. data/docs/architecture/adrs/003-ollama-embeddings.md +1 -1
  57. data/docs/architecture/adrs/010-redis-working-memory-rejected.md +2 -27
  58. data/docs/architecture/adrs/index.md +2 -13
  59. data/docs/architecture/hive-mind.md +165 -166
  60. data/docs/architecture/index.md +2 -2
  61. data/docs/architecture/overview.md +5 -171
  62. data/docs/architecture/two-tier-memory.md +1 -35
  63. data/docs/assets/images/adr-010-current-architecture.svg +37 -0
  64. data/docs/assets/images/adr-010-proposed-architecture.svg +48 -0
  65. data/docs/assets/images/adr-dependency-tree.svg +93 -0
  66. data/docs/assets/images/class-hierarchy.svg +55 -0
  67. data/docs/assets/images/exception-hierarchy.svg +45 -0
  68. data/docs/assets/images/htm-architecture-overview.svg +83 -0
  69. data/docs/assets/images/htm-complete-memory-flow.svg +160 -0
  70. data/docs/assets/images/htm-context-assembly-flow.svg +148 -0
  71. data/docs/assets/images/htm-eviction-process.svg +141 -0
  72. data/docs/assets/images/htm-memory-addition-flow.svg +138 -0
  73. data/docs/assets/images/htm-memory-recall-flow.svg +152 -0
  74. data/docs/assets/images/htm-node-states.svg +123 -0
  75. data/docs/assets/images/project-structure.svg +78 -0
  76. data/docs/assets/images/test-directory-structure.svg +38 -0
  77. data/{dbdoc → docs/database}/README.md +127 -125
  78. data/docs/database/public.file_sources.md +42 -0
  79. data/docs/database/public.file_sources.svg +211 -0
  80. data/{dbdoc → docs/database}/public.node_tags.md +7 -8
  81. data/docs/database/public.node_tags.svg +239 -0
  82. data/{dbdoc → docs/database}/public.nodes.md +22 -17
  83. data/docs/database/public.nodes.svg +271 -0
  84. data/docs/database/public.robot_nodes.md +46 -0
  85. data/docs/database/public.robot_nodes.svg +243 -0
  86. data/{dbdoc → docs/database}/public.robots.md +2 -3
  87. data/docs/database/public.robots.svg +161 -0
  88. data/docs/database/public.tags.svg +139 -0
  89. data/{dbdoc → docs/database}/schema.json +941 -630
  90. data/docs/database/schema.svg +282 -0
  91. data/docs/development/index.md +1 -29
  92. data/docs/development/schema.md +134 -309
  93. data/docs/development/testing.md +1 -9
  94. data/docs/getting-started/index.md +47 -0
  95. data/docs/{installation.md → getting-started/installation.md} +2 -2
  96. data/docs/{quick-start.md → getting-started/quick-start.md} +5 -5
  97. data/docs/guides/adding-memories.md +295 -643
  98. data/docs/guides/recalling-memories.md +36 -1
  99. data/docs/guides/search-strategies.md +85 -51
  100. data/docs/images/htm-er-diagram.svg +156 -0
  101. data/docs/index.md +16 -31
  102. data/docs/multi_framework_support.md +4 -4
  103. data/examples/README.md +280 -0
  104. data/examples/basic_usage.rb +18 -16
  105. data/examples/cli_app/htm_cli.rb +146 -8
  106. data/examples/cli_app/temp.log +93 -0
  107. data/examples/custom_llm_configuration.rb +1 -2
  108. data/examples/example_app/app.rb +11 -14
  109. data/examples/file_loader_usage.rb +177 -0
  110. data/examples/robot_groups/lib/robot_group.rb +419 -0
  111. data/examples/robot_groups/lib/working_memory_channel.rb +140 -0
  112. data/examples/robot_groups/multi_process.rb +286 -0
  113. data/examples/robot_groups/robot_worker.rb +136 -0
  114. data/examples/robot_groups/same_process.rb +229 -0
  115. data/examples/sinatra_app/Gemfile +1 -0
  116. data/examples/sinatra_app/Gemfile.lock +166 -0
  117. data/examples/sinatra_app/app.rb +219 -24
  118. data/examples/timeframe_demo.rb +276 -0
  119. data/lib/htm/active_record_config.rb +10 -3
  120. data/lib/htm/circuit_breaker.rb +202 -0
  121. data/lib/htm/configuration.rb +313 -80
  122. data/lib/htm/database.rb +67 -36
  123. data/lib/htm/embedding_service.rb +39 -2
  124. data/lib/htm/errors.rb +131 -11
  125. data/lib/htm/{sinatra.rb → integrations/sinatra.rb} +87 -12
  126. data/lib/htm/job_adapter.rb +10 -3
  127. data/lib/htm/jobs/generate_embedding_job.rb +5 -4
  128. data/lib/htm/jobs/generate_tags_job.rb +4 -0
  129. data/lib/htm/loaders/markdown_loader.rb +263 -0
  130. data/lib/htm/loaders/paragraph_chunker.rb +112 -0
  131. data/lib/htm/long_term_memory.rb +601 -321
  132. data/lib/htm/models/file_source.rb +99 -0
  133. data/lib/htm/models/node.rb +116 -12
  134. data/lib/htm/models/robot.rb +53 -4
  135. data/lib/htm/models/robot_node.rb +51 -0
  136. data/lib/htm/models/tag.rb +302 -0
  137. data/lib/htm/observability.rb +395 -0
  138. data/lib/htm/tag_service.rb +60 -3
  139. data/lib/htm/tasks.rb +29 -0
  140. data/lib/htm/timeframe.rb +194 -0
  141. data/lib/htm/timeframe_extractor.rb +307 -0
  142. data/lib/htm/version.rb +1 -1
  143. data/lib/htm/working_memory.rb +165 -70
  144. data/lib/htm.rb +352 -133
  145. data/lib/tasks/doc.rake +300 -0
  146. data/lib/tasks/files.rake +299 -0
  147. data/lib/tasks/htm.rake +188 -2
  148. data/lib/tasks/jobs.rake +10 -12
  149. data/lib/tasks/tags.rake +194 -0
  150. data/mkdocs.yml +91 -9
  151. data/notes/ARCHITECTURE_REVIEW.md +1167 -0
  152. data/notes/IMPLEMENTATION_SUMMARY.md +606 -0
  153. data/notes/MULTI_FRAMEWORK_IMPLEMENTATION.md +451 -0
  154. data/notes/next_steps.md +100 -0
  155. data/notes/plan.md +627 -0
  156. data/notes/tag_ontology_enhancement_ideas.md +222 -0
  157. data/notes/timescaledb_removal_summary.md +200 -0
  158. metadata +177 -37
  159. data/db/migrate/20250101000002_create_robots.rb +0 -14
  160. data/db/migrate/20250101000003_create_nodes.rb +0 -42
  161. data/db/migrate/20250101000005_create_tags.rb +0 -38
  162. data/db/migrate/20250101000007_add_node_vector_indexes.rb +0 -30
  163. data/dbdoc/public.node_tags.svg +0 -112
  164. data/dbdoc/public.nodes.svg +0 -118
  165. data/dbdoc/public.robots.svg +0 -90
  166. data/dbdoc/public.tags.svg +0 -60
  167. data/dbdoc/schema.svg +0 -154
  168. data/{dbdoc → docs/database}/public.node_stats.md +0 -0
  169. data/{dbdoc → docs/database}/public.node_stats.svg +0 -0
  170. data/{dbdoc → docs/database}/public.nodes_tags.md +0 -0
  171. data/{dbdoc → docs/database}/public.nodes_tags.svg +0 -0
  172. data/{dbdoc → docs/database}/public.ontology_structure.md +0 -0
  173. data/{dbdoc → docs/database}/public.ontology_structure.svg +0 -0
  174. data/{dbdoc → docs/database}/public.operations_log.md +0 -0
  175. data/{dbdoc → docs/database}/public.operations_log.svg +0 -0
  176. data/{dbdoc → docs/database}/public.relationships.md +0 -0
  177. data/{dbdoc → docs/database}/public.relationships.svg +0 -0
  178. data/{dbdoc → docs/database}/public.robot_activity.md +0 -0
  179. data/{dbdoc → docs/database}/public.robot_activity.svg +0 -0
  180. data/{dbdoc → docs/database}/public.schema_migrations.md +0 -0
  181. data/{dbdoc → docs/database}/public.schema_migrations.svg +0 -0
  182. data/{dbdoc → docs/database}/public.tags.md +3 -3
  183. /data/{dbdoc → docs/database}/public.topic_relationships.md +0 -0
  184. /data/{dbdoc → docs/database}/public.topic_relationships.svg +0 -0
@@ -6,10 +6,44 @@ require 'logger'
6
6
  class HTM
7
7
  # HTM Configuration
8
8
  #
9
- # Applications using HTM should configure LLM access by providing two methods:
10
- # 1. embedding_generator - Converts text to vector embeddings
11
- # 2. tag_extractor - Extracts hierarchical tags from text
12
- # 3. logger - Logger instance for HTM operations
9
+ # HTM uses RubyLLM for multi-provider LLM support. Supported providers:
10
+ # - :openai (OpenAI API)
11
+ # - :anthropic (Anthropic Claude)
12
+ # - :gemini (Google Gemini)
13
+ # - :azure (Azure OpenAI)
14
+ # - :ollama (Local Ollama - default)
15
+ # - :huggingface (HuggingFace Inference API)
16
+ # - :openrouter (OpenRouter)
17
+ # - :bedrock (AWS Bedrock)
18
+ # - :deepseek (DeepSeek)
19
+ #
20
+ # @example Configure with OpenAI
21
+ # HTM.configure do |config|
22
+ # config.embedding_provider = :openai
23
+ # config.embedding_model = 'text-embedding-3-small'
24
+ # config.tag_provider = :openai
25
+ # config.tag_model = 'gpt-4o-mini'
26
+ # config.openai_api_key = ENV['OPENAI_API_KEY']
27
+ # end
28
+ #
29
+ # @example Configure with Ollama (default)
30
+ # HTM.configure do |config|
31
+ # config.embedding_provider = :ollama
32
+ # config.embedding_model = 'nomic-embed-text'
33
+ # config.tag_provider = :ollama
34
+ # config.tag_model = 'llama3'
35
+ # config.ollama_url = 'http://localhost:11434'
36
+ # end
37
+ #
38
+ # @example Configure with Anthropic for tags, OpenAI for embeddings
39
+ # HTM.configure do |config|
40
+ # config.embedding_provider = :openai
41
+ # config.embedding_model = 'text-embedding-3-small'
42
+ # config.openai_api_key = ENV['OPENAI_API_KEY']
43
+ # config.tag_provider = :anthropic
44
+ # config.tag_model = 'claude-3-haiku-20240307'
45
+ # config.anthropic_api_key = ENV['ANTHROPIC_API_KEY']
46
+ # end
13
47
  #
14
48
  # @example Configure with custom methods
15
49
  # HTM.configure do |config|
@@ -19,39 +53,73 @@ class HTM
19
53
  # config.tag_extractor = ->(text, ontology) {
20
54
  # MyApp::LLMService.extract_tags(text, ontology) # Returns Array<String>
21
55
  # }
22
- # config.logger = Rails.logger # Use Rails logger
56
+ # config.logger = Rails.logger
23
57
  # end
24
58
  #
25
- # @example Use defaults with custom timeouts
26
- # HTM.configure do |config|
27
- # config.embedding_timeout = 60 # 1 minute for faster models
28
- # config.tag_timeout = 300 # 5 minutes for larger models
29
- # config.connection_timeout = 10 # 10 seconds connection timeout
30
- # config.reset_to_defaults # Apply default implementations with new timeouts
31
- # end
32
- #
33
- # @example Use defaults
34
- # HTM.configure # Uses default implementations
35
- #
36
59
  class Configuration
37
60
  attr_accessor :embedding_generator, :tag_extractor, :token_counter
38
61
  attr_accessor :embedding_model, :embedding_provider, :embedding_dimensions
39
62
  attr_accessor :tag_model, :tag_provider
40
- attr_accessor :ollama_url
41
63
  attr_accessor :embedding_timeout, :tag_timeout, :connection_timeout
42
64
  attr_accessor :logger
43
65
  attr_accessor :job_backend
66
+ attr_accessor :week_start
67
+
68
+ # Provider-specific API keys and endpoints
69
+ attr_accessor :openai_api_key, :openai_organization, :openai_project
70
+ attr_accessor :anthropic_api_key
71
+ attr_accessor :gemini_api_key
72
+ attr_accessor :azure_api_key, :azure_endpoint, :azure_api_version
73
+ attr_accessor :ollama_url
74
+ attr_accessor :huggingface_api_key
75
+ attr_accessor :openrouter_api_key
76
+ attr_accessor :bedrock_access_key, :bedrock_secret_key, :bedrock_region
77
+ attr_accessor :deepseek_api_key
78
+
79
+ # Supported providers
80
+ SUPPORTED_PROVIDERS = %i[
81
+ openai anthropic gemini azure ollama
82
+ huggingface openrouter bedrock deepseek
83
+ ].freeze
84
+
85
+ # Default embedding dimensions by provider/model
86
+ DEFAULT_DIMENSIONS = {
87
+ openai: 1536, # text-embedding-3-small
88
+ anthropic: 1024, # voyage embeddings
89
+ gemini: 768, # text-embedding-004
90
+ azure: 1536, # same as OpenAI
91
+ ollama: 768, # nomic-embed-text
92
+ huggingface: 768, # varies by model
93
+ openrouter: 1536, # varies by model
94
+ bedrock: 1536, # titan-embed-text
95
+ deepseek: 1536 # varies by model
96
+ }.freeze
44
97
 
45
98
  def initialize
46
- # Default configuration
99
+ # Default configuration - Ollama for local development
47
100
  @embedding_provider = :ollama
48
- @embedding_model = 'nomic-embed-text'
101
+ @embedding_model = 'nomic-embed-text:latest' # Include tag for Ollama models
49
102
  @embedding_dimensions = 768
50
103
 
51
104
  @tag_provider = :ollama
52
- @tag_model = 'llama3'
53
-
54
- @ollama_url = ENV['OLLAMA_URL'] || 'http://localhost:11434'
105
+ @tag_model = 'gemma3:latest' # Include tag for Ollama models
106
+
107
+ # Provider credentials from environment variables
108
+ @openai_api_key = ENV['OPENAI_API_KEY']
109
+ @openai_organization = ENV['OPENAI_ORGANIZATION']
110
+ @openai_project = ENV['OPENAI_PROJECT']
111
+ @anthropic_api_key = ENV['ANTHROPIC_API_KEY']
112
+ @gemini_api_key = ENV['GEMINI_API_KEY']
113
+ @azure_api_key = ENV['AZURE_OPENAI_API_KEY']
114
+ @azure_endpoint = ENV['AZURE_OPENAI_ENDPOINT']
115
+ @azure_api_version = ENV['AZURE_OPENAI_API_VERSION'] || '2024-02-01'
116
+ @ollama_url = ENV['OLLAMA_API_BASE'] || ENV['OLLAMA_URL'] || 'http://localhost:11434'
117
+ @huggingface_api_key = ENV['HUGGINGFACE_API_KEY']
118
+ @openrouter_api_key = ENV['OPENROUTER_API_KEY']
119
+ @bedrock_access_key = ENV['AWS_ACCESS_KEY_ID']
120
+ @bedrock_secret_key = ENV['AWS_SECRET_ACCESS_KEY']
121
+ @bedrock_region = ENV['AWS_REGION'] || 'us-east-1'
122
+ @deepseek_api_key = ENV['DEEPSEEK_API_KEY']
55
123
 
56
124
  # Timeout settings (in seconds) - apply to all LLM providers
57
125
  @embedding_timeout = 120 # 2 minutes for embedding generation
@@ -64,6 +132,14 @@ class HTM
64
132
  # Auto-detect job backend based on environment
65
133
  @job_backend = detect_job_backend
66
134
 
135
+ # Timeframe parsing configuration
136
+ # :sunday (default) or :monday for week start day
137
+ @week_start = :sunday
138
+
139
+ # Thread-safe Ollama model refresh tracking
140
+ @ollama_models_refreshed = false
141
+ @ollama_refresh_mutex = Mutex.new
142
+
67
143
  # Set default implementations
68
144
  reset_to_defaults
69
145
  end
@@ -96,6 +172,80 @@ class HTM
96
172
  unless [:active_job, :sidekiq, :inline, :thread].include?(@job_backend)
97
173
  raise HTM::ValidationError, "job_backend must be one of: :active_job, :sidekiq, :inline, :thread (got #{@job_backend.inspect})"
98
174
  end
175
+
176
+ unless [:sunday, :monday].include?(@week_start)
177
+ raise HTM::ValidationError, "week_start must be :sunday or :monday (got #{@week_start.inspect})"
178
+ end
179
+
180
+ # Validate provider if specified
181
+ if @embedding_provider && !SUPPORTED_PROVIDERS.include?(@embedding_provider)
182
+ raise HTM::ValidationError, "embedding_provider must be one of: #{SUPPORTED_PROVIDERS.join(', ')} (got #{@embedding_provider.inspect})"
183
+ end
184
+
185
+ if @tag_provider && !SUPPORTED_PROVIDERS.include?(@tag_provider)
186
+ raise HTM::ValidationError, "tag_provider must be one of: #{SUPPORTED_PROVIDERS.join(', ')} (got #{@tag_provider.inspect})"
187
+ end
188
+ end
189
+
190
+ # Normalize Ollama model name to include tag if missing
191
+ #
192
+ # Ollama models require a tag (e.g., :latest, :7b, :13b). If the user
193
+ # specifies a model without a tag, we append :latest by default.
194
+ #
195
+ # @param model_name [String] Original model name
196
+ # @return [String] Normalized model name with tag
197
+ #
198
+ def normalize_ollama_model(model_name)
199
+ return model_name if model_name.nil? || model_name.empty?
200
+ return model_name if model_name.include?(':')
201
+
202
+ "#{model_name}:latest"
203
+ end
204
+
205
+ # Configure RubyLLM with the appropriate provider credentials
206
+ #
207
+ # @param provider [Symbol] The provider to configure (:openai, :anthropic, etc.)
208
+ #
209
+ def configure_ruby_llm(provider = nil)
210
+ require 'ruby_llm' unless defined?(RubyLLM)
211
+
212
+ provider ||= @embedding_provider
213
+
214
+ RubyLLM.configure do |config|
215
+ case provider
216
+ when :openai
217
+ config.openai_api_key = @openai_api_key if @openai_api_key
218
+ config.openai_organization = @openai_organization if @openai_organization && config.respond_to?(:openai_organization=)
219
+ config.openai_project = @openai_project if @openai_project && config.respond_to?(:openai_project=)
220
+ when :anthropic
221
+ config.anthropic_api_key = @anthropic_api_key if @anthropic_api_key
222
+ when :gemini
223
+ config.gemini_api_key = @gemini_api_key if @gemini_api_key
224
+ when :azure
225
+ config.azure_api_key = @azure_api_key if @azure_api_key && config.respond_to?(:azure_api_key=)
226
+ config.azure_endpoint = @azure_endpoint if @azure_endpoint && config.respond_to?(:azure_endpoint=)
227
+ config.azure_api_version = @azure_api_version if @azure_api_version && config.respond_to?(:azure_api_version=)
228
+ when :ollama
229
+ # Ollama exposes OpenAI-compatible API at /v1
230
+ # Ensure URL has /v1 suffix (add if missing, don't duplicate if present)
231
+ ollama_api_base = if @ollama_url.end_with?('/v1') || @ollama_url.end_with?('/v1/')
232
+ @ollama_url.sub(%r{/+$}, '') # Just remove trailing slashes
233
+ else
234
+ "#{@ollama_url.sub(%r{/+$}, '')}/v1"
235
+ end
236
+ config.ollama_api_base = ollama_api_base
237
+ when :huggingface
238
+ config.huggingface_api_key = @huggingface_api_key if @huggingface_api_key && config.respond_to?(:huggingface_api_key=)
239
+ when :openrouter
240
+ config.openrouter_api_key = @openrouter_api_key if @openrouter_api_key && config.respond_to?(:openrouter_api_key=)
241
+ when :bedrock
242
+ config.bedrock_api_key = @bedrock_access_key if @bedrock_access_key && config.respond_to?(:bedrock_api_key=)
243
+ config.bedrock_secret_key = @bedrock_secret_key if @bedrock_secret_key && config.respond_to?(:bedrock_secret_key=)
244
+ config.bedrock_region = @bedrock_region if @bedrock_region && config.respond_to?(:bedrock_region=)
245
+ when :deepseek
246
+ config.deepseek_api_key = @deepseek_api_key if @deepseek_api_key && config.respond_to?(:deepseek_api_key=)
247
+ end
248
+ end
99
249
  end
100
250
 
101
251
  private
@@ -153,50 +303,101 @@ class HTM
153
303
  end
154
304
  end
155
305
 
156
- # Default embedding generator using Ollama HTTP API
306
+ # Default embedding generator using RubyLLM
157
307
  #
158
308
  # @return [Proc] Callable that takes text and returns embedding vector
159
309
  #
160
310
  def default_embedding_generator
161
311
  lambda do |text|
162
- require 'net/http'
163
- require 'json'
164
-
165
- case @embedding_provider
166
- when :ollama
167
- uri = URI("#{@ollama_url}/api/embeddings")
168
- request = Net::HTTP::Post.new(uri)
169
- request['Content-Type'] = 'application/json'
170
- request.body = { model: @embedding_model, prompt: text }.to_json
171
-
172
- response = Net::HTTP.start(uri.hostname, uri.port,
173
- read_timeout: @embedding_timeout,
174
- open_timeout: @connection_timeout) do |http|
175
- http.request(request)
312
+ require 'ruby_llm' unless defined?(RubyLLM)
313
+
314
+ # Configure RubyLLM for the embedding provider
315
+ configure_ruby_llm(@embedding_provider)
316
+
317
+ # Refresh models for Ollama to discover local models (thread-safe)
318
+ if @embedding_provider == :ollama
319
+ @ollama_refresh_mutex.synchronize do
320
+ unless @ollama_models_refreshed
321
+ RubyLLM.models.refresh!
322
+ @ollama_models_refreshed = true
323
+ end
176
324
  end
325
+ end
177
326
 
178
- data = JSON.parse(response.body)
179
- embedding = data['embedding']
327
+ # Normalize Ollama model name (ensure it has a tag like :latest)
328
+ model = @embedding_provider == :ollama ? normalize_ollama_model(@embedding_model) : @embedding_model
180
329
 
181
- unless embedding.is_a?(Array)
182
- raise HTM::EmbeddingError, "Invalid embedding response format"
183
- end
330
+ # Generate embedding using RubyLLM
331
+ response = RubyLLM.embed(text, model: model)
184
332
 
185
- embedding
186
- else
187
- raise HTM::EmbeddingError, "Unsupported embedding provider: #{@embedding_provider}. Only :ollama is currently supported."
333
+ # Extract embedding vector from response
334
+ embedding = extract_embedding_from_response(response)
335
+
336
+ unless embedding.is_a?(Array) && embedding.all? { |v| v.is_a?(Numeric) }
337
+ raise HTM::EmbeddingError, "Invalid embedding response format from #{@embedding_provider}"
188
338
  end
339
+
340
+ embedding
189
341
  end
190
342
  end
191
343
 
192
- # Default tag extractor using Ollama HTTP API
344
+ # Extract embedding vector from RubyLLM response
345
+ #
346
+ # @param response [Object] RubyLLM embed response
347
+ # @return [Array<Float>] Embedding vector
348
+ #
349
+ def extract_embedding_from_response(response)
350
+ return nil unless response
351
+
352
+ # Handle different response formats from RubyLLM
353
+ case response
354
+ when Array
355
+ # Direct array response
356
+ response
357
+ when ->(r) { r.respond_to?(:vectors) }
358
+ # RubyLLM::Embedding object with vectors method
359
+ vectors = response.vectors
360
+ vectors.is_a?(Array) && vectors.first.is_a?(Array) ? vectors.first : vectors
361
+ when ->(r) { r.respond_to?(:to_a) }
362
+ # Can be converted to array
363
+ response.to_a
364
+ when ->(r) { r.respond_to?(:embedding) }
365
+ # Has embedding attribute
366
+ response.embedding
367
+ else
368
+ # Try to extract vectors from instance variables
369
+ if response.respond_to?(:instance_variable_get)
370
+ vectors = response.instance_variable_get(:@vectors)
371
+ return vectors.first if vectors.is_a?(Array) && vectors.first.is_a?(Array)
372
+ return vectors if vectors.is_a?(Array)
373
+ end
374
+ raise HTM::EmbeddingError, "Cannot extract embedding from response: #{response.class}"
375
+ end
376
+ end
377
+
378
+ # Default tag extractor using RubyLLM chat
193
379
  #
194
380
  # @return [Proc] Callable that takes text and ontology, returns array of tags
195
381
  #
196
382
  def default_tag_extractor
197
383
  lambda do |text, existing_ontology = []|
198
- require 'net/http'
199
- require 'json'
384
+ require 'ruby_llm' unless defined?(RubyLLM)
385
+
386
+ # Configure RubyLLM for the tag provider
387
+ configure_ruby_llm(@tag_provider)
388
+
389
+ # Refresh models for Ollama to discover local models (thread-safe)
390
+ if @tag_provider == :ollama
391
+ @ollama_refresh_mutex.synchronize do
392
+ unless @ollama_models_refreshed
393
+ RubyLLM.models.refresh!
394
+ @ollama_models_refreshed = true
395
+ end
396
+ end
397
+ end
398
+
399
+ # Normalize Ollama model name (ensure it has a tag like :latest)
400
+ model = @tag_provider == :ollama ? normalize_ollama_model(@tag_model) : @tag_model
200
401
 
201
402
  # Build prompt
202
403
  ontology_context = if existing_ontology.any?
@@ -214,52 +415,84 @@ class HTM
214
415
 
215
416
  Rules:
216
417
  - Use lowercase letters, numbers, and hyphens only
217
- - Maximum depth: 5 levels
418
+ - Maximum depth: 4 levels (to prevent excessive nesting)
218
419
  - Return 2-5 tags per text
219
420
  - Tags should be reusable and consistent
220
421
  - Prefer existing ontology tags when applicable
221
422
  - Use hyphens for multi-word terms (e.g., natural-language-processing)
222
423
 
223
- Text: #{text}
424
+ CRITICAL CONSTRAINTS:
425
+ - NO CIRCULAR REFERENCES: A concept cannot appear at both the root and leaf of the same path
426
+ - NO REDUNDANT DUPLICATES: Do not create the same concept in multiple branches
427
+ Example (WRONG): database:postgresql vs database-management:relational-databases:postgresql
428
+ Example (RIGHT): Choose ONE primary location
429
+ - CONSISTENT DEPTH: Similar concept types should be at similar depth levels
430
+ Example (WRONG): age:numeric vs name:individual:specific-name:john
431
+ Example (RIGHT): Both should be at similar depths under personal-data
432
+ - NO SELF-CONTAINMENT: A parent concept should never contain itself as a descendant
433
+ Example (WRONG): age:personal-information:personal-data:age
434
+ Example (RIGHT): personal-information:personal-data:age
435
+ - AVOID AMBIGUOUS CROSS-DOMAIN CONCEPTS: Each concept should have ONE primary parent
436
+ If a concept truly belongs in multiple domains, use the most specific/primary domain
437
+
438
+ TEXT: #{text}
224
439
 
225
440
  Return ONLY the topic tags, one per line, no explanations.
226
441
  PROMPT
227
442
 
228
- case @tag_provider
229
- when :ollama
230
- uri = URI("#{@ollama_url}/api/generate")
231
- request = Net::HTTP::Post.new(uri)
232
- request['Content-Type'] = 'application/json'
233
- request.body = {
234
- model: @tag_model,
235
- prompt: prompt,
236
- system: 'You are a precise topic extraction system. Output only topic tags in hierarchical format: root:subtopic:detail',
237
- stream: false,
238
- options: { temperature: 0 }
239
- }.to_json
240
-
241
- response = Net::HTTP.start(uri.hostname, uri.port,
242
- read_timeout: @tag_timeout,
243
- open_timeout: @connection_timeout) do |http|
244
- http.request(request)
245
- end
443
+ system_prompt = <<~SYSTEM.strip
444
+ You are a precise topic extraction system that prevents ontological errors.
246
445
 
247
- data = JSON.parse(response.body)
248
- response_text = data['response']
446
+ Your job is to:
447
+ 1. Extract hierarchical tags in format: root:subtopic:detail
448
+ 2. Maintain consistency with existing ontology (no duplicates)
449
+ 3. Prevent circular references and self-containing concepts
450
+ 4. Keep hierarchies at consistent depth levels
451
+ 5. Choose PRIMARY locations for concepts (no multi-parent confusion)
249
452
 
250
- # Parse and validate tags
251
- tags = response_text.to_s.split("\n").map(&:strip).reject(&:empty?)
453
+ Output ONLY topic tags, one per line.
454
+ SYSTEM
252
455
 
253
- # Validate format: lowercase alphanumeric + hyphens + colons
254
- valid_tags = tags.select do |tag|
255
- tag =~ /^[a-z0-9\-]+(:[a-z0-9\-]+)*$/
256
- end
456
+ # Use RubyLLM chat for tag extraction
457
+ chat = RubyLLM.chat(model: model)
458
+ chat.with_instructions(system_prompt)
459
+ response = chat.ask(prompt)
257
460
 
258
- # Limit depth to 5 levels (4 colons maximum)
259
- valid_tags.select { |tag| tag.count(':') < 5 }
260
- else
261
- raise HTM::TagError, "Unsupported tag provider: #{@tag_provider}. Only :ollama is currently supported."
461
+ # Extract text from response
462
+ response_text = extract_text_from_response(response)
463
+
464
+ # Parse and validate tags
465
+ tags = response_text.to_s.split("\n").map(&:strip).reject(&:empty?)
466
+
467
+ # Validate format: lowercase alphanumeric + hyphens + colons
468
+ valid_tags = tags.select do |tag|
469
+ tag =~ /^[a-z0-9\-]+(:[a-z0-9\-]+)*$/
262
470
  end
471
+
472
+ # Limit depth to 4 levels (3 colons maximum)
473
+ valid_tags.select { |tag| tag.count(':') < 4 }
474
+ end
475
+ end
476
+
477
+ # Extract text content from RubyLLM chat response
478
+ #
479
+ # @param response [Object] RubyLLM chat response
480
+ # @return [String] Response text
481
+ #
482
+ def extract_text_from_response(response)
483
+ return '' unless response
484
+
485
+ case response
486
+ when String
487
+ response
488
+ when ->(r) { r.respond_to?(:content) }
489
+ response.content.to_s
490
+ when ->(r) { r.respond_to?(:text) }
491
+ response.text.to_s
492
+ when ->(r) { r.respond_to?(:to_s) }
493
+ response.to_s
494
+ else
495
+ ''
263
496
  end
264
497
  end
265
498
  end
data/lib/htm/database.rb CHANGED
@@ -116,7 +116,7 @@ class HTM
116
116
 
117
117
  conn = PG.connect(config)
118
118
 
119
- tables = ['nodes', 'node_tags', 'tags', 'robots', 'operations_log', 'schema_migrations']
119
+ tables = ['nodes', 'node_tags', 'tags', 'robots', 'robot_nodes', 'file_sources', 'schema_migrations']
120
120
 
121
121
  puts "Dropping HTM tables..."
122
122
  tables.each do |table|
@@ -282,7 +282,8 @@ class HTM
282
282
 
283
283
  # Generate database documentation using tbls
284
284
  #
285
- # Creates comprehensive database documentation in dbdoc/ directory including:
285
+ # Uses .tbls.yml configuration file for output directory and settings.
286
+ # Creates comprehensive database documentation including:
286
287
  # - Entity-relationship diagrams
287
288
  # - Table schemas with comments
288
289
  # - Index information
@@ -292,23 +293,6 @@ class HTM
292
293
  # @return [void]
293
294
  #
294
295
  def generate_docs(db_url = nil)
295
- config = parse_connection_url(db_url || ENV['HTM_DBURL'])
296
- raise "Database configuration not found" unless config
297
-
298
- dbdoc_dir = File.expand_path('../../dbdoc', __dir__)
299
-
300
- puts "Generating database documentation in #{dbdoc_dir}..."
301
-
302
- # Create dbdoc directory if it doesn't exist
303
- Dir.mkdir(dbdoc_dir) unless Dir.exist?(dbdoc_dir)
304
-
305
- # Build PostgreSQL connection string for tbls
306
- pg_url = if config[:password]
307
- "postgresql://#{config[:user]}:#{config[:password]}@#{config[:host]}:#{config[:port]}/#{config[:dbname]}?sslmode=#{config[:sslmode] || 'prefer'}"
308
- else
309
- "postgresql://#{config[:user]}@#{config[:host]}:#{config[:port]}/#{config[:dbname]}?sslmode=#{config[:sslmode] || 'prefer'}"
310
- end
311
-
312
296
  # Check if tbls is installed
313
297
  unless system('which tbls > /dev/null 2>&1')
314
298
  puts "✗ Error: 'tbls' is not installed"
@@ -322,9 +306,31 @@ class HTM
322
306
  exit 1
323
307
  end
324
308
 
325
- # Run tbls doc command with --force to allow updates
309
+ # Find the project root (where .tbls.yml should be)
310
+ project_root = File.expand_path('../..', __dir__)
311
+ tbls_config = File.join(project_root, '.tbls.yml')
312
+
313
+ unless File.exist?(tbls_config)
314
+ puts "✗ Error: .tbls.yml not found at #{tbls_config}"
315
+ exit 1
316
+ end
317
+
318
+ # Get database URL
319
+ dsn = db_url || ENV['HTM_DBURL']
320
+ raise "Database configuration not found. Set HTM_DBURL environment variable." unless dsn
321
+
322
+ # Ensure sslmode is set for local development (tbls requires it)
323
+ unless dsn.include?('sslmode=')
324
+ separator = dsn.include?('?') ? '&' : '?'
325
+ dsn = "#{dsn}#{separator}sslmode=disable"
326
+ end
327
+
328
+ puts "Generating database documentation using #{tbls_config}..."
329
+
330
+ # Run tbls doc command with config file and DSN override
331
+ # The --dsn flag overrides the dsn in .tbls.yml but other settings are preserved
326
332
  require 'open3'
327
- cmd = ['tbls', 'doc', '--force', pg_url, dbdoc_dir]
333
+ cmd = ['tbls', 'doc', '--config', tbls_config, '--dsn', dsn, '--force']
328
334
 
329
335
  stdout, stderr, status = Open3.capture3(*cmd)
330
336
 
@@ -336,15 +342,18 @@ class HTM
336
342
  end
337
343
 
338
344
  puts stdout if stdout && !stdout.empty?
345
+
346
+ # Read docPath from config to show correct output location
347
+ doc_path = 'docs/database' # default from .tbls.yml
339
348
  puts "✓ Database documentation generated successfully"
340
349
  puts ""
341
350
  puts "Documentation files:"
342
- puts " #{dbdoc_dir}/README.md - Main documentation"
343
- puts " #{dbdoc_dir}/schema.svg - ER diagram (if generated)"
344
- puts " #{dbdoc_dir}/*.md - Individual table documentation"
351
+ puts " #{doc_path}/README.md - Main documentation"
352
+ puts " #{doc_path}/schema.svg - ER diagram"
353
+ puts " #{doc_path}/*.md - Individual table documentation"
345
354
  puts ""
346
355
  puts "View documentation:"
347
- puts " open #{dbdoc_dir}/README.md"
356
+ puts " open #{doc_path}/README.md"
348
357
  end
349
358
 
350
359
  # Show database info
@@ -382,7 +391,7 @@ class HTM
382
391
 
383
392
  # Table info
384
393
  puts "\nHTM Tables:"
385
- tables = ['nodes', 'tags', 'robots', 'operations_log', 'schema_migrations']
394
+ tables = ['nodes', 'node_tags', 'tags', 'robots', 'robot_nodes', 'file_sources', 'schema_migrations']
386
395
  tables.each do |table|
387
396
  begin
388
397
  count = conn.exec("SELECT COUNT(*) FROM #{table}").first['count']
@@ -405,23 +414,41 @@ class HTM
405
414
 
406
415
  # Parse database connection URL
407
416
  #
408
- # @param url [String] Connection URL
417
+ # @param url [String] Connection URL (e.g., postgresql://user:pass@host:port/dbname)
409
418
  # @return [Hash, nil] Connection configuration hash
419
+ # @raise [ArgumentError] If URL format is invalid
410
420
  #
411
421
  def parse_connection_url(url)
412
422
  return nil unless url
413
423
 
414
424
  uri = URI.parse(url)
425
+
426
+ # Validate URL format
427
+ unless uri.scheme&.match?(/\Apostgres(?:ql)?\z/i)
428
+ raise ArgumentError, "Invalid database URL scheme: #{uri.scheme}. Expected 'postgresql' or 'postgres'."
429
+ end
430
+
431
+ unless uri.host && !uri.host.empty?
432
+ raise ArgumentError, "Database URL must include a host"
433
+ end
434
+
435
+ dbname = uri.path&.slice(1..-1) # Remove leading /
436
+ if dbname.nil? || dbname.empty?
437
+ raise ArgumentError, "Database URL must include a database name (path segment)"
438
+ end
439
+
415
440
  params = URI.decode_www_form(uri.query || '').to_h
416
441
 
417
442
  {
418
443
  host: uri.host,
419
- port: uri.port,
420
- dbname: uri.path[1..-1], # Remove leading /
444
+ port: uri.port || 5432,
445
+ dbname: dbname,
421
446
  user: uri.user,
422
447
  password: uri.password,
423
448
  sslmode: params['sslmode'] || 'prefer'
424
449
  }
450
+ rescue URI::InvalidURIError => e
451
+ raise ArgumentError, "Invalid database URL format: #{e.message}"
425
452
  end
426
453
 
427
454
  # Build config from individual environment variables
@@ -432,12 +459,12 @@ class HTM
432
459
  return nil unless ENV['HTM_DBNAME']
433
460
 
434
461
  {
435
- host: ENV['HTM_DBHOST'] || 'cw7rxj91bm.srbbwwxn56.tsdb.cloud.timescale.com',
436
- port: (ENV['HTM_DBPORT'] || 37807).to_i,
462
+ host: ENV['HTM_DBHOST'] || 'localhost',
463
+ port: (ENV['HTM_DBPORT'] || 5432).to_i,
437
464
  dbname: ENV['HTM_DBNAME'],
438
465
  user: ENV['HTM_DBUSER'],
439
466
  password: ENV['HTM_DBPASS'],
440
- sslmode: 'require'
467
+ sslmode: ENV['HTM_DBSSLMODE'] || 'prefer'
441
468
  }
442
469
  end
443
470
 
@@ -506,9 +533,11 @@ class HTM
506
533
  version = File.basename(file).split('_').first
507
534
  name = File.basename(file, '.rb')
508
535
 
509
- # Check if already run
536
+ # Check if already run (use parameterized query to prevent SQL injection)
510
537
  already_run = conn.select_value(
511
- "SELECT COUNT(*) FROM schema_migrations WHERE version = '#{version}'"
538
+ ActiveRecord::Base.sanitize_sql_array(
539
+ ["SELECT COUNT(*) FROM schema_migrations WHERE version = ?", version]
540
+ )
512
541
  ).to_i > 0
513
542
 
514
543
  if already_run
@@ -525,9 +554,11 @@ class HTM
525
554
  migration = migration_class.new
526
555
  migration.migrate(:up)
527
556
 
528
- # Record in schema_migrations
557
+ # Record in schema_migrations (use parameterized query to prevent SQL injection)
529
558
  conn.execute(
530
- "INSERT INTO schema_migrations (version) VALUES ('#{version}')"
559
+ ActiveRecord::Base.sanitize_sql_array(
560
+ ["INSERT INTO schema_migrations (version) VALUES (?)", version]
561
+ )
531
562
  )
532
563
 
533
564
  puts " ✓ Completed"