htm 0.0.31 → 0.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.irbrc +2 -3
- data/.rubocop.yml +184 -0
- data/CHANGELOG.md +46 -0
- data/README.md +2 -0
- data/Rakefile +93 -12
- data/db/migrate/00008_create_node_relationships.rb +54 -0
- data/db/migrate/00009_fix_node_relationships_column_types.rb +17 -0
- data/db/schema.sql +124 -1
- data/docs/api/database.md +35 -57
- data/docs/api/embedding-service.md +1 -1
- data/docs/api/index.md +26 -15
- data/docs/api/working-memory.md +8 -8
- data/docs/architecture/index.md +5 -7
- data/docs/architecture/overview.md +5 -8
- data/docs/assets/images/htm-architecture-overview.svg +1 -1
- data/docs/assets/images/htm-context-assembly-flow.svg +2 -2
- data/docs/assets/images/htm-layered-architecture.svg +3 -3
- data/docs/assets/images/two-tier-memory-architecture.svg +1 -1
- data/docs/database/README.md +1 -0
- data/docs/database_rake_tasks.md +20 -28
- data/docs/development/contributing.md +5 -5
- data/docs/development/index.md +4 -7
- data/docs/development/schema.md +71 -1
- data/docs/development/setup.md +40 -82
- data/docs/development/testing.md +1 -1
- data/docs/examples/file-loading.md +4 -4
- data/docs/examples/mcp-client.md +1 -1
- data/docs/getting-started/quick-start.md +4 -4
- data/docs/guides/adding-memories.md +14 -1
- data/docs/guides/configuration.md +5 -5
- data/docs/guides/context-assembly.md +4 -4
- data/docs/guides/file-loading.md +12 -12
- data/docs/guides/getting-started.md +2 -2
- data/docs/guides/long-term-memory.md +7 -27
- data/docs/guides/propositions.md +20 -19
- data/docs/guides/recalling-memories.md +5 -5
- data/docs/guides/tags.md +18 -13
- data/docs/multi_framework_support.md +1 -1
- data/docs/robots/hive-mind.md +1 -1
- data/docs/robots/multi-robot.md +2 -2
- data/docs/robots/robot-groups.md +1 -1
- data/docs/robots/two-tier-memory.md +72 -94
- data/docs/setup_local_database.md +8 -54
- data/docs/using_rake_tasks_in_your_app.md +6 -6
- data/examples/01_basic_usage.rb +1 -0
- data/examples/03_custom_llm_configuration.rb +1 -0
- data/examples/04_file_loader_usage.rb +1 -0
- data/examples/05_timeframe_demo.rb +1 -0
- data/examples/06_example_app/app.rb +1 -0
- data/examples/07_cli_app/htm_cli.rb +1 -0
- data/examples/09_mcp_client.rb +1 -0
- data/examples/10_telemetry/demo.rb +1 -0
- data/examples/11_robot_groups/multi_process.rb +1 -0
- data/examples/11_robot_groups/same_process.rb +1 -0
- data/examples/12_rails_app/.envrc +12 -0
- data/examples/12_rails_app/Gemfile +8 -3
- data/examples/12_rails_app/Gemfile.lock +94 -89
- data/examples/12_rails_app/README.md +70 -19
- data/examples/12_rails_app/app/controllers/application_controller.rb +6 -0
- data/examples/12_rails_app/app/controllers/chats_controller.rb +305 -0
- data/examples/12_rails_app/app/controllers/dashboard_controller.rb +3 -0
- data/examples/12_rails_app/app/controllers/files_controller.rb +17 -2
- data/examples/12_rails_app/app/controllers/home_controller.rb +8 -0
- data/examples/12_rails_app/app/controllers/memories_controller.rb +9 -4
- data/examples/12_rails_app/app/controllers/messages_controller.rb +214 -0
- data/examples/12_rails_app/app/controllers/robots_controller.rb +11 -1
- data/examples/12_rails_app/app/controllers/tags_controller.rb +14 -1
- data/examples/12_rails_app/app/javascript/application.js +1 -1
- data/examples/12_rails_app/app/models/application_record.rb +5 -0
- data/examples/12_rails_app/app/models/chat.rb +36 -0
- data/examples/12_rails_app/app/models/message.rb +5 -0
- data/examples/12_rails_app/app/models/model.rb +5 -0
- data/examples/12_rails_app/app/models/tool_call.rb +5 -0
- data/examples/12_rails_app/app/views/chats/index.html.erb +61 -0
- data/examples/12_rails_app/app/views/chats/show.html.erb +213 -0
- data/examples/12_rails_app/app/views/dashboard/index.html.erb +3 -0
- data/examples/12_rails_app/app/views/files/index.html.erb +10 -5
- data/examples/12_rails_app/app/views/files/new.html.erb +4 -2
- data/examples/12_rails_app/app/views/files/show.html.erb +19 -3
- data/examples/12_rails_app/app/views/home/index.html.erb +45 -0
- data/examples/12_rails_app/app/views/layouts/application.html.erb +20 -18
- data/examples/12_rails_app/app/views/memories/_memory_card.html.erb +1 -1
- data/examples/12_rails_app/app/views/memories/deleted.html.erb +3 -1
- data/examples/12_rails_app/app/views/memories/edit.html.erb +2 -0
- data/examples/12_rails_app/app/views/memories/index.html.erb +2 -0
- data/examples/12_rails_app/app/views/memories/new.html.erb +2 -0
- data/examples/12_rails_app/app/views/memories/show.html.erb +4 -2
- data/examples/12_rails_app/app/views/messages/_message.html.erb +20 -0
- data/examples/12_rails_app/app/views/robots/index.html.erb +2 -0
- data/examples/12_rails_app/app/views/robots/new.html.erb +2 -0
- data/examples/12_rails_app/app/views/robots/show.html.erb +2 -0
- data/examples/12_rails_app/app/views/search/index.html.erb +59 -8
- data/examples/12_rails_app/app/views/shared/_navbar.html.erb +75 -29
- data/examples/12_rails_app/app/views/tags/index.html.erb +2 -0
- data/examples/12_rails_app/app/views/tags/show.html.erb +3 -1
- data/examples/12_rails_app/config/application.rb +1 -1
- data/examples/12_rails_app/config/database.yml +9 -5
- data/examples/12_rails_app/config/importmap.rb +1 -1
- data/examples/12_rails_app/config/initializers/htm.rb +9 -2
- data/examples/12_rails_app/config/initializers/ruby_llm.rb +33 -0
- data/examples/12_rails_app/config/routes.rb +39 -23
- data/examples/12_rails_app/db/migrate/20250124000001_create_ruby_llm_tables.rb +34 -0
- data/examples/12_rails_app/db/migrate/20250124000002_create_models_table.rb +28 -0
- data/examples/12_rails_app/db/schema.rb +67 -0
- data/examples/examples_helper.rb +25 -0
- data/lib/htm/circuit_breaker.rb +5 -6
- data/lib/htm/config/builder.rb +12 -12
- data/lib/htm/config/database.rb +21 -27
- data/lib/htm/config/validator.rb +12 -18
- data/lib/htm/config.rb +76 -65
- data/lib/htm/database.rb +193 -199
- data/lib/htm/embedding_service.rb +4 -9
- data/lib/htm/integrations/sinatra.rb +7 -7
- data/lib/htm/job_adapter.rb +14 -21
- data/lib/htm/jobs/generate_embedding_job.rb +28 -44
- data/lib/htm/jobs/generate_propositions_job.rb +29 -55
- data/lib/htm/jobs/generate_relationships_job.rb +137 -0
- data/lib/htm/jobs/generate_tags_job.rb +45 -67
- data/lib/htm/loaders/markdown_loader.rb +65 -112
- data/lib/htm/long_term_memory/fulltext_search.rb +1 -1
- data/lib/htm/long_term_memory/hybrid_search.rb +300 -128
- data/lib/htm/long_term_memory/node_operations.rb +2 -2
- data/lib/htm/long_term_memory/relevance_scorer.rb +100 -68
- data/lib/htm/long_term_memory/tag_operations.rb +87 -120
- data/lib/htm/long_term_memory/vector_search.rb +1 -1
- data/lib/htm/long_term_memory.rb +2 -1
- data/lib/htm/mcp/cli.rb +59 -58
- data/lib/htm/mcp/server.rb +5 -6
- data/lib/htm/mcp/tools.rb +30 -36
- data/lib/htm/migration.rb +10 -10
- data/lib/htm/models/node.rb +2 -3
- data/lib/htm/models/node_relationship.rb +72 -0
- data/lib/htm/models/node_tag.rb +2 -2
- data/lib/htm/models/robot_node.rb +2 -2
- data/lib/htm/models/tag.rb +41 -28
- data/lib/htm/observability.rb +45 -51
- data/lib/htm/proposition_service.rb +3 -7
- data/lib/htm/query_cache.rb +13 -15
- data/lib/htm/railtie.rb +1 -2
- data/lib/htm/robot_group.rb +9 -9
- data/lib/htm/sequel_config.rb +1 -0
- data/lib/htm/sql_builder.rb +1 -1
- data/lib/htm/tag_service.rb +2 -6
- data/lib/htm/timeframe.rb +4 -5
- data/lib/htm/timeframe_extractor.rb +42 -83
- data/lib/htm/version.rb +1 -1
- data/lib/htm/workflows/remember_workflow.rb +112 -115
- data/lib/htm/working_memory.rb +21 -26
- data/lib/htm.rb +103 -116
- data/lib/tasks/db.rake +0 -2
- data/lib/tasks/doc.rake +14 -13
- data/lib/tasks/files.rake +5 -12
- data/lib/tasks/htm.rake +70 -71
- data/lib/tasks/jobs.rake +41 -47
- data/lib/tasks/tags.rake +3 -8
- metadata +25 -100
data/examples/examples_helper.rb
CHANGED
|
@@ -72,6 +72,31 @@ end
|
|
|
72
72
|
|
|
73
73
|
# Module with helper methods for examples
|
|
74
74
|
module ExamplesHelper
|
|
75
|
+
# Reset examples database if --reset flag is present in ARGV
|
|
76
|
+
#
|
|
77
|
+
# Drops and recreates htm_examples, then runs all migrations.
|
|
78
|
+
# Removes --reset from ARGV so downstream option parsers are unaffected.
|
|
79
|
+
#
|
|
80
|
+
# @return [void]
|
|
81
|
+
def self.reset_if_requested!
|
|
82
|
+
return unless ARGV.delete('--reset')
|
|
83
|
+
|
|
84
|
+
db_name = "#{ENV['HTM_SERVICE__NAME'] || 'htm'}_examples"
|
|
85
|
+
puts "Resetting examples database '#{db_name}'..."
|
|
86
|
+
|
|
87
|
+
HTM::SequelConfig.disconnect! if HTM::SequelConfig.db
|
|
88
|
+
|
|
89
|
+
abort "ERROR: Failed to drop '#{db_name}'" unless system("dropdb --if-exists #{db_name}")
|
|
90
|
+
abort "ERROR: Failed to create '#{db_name}'" unless system("createdb #{db_name}")
|
|
91
|
+
|
|
92
|
+
HTM::SequelConfig.establish_connection!(load_models: false)
|
|
93
|
+
HTM::Database.setup
|
|
94
|
+
HTM::SequelConfig.ensure_models_loaded!
|
|
95
|
+
|
|
96
|
+
puts "Database reset complete."
|
|
97
|
+
puts
|
|
98
|
+
end
|
|
99
|
+
|
|
75
100
|
# Check if database is available and configured
|
|
76
101
|
#
|
|
77
102
|
# @return [Boolean] true if database is ready
|
data/lib/htm/circuit_breaker.rb
CHANGED
|
@@ -158,7 +158,7 @@ class HTM
|
|
|
158
158
|
end
|
|
159
159
|
when :closed
|
|
160
160
|
# Reset failure count on success in closed state
|
|
161
|
-
@failure_count = 0 if @failure_count
|
|
161
|
+
@failure_count = 0 if @failure_count.positive?
|
|
162
162
|
end
|
|
163
163
|
end
|
|
164
164
|
end
|
|
@@ -190,11 +190,10 @@ class HTM
|
|
|
190
190
|
return unless @state == :open && @last_failure_time
|
|
191
191
|
|
|
192
192
|
elapsed = Time.now - @last_failure_time
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
end
|
|
193
|
+
return unless elapsed >= @reset_timeout
|
|
194
|
+
@state = :half_open
|
|
195
|
+
@success_count = 0
|
|
196
|
+
HTM.logger.info "CircuitBreaker[#{@name}]: Reset timeout elapsed (#{@reset_timeout}s), circuit HALF-OPEN"
|
|
198
197
|
end
|
|
199
198
|
end
|
|
200
199
|
end
|
data/lib/htm/config/builder.rb
CHANGED
|
@@ -31,7 +31,7 @@ class HTM
|
|
|
31
31
|
response = RubyLLM.embed(text, model: model)
|
|
32
32
|
embedding = extract_embedding_from_response(response)
|
|
33
33
|
|
|
34
|
-
unless embedding.is_a?(Array) && embedding.all?
|
|
34
|
+
unless embedding.is_a?(Array) && embedding.all?(Numeric)
|
|
35
35
|
raise HTM::EmbeddingError, "Invalid embedding response format from #{embedding_provider}"
|
|
36
36
|
end
|
|
37
37
|
|
|
@@ -119,17 +119,17 @@ class HTM
|
|
|
119
119
|
|
|
120
120
|
def parse_tag_response(text)
|
|
121
121
|
tags = text.to_s.split("\n").map(&:strip).reject(&:empty?)
|
|
122
|
-
valid_tags = tags.
|
|
122
|
+
valid_tags = tags.grep(/^[a-z0-9-]+(:[a-z0-9-]+)*$/)
|
|
123
123
|
valid_tags.select { |tag| tag.count(':') < max_tag_depth }
|
|
124
124
|
end
|
|
125
125
|
|
|
126
126
|
def parse_proposition_response(text)
|
|
127
127
|
text.to_s
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
128
|
+
.split("\n")
|
|
129
|
+
.map(&:strip)
|
|
130
|
+
.map { |line| line.sub(/^[-*]\s*/, '') }
|
|
131
|
+
.map(&:strip)
|
|
132
|
+
.reject(&:empty?)
|
|
133
133
|
end
|
|
134
134
|
|
|
135
135
|
# ==========================================================================
|
|
@@ -138,11 +138,11 @@ class HTM
|
|
|
138
138
|
|
|
139
139
|
def build_tag_extraction_prompt(text, existing_ontology)
|
|
140
140
|
taxonomy_context = if existing_ontology.any?
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
141
|
+
sample_tags = existing_ontology.sample([existing_ontology.size, 20].min)
|
|
142
|
+
tag.taxonomy_context_existing % { sample_tags: sample_tags.join(', ') }
|
|
143
|
+
else
|
|
144
|
+
tag.taxonomy_context_empty
|
|
145
|
+
end
|
|
146
146
|
|
|
147
147
|
tag.user_prompt_template % {
|
|
148
148
|
text: text,
|
data/lib/htm/config/database.rb
CHANGED
|
@@ -87,9 +87,9 @@ class HTM
|
|
|
87
87
|
|
|
88
88
|
unless database_configured?
|
|
89
89
|
raise HTM::ConfigurationError,
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
90
|
+
"No database configured for environment '#{environment}'. " \
|
|
91
|
+
"Set HTM_DATABASE__URL or HTM_DATABASE__NAME, " \
|
|
92
|
+
"or add database.name to the '#{environment}:' section in your config."
|
|
93
93
|
end
|
|
94
94
|
|
|
95
95
|
true
|
|
@@ -165,16 +165,16 @@ class HTM
|
|
|
165
165
|
return true if actual == expected
|
|
166
166
|
|
|
167
167
|
raise HTM::ConfigurationError,
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
168
|
+
"Database name '#{actual}' does not match expected '#{expected}'.\n" \
|
|
169
|
+
"Database names must follow the convention: {service_name}_{environment}\n " \
|
|
170
|
+
"Service name: #{service_name}\n " \
|
|
171
|
+
"Environment: #{environment}\n " \
|
|
172
|
+
"Expected: #{expected}\n " \
|
|
173
|
+
"Actual: #{actual}\n\n" \
|
|
174
|
+
"Either:\n " \
|
|
175
|
+
"- Set HTM_DATABASE__URL to point to '#{expected}'\n " \
|
|
176
|
+
"- Set HTM_DATABASE__NAME=#{expected}\n " \
|
|
177
|
+
"- Change HTM_ENV to match the database suffix"
|
|
178
178
|
end
|
|
179
179
|
|
|
180
180
|
# Check if the database name matches the expected convention
|
|
@@ -216,23 +216,17 @@ class HTM
|
|
|
216
216
|
|
|
217
217
|
def build_database_url
|
|
218
218
|
return nil unless database.name && !database.name.empty?
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
database.password && !database.password.empty? ? "#{database.user}:#{database.password}@" : "#{database.user}@"
|
|
222
|
-
else
|
|
223
|
-
''
|
|
224
|
-
end
|
|
225
|
-
|
|
226
|
-
url = "postgresql://#{auth}#{database.host}:#{database.port}/#{database.name}"
|
|
227
|
-
|
|
228
|
-
# Add sslmode as query parameter if set
|
|
229
|
-
if database.sslmode && !database.sslmode.empty?
|
|
230
|
-
url += "?sslmode=#{database.sslmode}"
|
|
231
|
-
end
|
|
232
|
-
|
|
219
|
+
url = "postgresql://#{database_auth_segment}#{database.host}:#{database.port}/#{database.name}"
|
|
220
|
+
url += "?sslmode=#{database.sslmode}" if database.sslmode && !database.sslmode.empty?
|
|
233
221
|
url
|
|
234
222
|
end
|
|
235
223
|
|
|
224
|
+
def database_auth_segment
|
|
225
|
+
return '' unless database.user && !database.user.empty?
|
|
226
|
+
return "#{database.user}@" unless database.password && !database.password.empty?
|
|
227
|
+
"#{database.user}:#{database.password}@"
|
|
228
|
+
end
|
|
229
|
+
|
|
236
230
|
# ==========================================================================
|
|
237
231
|
# Database Configuration Reconciliation
|
|
238
232
|
# ==========================================================================
|
data/lib/htm/config/validator.rb
CHANGED
|
@@ -27,32 +27,28 @@ class HTM
|
|
|
27
27
|
def validate_provider(name, value)
|
|
28
28
|
return if value.nil?
|
|
29
29
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
end
|
|
30
|
+
return if SUPPORTED_PROVIDERS.include?(value)
|
|
31
|
+
raise_validation_error("#{name} must be one of: #{SUPPORTED_PROVIDERS.join(', ')} (got #{value.inspect})")
|
|
33
32
|
end
|
|
34
33
|
|
|
35
34
|
def validate_job_backend
|
|
36
35
|
return unless job_backend
|
|
37
36
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
end
|
|
37
|
+
return if SUPPORTED_JOB_BACKENDS.include?(job_backend)
|
|
38
|
+
raise_validation_error("job.backend must be one of: #{SUPPORTED_JOB_BACKENDS.join(', ')} (got #{job_backend.inspect})")
|
|
41
39
|
end
|
|
42
40
|
|
|
43
41
|
def validate_week_start
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
end
|
|
42
|
+
return if SUPPORTED_WEEK_STARTS.include?(week_start)
|
|
43
|
+
raise_validation_error("week_start must be one of: #{SUPPORTED_WEEK_STARTS.join(', ')} (got #{week_start.inspect})")
|
|
47
44
|
end
|
|
48
45
|
|
|
49
46
|
def validate_relevance_weights
|
|
50
47
|
total = relevance_semantic_weight + relevance_tag_weight +
|
|
51
48
|
relevance_recency_weight + relevance_access_weight
|
|
52
49
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
end
|
|
50
|
+
return if (0.99..1.01).cover?(total)
|
|
51
|
+
raise_validation_error("relevance weights must sum to 1.0 (got #{total})")
|
|
56
52
|
end
|
|
57
53
|
|
|
58
54
|
def validate_callables
|
|
@@ -68,15 +64,13 @@ class HTM
|
|
|
68
64
|
raise HTM::ValidationError, "proposition_extractor must be callable"
|
|
69
65
|
end
|
|
70
66
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
end
|
|
67
|
+
return if @token_counter.respond_to?(:call)
|
|
68
|
+
raise HTM::ValidationError, "token_counter must be callable"
|
|
74
69
|
end
|
|
75
70
|
|
|
76
71
|
def validate_logger
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
end
|
|
72
|
+
return if @logger.respond_to?(:info) && @logger.respond_to?(:warn) && @logger.respond_to?(:error)
|
|
73
|
+
raise HTM::ValidationError, "logger must respond to :info, :warn, and :error"
|
|
80
74
|
end
|
|
81
75
|
end
|
|
82
76
|
end
|
data/lib/htm/config.rb
CHANGED
|
@@ -99,8 +99,7 @@ class HTM
|
|
|
99
99
|
# Callable Accessors (not loaded from config sources)
|
|
100
100
|
# ==========================================================================
|
|
101
101
|
|
|
102
|
-
attr_accessor :embedding_generator, :tag_extractor, :proposition_extractor
|
|
103
|
-
attr_accessor :token_counter, :logger
|
|
102
|
+
attr_accessor :embedding_generator, :tag_extractor, :proposition_extractor, :token_counter, :logger
|
|
104
103
|
|
|
105
104
|
# ==========================================================================
|
|
106
105
|
# Instance Methods
|
|
@@ -295,7 +294,7 @@ class HTM
|
|
|
295
294
|
# Environment Helpers
|
|
296
295
|
# ==========================================================================
|
|
297
296
|
|
|
298
|
-
#
|
|
297
|
+
# NOTE: test?, development?, production? are auto-generated by MywayConfig::Base
|
|
299
298
|
# based on environment keys in defaults.yml
|
|
300
299
|
|
|
301
300
|
def environment
|
|
@@ -327,11 +326,11 @@ class HTM
|
|
|
327
326
|
current = env
|
|
328
327
|
return true if valid_environment?
|
|
329
328
|
|
|
330
|
-
valid = valid_environments.
|
|
329
|
+
valid = valid_environments.join(', ')
|
|
331
330
|
raise HTM::ConfigurationError,
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
331
|
+
"Invalid environment '#{current}'. " \
|
|
332
|
+
"Valid environments are: #{valid}. " \
|
|
333
|
+
"Set HTM_ENV to a valid environment or add a '#{current}:' section to your config."
|
|
335
334
|
end
|
|
336
335
|
|
|
337
336
|
# Instance method delegates
|
|
@@ -352,12 +351,12 @@ class HTM
|
|
|
352
351
|
end
|
|
353
352
|
|
|
354
353
|
def self.xdg_config_file
|
|
355
|
-
xdg_home = ENV
|
|
354
|
+
xdg_home = ENV.fetch('XDG_CONFIG_HOME', nil)
|
|
356
355
|
base = if xdg_home && !xdg_home.empty?
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
356
|
+
xdg_home
|
|
357
|
+
else
|
|
358
|
+
File.expand_path('~/.config')
|
|
359
|
+
end
|
|
361
360
|
File.join(base, 'htm', 'htm.yml')
|
|
362
361
|
end
|
|
363
362
|
|
|
@@ -378,42 +377,8 @@ class HTM
|
|
|
378
377
|
|
|
379
378
|
def configure_ruby_llm(provider = nil)
|
|
380
379
|
require 'ruby_llm'
|
|
381
|
-
|
|
382
380
|
provider ||= embedding_provider
|
|
383
|
-
|
|
384
|
-
RubyLLM.configure do |config|
|
|
385
|
-
case provider
|
|
386
|
-
when :openai
|
|
387
|
-
config.openai_api_key = openai_api_key if openai_api_key
|
|
388
|
-
config.openai_organization = openai_organization if openai_organization && config.respond_to?(:openai_organization=)
|
|
389
|
-
config.openai_project = openai_project if openai_project && config.respond_to?(:openai_project=)
|
|
390
|
-
when :anthropic
|
|
391
|
-
config.anthropic_api_key = anthropic_api_key if anthropic_api_key
|
|
392
|
-
when :gemini
|
|
393
|
-
config.gemini_api_key = gemini_api_key if gemini_api_key
|
|
394
|
-
when :azure
|
|
395
|
-
config.azure_api_key = azure_api_key if azure_api_key && config.respond_to?(:azure_api_key=)
|
|
396
|
-
config.azure_endpoint = azure_endpoint if azure_endpoint && config.respond_to?(:azure_endpoint=)
|
|
397
|
-
config.azure_api_version = azure_api_version if azure_api_version && config.respond_to?(:azure_api_version=)
|
|
398
|
-
when :ollama
|
|
399
|
-
ollama_api_base = if ollama_url.end_with?('/v1') || ollama_url.end_with?('/v1/')
|
|
400
|
-
ollama_url.sub(%r{/+$}, '')
|
|
401
|
-
else
|
|
402
|
-
"#{ollama_url.sub(%r{/+$}, '')}/v1"
|
|
403
|
-
end
|
|
404
|
-
config.ollama_api_base = ollama_api_base
|
|
405
|
-
when :huggingface
|
|
406
|
-
config.huggingface_api_key = huggingface_api_key if huggingface_api_key && config.respond_to?(:huggingface_api_key=)
|
|
407
|
-
when :openrouter
|
|
408
|
-
config.openrouter_api_key = openrouter_api_key if openrouter_api_key && config.respond_to?(:openrouter_api_key=)
|
|
409
|
-
when :bedrock
|
|
410
|
-
config.bedrock_api_key = bedrock_access_key if bedrock_access_key && config.respond_to?(:bedrock_api_key=)
|
|
411
|
-
config.bedrock_secret_key = bedrock_secret_key if bedrock_secret_key && config.respond_to?(:bedrock_secret_key=)
|
|
412
|
-
config.bedrock_region = bedrock_region if bedrock_region && config.respond_to?(:bedrock_region=)
|
|
413
|
-
when :deepseek
|
|
414
|
-
config.deepseek_api_key = deepseek_api_key if deepseek_api_key && config.respond_to?(:deepseek_api_key=)
|
|
415
|
-
end
|
|
416
|
-
end
|
|
381
|
+
RubyLLM.configure { |config| apply_provider_config(config, provider) }
|
|
417
382
|
end
|
|
418
383
|
|
|
419
384
|
def refresh_ollama_models!
|
|
@@ -453,27 +418,73 @@ class HTM
|
|
|
453
418
|
# ==========================================================================
|
|
454
419
|
|
|
455
420
|
def coerce_nested_types
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
%i[openai anthropic gemini azure ollama huggingface openrouter bedrock deepseek].each do |provider|
|
|
460
|
-
value = providers[provider]
|
|
461
|
-
providers[provider] = MywayConfig::ConfigSection.new(value) if value.is_a?(Hash)
|
|
462
|
-
end
|
|
463
|
-
end
|
|
421
|
+
coerce_provider_sections
|
|
422
|
+
coerce_database_integers
|
|
423
|
+
end
|
|
464
424
|
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
database.pool_size = database.pool_size.to_i
|
|
471
|
-
end
|
|
472
|
-
if database&.timeout && !database.timeout.is_a?(Integer)
|
|
473
|
-
database.timeout = database.timeout.to_i
|
|
425
|
+
def coerce_provider_sections
|
|
426
|
+
return unless providers.is_a?(MywayConfig::ConfigSection)
|
|
427
|
+
%i[openai anthropic gemini azure ollama huggingface openrouter bedrock deepseek].each do |provider|
|
|
428
|
+
value = providers[provider]
|
|
429
|
+
providers[provider] = MywayConfig::ConfigSection.new(value) if value.is_a?(Hash)
|
|
474
430
|
end
|
|
475
431
|
end
|
|
476
432
|
|
|
433
|
+
def coerce_database_integers
|
|
434
|
+
return unless database
|
|
435
|
+
database.port = database.port.to_i if database.port && !database.port.is_a?(Integer)
|
|
436
|
+
database.pool_size = database.pool_size.to_i if database.pool_size && !database.pool_size.is_a?(Integer)
|
|
437
|
+
database.timeout = database.timeout.to_i if database.timeout && !database.timeout.is_a?(Integer)
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
def apply_provider_config(config, provider)
|
|
441
|
+
handler = :"apply_#{provider}_provider_config"
|
|
442
|
+
send(handler, config) if respond_to?(handler, true)
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
def apply_openai_provider_config(config)
|
|
446
|
+
config.openai_api_key = openai_api_key if openai_api_key
|
|
447
|
+
config.openai_organization = openai_organization if openai_organization && config.respond_to?(:openai_organization=)
|
|
448
|
+
config.openai_project = openai_project if openai_project && config.respond_to?(:openai_project=)
|
|
449
|
+
end
|
|
450
|
+
|
|
451
|
+
def apply_anthropic_provider_config(config)
|
|
452
|
+
config.anthropic_api_key = anthropic_api_key if anthropic_api_key
|
|
453
|
+
end
|
|
454
|
+
|
|
455
|
+
def apply_gemini_provider_config(config)
|
|
456
|
+
config.gemini_api_key = gemini_api_key if gemini_api_key
|
|
457
|
+
end
|
|
458
|
+
|
|
459
|
+
def apply_azure_provider_config(config)
|
|
460
|
+
config.azure_api_key = azure_api_key if azure_api_key && config.respond_to?(:azure_api_key=)
|
|
461
|
+
config.azure_endpoint = azure_endpoint if azure_endpoint && config.respond_to?(:azure_endpoint=)
|
|
462
|
+
config.azure_api_version = azure_api_version if azure_api_version && config.respond_to?(:azure_api_version=)
|
|
463
|
+
end
|
|
464
|
+
|
|
465
|
+
def apply_ollama_provider_config(config)
|
|
466
|
+
base = ollama_url.sub(%r{/+$}, '')
|
|
467
|
+
config.ollama_api_base = base.end_with?('/v1') ? base : "#{base}/v1"
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
def apply_huggingface_provider_config(config)
|
|
471
|
+
config.huggingface_api_key = huggingface_api_key if huggingface_api_key && config.respond_to?(:huggingface_api_key=)
|
|
472
|
+
end
|
|
473
|
+
|
|
474
|
+
def apply_openrouter_provider_config(config)
|
|
475
|
+
config.openrouter_api_key = openrouter_api_key if openrouter_api_key && config.respond_to?(:openrouter_api_key=)
|
|
476
|
+
end
|
|
477
|
+
|
|
478
|
+
def apply_bedrock_provider_config(config)
|
|
479
|
+
config.bedrock_api_key = bedrock_access_key if bedrock_access_key && config.respond_to?(:bedrock_api_key=)
|
|
480
|
+
config.bedrock_secret_key = bedrock_secret_key if bedrock_secret_key && config.respond_to?(:bedrock_secret_key=)
|
|
481
|
+
config.bedrock_region = bedrock_region if bedrock_region && config.respond_to?(:bedrock_region=)
|
|
482
|
+
end
|
|
483
|
+
|
|
484
|
+
def apply_deepseek_provider_config(config)
|
|
485
|
+
config.deepseek_api_key = deepseek_api_key if deepseek_api_key && config.respond_to?(:deepseek_api_key=)
|
|
486
|
+
end
|
|
487
|
+
|
|
477
488
|
# ==========================================================================
|
|
478
489
|
# Setup Defaults Callback
|
|
479
490
|
# ==========================================================================
|
|
@@ -484,7 +495,7 @@ class HTM
|
|
|
484
495
|
@embedding_generator ||= build_default_embedding_generator
|
|
485
496
|
@tag_extractor ||= build_default_tag_extractor
|
|
486
497
|
@proposition_extractor ||= build_default_proposition_extractor
|
|
487
|
-
@token_counter
|
|
498
|
+
@token_counter = build_default_token_counter if @token_counter.nil?
|
|
488
499
|
end
|
|
489
500
|
|
|
490
501
|
def detect_job_backend
|