maglev-rb 0.1.1 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.ja.md +618 -0
  4. data/README.md +533 -248
  5. data/README.zh-CN.md +457 -250
  6. data/lib/generators/maglev/install/install_generator.rb +20 -0
  7. data/lib/generators/maglev/upgrade_index_version/upgrade_index_version_generator.rb +27 -0
  8. data/lib/generators/maglev/upgrade_source_identity/upgrade_source_identity_generator.rb +52 -0
  9. data/lib/maglev/active_record_extension.rb +141 -24
  10. data/lib/maglev/adapters/faraday_client.rb +94 -0
  11. data/lib/maglev/adapters/faraday_embedding.rb +51 -0
  12. data/lib/maglev/adapters/faraday_generation.rb +49 -0
  13. data/lib/maglev/adapters/faraday_planner.rb +88 -0
  14. data/lib/maglev/answerer.rb +30 -12
  15. data/lib/maglev/chunker.rb +39 -4
  16. data/lib/maglev/configuration.rb +66 -1
  17. data/lib/maglev/content_source_graph.rb +17 -11
  18. data/lib/maglev/dependency_graph.rb +72 -13
  19. data/lib/maglev/embedding_adapter.rb +10 -0
  20. data/lib/maglev/hybrid_candidate_set.rb +25 -0
  21. data/lib/maglev/hybrid_coordinator.rb +112 -0
  22. data/lib/maglev/hybrid_result.rb +25 -0
  23. data/lib/maglev/index_diagnostics.rb +83 -0
  24. data/lib/maglev/index_identity.rb +70 -0
  25. data/lib/maglev/index_state.rb +9 -0
  26. data/lib/maglev/indexer.rb +185 -35
  27. data/lib/maglev/knowledge_config.rb +27 -5
  28. data/lib/maglev/knowledge_registry.rb +33 -0
  29. data/lib/maglev/planner.rb +172 -0
  30. data/lib/maglev/planner_adapter.rb +25 -0
  31. data/lib/maglev/planner_evaluation.rb +49 -0
  32. data/lib/maglev/query_compiler.rb +197 -0
  33. data/lib/maglev/query_ir.rb +143 -0
  34. data/lib/maglev/query_validator.rb +311 -0
  35. data/lib/maglev/railtie.rb +9 -0
  36. data/lib/maglev/registry.rb +72 -0
  37. data/lib/maglev/reindex_job.rb +34 -2
  38. data/lib/maglev/relation_order.rb +16 -0
  39. data/lib/maglev/request.rb +22 -0
  40. data/lib/maglev/request_executor.rb +101 -0
  41. data/lib/maglev/resource_config.rb +222 -0
  42. data/lib/maglev/response.rb +2 -2
  43. data/lib/maglev/result.rb +30 -0
  44. data/lib/maglev/retrieval_outcome.rb +52 -0
  45. data/lib/maglev/retrieval_result.rb +25 -0
  46. data/lib/maglev/retriever.rb +282 -27
  47. data/lib/maglev/router.rb +77 -0
  48. data/lib/maglev/routing_adapter.rb +25 -0
  49. data/lib/maglev/schema_compiler.rb +17 -4
  50. data/lib/maglev/schema_snapshot.rb +159 -0
  51. data/lib/maglev/search_result.rb +7 -3
  52. data/lib/maglev/snapshot.rb +21 -1
  53. data/lib/maglev/snapshot_budget.rb +57 -0
  54. data/lib/maglev/snapshot_builder.rb +89 -11
  55. data/lib/maglev/source_extractor.rb +43 -0
  56. data/lib/maglev/source_fragment.rb +9 -0
  57. data/lib/maglev/structured_answer_composer.rb +67 -0
  58. data/lib/maglev/structured_evidence_builder.rb +56 -0
  59. data/lib/maglev/structured_executor.rb +157 -0
  60. data/lib/maglev/structured_result.rb +97 -0
  61. data/lib/maglev/trace.rb +56 -0
  62. data/lib/maglev/vector_stores/base.rb +12 -0
  63. data/lib/maglev/vector_stores/document.rb +14 -5
  64. data/lib/maglev/vector_stores/document_id.rb +27 -0
  65. data/lib/maglev/vector_stores/memory.rb +68 -6
  66. data/lib/maglev/vector_stores/metadata_filter.rb +55 -0
  67. data/lib/maglev/vector_stores/pgvector.rb +94 -7
  68. data/lib/maglev/version.rb +1 -1
  69. data/lib/maglev-rb.rb +3 -0
  70. data/lib/maglev.rb +36 -3
  71. data/lib/tasks/maglev.rake +43 -0
  72. metadata +71 -11
  73. data/lib/maglev/adapters/ruby_llm_attachment_extractor.rb +0 -15
  74. data/lib/maglev/adapters/ruby_llm_embedding.rb +0 -22
  75. data/lib/maglev/adapters/ruby_llm_generation.rb +0 -22
  76. data/lib/maglev/adapters/ruby_llm_provider.rb +0 -64
@@ -0,0 +1,311 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "date"
4
+ require "time"
5
+ require_relative "query_ir"
6
+ require_relative "schema_snapshot"
7
+
8
+ module Maglev
9
+ class QueryValidator
10
+ attr_reader :policy_limits
11
+ ROOT_KEYS = %w[version root operation scopes filters joins sort distinct limit aggregate].freeze
12
+ OPERATIONS = %w[records aggregate].freeze
13
+ OPERATORS = %w[eq not_eq gt gte lt lte in not_in is_null is_not_null between].freeze
14
+ COMPARISON_OPERATORS = %w[gt gte lt lte between].freeze
15
+ ARRAY_OPERATORS = %w[in not_in between].freeze
16
+ NULL_OPERATORS = %w[is_null is_not_null].freeze
17
+ NUMERIC_TYPES = %i[integer float decimal].freeze
18
+ TIME_TYPES = %i[date datetime timestamp time].freeze
19
+ DEFAULT_LIMITS = {rows: 100, operations: 30, joins: 2, complexity: 100}.freeze
20
+
21
+ Error = Struct.new(:code, :message, :path, :details) do
22
+ def initialize(**attributes)
23
+ attributes[:path] = Array(attributes[:path]).freeze
24
+ attributes[:details] = (attributes[:details] || {}).freeze
25
+ super
26
+ freeze
27
+ end
28
+ end
29
+ Result = Struct.new(:ir, :errors, :explanation, :snapshot) do
30
+ def initialize(**attributes)
31
+ attributes[:errors] = Array(attributes[:errors]).freeze
32
+ super
33
+ freeze
34
+ end
35
+
36
+ def valid? = errors.empty?
37
+ end
38
+
39
+ def initialize(snapshot:, root:, limits: {})
40
+ @snapshot = snapshot
41
+ @root = root.to_s
42
+ @resources = snapshot.resources.to_h { |resource| [resource.identifier, resource] }
43
+ resource_limits = @resources[@root]&.limits || {}
44
+ requested = limits.transform_keys(&:to_sym)
45
+ @limits = DEFAULT_LIMITS.merge(resource_limits).merge(requested) do |_key, left, right|
46
+ [left, right].min
47
+ end
48
+ @policy_limits = @limits.freeze
49
+ end
50
+
51
+ def call(input)
52
+ @errors = []
53
+ unless input.is_a?(Hash) && input.keys.all? { |key| key.is_a?(String) }
54
+ error(:invalid_ir, "Query IR must be an object with string keys", [])
55
+ return result
56
+ end
57
+ unknown = input.keys - ROOT_KEYS
58
+ error(:invalid_ir, "Query IR contains unknown keys", [], keys: unknown.sort) if unknown.any?
59
+ error(:invalid_ir, "Unsupported Query IR version", ["version"]) unless input["version"] == 1
60
+ error(:unregistered, "The requested root resource is unavailable", ["root"], resource: @root) unless input["root"] == @root && @resources.key?(@root)
61
+ error(:invalid_ir, "Unknown operation", ["operation"]) unless OPERATIONS.include?(input["operation"])
62
+ return result if @errors.any?
63
+
64
+ scopes = validate_scopes(input.fetch("scopes", []))
65
+ joins = validate_joins(input.fetch("joins", []))
66
+ filters = validate_filters(input.fetch("filters", []), joins)
67
+ sort = validate_sort(input.fetch("sort", []), joins)
68
+ distinct = input.fetch("distinct", false)
69
+ error(:invalid_ir, "Distinct must be boolean", ["distinct"]) unless [true, false].include?(distinct)
70
+ limit = validate_limit(input.fetch("limit", @limits[:rows]))
71
+ aggregate = validate_aggregate(input["operation"], input["aggregate"], joins)
72
+ operation_count = scopes.length + filters.length + joins.length + sort.length + (distinct ? 1 : 0) + (aggregate ? 1 : 0)
73
+ error(:limit_exceeded, "Operation limit exceeded", [], limit: @limits[:operations]) if operation_count > @limits[:operations]
74
+ complexity = filters.sum { |filter| 1 + filter.field.segments.length } + scopes.length * 2 + joins.sum { |join| join.segments.length * 3 } + sort.length + (aggregate ? 2 : 0)
75
+ error(:limit_exceeded, "Query complexity exceeded", [], limit: @limits[:complexity]) if complexity > @limits[:complexity]
76
+ return result if @errors.any?
77
+
78
+ ir = QueryIR::Query.new(version: 1, root: @root, operation: input["operation"], scopes: scopes,
79
+ filters: filters, joins: joins, sort: sort, distinct: distinct, limit: limit, aggregate: aggregate)
80
+ Result.new(ir: ir, errors: [], explanation: explain(ir), snapshot: @snapshot)
81
+ rescue KeyError, TypeError, ArgumentError
82
+ error(:invalid_ir, "Malformed Query IR", [])
83
+ result
84
+ end
85
+
86
+ private
87
+
88
+ def validate_scopes(values)
89
+ return invalid_collection("Scopes", ["scopes"]) unless values.is_a?(Array)
90
+ schemas = @resources.fetch(@root).scopes.to_h { |scope| [scope.fetch(:name), scope] }
91
+ values.each_with_index.filter_map do |value, index|
92
+ unless exact_hash?(value, %w[name parameters]) && value["name"].is_a?(String) && value.fetch("parameters", {}).is_a?(Hash)
93
+ error(:invalid_ir, "Invalid scope", ["scopes", index])
94
+ next
95
+ end
96
+ schema = schemas[value["name"]]
97
+ unless schema
98
+ error(:unregistered, "The requested scope is not registered", ["scopes", index, "name"], resource: @root)
99
+ next
100
+ end
101
+ parameters = validate_parameters(value.fetch("parameters", {}), schema.fetch(:parameters), ["scopes", index, "parameters"])
102
+ QueryIR::Scope.new(name: value["name"], parameters: parameters)
103
+ end
104
+ end
105
+
106
+ def validate_parameters(values, schemas, path)
107
+ unknown = values.keys - schemas.keys
108
+ error(:invalid_ir, "Unknown scope parameters", path, keys: unknown.sort) if unknown.any?
109
+ schemas.each_with_object({}) do |(name, schema), result|
110
+ if !values.key?(name)
111
+ error(:invalid_ir, "Required scope parameter is missing", path + [name]) if schema.fetch(:required)
112
+ else
113
+ result[name] = coerce(values[name], schema.fetch(:type), schema, path + [name])
114
+ end
115
+ end.freeze
116
+ end
117
+
118
+ def validate_joins(values)
119
+ return invalid_collection("Joins", ["joins"]) unless values.is_a?(Array)
120
+ error(:limit_exceeded, "Join limit exceeded", ["joins"], limit: @limits[:joins]) if values.length > @limits[:joins]
121
+ values.each_with_index.filter_map do |value, index|
122
+ unless value.is_a?(String) && value.split(".").length <= 2 && @snapshot.paths.include?("#{@root}.#{value}")
123
+ error(:unregistered, "The requested association path is not registered", ["joins", index], resource: @root)
124
+ next
125
+ end
126
+ QueryIR::Path.new(value)
127
+ end
128
+ end
129
+
130
+ def validate_filters(values, joins)
131
+ return invalid_collection("Filters", ["filters"]) unless values.is_a?(Array)
132
+ values.each_with_index.filter_map do |value, index|
133
+ unless value.is_a?(Hash) && value.keys.all? { |key| %w[field operator value].include?(key) } && value["field"].is_a?(String) && OPERATORS.include?(value["operator"])
134
+ error(:invalid_ir, "Invalid filter", ["filters", index])
135
+ next
136
+ end
137
+ field = resolve_field(value["field"], joins, ["filters", index, "field"])
138
+ next unless field
139
+ operator = value["operator"]
140
+ if NULL_OPERATORS.include?(operator)
141
+ error(:invalid_ir, "Null operator does not accept a value", ["filters", index, "value"]) if value.key?("value")
142
+ error(:invalid_ir, "Null check is invalid for a non-null field", ["filters", index, "operator"]) unless field.null
143
+ typed = nil
144
+ else
145
+ error(:invalid_ir, "Filter value is required", ["filters", index, "value"]) unless value.key?("value")
146
+ if COMPARISON_OPERATORS.include?(operator) && !comparable_type?(field.type)
147
+ error(:invalid_ir, "Operator is incompatible with field type", ["filters", index, "operator"])
148
+ end
149
+ typed = coerce_filter(value["value"], field, operator, ["filters", index, "value"])
150
+ end
151
+ QueryIR::Predicate.new(field: QueryIR::Path.new(value["field"]), operator: operator.to_sym, value: typed)
152
+ end
153
+ end
154
+
155
+ def validate_sort(values, joins)
156
+ return invalid_collection("Sort", ["sort"]) unless values.is_a?(Array)
157
+ values.each_with_index.filter_map do |value, index|
158
+ unless exact_hash?(value, %w[field direction]) && %w[asc desc].include?(value["direction"])
159
+ error(:invalid_ir, "Invalid sort", ["sort", index])
160
+ next
161
+ end
162
+ next unless resolve_field(value["field"], joins, ["sort", index, "field"])
163
+ QueryIR::Sort.new(field: QueryIR::Path.new(value["field"]), direction: value["direction"].to_sym)
164
+ end
165
+ end
166
+
167
+ def validate_limit(value)
168
+ unless value.is_a?(Integer) && value.positive? && value <= @limits[:rows]
169
+ error(:limit_exceeded, "Result limit exceeded", ["limit"], limit: @limits[:rows])
170
+ end
171
+ value
172
+ end
173
+
174
+ def validate_aggregate(operation, value, joins)
175
+ if operation == "records"
176
+ error(:invalid_ir, "Aggregate is only valid for aggregate operations", ["aggregate"]) if value
177
+ return
178
+ end
179
+ unless value.is_a?(Hash) && value.keys.all? { |key| %w[function field].include?(key) } && value["function"].is_a?(String)
180
+ error(:invalid_ir, "Aggregate is required", ["aggregate"])
181
+ return
182
+ end
183
+ function = value["function"].to_sym
184
+ permission = @resources.fetch(@root).aggregates[function]
185
+ unless permission
186
+ error(:unregistered, "The requested aggregate is not registered", ["aggregate", "function"], resource: @root)
187
+ return
188
+ end
189
+ field_path = value["field"]
190
+ if function == :count
191
+ error(:invalid_ir, "Count does not accept a field", ["aggregate", "field"]) if field_path
192
+ elsif !field_path || !resolve_field(field_path, joins, ["aggregate", "field"]) || permission != true && !permission.include?(field_path)
193
+ error(:unregistered, "The aggregate field is not registered", ["aggregate", "field"], resource: @root)
194
+ return
195
+ end
196
+ QueryIR::Aggregate.new(function: function, field: field_path && QueryIR::Path.new(field_path))
197
+ end
198
+
199
+ def resolve_field(path, joins, error_path)
200
+ unless path.is_a?(String) && path.split(".").length <= 3
201
+ error(:invalid_ir, "Invalid field path", error_path)
202
+ return
203
+ end
204
+ segments = path.split(".")
205
+ resource = @resources.fetch(@root)
206
+ if segments.length > 1
207
+ join = segments[0...-1].join(".")
208
+ unless joins.any? { |item| item.to_s == join }
209
+ error(:unregistered, "The field path is not joined", error_path, resource: @root)
210
+ return
211
+ end
212
+ missing_association = false
213
+ segments[0...-1].each do |association_name|
214
+ association = resource.associations.find { |item| item.name == association_name }
215
+ unless association
216
+ error(:unregistered, "The field path is not registered", error_path, resource: @root)
217
+ missing_association = true
218
+ break
219
+ end
220
+ resource = @resources[association.resource]
221
+ end
222
+ return if missing_association
223
+ end
224
+ field = resource&.fields&.find { |candidate| candidate.name == segments.last }
225
+ return field if field
226
+
227
+ error(:unregistered, "The requested field is not registered", error_path, resource: @root)
228
+ nil
229
+ end
230
+
231
+ def coerce_filter(value, field, operator, path)
232
+ if ARRAY_OPERATORS.include?(operator)
233
+ expected = (operator == "between") ? 2 : nil
234
+ valid_array = value.is_a?(Array) && !value.empty?
235
+ valid_array &&= value.length == expected if expected
236
+ unless valid_array
237
+ error(:invalid_ir, "Operator requires a bounded array", path)
238
+ return
239
+ end
240
+ return value.map { |item| coerce(item, field.type, {enum_values: field.enum_values, nullable: field.null}, path) }.freeze
241
+ end
242
+ coerce(value, field.type, {enum_values: field.enum_values, nullable: field.null}, path)
243
+ end
244
+
245
+ def coerce(value, type, schema, path)
246
+ if value.nil?
247
+ error(:invalid_ir, "Null is not allowed", path) unless schema[:nullable]
248
+ return QueryIR::Literal.new(value: nil, type: type)
249
+ end
250
+ if TIME_TYPES.include?(type.to_sym) && value.is_a?(Hash)
251
+ relative = value["relative"]
252
+ unless valid_relative_time?(value, relative)
253
+ error(:invalid_ir, "Invalid relative time", path)
254
+ return
255
+ end
256
+ return QueryIR::RelativeTime.new(amount: relative["amount"], unit: relative["unit"], direction: relative["direction"])
257
+ end
258
+ valid = case type.to_sym
259
+ when :integer then value.is_a?(Integer)
260
+ when :float, :decimal then value.is_a?(Numeric) && value.finite?
261
+ when :boolean then [true, false].include?(value)
262
+ when :date then value.is_a?(String) && Date.iso8601(value).to_s == value
263
+ when :datetime, :timestamp, :time then valid_timestamp?(value)
264
+ else value.is_a?(String)
265
+ end
266
+ enum_values = schema[:enum_values] || []
267
+ valid &&= enum_values.include?(value) if enum_values.any?
268
+ if schema[:minimum] && value.respond_to?(:<) then valid &&= value >= schema[:minimum] end
269
+ if schema[:maximum] && value.respond_to?(:>) then valid &&= value <= schema[:maximum] end
270
+ error(:invalid_ir, "Value is incompatible with the registered type", path) unless valid
271
+ QueryIR::Literal.new(value: value, type: type)
272
+ rescue ArgumentError
273
+ error(:invalid_ir, "Value is incompatible with the registered type", path)
274
+ QueryIR::Literal.new(value: value, type: type)
275
+ end
276
+
277
+ def valid_timestamp?(value)
278
+ return false unless value.is_a?(String) && value.match?(/(?:Z|[+-]\d{2}:\d{2})\z/)
279
+ Time.iso8601(value)
280
+ true
281
+ rescue ArgumentError
282
+ false
283
+ end
284
+
285
+ def valid_relative_time?(value, relative)
286
+ exact_hash?(value, ["relative"]) &&
287
+ exact_hash?(relative, %w[amount unit direction]) &&
288
+ relative["amount"].is_a?(Integer) &&
289
+ relative["amount"].positive? &&
290
+ QueryIR::RelativeTime::UNITS.include?(relative["unit"]) &&
291
+ QueryIR::RelativeTime::DIRECTIONS.include?(relative["direction"])
292
+ end
293
+
294
+ def comparable_type?(type) = NUMERIC_TYPES.include?(type.to_sym) || TIME_TYPES.include?(type.to_sym) || type.to_sym == :string
295
+ def exact_hash?(value, keys) = value.is_a?(Hash) && value.keys.sort == keys.sort
296
+ def invalid_collection(name, path) = error(:invalid_ir, "#{name} must be an array", path) && []
297
+ def error(code, message, path, details = {}) = @errors << Error.new(code: code, message: message, path: path, details: details)
298
+ def result = Result.new(ir: nil, errors: @errors, explanation: nil, snapshot: @snapshot)
299
+
300
+ def explain(ir)
301
+ parts = [(ir.operation == :records) ? "Select records from #{ir.root}" : "Aggregate #{ir.root} using #{ir.aggregate.function}"]
302
+ parts << "apply scope #{ir.scopes.map(&:name).join(", ")}" if ir.scopes.any?
303
+ parts << "filter #{ir.filters.map { |filter| "#{filter.field} #{filter.operator}" }.join(", ")}" if ir.filters.any?
304
+ parts << "join #{ir.joins.join(", ")}" if ir.joins.any?
305
+ parts << "sort #{ir.sort.map { |sort| "#{sort.field} #{sort.direction}" }.join(", ")}" if ir.sort.any?
306
+ parts << "use distinct records" if ir.distinct
307
+ parts << "limit #{ir.limit}"
308
+ parts.join("; ")
309
+ end
310
+ end
311
+ end
@@ -9,6 +9,15 @@ module Maglev
9
9
  require "maglev/active_record_extension"
10
10
 
11
11
  include Maglev::ActiveRecordExtension
12
+ ActiveRecord::Relation.include(Maglev::RelationExtension)
13
+ end
14
+ end
15
+
16
+ initializer "maglev.reloader" do
17
+ config.to_prepare do
18
+ Maglev::DependencyGraph.reset!
19
+ Maglev::KnowledgeRegistry.rebuild!
20
+ Maglev::Registry.rebuild!
12
21
  end
13
22
  end
14
23
 
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "resource_config"
4
+ require_relative "schema_snapshot"
5
+
6
+ module Maglev
7
+ class Registry
8
+ class << self
9
+ def register(entry)
10
+ mutex.synchronize do
11
+ @entries ||= {}
12
+ existing = @entries[entry.identifier]
13
+ if existing && existing.model_class != entry.model_class && existing.model_class.name != entry.model_class.name
14
+ raise ConfigurationError, "Maglev resource #{entry.identifier} is already registered"
15
+ end
16
+ @entries[entry.identifier] = entry
17
+ @snapshot_cache = {}
18
+ end
19
+ entry
20
+ end
21
+
22
+ def fetch(identifier)
23
+ mutex.synchronize { (@entries || {})[identifier.to_s] }
24
+ end
25
+
26
+ def entries
27
+ mutex.synchronize { (@entries || {}).values.sort_by(&:identifier).freeze }
28
+ end
29
+
30
+ def snapshot(resources:, user: nil, authorizer: nil, limits: {})
31
+ identifiers = Array(resources).map(&:to_s).uniq.sort
32
+ selected = identifiers.filter_map { |identifier| fetch(identifier) }.select(&:queryable)
33
+ selected.select! do |entry|
34
+ entry.queryable.authorization == :public || authorizer&.call(entry, user)
35
+ end
36
+
37
+ cache_key = [identifiers, limits.sort].freeze if user.nil? && authorizer.nil?
38
+ mutex.synchronize { return @snapshot_cache[cache_key] if cache_key && @snapshot_cache&.key?(cache_key) }
39
+
40
+ result = SchemaSnapshot::Builder.new(selected, limits: limits).build
41
+ mutex.synchronize { (@snapshot_cache ||= {})[cache_key] = result } if cache_key
42
+ result
43
+ end
44
+
45
+ def invalidate!
46
+ mutex.synchronize { @snapshot_cache = {} }
47
+ end
48
+
49
+ def reset!
50
+ mutex.synchronize do
51
+ @entries = {}
52
+ @snapshot_cache = {}
53
+ end
54
+ end
55
+
56
+ def rebuild!
57
+ reset!
58
+ KnowledgeRegistry.model_names.each do |model_name|
59
+ model = model_name.safe_constantize
60
+ model.rebuild_maglev_registry_registration if model&.respond_to?(:rebuild_maglev_registry_registration)
61
+ end
62
+ invalidate!
63
+ end
64
+
65
+ private
66
+
67
+ def mutex
68
+ @mutex ||= Mutex.new
69
+ end
70
+ end
71
+ end
72
+ end
@@ -9,8 +9,40 @@ module Maglev
9
9
  queue_as :default
10
10
 
11
11
  def perform(owner_class_name, owner_id)
12
- owner = owner_class_name.constantize.find(owner_id)
13
- Indexer.new(owner).index
12
+ owner_class = owner_class_name.constantize
13
+ owner = owner_class.find(owner_id)
14
+
15
+ begin
16
+ provider_call = ProviderCall.new(max_attempts: 1)
17
+ Indexer.new(owner, provider_call: provider_call).index
18
+ rescue RetryableProviderError => error
19
+ if executions < Maglev.configuration.provider_max_attempts
20
+ instrument_reindex("maglev.reindex.retry", owner_class_name, owner_id, error)
21
+ retry_job(wait: 0)
22
+ else
23
+ instrument_reindex("maglev.reindex.exhausted", owner_class_name, owner_id, error)
24
+ raise
25
+ end
26
+ rescue PermanentProviderError => error
27
+ instrument_reindex("maglev.reindex.discard", owner_class_name, owner_id, error)
28
+ end
29
+ rescue ActiveRecord::RecordNotFound => error
30
+ instrument_reindex("maglev.reindex.discard", owner_class_name, owner_id, error)
31
+ end
32
+
33
+ private
34
+
35
+ def instrument_reindex(name, owner_class_name, owner_id, error)
36
+ ActiveSupport::Notifications.instrument(
37
+ name,
38
+ job_class: self.class.name,
39
+ owner_class: owner_class_name,
40
+ owner_id: owner_id,
41
+ error_class: error.class.name,
42
+ execution_count: executions,
43
+ attempt: executions,
44
+ max_attempts: Maglev.configuration.provider_max_attempts
45
+ )
14
46
  end
15
47
  end
16
48
  end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Maglev
4
+ module RelationOrder
5
+ module_function
6
+
7
+ def with_primary_key(order, model_class)
8
+ return nil unless order
9
+
10
+ normalized = order.dup
11
+ primary_key = model_class.primary_key&.to_sym
12
+ normalized[primary_key] = :asc if primary_key && !normalized.key?(primary_key)
13
+ normalized.freeze
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Maglev
4
+ class Request
5
+ MODES = %i[auto structured rag hybrid].freeze
6
+
7
+ attr_reader :question, :mode, :resources, :base_relation, :user, :options
8
+
9
+ def initialize(question:, mode: :auto, resources: [], base_relation: nil, user: nil, **options)
10
+ normalized_mode = mode.to_sym
11
+ raise ArgumentError, "invalid request mode" unless MODES.include?(normalized_mode)
12
+
13
+ @question = question.to_s.freeze
14
+ @mode = normalized_mode
15
+ @resources = Array(resources).map { |resource| resource.to_s.freeze }.uniq.freeze
16
+ @base_relation = base_relation
17
+ @user = user
18
+ @options = options.freeze
19
+ freeze
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,101 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "securerandom"
4
+
5
+ module Maglev
6
+ class RequestExecutor
7
+ def initialize(router:, retriever_factory: ->(model) { Retriever.new(model) },
8
+ answerer_factory: ->(model) { Answerer.new(model) })
9
+ @router = router
10
+ @retriever_factory = retriever_factory
11
+ @answerer_factory = answerer_factory
12
+ end
13
+
14
+ def call(request)
15
+ decision = @router.route(request)
16
+ case decision.route
17
+ when :unsupported, :clarification_required
18
+ Result.new(status: decision.route, route: decision.route, kind: :none,
19
+ trace_id: SecureRandom.uuid, confidence: decision.confidence, reasons: decision.reasons)
20
+ when :hybrid
21
+ HybridCoordinator.new(retriever_factory: @retriever_factory).call(request, decision)
22
+ when :structured then execute_structured(request, decision)
23
+ when :rag then execute_rag(request, decision)
24
+ end
25
+ end
26
+
27
+ private
28
+
29
+ def execute_structured(request, decision)
30
+ entry = single_entry(request, capability: :queryable)
31
+ relation = request.base_relation
32
+ if relation.nil? && entry.queryable.allow_unscoped_model_queries
33
+ relation = entry.model_class.all
34
+ end
35
+ raise ConfigurationError, "structured requests require an authorized base relation" unless relation
36
+
37
+ plan = Maglev.plan(request.question, resource: entry.identifier, resources: request.resources,
38
+ base_relation: relation, user: request.user, authorizer: request.options[:authorizer],
39
+ constraints: request.options.fetch(:constraints, {}),
40
+ adapter: request.options[:planner_adapter] || Maglev.configuration.planner_adapter)
41
+ wrap_structured(Maglev.execute(plan), decision)
42
+ end
43
+
44
+ def execute_rag(request, decision)
45
+ entry = single_entry(request, capability: :knowledge)
46
+ limit = request.options.fetch(:limit, 10)
47
+ if request.options[:answer]
48
+ response = @answerer_factory.call(entry.model_class).ask(request.question, limit: limit,
49
+ user: request.user, minimum_similarity: request.options[:minimum_similarity],
50
+ chunks_per_owner: request.options[:chunks_per_owner])
51
+ Result.new(status: :succeeded, route: :rag, kind: :rag_answer, value: response,
52
+ evidence: response.sources, trace_id: response.metadata[:trace_id] || SecureRandom.uuid,
53
+ confidence: decision.confidence, reasons: decision.reasons, metadata: response.metadata)
54
+ else
55
+ retrieval = @retriever_factory.call(entry.model_class).retrieve(request.question, limit: limit,
56
+ user: request.user, minimum_similarity: request.options[:minimum_similarity],
57
+ chunks_per_owner: request.options.fetch(:chunks_per_owner, 1))
58
+ Result.new(status: :succeeded, route: :rag, kind: :semantic_matches, value: retrieval,
59
+ evidence: retrieval.selected, trace_id: retrieval.trace_id, confidence: decision.confidence,
60
+ reasons: decision.reasons, metadata: retrieval.metadata)
61
+ end
62
+ end
63
+
64
+ def single_entry(request, capability:)
65
+ entries = request.resources.filter_map { |identifier| Registry.fetch(identifier) }
66
+ .select { |entry| entry.public_send(capability) }
67
+ unless entries.one?
68
+ raise ConfigurationError, "#{capability} routing requires exactly one registered resource"
69
+ end
70
+ entries.first
71
+ end
72
+
73
+ def wrap_structured(result, decision)
74
+ Result.new(status: result.status, route: :structured, kind: result.kind, value: result.value,
75
+ evidence: result.evidence, warnings: result.warnings, trace_id: result.trace_id,
76
+ confidence: decision.confidence, reasons: decision.reasons, metadata: {plan: result.plan}.freeze)
77
+ end
78
+ end
79
+
80
+ def self.request(question, mode: :auto, resources: nil, models: nil, base_relation: nil,
81
+ user: nil, router: nil, retriever_factory: nil, answerer_factory: nil, **options)
82
+ entries = Array(models).filter_map do |model|
83
+ Registry.entries.find { |entry| entry.model_class == model }
84
+ end
85
+ identifiers = [*Array(resources), *entries.map(&:identifier)].compact.map(&:to_s).uniq
86
+ if identifiers.empty? && base_relation
87
+ entry = Registry.entries.find { |candidate| candidate.model_class == base_relation.klass }
88
+ identifiers << entry.identifier if entry
89
+ end
90
+ if identifiers.empty?
91
+ raise ConfigurationError, "an explicit resource, model, or base relation is required"
92
+ end
93
+
94
+ router ||= Router.new(classifier: Maglev.configuration.routing_adapter)
95
+ executor_options = {router: router}
96
+ executor_options[:retriever_factory] = retriever_factory if retriever_factory
97
+ executor_options[:answerer_factory] = answerer_factory if answerer_factory
98
+ RequestExecutor.new(**executor_options).call(Request.new(question: question, mode: mode,
99
+ resources: identifiers, base_relation: base_relation, user: user, **options))
100
+ end
101
+ end