maglev-rb 0.1.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.ja.md +618 -0
- data/README.md +533 -248
- data/README.zh-CN.md +457 -250
- data/lib/generators/maglev/install/install_generator.rb +20 -0
- data/lib/generators/maglev/upgrade_index_version/upgrade_index_version_generator.rb +27 -0
- data/lib/generators/maglev/upgrade_source_identity/upgrade_source_identity_generator.rb +52 -0
- data/lib/maglev/active_record_extension.rb +141 -24
- data/lib/maglev/adapters/faraday_client.rb +94 -0
- data/lib/maglev/adapters/faraday_embedding.rb +51 -0
- data/lib/maglev/adapters/faraday_generation.rb +49 -0
- data/lib/maglev/adapters/faraday_planner.rb +88 -0
- data/lib/maglev/answerer.rb +30 -12
- data/lib/maglev/chunker.rb +39 -4
- data/lib/maglev/configuration.rb +66 -1
- data/lib/maglev/content_source_graph.rb +17 -11
- data/lib/maglev/dependency_graph.rb +72 -13
- data/lib/maglev/embedding_adapter.rb +10 -0
- data/lib/maglev/hybrid_candidate_set.rb +25 -0
- data/lib/maglev/hybrid_coordinator.rb +112 -0
- data/lib/maglev/hybrid_result.rb +25 -0
- data/lib/maglev/index_diagnostics.rb +83 -0
- data/lib/maglev/index_identity.rb +70 -0
- data/lib/maglev/index_state.rb +9 -0
- data/lib/maglev/indexer.rb +185 -35
- data/lib/maglev/knowledge_config.rb +27 -5
- data/lib/maglev/knowledge_registry.rb +33 -0
- data/lib/maglev/planner.rb +172 -0
- data/lib/maglev/planner_adapter.rb +25 -0
- data/lib/maglev/planner_evaluation.rb +49 -0
- data/lib/maglev/query_compiler.rb +197 -0
- data/lib/maglev/query_ir.rb +143 -0
- data/lib/maglev/query_validator.rb +311 -0
- data/lib/maglev/railtie.rb +9 -0
- data/lib/maglev/registry.rb +72 -0
- data/lib/maglev/reindex_job.rb +34 -2
- data/lib/maglev/relation_order.rb +16 -0
- data/lib/maglev/request.rb +22 -0
- data/lib/maglev/request_executor.rb +101 -0
- data/lib/maglev/resource_config.rb +222 -0
- data/lib/maglev/response.rb +2 -2
- data/lib/maglev/result.rb +30 -0
- data/lib/maglev/retrieval_outcome.rb +52 -0
- data/lib/maglev/retrieval_result.rb +25 -0
- data/lib/maglev/retriever.rb +282 -27
- data/lib/maglev/router.rb +77 -0
- data/lib/maglev/routing_adapter.rb +25 -0
- data/lib/maglev/schema_compiler.rb +17 -4
- data/lib/maglev/schema_snapshot.rb +159 -0
- data/lib/maglev/search_result.rb +7 -3
- data/lib/maglev/snapshot.rb +21 -1
- data/lib/maglev/snapshot_budget.rb +57 -0
- data/lib/maglev/snapshot_builder.rb +89 -11
- data/lib/maglev/source_extractor.rb +43 -0
- data/lib/maglev/source_fragment.rb +9 -0
- data/lib/maglev/structured_answer_composer.rb +67 -0
- data/lib/maglev/structured_evidence_builder.rb +56 -0
- data/lib/maglev/structured_executor.rb +157 -0
- data/lib/maglev/structured_result.rb +97 -0
- data/lib/maglev/trace.rb +56 -0
- data/lib/maglev/vector_stores/base.rb +12 -0
- data/lib/maglev/vector_stores/document.rb +14 -5
- data/lib/maglev/vector_stores/document_id.rb +27 -0
- data/lib/maglev/vector_stores/memory.rb +68 -6
- data/lib/maglev/vector_stores/metadata_filter.rb +55 -0
- data/lib/maglev/vector_stores/pgvector.rb +94 -7
- data/lib/maglev/version.rb +1 -1
- data/lib/maglev-rb.rb +3 -0
- data/lib/maglev.rb +36 -3
- data/lib/tasks/maglev.rake +43 -0
- metadata +71 -11
- data/lib/maglev/adapters/ruby_llm_attachment_extractor.rb +0 -15
- data/lib/maglev/adapters/ruby_llm_embedding.rb +0 -22
- data/lib/maglev/adapters/ruby_llm_generation.rb +0 -22
- data/lib/maglev/adapters/ruby_llm_provider.rb +0 -64
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
4
|
+
require "time"
|
|
5
|
+
require_relative "query_ir"
|
|
6
|
+
require_relative "schema_snapshot"
|
|
7
|
+
|
|
8
|
+
module Maglev
|
|
9
|
+
class QueryValidator
|
|
10
|
+
attr_reader :policy_limits
|
|
11
|
+
ROOT_KEYS = %w[version root operation scopes filters joins sort distinct limit aggregate].freeze
|
|
12
|
+
OPERATIONS = %w[records aggregate].freeze
|
|
13
|
+
OPERATORS = %w[eq not_eq gt gte lt lte in not_in is_null is_not_null between].freeze
|
|
14
|
+
COMPARISON_OPERATORS = %w[gt gte lt lte between].freeze
|
|
15
|
+
ARRAY_OPERATORS = %w[in not_in between].freeze
|
|
16
|
+
NULL_OPERATORS = %w[is_null is_not_null].freeze
|
|
17
|
+
NUMERIC_TYPES = %i[integer float decimal].freeze
|
|
18
|
+
TIME_TYPES = %i[date datetime timestamp time].freeze
|
|
19
|
+
DEFAULT_LIMITS = {rows: 100, operations: 30, joins: 2, complexity: 100}.freeze
|
|
20
|
+
|
|
21
|
+
Error = Struct.new(:code, :message, :path, :details) do
|
|
22
|
+
def initialize(**attributes)
|
|
23
|
+
attributes[:path] = Array(attributes[:path]).freeze
|
|
24
|
+
attributes[:details] = (attributes[:details] || {}).freeze
|
|
25
|
+
super
|
|
26
|
+
freeze
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
Result = Struct.new(:ir, :errors, :explanation, :snapshot) do
|
|
30
|
+
def initialize(**attributes)
|
|
31
|
+
attributes[:errors] = Array(attributes[:errors]).freeze
|
|
32
|
+
super
|
|
33
|
+
freeze
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def valid? = errors.empty?
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def initialize(snapshot:, root:, limits: {})
|
|
40
|
+
@snapshot = snapshot
|
|
41
|
+
@root = root.to_s
|
|
42
|
+
@resources = snapshot.resources.to_h { |resource| [resource.identifier, resource] }
|
|
43
|
+
resource_limits = @resources[@root]&.limits || {}
|
|
44
|
+
requested = limits.transform_keys(&:to_sym)
|
|
45
|
+
@limits = DEFAULT_LIMITS.merge(resource_limits).merge(requested) do |_key, left, right|
|
|
46
|
+
[left, right].min
|
|
47
|
+
end
|
|
48
|
+
@policy_limits = @limits.freeze
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def call(input)
|
|
52
|
+
@errors = []
|
|
53
|
+
unless input.is_a?(Hash) && input.keys.all? { |key| key.is_a?(String) }
|
|
54
|
+
error(:invalid_ir, "Query IR must be an object with string keys", [])
|
|
55
|
+
return result
|
|
56
|
+
end
|
|
57
|
+
unknown = input.keys - ROOT_KEYS
|
|
58
|
+
error(:invalid_ir, "Query IR contains unknown keys", [], keys: unknown.sort) if unknown.any?
|
|
59
|
+
error(:invalid_ir, "Unsupported Query IR version", ["version"]) unless input["version"] == 1
|
|
60
|
+
error(:unregistered, "The requested root resource is unavailable", ["root"], resource: @root) unless input["root"] == @root && @resources.key?(@root)
|
|
61
|
+
error(:invalid_ir, "Unknown operation", ["operation"]) unless OPERATIONS.include?(input["operation"])
|
|
62
|
+
return result if @errors.any?
|
|
63
|
+
|
|
64
|
+
scopes = validate_scopes(input.fetch("scopes", []))
|
|
65
|
+
joins = validate_joins(input.fetch("joins", []))
|
|
66
|
+
filters = validate_filters(input.fetch("filters", []), joins)
|
|
67
|
+
sort = validate_sort(input.fetch("sort", []), joins)
|
|
68
|
+
distinct = input.fetch("distinct", false)
|
|
69
|
+
error(:invalid_ir, "Distinct must be boolean", ["distinct"]) unless [true, false].include?(distinct)
|
|
70
|
+
limit = validate_limit(input.fetch("limit", @limits[:rows]))
|
|
71
|
+
aggregate = validate_aggregate(input["operation"], input["aggregate"], joins)
|
|
72
|
+
operation_count = scopes.length + filters.length + joins.length + sort.length + (distinct ? 1 : 0) + (aggregate ? 1 : 0)
|
|
73
|
+
error(:limit_exceeded, "Operation limit exceeded", [], limit: @limits[:operations]) if operation_count > @limits[:operations]
|
|
74
|
+
complexity = filters.sum { |filter| 1 + filter.field.segments.length } + scopes.length * 2 + joins.sum { |join| join.segments.length * 3 } + sort.length + (aggregate ? 2 : 0)
|
|
75
|
+
error(:limit_exceeded, "Query complexity exceeded", [], limit: @limits[:complexity]) if complexity > @limits[:complexity]
|
|
76
|
+
return result if @errors.any?
|
|
77
|
+
|
|
78
|
+
ir = QueryIR::Query.new(version: 1, root: @root, operation: input["operation"], scopes: scopes,
|
|
79
|
+
filters: filters, joins: joins, sort: sort, distinct: distinct, limit: limit, aggregate: aggregate)
|
|
80
|
+
Result.new(ir: ir, errors: [], explanation: explain(ir), snapshot: @snapshot)
|
|
81
|
+
rescue KeyError, TypeError, ArgumentError
|
|
82
|
+
error(:invalid_ir, "Malformed Query IR", [])
|
|
83
|
+
result
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
def validate_scopes(values)
|
|
89
|
+
return invalid_collection("Scopes", ["scopes"]) unless values.is_a?(Array)
|
|
90
|
+
schemas = @resources.fetch(@root).scopes.to_h { |scope| [scope.fetch(:name), scope] }
|
|
91
|
+
values.each_with_index.filter_map do |value, index|
|
|
92
|
+
unless exact_hash?(value, %w[name parameters]) && value["name"].is_a?(String) && value.fetch("parameters", {}).is_a?(Hash)
|
|
93
|
+
error(:invalid_ir, "Invalid scope", ["scopes", index])
|
|
94
|
+
next
|
|
95
|
+
end
|
|
96
|
+
schema = schemas[value["name"]]
|
|
97
|
+
unless schema
|
|
98
|
+
error(:unregistered, "The requested scope is not registered", ["scopes", index, "name"], resource: @root)
|
|
99
|
+
next
|
|
100
|
+
end
|
|
101
|
+
parameters = validate_parameters(value.fetch("parameters", {}), schema.fetch(:parameters), ["scopes", index, "parameters"])
|
|
102
|
+
QueryIR::Scope.new(name: value["name"], parameters: parameters)
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def validate_parameters(values, schemas, path)
|
|
107
|
+
unknown = values.keys - schemas.keys
|
|
108
|
+
error(:invalid_ir, "Unknown scope parameters", path, keys: unknown.sort) if unknown.any?
|
|
109
|
+
schemas.each_with_object({}) do |(name, schema), result|
|
|
110
|
+
if !values.key?(name)
|
|
111
|
+
error(:invalid_ir, "Required scope parameter is missing", path + [name]) if schema.fetch(:required)
|
|
112
|
+
else
|
|
113
|
+
result[name] = coerce(values[name], schema.fetch(:type), schema, path + [name])
|
|
114
|
+
end
|
|
115
|
+
end.freeze
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def validate_joins(values)
|
|
119
|
+
return invalid_collection("Joins", ["joins"]) unless values.is_a?(Array)
|
|
120
|
+
error(:limit_exceeded, "Join limit exceeded", ["joins"], limit: @limits[:joins]) if values.length > @limits[:joins]
|
|
121
|
+
values.each_with_index.filter_map do |value, index|
|
|
122
|
+
unless value.is_a?(String) && value.split(".").length <= 2 && @snapshot.paths.include?("#{@root}.#{value}")
|
|
123
|
+
error(:unregistered, "The requested association path is not registered", ["joins", index], resource: @root)
|
|
124
|
+
next
|
|
125
|
+
end
|
|
126
|
+
QueryIR::Path.new(value)
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def validate_filters(values, joins)
|
|
131
|
+
return invalid_collection("Filters", ["filters"]) unless values.is_a?(Array)
|
|
132
|
+
values.each_with_index.filter_map do |value, index|
|
|
133
|
+
unless value.is_a?(Hash) && value.keys.all? { |key| %w[field operator value].include?(key) } && value["field"].is_a?(String) && OPERATORS.include?(value["operator"])
|
|
134
|
+
error(:invalid_ir, "Invalid filter", ["filters", index])
|
|
135
|
+
next
|
|
136
|
+
end
|
|
137
|
+
field = resolve_field(value["field"], joins, ["filters", index, "field"])
|
|
138
|
+
next unless field
|
|
139
|
+
operator = value["operator"]
|
|
140
|
+
if NULL_OPERATORS.include?(operator)
|
|
141
|
+
error(:invalid_ir, "Null operator does not accept a value", ["filters", index, "value"]) if value.key?("value")
|
|
142
|
+
error(:invalid_ir, "Null check is invalid for a non-null field", ["filters", index, "operator"]) unless field.null
|
|
143
|
+
typed = nil
|
|
144
|
+
else
|
|
145
|
+
error(:invalid_ir, "Filter value is required", ["filters", index, "value"]) unless value.key?("value")
|
|
146
|
+
if COMPARISON_OPERATORS.include?(operator) && !comparable_type?(field.type)
|
|
147
|
+
error(:invalid_ir, "Operator is incompatible with field type", ["filters", index, "operator"])
|
|
148
|
+
end
|
|
149
|
+
typed = coerce_filter(value["value"], field, operator, ["filters", index, "value"])
|
|
150
|
+
end
|
|
151
|
+
QueryIR::Predicate.new(field: QueryIR::Path.new(value["field"]), operator: operator.to_sym, value: typed)
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def validate_sort(values, joins)
|
|
156
|
+
return invalid_collection("Sort", ["sort"]) unless values.is_a?(Array)
|
|
157
|
+
values.each_with_index.filter_map do |value, index|
|
|
158
|
+
unless exact_hash?(value, %w[field direction]) && %w[asc desc].include?(value["direction"])
|
|
159
|
+
error(:invalid_ir, "Invalid sort", ["sort", index])
|
|
160
|
+
next
|
|
161
|
+
end
|
|
162
|
+
next unless resolve_field(value["field"], joins, ["sort", index, "field"])
|
|
163
|
+
QueryIR::Sort.new(field: QueryIR::Path.new(value["field"]), direction: value["direction"].to_sym)
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def validate_limit(value)
|
|
168
|
+
unless value.is_a?(Integer) && value.positive? && value <= @limits[:rows]
|
|
169
|
+
error(:limit_exceeded, "Result limit exceeded", ["limit"], limit: @limits[:rows])
|
|
170
|
+
end
|
|
171
|
+
value
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def validate_aggregate(operation, value, joins)
|
|
175
|
+
if operation == "records"
|
|
176
|
+
error(:invalid_ir, "Aggregate is only valid for aggregate operations", ["aggregate"]) if value
|
|
177
|
+
return
|
|
178
|
+
end
|
|
179
|
+
unless value.is_a?(Hash) && value.keys.all? { |key| %w[function field].include?(key) } && value["function"].is_a?(String)
|
|
180
|
+
error(:invalid_ir, "Aggregate is required", ["aggregate"])
|
|
181
|
+
return
|
|
182
|
+
end
|
|
183
|
+
function = value["function"].to_sym
|
|
184
|
+
permission = @resources.fetch(@root).aggregates[function]
|
|
185
|
+
unless permission
|
|
186
|
+
error(:unregistered, "The requested aggregate is not registered", ["aggregate", "function"], resource: @root)
|
|
187
|
+
return
|
|
188
|
+
end
|
|
189
|
+
field_path = value["field"]
|
|
190
|
+
if function == :count
|
|
191
|
+
error(:invalid_ir, "Count does not accept a field", ["aggregate", "field"]) if field_path
|
|
192
|
+
elsif !field_path || !resolve_field(field_path, joins, ["aggregate", "field"]) || permission != true && !permission.include?(field_path)
|
|
193
|
+
error(:unregistered, "The aggregate field is not registered", ["aggregate", "field"], resource: @root)
|
|
194
|
+
return
|
|
195
|
+
end
|
|
196
|
+
QueryIR::Aggregate.new(function: function, field: field_path && QueryIR::Path.new(field_path))
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def resolve_field(path, joins, error_path)
|
|
200
|
+
unless path.is_a?(String) && path.split(".").length <= 3
|
|
201
|
+
error(:invalid_ir, "Invalid field path", error_path)
|
|
202
|
+
return
|
|
203
|
+
end
|
|
204
|
+
segments = path.split(".")
|
|
205
|
+
resource = @resources.fetch(@root)
|
|
206
|
+
if segments.length > 1
|
|
207
|
+
join = segments[0...-1].join(".")
|
|
208
|
+
unless joins.any? { |item| item.to_s == join }
|
|
209
|
+
error(:unregistered, "The field path is not joined", error_path, resource: @root)
|
|
210
|
+
return
|
|
211
|
+
end
|
|
212
|
+
missing_association = false
|
|
213
|
+
segments[0...-1].each do |association_name|
|
|
214
|
+
association = resource.associations.find { |item| item.name == association_name }
|
|
215
|
+
unless association
|
|
216
|
+
error(:unregistered, "The field path is not registered", error_path, resource: @root)
|
|
217
|
+
missing_association = true
|
|
218
|
+
break
|
|
219
|
+
end
|
|
220
|
+
resource = @resources[association.resource]
|
|
221
|
+
end
|
|
222
|
+
return if missing_association
|
|
223
|
+
end
|
|
224
|
+
field = resource&.fields&.find { |candidate| candidate.name == segments.last }
|
|
225
|
+
return field if field
|
|
226
|
+
|
|
227
|
+
error(:unregistered, "The requested field is not registered", error_path, resource: @root)
|
|
228
|
+
nil
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def coerce_filter(value, field, operator, path)
|
|
232
|
+
if ARRAY_OPERATORS.include?(operator)
|
|
233
|
+
expected = (operator == "between") ? 2 : nil
|
|
234
|
+
valid_array = value.is_a?(Array) && !value.empty?
|
|
235
|
+
valid_array &&= value.length == expected if expected
|
|
236
|
+
unless valid_array
|
|
237
|
+
error(:invalid_ir, "Operator requires a bounded array", path)
|
|
238
|
+
return
|
|
239
|
+
end
|
|
240
|
+
return value.map { |item| coerce(item, field.type, {enum_values: field.enum_values, nullable: field.null}, path) }.freeze
|
|
241
|
+
end
|
|
242
|
+
coerce(value, field.type, {enum_values: field.enum_values, nullable: field.null}, path)
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def coerce(value, type, schema, path)
|
|
246
|
+
if value.nil?
|
|
247
|
+
error(:invalid_ir, "Null is not allowed", path) unless schema[:nullable]
|
|
248
|
+
return QueryIR::Literal.new(value: nil, type: type)
|
|
249
|
+
end
|
|
250
|
+
if TIME_TYPES.include?(type.to_sym) && value.is_a?(Hash)
|
|
251
|
+
relative = value["relative"]
|
|
252
|
+
unless valid_relative_time?(value, relative)
|
|
253
|
+
error(:invalid_ir, "Invalid relative time", path)
|
|
254
|
+
return
|
|
255
|
+
end
|
|
256
|
+
return QueryIR::RelativeTime.new(amount: relative["amount"], unit: relative["unit"], direction: relative["direction"])
|
|
257
|
+
end
|
|
258
|
+
valid = case type.to_sym
|
|
259
|
+
when :integer then value.is_a?(Integer)
|
|
260
|
+
when :float, :decimal then value.is_a?(Numeric) && value.finite?
|
|
261
|
+
when :boolean then [true, false].include?(value)
|
|
262
|
+
when :date then value.is_a?(String) && Date.iso8601(value).to_s == value
|
|
263
|
+
when :datetime, :timestamp, :time then valid_timestamp?(value)
|
|
264
|
+
else value.is_a?(String)
|
|
265
|
+
end
|
|
266
|
+
enum_values = schema[:enum_values] || []
|
|
267
|
+
valid &&= enum_values.include?(value) if enum_values.any?
|
|
268
|
+
if schema[:minimum] && value.respond_to?(:<) then valid &&= value >= schema[:minimum] end
|
|
269
|
+
if schema[:maximum] && value.respond_to?(:>) then valid &&= value <= schema[:maximum] end
|
|
270
|
+
error(:invalid_ir, "Value is incompatible with the registered type", path) unless valid
|
|
271
|
+
QueryIR::Literal.new(value: value, type: type)
|
|
272
|
+
rescue ArgumentError
|
|
273
|
+
error(:invalid_ir, "Value is incompatible with the registered type", path)
|
|
274
|
+
QueryIR::Literal.new(value: value, type: type)
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
def valid_timestamp?(value)
|
|
278
|
+
return false unless value.is_a?(String) && value.match?(/(?:Z|[+-]\d{2}:\d{2})\z/)
|
|
279
|
+
Time.iso8601(value)
|
|
280
|
+
true
|
|
281
|
+
rescue ArgumentError
|
|
282
|
+
false
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
def valid_relative_time?(value, relative)
|
|
286
|
+
exact_hash?(value, ["relative"]) &&
|
|
287
|
+
exact_hash?(relative, %w[amount unit direction]) &&
|
|
288
|
+
relative["amount"].is_a?(Integer) &&
|
|
289
|
+
relative["amount"].positive? &&
|
|
290
|
+
QueryIR::RelativeTime::UNITS.include?(relative["unit"]) &&
|
|
291
|
+
QueryIR::RelativeTime::DIRECTIONS.include?(relative["direction"])
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
def comparable_type?(type) = NUMERIC_TYPES.include?(type.to_sym) || TIME_TYPES.include?(type.to_sym) || type.to_sym == :string
|
|
295
|
+
def exact_hash?(value, keys) = value.is_a?(Hash) && value.keys.sort == keys.sort
|
|
296
|
+
def invalid_collection(name, path) = error(:invalid_ir, "#{name} must be an array", path) && []
|
|
297
|
+
def error(code, message, path, details = {}) = @errors << Error.new(code: code, message: message, path: path, details: details)
|
|
298
|
+
def result = Result.new(ir: nil, errors: @errors, explanation: nil, snapshot: @snapshot)
|
|
299
|
+
|
|
300
|
+
def explain(ir)
|
|
301
|
+
parts = [(ir.operation == :records) ? "Select records from #{ir.root}" : "Aggregate #{ir.root} using #{ir.aggregate.function}"]
|
|
302
|
+
parts << "apply scope #{ir.scopes.map(&:name).join(", ")}" if ir.scopes.any?
|
|
303
|
+
parts << "filter #{ir.filters.map { |filter| "#{filter.field} #{filter.operator}" }.join(", ")}" if ir.filters.any?
|
|
304
|
+
parts << "join #{ir.joins.join(", ")}" if ir.joins.any?
|
|
305
|
+
parts << "sort #{ir.sort.map { |sort| "#{sort.field} #{sort.direction}" }.join(", ")}" if ir.sort.any?
|
|
306
|
+
parts << "use distinct records" if ir.distinct
|
|
307
|
+
parts << "limit #{ir.limit}"
|
|
308
|
+
parts.join("; ")
|
|
309
|
+
end
|
|
310
|
+
end
|
|
311
|
+
end
|
data/lib/maglev/railtie.rb
CHANGED
|
@@ -9,6 +9,15 @@ module Maglev
|
|
|
9
9
|
require "maglev/active_record_extension"
|
|
10
10
|
|
|
11
11
|
include Maglev::ActiveRecordExtension
|
|
12
|
+
ActiveRecord::Relation.include(Maglev::RelationExtension)
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
initializer "maglev.reloader" do
|
|
17
|
+
config.to_prepare do
|
|
18
|
+
Maglev::DependencyGraph.reset!
|
|
19
|
+
Maglev::KnowledgeRegistry.rebuild!
|
|
20
|
+
Maglev::Registry.rebuild!
|
|
12
21
|
end
|
|
13
22
|
end
|
|
14
23
|
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "resource_config"
|
|
4
|
+
require_relative "schema_snapshot"
|
|
5
|
+
|
|
6
|
+
module Maglev
|
|
7
|
+
class Registry
|
|
8
|
+
class << self
|
|
9
|
+
def register(entry)
|
|
10
|
+
mutex.synchronize do
|
|
11
|
+
@entries ||= {}
|
|
12
|
+
existing = @entries[entry.identifier]
|
|
13
|
+
if existing && existing.model_class != entry.model_class && existing.model_class.name != entry.model_class.name
|
|
14
|
+
raise ConfigurationError, "Maglev resource #{entry.identifier} is already registered"
|
|
15
|
+
end
|
|
16
|
+
@entries[entry.identifier] = entry
|
|
17
|
+
@snapshot_cache = {}
|
|
18
|
+
end
|
|
19
|
+
entry
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def fetch(identifier)
|
|
23
|
+
mutex.synchronize { (@entries || {})[identifier.to_s] }
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def entries
|
|
27
|
+
mutex.synchronize { (@entries || {}).values.sort_by(&:identifier).freeze }
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def snapshot(resources:, user: nil, authorizer: nil, limits: {})
|
|
31
|
+
identifiers = Array(resources).map(&:to_s).uniq.sort
|
|
32
|
+
selected = identifiers.filter_map { |identifier| fetch(identifier) }.select(&:queryable)
|
|
33
|
+
selected.select! do |entry|
|
|
34
|
+
entry.queryable.authorization == :public || authorizer&.call(entry, user)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
cache_key = [identifiers, limits.sort].freeze if user.nil? && authorizer.nil?
|
|
38
|
+
mutex.synchronize { return @snapshot_cache[cache_key] if cache_key && @snapshot_cache&.key?(cache_key) }
|
|
39
|
+
|
|
40
|
+
result = SchemaSnapshot::Builder.new(selected, limits: limits).build
|
|
41
|
+
mutex.synchronize { (@snapshot_cache ||= {})[cache_key] = result } if cache_key
|
|
42
|
+
result
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def invalidate!
|
|
46
|
+
mutex.synchronize { @snapshot_cache = {} }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def reset!
|
|
50
|
+
mutex.synchronize do
|
|
51
|
+
@entries = {}
|
|
52
|
+
@snapshot_cache = {}
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def rebuild!
|
|
57
|
+
reset!
|
|
58
|
+
KnowledgeRegistry.model_names.each do |model_name|
|
|
59
|
+
model = model_name.safe_constantize
|
|
60
|
+
model.rebuild_maglev_registry_registration if model&.respond_to?(:rebuild_maglev_registry_registration)
|
|
61
|
+
end
|
|
62
|
+
invalidate!
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
def mutex
|
|
68
|
+
@mutex ||= Mutex.new
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
data/lib/maglev/reindex_job.rb
CHANGED
|
@@ -9,8 +9,40 @@ module Maglev
|
|
|
9
9
|
queue_as :default
|
|
10
10
|
|
|
11
11
|
def perform(owner_class_name, owner_id)
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
owner_class = owner_class_name.constantize
|
|
13
|
+
owner = owner_class.find(owner_id)
|
|
14
|
+
|
|
15
|
+
begin
|
|
16
|
+
provider_call = ProviderCall.new(max_attempts: 1)
|
|
17
|
+
Indexer.new(owner, provider_call: provider_call).index
|
|
18
|
+
rescue RetryableProviderError => error
|
|
19
|
+
if executions < Maglev.configuration.provider_max_attempts
|
|
20
|
+
instrument_reindex("maglev.reindex.retry", owner_class_name, owner_id, error)
|
|
21
|
+
retry_job(wait: 0)
|
|
22
|
+
else
|
|
23
|
+
instrument_reindex("maglev.reindex.exhausted", owner_class_name, owner_id, error)
|
|
24
|
+
raise
|
|
25
|
+
end
|
|
26
|
+
rescue PermanentProviderError => error
|
|
27
|
+
instrument_reindex("maglev.reindex.discard", owner_class_name, owner_id, error)
|
|
28
|
+
end
|
|
29
|
+
rescue ActiveRecord::RecordNotFound => error
|
|
30
|
+
instrument_reindex("maglev.reindex.discard", owner_class_name, owner_id, error)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def instrument_reindex(name, owner_class_name, owner_id, error)
|
|
36
|
+
ActiveSupport::Notifications.instrument(
|
|
37
|
+
name,
|
|
38
|
+
job_class: self.class.name,
|
|
39
|
+
owner_class: owner_class_name,
|
|
40
|
+
owner_id: owner_id,
|
|
41
|
+
error_class: error.class.name,
|
|
42
|
+
execution_count: executions,
|
|
43
|
+
attempt: executions,
|
|
44
|
+
max_attempts: Maglev.configuration.provider_max_attempts
|
|
45
|
+
)
|
|
14
46
|
end
|
|
15
47
|
end
|
|
16
48
|
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Maglev
|
|
4
|
+
module RelationOrder
|
|
5
|
+
module_function
|
|
6
|
+
|
|
7
|
+
def with_primary_key(order, model_class)
|
|
8
|
+
return nil unless order
|
|
9
|
+
|
|
10
|
+
normalized = order.dup
|
|
11
|
+
primary_key = model_class.primary_key&.to_sym
|
|
12
|
+
normalized[primary_key] = :asc if primary_key && !normalized.key?(primary_key)
|
|
13
|
+
normalized.freeze
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Maglev
|
|
4
|
+
class Request
|
|
5
|
+
MODES = %i[auto structured rag hybrid].freeze
|
|
6
|
+
|
|
7
|
+
attr_reader :question, :mode, :resources, :base_relation, :user, :options
|
|
8
|
+
|
|
9
|
+
def initialize(question:, mode: :auto, resources: [], base_relation: nil, user: nil, **options)
|
|
10
|
+
normalized_mode = mode.to_sym
|
|
11
|
+
raise ArgumentError, "invalid request mode" unless MODES.include?(normalized_mode)
|
|
12
|
+
|
|
13
|
+
@question = question.to_s.freeze
|
|
14
|
+
@mode = normalized_mode
|
|
15
|
+
@resources = Array(resources).map { |resource| resource.to_s.freeze }.uniq.freeze
|
|
16
|
+
@base_relation = base_relation
|
|
17
|
+
@user = user
|
|
18
|
+
@options = options.freeze
|
|
19
|
+
freeze
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "securerandom"
|
|
4
|
+
|
|
5
|
+
module Maglev
|
|
6
|
+
class RequestExecutor
|
|
7
|
+
def initialize(router:, retriever_factory: ->(model) { Retriever.new(model) },
|
|
8
|
+
answerer_factory: ->(model) { Answerer.new(model) })
|
|
9
|
+
@router = router
|
|
10
|
+
@retriever_factory = retriever_factory
|
|
11
|
+
@answerer_factory = answerer_factory
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def call(request)
|
|
15
|
+
decision = @router.route(request)
|
|
16
|
+
case decision.route
|
|
17
|
+
when :unsupported, :clarification_required
|
|
18
|
+
Result.new(status: decision.route, route: decision.route, kind: :none,
|
|
19
|
+
trace_id: SecureRandom.uuid, confidence: decision.confidence, reasons: decision.reasons)
|
|
20
|
+
when :hybrid
|
|
21
|
+
HybridCoordinator.new(retriever_factory: @retriever_factory).call(request, decision)
|
|
22
|
+
when :structured then execute_structured(request, decision)
|
|
23
|
+
when :rag then execute_rag(request, decision)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
def execute_structured(request, decision)
|
|
30
|
+
entry = single_entry(request, capability: :queryable)
|
|
31
|
+
relation = request.base_relation
|
|
32
|
+
if relation.nil? && entry.queryable.allow_unscoped_model_queries
|
|
33
|
+
relation = entry.model_class.all
|
|
34
|
+
end
|
|
35
|
+
raise ConfigurationError, "structured requests require an authorized base relation" unless relation
|
|
36
|
+
|
|
37
|
+
plan = Maglev.plan(request.question, resource: entry.identifier, resources: request.resources,
|
|
38
|
+
base_relation: relation, user: request.user, authorizer: request.options[:authorizer],
|
|
39
|
+
constraints: request.options.fetch(:constraints, {}),
|
|
40
|
+
adapter: request.options[:planner_adapter] || Maglev.configuration.planner_adapter)
|
|
41
|
+
wrap_structured(Maglev.execute(plan), decision)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def execute_rag(request, decision)
|
|
45
|
+
entry = single_entry(request, capability: :knowledge)
|
|
46
|
+
limit = request.options.fetch(:limit, 10)
|
|
47
|
+
if request.options[:answer]
|
|
48
|
+
response = @answerer_factory.call(entry.model_class).ask(request.question, limit: limit,
|
|
49
|
+
user: request.user, minimum_similarity: request.options[:minimum_similarity],
|
|
50
|
+
chunks_per_owner: request.options[:chunks_per_owner])
|
|
51
|
+
Result.new(status: :succeeded, route: :rag, kind: :rag_answer, value: response,
|
|
52
|
+
evidence: response.sources, trace_id: response.metadata[:trace_id] || SecureRandom.uuid,
|
|
53
|
+
confidence: decision.confidence, reasons: decision.reasons, metadata: response.metadata)
|
|
54
|
+
else
|
|
55
|
+
retrieval = @retriever_factory.call(entry.model_class).retrieve(request.question, limit: limit,
|
|
56
|
+
user: request.user, minimum_similarity: request.options[:minimum_similarity],
|
|
57
|
+
chunks_per_owner: request.options.fetch(:chunks_per_owner, 1))
|
|
58
|
+
Result.new(status: :succeeded, route: :rag, kind: :semantic_matches, value: retrieval,
|
|
59
|
+
evidence: retrieval.selected, trace_id: retrieval.trace_id, confidence: decision.confidence,
|
|
60
|
+
reasons: decision.reasons, metadata: retrieval.metadata)
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def single_entry(request, capability:)
|
|
65
|
+
entries = request.resources.filter_map { |identifier| Registry.fetch(identifier) }
|
|
66
|
+
.select { |entry| entry.public_send(capability) }
|
|
67
|
+
unless entries.one?
|
|
68
|
+
raise ConfigurationError, "#{capability} routing requires exactly one registered resource"
|
|
69
|
+
end
|
|
70
|
+
entries.first
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def wrap_structured(result, decision)
|
|
74
|
+
Result.new(status: result.status, route: :structured, kind: result.kind, value: result.value,
|
|
75
|
+
evidence: result.evidence, warnings: result.warnings, trace_id: result.trace_id,
|
|
76
|
+
confidence: decision.confidence, reasons: decision.reasons, metadata: {plan: result.plan}.freeze)
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def self.request(question, mode: :auto, resources: nil, models: nil, base_relation: nil,
|
|
81
|
+
user: nil, router: nil, retriever_factory: nil, answerer_factory: nil, **options)
|
|
82
|
+
entries = Array(models).filter_map do |model|
|
|
83
|
+
Registry.entries.find { |entry| entry.model_class == model }
|
|
84
|
+
end
|
|
85
|
+
identifiers = [*Array(resources), *entries.map(&:identifier)].compact.map(&:to_s).uniq
|
|
86
|
+
if identifiers.empty? && base_relation
|
|
87
|
+
entry = Registry.entries.find { |candidate| candidate.model_class == base_relation.klass }
|
|
88
|
+
identifiers << entry.identifier if entry
|
|
89
|
+
end
|
|
90
|
+
if identifiers.empty?
|
|
91
|
+
raise ConfigurationError, "an explicit resource, model, or base relation is required"
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
router ||= Router.new(classifier: Maglev.configuration.routing_adapter)
|
|
95
|
+
executor_options = {router: router}
|
|
96
|
+
executor_options[:retriever_factory] = retriever_factory if retriever_factory
|
|
97
|
+
executor_options[:answerer_factory] = answerer_factory if answerer_factory
|
|
98
|
+
RequestExecutor.new(**executor_options).call(Request.new(question: question, mode: mode,
|
|
99
|
+
resources: identifiers, base_relation: base_relation, user: user, **options))
|
|
100
|
+
end
|
|
101
|
+
end
|