maglev-rb 0.1.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.ja.md +618 -0
- data/README.md +533 -248
- data/README.zh-CN.md +457 -250
- data/lib/generators/maglev/install/install_generator.rb +20 -0
- data/lib/generators/maglev/upgrade_index_version/upgrade_index_version_generator.rb +27 -0
- data/lib/generators/maglev/upgrade_source_identity/upgrade_source_identity_generator.rb +52 -0
- data/lib/maglev/active_record_extension.rb +141 -24
- data/lib/maglev/adapters/faraday_client.rb +94 -0
- data/lib/maglev/adapters/faraday_embedding.rb +51 -0
- data/lib/maglev/adapters/faraday_generation.rb +49 -0
- data/lib/maglev/adapters/faraday_planner.rb +88 -0
- data/lib/maglev/answerer.rb +30 -12
- data/lib/maglev/chunker.rb +39 -4
- data/lib/maglev/configuration.rb +66 -1
- data/lib/maglev/content_source_graph.rb +17 -11
- data/lib/maglev/dependency_graph.rb +72 -13
- data/lib/maglev/embedding_adapter.rb +10 -0
- data/lib/maglev/hybrid_candidate_set.rb +25 -0
- data/lib/maglev/hybrid_coordinator.rb +112 -0
- data/lib/maglev/hybrid_result.rb +25 -0
- data/lib/maglev/index_diagnostics.rb +83 -0
- data/lib/maglev/index_identity.rb +70 -0
- data/lib/maglev/index_state.rb +9 -0
- data/lib/maglev/indexer.rb +185 -35
- data/lib/maglev/knowledge_config.rb +27 -5
- data/lib/maglev/knowledge_registry.rb +33 -0
- data/lib/maglev/planner.rb +172 -0
- data/lib/maglev/planner_adapter.rb +25 -0
- data/lib/maglev/planner_evaluation.rb +49 -0
- data/lib/maglev/query_compiler.rb +197 -0
- data/lib/maglev/query_ir.rb +143 -0
- data/lib/maglev/query_validator.rb +311 -0
- data/lib/maglev/railtie.rb +9 -0
- data/lib/maglev/registry.rb +72 -0
- data/lib/maglev/reindex_job.rb +34 -2
- data/lib/maglev/relation_order.rb +16 -0
- data/lib/maglev/request.rb +22 -0
- data/lib/maglev/request_executor.rb +101 -0
- data/lib/maglev/resource_config.rb +222 -0
- data/lib/maglev/response.rb +2 -2
- data/lib/maglev/result.rb +30 -0
- data/lib/maglev/retrieval_outcome.rb +52 -0
- data/lib/maglev/retrieval_result.rb +25 -0
- data/lib/maglev/retriever.rb +282 -27
- data/lib/maglev/router.rb +77 -0
- data/lib/maglev/routing_adapter.rb +25 -0
- data/lib/maglev/schema_compiler.rb +17 -4
- data/lib/maglev/schema_snapshot.rb +159 -0
- data/lib/maglev/search_result.rb +7 -3
- data/lib/maglev/snapshot.rb +21 -1
- data/lib/maglev/snapshot_budget.rb +57 -0
- data/lib/maglev/snapshot_builder.rb +89 -11
- data/lib/maglev/source_extractor.rb +43 -0
- data/lib/maglev/source_fragment.rb +9 -0
- data/lib/maglev/structured_answer_composer.rb +67 -0
- data/lib/maglev/structured_evidence_builder.rb +56 -0
- data/lib/maglev/structured_executor.rb +157 -0
- data/lib/maglev/structured_result.rb +97 -0
- data/lib/maglev/trace.rb +56 -0
- data/lib/maglev/vector_stores/base.rb +12 -0
- data/lib/maglev/vector_stores/document.rb +14 -5
- data/lib/maglev/vector_stores/document_id.rb +27 -0
- data/lib/maglev/vector_stores/memory.rb +68 -6
- data/lib/maglev/vector_stores/metadata_filter.rb +55 -0
- data/lib/maglev/vector_stores/pgvector.rb +94 -7
- data/lib/maglev/version.rb +1 -1
- data/lib/maglev-rb.rb +3 -0
- data/lib/maglev.rb +36 -3
- data/lib/tasks/maglev.rake +43 -0
- metadata +71 -11
- data/lib/maglev/adapters/ruby_llm_attachment_extractor.rb +0 -15
- data/lib/maglev/adapters/ruby_llm_embedding.rb +0 -22
- data/lib/maglev/adapters/ruby_llm_generation.rb +0 -22
- data/lib/maglev/adapters/ruby_llm_provider.rb +0 -64
data/lib/tasks/maglev.rake
CHANGED
|
@@ -1,6 +1,37 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
namespace :maglev do
|
|
4
|
+
desc "Inspect a built gem before release: rake maglev:release_audit[path]"
|
|
5
|
+
task :release_audit, [:path] do |_task, args|
|
|
6
|
+
require "rubygems/package"
|
|
7
|
+
require "stringio"
|
|
8
|
+
require "zlib"
|
|
9
|
+
|
|
10
|
+
path = args[:path] || Dir[File.expand_path("../../pkg/maglev-rb-*.gem", __dir__)].max
|
|
11
|
+
raise "built gem not found" unless path && File.file?(path)
|
|
12
|
+
|
|
13
|
+
package = Gem::Package.new(path)
|
|
14
|
+
specification = package.spec
|
|
15
|
+
raise "gem version mismatch" unless specification.version.to_s == Maglev::VERSION
|
|
16
|
+
|
|
17
|
+
files = []
|
|
18
|
+
File.open(path, "rb") do |io|
|
|
19
|
+
Gem::Package::TarReader.new(io) do |outer|
|
|
20
|
+
outer.each do |entry|
|
|
21
|
+
next unless entry.full_name == "data.tar.gz"
|
|
22
|
+
|
|
23
|
+
Zlib::GzipReader.wrap(StringIO.new(entry.read)) do |gzip|
|
|
24
|
+
Gem::Package::TarReader.new(gzip) { |inner| inner.each { |item| files << item.full_name } }
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
forbidden = files.grep(%r{(?:^|/)(?:tmp|log|spec)/|AGENTS|IMPLEMENTATION_PLAN|TODO|local_secret})
|
|
30
|
+
raise "forbidden packaged files: #{forbidden.join(", ")}" if forbidden.any?
|
|
31
|
+
|
|
32
|
+
puts "Release audit passed: #{specification.name} #{specification.version} (#{files.size} files)"
|
|
33
|
+
end
|
|
34
|
+
|
|
4
35
|
desc "Reindex one Maglev model: rails maglev:reindex[ModelName]"
|
|
5
36
|
task :reindex, [:model_name] => :environment do |_task, args|
|
|
6
37
|
model_name = args[:model_name]
|
|
@@ -28,4 +59,16 @@ namespace :maglev do
|
|
|
28
59
|
puts "- #{schema.fetch(:model)} fields=#{schema.fetch(:exposed_attributes).join(",")}"
|
|
29
60
|
end
|
|
30
61
|
end
|
|
62
|
+
|
|
63
|
+
desc "Score a provider-free Maglev planner evaluation corpus"
|
|
64
|
+
task :evaluate_planner, [:path] do |_task, args|
|
|
65
|
+
path = args[:path] || File.expand_path("../../spec/dummy/evaluations/planner_v1.json", __dir__)
|
|
66
|
+
corpus = Maglev::PlannerEvaluation.load(path)
|
|
67
|
+
report = Maglev::PlannerEvaluation.score(corpus.fetch("cases"))
|
|
68
|
+
percentage = (report.fetch(:score) * 100).round(1)
|
|
69
|
+
puts "Planner evaluation v#{corpus.fetch("version")}: #{report.fetch(:passed)}/#{report.fetch(:total)} passed (#{percentage}%)"
|
|
70
|
+
report.fetch(:cases).reject { |item| item.fetch(:passed) }.each do |item|
|
|
71
|
+
puts "- #{item.fetch(:id)}: #{item.fetch(:failure_class)}"
|
|
72
|
+
end
|
|
73
|
+
end
|
|
31
74
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: maglev-rb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Maglev contributors
|
|
@@ -44,19 +44,19 @@ dependencies:
|
|
|
44
44
|
- !ruby/object:Gem::Version
|
|
45
45
|
version: '9.0'
|
|
46
46
|
- !ruby/object:Gem::Dependency
|
|
47
|
-
name:
|
|
47
|
+
name: faraday
|
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
|
49
49
|
requirements:
|
|
50
50
|
- - "~>"
|
|
51
51
|
- !ruby/object:Gem::Version
|
|
52
|
-
version: '
|
|
52
|
+
version: '2.0'
|
|
53
53
|
type: :runtime
|
|
54
54
|
prerelease: false
|
|
55
55
|
version_requirements: !ruby/object:Gem::Requirement
|
|
56
56
|
requirements:
|
|
57
57
|
- - "~>"
|
|
58
58
|
- !ruby/object:Gem::Version
|
|
59
|
-
version: '
|
|
59
|
+
version: '2.0'
|
|
60
60
|
- !ruby/object:Gem::Dependency
|
|
61
61
|
name: pg
|
|
62
62
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -71,6 +71,26 @@ dependencies:
|
|
|
71
71
|
- - "~>"
|
|
72
72
|
- !ruby/object:Gem::Version
|
|
73
73
|
version: '1.5'
|
|
74
|
+
- !ruby/object:Gem::Dependency
|
|
75
|
+
name: parallel
|
|
76
|
+
requirement: !ruby/object:Gem::Requirement
|
|
77
|
+
requirements:
|
|
78
|
+
- - ">="
|
|
79
|
+
- !ruby/object:Gem::Version
|
|
80
|
+
version: '1.10'
|
|
81
|
+
- - "<"
|
|
82
|
+
- !ruby/object:Gem::Version
|
|
83
|
+
version: '2.0'
|
|
84
|
+
type: :development
|
|
85
|
+
prerelease: false
|
|
86
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
87
|
+
requirements:
|
|
88
|
+
- - ">="
|
|
89
|
+
- !ruby/object:Gem::Version
|
|
90
|
+
version: '1.10'
|
|
91
|
+
- - "<"
|
|
92
|
+
- !ruby/object:Gem::Version
|
|
93
|
+
version: '2.0'
|
|
74
94
|
- !ruby/object:Gem::Dependency
|
|
75
95
|
name: rake
|
|
76
96
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -127,21 +147,27 @@ dependencies:
|
|
|
127
147
|
- - "~>"
|
|
128
148
|
- !ruby/object:Gem::Version
|
|
129
149
|
version: '1.44'
|
|
130
|
-
description: Maglev
|
|
150
|
+
description: Maglev provides safe structured queries, semantic retrieval, and bounded
|
|
151
|
+
hybrid workflows over registered ActiveRecord resources.
|
|
131
152
|
executables: []
|
|
132
153
|
extensions: []
|
|
133
154
|
extra_rdoc_files: []
|
|
134
155
|
files:
|
|
156
|
+
- CHANGELOG.md
|
|
135
157
|
- LICENSE.txt
|
|
158
|
+
- README.ja.md
|
|
136
159
|
- README.md
|
|
137
160
|
- README.zh-CN.md
|
|
138
161
|
- lib/generators/maglev/install/install_generator.rb
|
|
162
|
+
- lib/generators/maglev/upgrade_index_version/upgrade_index_version_generator.rb
|
|
163
|
+
- lib/generators/maglev/upgrade_source_identity/upgrade_source_identity_generator.rb
|
|
164
|
+
- lib/maglev-rb.rb
|
|
139
165
|
- lib/maglev.rb
|
|
140
166
|
- lib/maglev/active_record_extension.rb
|
|
141
|
-
- lib/maglev/adapters/
|
|
142
|
-
- lib/maglev/adapters/
|
|
143
|
-
- lib/maglev/adapters/
|
|
144
|
-
- lib/maglev/adapters/
|
|
167
|
+
- lib/maglev/adapters/faraday_client.rb
|
|
168
|
+
- lib/maglev/adapters/faraday_embedding.rb
|
|
169
|
+
- lib/maglev/adapters/faraday_generation.rb
|
|
170
|
+
- lib/maglev/adapters/faraday_planner.rb
|
|
145
171
|
- lib/maglev/answerer.rb
|
|
146
172
|
- lib/maglev/attachment_extractor.rb
|
|
147
173
|
- lib/maglev/authorization.rb
|
|
@@ -156,22 +182,56 @@ files:
|
|
|
156
182
|
- lib/maglev/errors.rb
|
|
157
183
|
- lib/maglev/extracted_document.rb
|
|
158
184
|
- lib/maglev/generation_adapter.rb
|
|
185
|
+
- lib/maglev/hybrid_candidate_set.rb
|
|
186
|
+
- lib/maglev/hybrid_coordinator.rb
|
|
187
|
+
- lib/maglev/hybrid_result.rb
|
|
188
|
+
- lib/maglev/index_diagnostics.rb
|
|
189
|
+
- lib/maglev/index_identity.rb
|
|
190
|
+
- lib/maglev/index_state.rb
|
|
159
191
|
- lib/maglev/indexer.rb
|
|
160
192
|
- lib/maglev/knowledge_config.rb
|
|
193
|
+
- lib/maglev/knowledge_registry.rb
|
|
194
|
+
- lib/maglev/planner.rb
|
|
195
|
+
- lib/maglev/planner_adapter.rb
|
|
196
|
+
- lib/maglev/planner_evaluation.rb
|
|
161
197
|
- lib/maglev/prompt_builder.rb
|
|
162
198
|
- lib/maglev/provider_call.rb
|
|
163
199
|
- lib/maglev/provider_configuration.rb
|
|
200
|
+
- lib/maglev/query_compiler.rb
|
|
201
|
+
- lib/maglev/query_ir.rb
|
|
202
|
+
- lib/maglev/query_validator.rb
|
|
164
203
|
- lib/maglev/railtie.rb
|
|
204
|
+
- lib/maglev/registry.rb
|
|
165
205
|
- lib/maglev/reindex_job.rb
|
|
206
|
+
- lib/maglev/relation_order.rb
|
|
207
|
+
- lib/maglev/request.rb
|
|
208
|
+
- lib/maglev/request_executor.rb
|
|
209
|
+
- lib/maglev/resource_config.rb
|
|
166
210
|
- lib/maglev/response.rb
|
|
211
|
+
- lib/maglev/result.rb
|
|
212
|
+
- lib/maglev/retrieval_outcome.rb
|
|
213
|
+
- lib/maglev/retrieval_result.rb
|
|
167
214
|
- lib/maglev/retriever.rb
|
|
215
|
+
- lib/maglev/router.rb
|
|
216
|
+
- lib/maglev/routing_adapter.rb
|
|
168
217
|
- lib/maglev/schema_compiler.rb
|
|
218
|
+
- lib/maglev/schema_snapshot.rb
|
|
169
219
|
- lib/maglev/search_result.rb
|
|
170
220
|
- lib/maglev/snapshot.rb
|
|
221
|
+
- lib/maglev/snapshot_budget.rb
|
|
171
222
|
- lib/maglev/snapshot_builder.rb
|
|
223
|
+
- lib/maglev/source_extractor.rb
|
|
224
|
+
- lib/maglev/source_fragment.rb
|
|
225
|
+
- lib/maglev/structured_answer_composer.rb
|
|
226
|
+
- lib/maglev/structured_evidence_builder.rb
|
|
227
|
+
- lib/maglev/structured_executor.rb
|
|
228
|
+
- lib/maglev/structured_result.rb
|
|
229
|
+
- lib/maglev/trace.rb
|
|
172
230
|
- lib/maglev/vector_stores/base.rb
|
|
173
231
|
- lib/maglev/vector_stores/document.rb
|
|
232
|
+
- lib/maglev/vector_stores/document_id.rb
|
|
174
233
|
- lib/maglev/vector_stores/memory.rb
|
|
234
|
+
- lib/maglev/vector_stores/metadata_filter.rb
|
|
175
235
|
- lib/maglev/vector_stores/pgvector.rb
|
|
176
236
|
- lib/maglev/version.rb
|
|
177
237
|
- lib/tasks/maglev.rake
|
|
@@ -189,7 +249,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
189
249
|
requirements:
|
|
190
250
|
- - ">="
|
|
191
251
|
- !ruby/object:Gem::Version
|
|
192
|
-
version: 3.
|
|
252
|
+
version: 3.3.0
|
|
193
253
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
194
254
|
requirements:
|
|
195
255
|
- - ">="
|
|
@@ -198,5 +258,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
198
258
|
requirements: []
|
|
199
259
|
rubygems_version: 4.0.10
|
|
200
260
|
specification_version: 4
|
|
201
|
-
summary: Rails-native
|
|
261
|
+
summary: Rails-native knowledge and query layer for ActiveRecord applications.
|
|
202
262
|
test_files: []
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require_relative "../attachment_extractor"
|
|
4
|
-
|
|
5
|
-
module Maglev
|
|
6
|
-
module Adapters
|
|
7
|
-
class RubyLLMAttachmentExtractor < AttachmentExtractor
|
|
8
|
-
def extract(blob, source_name:)
|
|
9
|
-
require "ruby_llm"
|
|
10
|
-
|
|
11
|
-
super
|
|
12
|
-
end
|
|
13
|
-
end
|
|
14
|
-
end
|
|
15
|
-
end
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require_relative "../embedding_adapter"
|
|
4
|
-
require_relative "../provider_configuration"
|
|
5
|
-
require_relative "ruby_llm_provider"
|
|
6
|
-
|
|
7
|
-
module Maglev
|
|
8
|
-
module Adapters
|
|
9
|
-
class RubyLLMEmbedding < EmbeddingAdapter
|
|
10
|
-
def initialize(provider: Maglev.configuration.embedding_provider, model: nil, dimensions: nil)
|
|
11
|
-
configuration = provider.to_h
|
|
12
|
-
configuration[:model] = model if model
|
|
13
|
-
configuration[:dimensions] = dimensions if dimensions
|
|
14
|
-
@client = RubyLLMProvider.new(ProviderConfiguration.new(**configuration))
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
def embed(text)
|
|
18
|
-
@client.embed(text)
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
end
|
|
22
|
-
end
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require_relative "../configuration"
|
|
4
|
-
require_relative "../generation_adapter"
|
|
5
|
-
require_relative "../provider_configuration"
|
|
6
|
-
require_relative "ruby_llm_provider"
|
|
7
|
-
|
|
8
|
-
module Maglev
|
|
9
|
-
module Adapters
|
|
10
|
-
class RubyLLMGeneration < GenerationAdapter
|
|
11
|
-
def initialize(provider: Maglev.configuration.generation_provider, model: nil)
|
|
12
|
-
configuration = provider.to_h
|
|
13
|
-
configuration[:model] = model if model
|
|
14
|
-
@client = RubyLLMProvider.new(ProviderConfiguration.new(**configuration))
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
def generate(prompt)
|
|
18
|
-
@client.generate(prompt)
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
end
|
|
22
|
-
end
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require_relative "../errors"
|
|
4
|
-
|
|
5
|
-
module Maglev
|
|
6
|
-
module Adapters
|
|
7
|
-
class RubyLLMProvider
|
|
8
|
-
def initialize(provider)
|
|
9
|
-
@provider = provider
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def embed(text)
|
|
13
|
-
result = translate_errors do
|
|
14
|
-
context.embed(
|
|
15
|
-
text,
|
|
16
|
-
model: @provider.model,
|
|
17
|
-
provider: :openai,
|
|
18
|
-
assume_model_exists: true,
|
|
19
|
-
dimensions: @provider.dimensions
|
|
20
|
-
)
|
|
21
|
-
end
|
|
22
|
-
result.vectors.first.is_a?(Array) ? result.vectors.first : result.vectors
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
def generate(prompt)
|
|
26
|
-
response = translate_errors do
|
|
27
|
-
context.chat(
|
|
28
|
-
model: @provider.model,
|
|
29
|
-
provider: :openai,
|
|
30
|
-
assume_model_exists: true
|
|
31
|
-
).ask(prompt)
|
|
32
|
-
end
|
|
33
|
-
response.respond_to?(:content) ? response.content : response.to_s
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
private
|
|
37
|
-
|
|
38
|
-
def context
|
|
39
|
-
@context ||= begin
|
|
40
|
-
require "ruby_llm"
|
|
41
|
-
|
|
42
|
-
RubyLLM.context do |configuration|
|
|
43
|
-
configuration.openai_api_base = @provider.url
|
|
44
|
-
configuration.openai_api_key = @provider.api_key
|
|
45
|
-
configuration.request_timeout = Maglev.configuration.provider_timeout
|
|
46
|
-
configuration.max_retries = 0
|
|
47
|
-
configuration.logger = Maglev.configuration.logger
|
|
48
|
-
end
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
def translate_errors
|
|
53
|
-
yield
|
|
54
|
-
rescue RubyLLM::RateLimitError, RubyLLM::OverloadedError,
|
|
55
|
-
RubyLLM::ServerError, RubyLLM::ServiceUnavailableError,
|
|
56
|
-
Faraday::TimeoutError, Faraday::ConnectionFailed => error
|
|
57
|
-
raise RetryableProviderError, error.message
|
|
58
|
-
rescue RubyLLM::Error, RubyLLM::ConfigurationError,
|
|
59
|
-
RubyLLM::ModelNotFoundError => error
|
|
60
|
-
raise PermanentProviderError, error.message
|
|
61
|
-
end
|
|
62
|
-
end
|
|
63
|
-
end
|
|
64
|
-
end
|