lex-knowledge 0.6.3 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/legion/extensions/knowledge/actors/corpus_ingest.rb +3 -3
- data/lib/legion/extensions/knowledge/actors/corpus_watcher.rb +2 -2
- data/lib/legion/extensions/knowledge/actors/maintenance_runner.rb +2 -2
- data/lib/legion/extensions/knowledge/helpers/chunker.rb +2 -2
- data/lib/legion/extensions/knowledge/helpers/manifest_store.rb +2 -2
- data/lib/legion/extensions/knowledge/runners/corpus.rb +1 -1
- data/lib/legion/extensions/knowledge/runners/ingest.rb +3 -3
- data/lib/legion/extensions/knowledge/runners/maintenance.rb +13 -13
- data/lib/legion/extensions/knowledge/runners/monitor.rb +6 -6
- data/lib/legion/extensions/knowledge/runners/query.rb +5 -5
- data/lib/legion/extensions/knowledge/version.rb +1 -1
- data/lib/legion/extensions/knowledge.rb +4 -4
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 02e8d24d0e0762a0845d902fbbf0b4b1b40ba181ca7656307f8a15ba31d5419e
|
|
4
|
+
data.tar.gz: bb5b158d9c815dade808201d81a228d144d2aa8e66dedf9df1366c3d7427df6c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7e3a4e56320e2b30f78ac3323e124cdcd0650fbd2dde431592d71a3bf55182ba1d8deafc1b89ce6bf3ef7828b8a75e210b0390f018791ab009e4a022fa58ea33
|
|
7
|
+
data.tar.gz: 7c39e020cfa3df824f388c9b3e5c1f5aa4168e08142a7f4b7ce3bd18a1af2ae96276c383dddf5e5c56ee71b320298db805908396fb1f4e566fc0aabdc248538c
|
|
@@ -10,10 +10,10 @@ module Legion
|
|
|
10
10
|
def check_subtask? = false
|
|
11
11
|
def generate_task? = false
|
|
12
12
|
|
|
13
|
-
def enabled?
|
|
14
|
-
|
|
13
|
+
def enabled? # rubocop:disable Legion/Extension/ActorEnabledSideEffects
|
|
14
|
+
Legion.const_defined?(:Transport, false) &&
|
|
15
15
|
defined?(Legion::Extensions::Knowledge::Runners::Ingest)
|
|
16
|
-
rescue StandardError
|
|
16
|
+
rescue StandardError => _e
|
|
17
17
|
false
|
|
18
18
|
end
|
|
19
19
|
end
|
|
@@ -4,7 +4,7 @@ module Legion
|
|
|
4
4
|
module Extensions
|
|
5
5
|
module Knowledge
|
|
6
6
|
module Actor
|
|
7
|
-
class CorpusWatcher < Legion::Extensions::Actors::Every
|
|
7
|
+
class CorpusWatcher < Legion::Extensions::Actors::Every # rubocop:disable Legion/Extension/EveryActorRequiresTime
|
|
8
8
|
def runner_class = 'Legion::Extensions::Knowledge::Runners::Ingest'
|
|
9
9
|
def runner_function = 'ingest_corpus'
|
|
10
10
|
def check_subtask? = false
|
|
@@ -21,7 +21,7 @@ module Legion
|
|
|
21
21
|
300
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
-
def enabled?
|
|
24
|
+
def enabled? # rubocop:disable Legion/Extension/ActorEnabledSideEffects
|
|
25
25
|
resolve_monitors.any?
|
|
26
26
|
rescue StandardError => e
|
|
27
27
|
log.warn(e.message)
|
|
@@ -4,7 +4,7 @@ module Legion
|
|
|
4
4
|
module Extensions
|
|
5
5
|
module Knowledge
|
|
6
6
|
module Actor
|
|
7
|
-
class MaintenanceRunner < Legion::Extensions::Actors::Every
|
|
7
|
+
class MaintenanceRunner < Legion::Extensions::Actors::Every # rubocop:disable Legion/Extension/EveryActorRequiresTime
|
|
8
8
|
def runner_class = 'Legion::Extensions::Knowledge::Runners::Maintenance'
|
|
9
9
|
def runner_function = 'health'
|
|
10
10
|
def check_subtask? = false
|
|
@@ -21,7 +21,7 @@ module Legion
|
|
|
21
21
|
21_600
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
-
def enabled?
|
|
24
|
+
def enabled? # rubocop:disable Legion/Extension/ActorEnabledSideEffects
|
|
25
25
|
return false unless corpus_path && !corpus_path.empty?
|
|
26
26
|
|
|
27
27
|
true
|
|
@@ -79,7 +79,7 @@ module Legion
|
|
|
79
79
|
return nil unless defined?(Legion::Settings)
|
|
80
80
|
|
|
81
81
|
Legion::Settings.dig(:knowledge, :chunker, :max_tokens)
|
|
82
|
-
rescue StandardError
|
|
82
|
+
rescue StandardError => _e
|
|
83
83
|
nil
|
|
84
84
|
end
|
|
85
85
|
private_class_method :settings_max_tokens
|
|
@@ -88,7 +88,7 @@ module Legion
|
|
|
88
88
|
return nil unless defined?(Legion::Settings)
|
|
89
89
|
|
|
90
90
|
Legion::Settings.dig(:knowledge, :chunker, :overlap_tokens)
|
|
91
|
-
rescue StandardError
|
|
91
|
+
rescue StandardError => _e
|
|
92
92
|
nil
|
|
93
93
|
end
|
|
94
94
|
private_class_method :settings_overlap_tokens
|
|
@@ -20,7 +20,7 @@ module Legion
|
|
|
20
20
|
|
|
21
21
|
raw = ::File.read(path, encoding: 'utf-8')
|
|
22
22
|
::JSON.parse(raw, symbolize_names: true)
|
|
23
|
-
rescue StandardError
|
|
23
|
+
rescue StandardError => _e
|
|
24
24
|
[]
|
|
25
25
|
end
|
|
26
26
|
|
|
@@ -31,7 +31,7 @@ module Legion
|
|
|
31
31
|
::File.write(tmp, ::JSON.generate(manifest.map { |e| serialize_entry(e) }))
|
|
32
32
|
::File.rename(tmp, path)
|
|
33
33
|
true
|
|
34
|
-
rescue StandardError
|
|
34
|
+
rescue StandardError => _e
|
|
35
35
|
false
|
|
36
36
|
end
|
|
37
37
|
|
|
@@ -6,7 +6,7 @@ module Legion
|
|
|
6
6
|
module Extensions
|
|
7
7
|
module Knowledge
|
|
8
8
|
module Runners
|
|
9
|
-
module Ingest
|
|
9
|
+
module Ingest # rubocop:disable Legion/Extension/RunnerIncludeHelpers
|
|
10
10
|
module_function
|
|
11
11
|
|
|
12
12
|
def log
|
|
@@ -196,7 +196,7 @@ module Legion
|
|
|
196
196
|
private_class_method :paired_without_embed
|
|
197
197
|
|
|
198
198
|
def build_embed_map(needs_embed)
|
|
199
|
-
results = Legion::LLM.embed_batch(needs_embed.map { |c| c[:content] })
|
|
199
|
+
results = Legion::LLM.embed_batch(needs_embed.map { |c| c[:content] }) # rubocop:disable Legion/HelperMigration/DirectLlm
|
|
200
200
|
results.each_with_object({}) do |r, h|
|
|
201
201
|
h[needs_embed[r[:index]][:content_hash]] = r[:vector] unless r[:error]
|
|
202
202
|
end
|
|
@@ -258,7 +258,7 @@ module Legion
|
|
|
258
258
|
return unless defined?(Legion::Apollo)
|
|
259
259
|
return unless Legion::Apollo.respond_to?(:ingest) && Legion::Apollo.started?
|
|
260
260
|
|
|
261
|
-
Legion::Apollo.ingest(
|
|
261
|
+
Legion::Apollo.ingest( # rubocop:disable Legion/HelperMigration/DirectKnowledge
|
|
262
262
|
content: file_path,
|
|
263
263
|
content_type: 'document_retired',
|
|
264
264
|
tags: [file_path, 'retired', 'document_chunk'].uniq,
|
|
@@ -4,7 +4,7 @@ module Legion
|
|
|
4
4
|
module Extensions
|
|
5
5
|
module Knowledge
|
|
6
6
|
module Runners
|
|
7
|
-
module Maintenance
|
|
7
|
+
module Maintenance # rubocop:disable Legion/Extension/RunnerIncludeHelpers
|
|
8
8
|
module_function
|
|
9
9
|
|
|
10
10
|
def detect_orphans(path:)
|
|
@@ -99,7 +99,7 @@ module Legion
|
|
|
99
99
|
|
|
100
100
|
rows = base.select(:confidence, :status, :access_count, :embedding, :created_at).all
|
|
101
101
|
apollo_stats_from_rows(base, rows, total)
|
|
102
|
-
rescue StandardError
|
|
102
|
+
rescue StandardError => _e
|
|
103
103
|
apollo_defaults
|
|
104
104
|
end
|
|
105
105
|
private_class_method :build_apollo_stats
|
|
@@ -155,7 +155,7 @@ module Legion
|
|
|
155
155
|
|
|
156
156
|
def load_manifest_files(path)
|
|
157
157
|
manifest = Helpers::ManifestStore.load(corpus_path: path)
|
|
158
|
-
manifest.
|
|
158
|
+
manifest.filter_map { |e| e[:path] }.uniq
|
|
159
159
|
end
|
|
160
160
|
private_class_method :load_manifest_files
|
|
161
161
|
|
|
@@ -168,7 +168,7 @@ module Legion
|
|
|
168
168
|
.select_map(Sequel.lit("source_context->>'source_file'"))
|
|
169
169
|
.compact
|
|
170
170
|
.uniq
|
|
171
|
-
rescue StandardError
|
|
171
|
+
rescue StandardError => _e
|
|
172
172
|
[]
|
|
173
173
|
end
|
|
174
174
|
private_class_method :load_apollo_source_files
|
|
@@ -180,7 +180,7 @@ module Legion
|
|
|
180
180
|
.where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
|
|
181
181
|
.exclude(status: 'archived')
|
|
182
182
|
.count
|
|
183
|
-
rescue StandardError
|
|
183
|
+
rescue StandardError => _e
|
|
184
184
|
0
|
|
185
185
|
end
|
|
186
186
|
private_class_method :count_apollo_chunks
|
|
@@ -208,7 +208,7 @@ module Legion
|
|
|
208
208
|
.select_map([:id, :access_count, :confidence,
|
|
209
209
|
Sequel.lit("source_context->>'source_file' AS source_file")])
|
|
210
210
|
.map { |r| { id: r[0], access_count: r[1], confidence: r[2], source_file: r[3] } }
|
|
211
|
-
rescue StandardError
|
|
211
|
+
rescue StandardError => _e
|
|
212
212
|
[]
|
|
213
213
|
end
|
|
214
214
|
private_class_method :hot_chunks
|
|
@@ -229,7 +229,7 @@ module Legion
|
|
|
229
229
|
.select_map([:id, :confidence, :created_at,
|
|
230
230
|
Sequel.lit("source_context->>'source_file' AS source_file")])
|
|
231
231
|
.map { |r| { id: r[0], confidence: r[1], created_at: r[2]&.iso8601, source_file: r[3] } }
|
|
232
|
-
rescue StandardError
|
|
232
|
+
rescue StandardError => _e
|
|
233
233
|
[]
|
|
234
234
|
end
|
|
235
235
|
private_class_method :cold_chunks
|
|
@@ -248,7 +248,7 @@ module Legion
|
|
|
248
248
|
.select_map([:id, :confidence, :access_count,
|
|
249
249
|
Sequel.lit("source_context->>'source_file' AS source_file")])
|
|
250
250
|
.map { |r| { id: r[0], confidence: r[1], access_count: r[2], source_file: r[3] } }
|
|
251
|
-
rescue StandardError
|
|
251
|
+
rescue StandardError => _e
|
|
252
252
|
[]
|
|
253
253
|
end
|
|
254
254
|
private_class_method :low_confidence_chunks
|
|
@@ -268,7 +268,7 @@ module Legion
|
|
|
268
268
|
chunks_never_accessed: base.where(access_count: 0).count,
|
|
269
269
|
chunks_below_threshold: base.where { confidence < settings_stale_threshold }.count
|
|
270
270
|
}
|
|
271
|
-
rescue StandardError
|
|
271
|
+
rescue StandardError => _e
|
|
272
272
|
defaults
|
|
273
273
|
end
|
|
274
274
|
private_class_method :quality_summary
|
|
@@ -277,7 +277,7 @@ module Legion
|
|
|
277
277
|
return 0 unless defined?(Legion::Data::Model::ApolloAccessLog)
|
|
278
278
|
|
|
279
279
|
Legion::Data::Model::ApolloAccessLog.where(action: 'knowledge_query').count
|
|
280
|
-
rescue StandardError
|
|
280
|
+
rescue StandardError => _e
|
|
281
281
|
0
|
|
282
282
|
end
|
|
283
283
|
private_class_method :query_count
|
|
@@ -286,7 +286,7 @@ module Legion
|
|
|
286
286
|
return 0.3 unless defined?(Legion::Settings)
|
|
287
287
|
|
|
288
288
|
Legion::Settings.dig(:knowledge, :maintenance, :stale_threshold) || 0.3
|
|
289
|
-
rescue StandardError
|
|
289
|
+
rescue StandardError => _e
|
|
290
290
|
0.3
|
|
291
291
|
end
|
|
292
292
|
private_class_method :settings_stale_threshold
|
|
@@ -295,7 +295,7 @@ module Legion
|
|
|
295
295
|
return 7 unless defined?(Legion::Settings)
|
|
296
296
|
|
|
297
297
|
Legion::Settings.dig(:knowledge, :maintenance, :cold_chunk_days) || 7
|
|
298
|
-
rescue StandardError
|
|
298
|
+
rescue StandardError => _e
|
|
299
299
|
7
|
|
300
300
|
end
|
|
301
301
|
private_class_method :settings_cold_chunk_days
|
|
@@ -304,7 +304,7 @@ module Legion
|
|
|
304
304
|
return 10 unless defined?(Legion::Settings)
|
|
305
305
|
|
|
306
306
|
Legion::Settings.dig(:knowledge, :maintenance, :quality_report_limit) || 10
|
|
307
|
-
rescue StandardError
|
|
307
|
+
rescue StandardError => _e
|
|
308
308
|
10
|
|
309
309
|
end
|
|
310
310
|
private_class_method :settings_quality_limit
|
|
@@ -4,7 +4,7 @@ module Legion
|
|
|
4
4
|
module Extensions
|
|
5
5
|
module Knowledge
|
|
6
6
|
module Runners
|
|
7
|
-
module Monitor
|
|
7
|
+
module Monitor # rubocop:disable Legion/Extension/RunnerIncludeHelpers
|
|
8
8
|
module_function
|
|
9
9
|
|
|
10
10
|
DEFAULT_EXTENSIONS = %w[.md .txt].freeze
|
|
@@ -18,7 +18,7 @@ module Legion
|
|
|
18
18
|
end
|
|
19
19
|
|
|
20
20
|
monitors
|
|
21
|
-
rescue StandardError
|
|
21
|
+
rescue StandardError => _e
|
|
22
22
|
[]
|
|
23
23
|
end
|
|
24
24
|
|
|
@@ -70,7 +70,7 @@ module Legion
|
|
|
70
70
|
monitors.each do |m|
|
|
71
71
|
scan = Helpers::Manifest.scan(path: m[:path], extensions: m[:extensions])
|
|
72
72
|
total_files += scan.size
|
|
73
|
-
rescue StandardError
|
|
73
|
+
rescue StandardError => _e
|
|
74
74
|
next
|
|
75
75
|
end
|
|
76
76
|
|
|
@@ -85,7 +85,7 @@ module Legion
|
|
|
85
85
|
return nil unless defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
|
|
86
86
|
|
|
87
87
|
Legion::Settings.dig(:knowledge, :monitors)
|
|
88
|
-
rescue StandardError
|
|
88
|
+
rescue StandardError => _e
|
|
89
89
|
nil
|
|
90
90
|
end
|
|
91
91
|
private_class_method :read_monitors_setting
|
|
@@ -94,7 +94,7 @@ module Legion
|
|
|
94
94
|
return nil unless defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
|
|
95
95
|
|
|
96
96
|
Legion::Settings.dig(:knowledge, :corpus_path)
|
|
97
|
-
rescue StandardError
|
|
97
|
+
rescue StandardError => _e
|
|
98
98
|
nil
|
|
99
99
|
end
|
|
100
100
|
private_class_method :read_legacy_corpus_path
|
|
@@ -107,7 +107,7 @@ module Legion
|
|
|
107
107
|
knowledge[:monitors] = monitors
|
|
108
108
|
loader.settings[:knowledge] = knowledge
|
|
109
109
|
true
|
|
110
|
-
rescue StandardError
|
|
110
|
+
rescue StandardError => _e
|
|
111
111
|
false
|
|
112
112
|
end
|
|
113
113
|
private_class_method :persist_monitors
|
|
@@ -6,7 +6,7 @@ module Legion
|
|
|
6
6
|
module Extensions
|
|
7
7
|
module Knowledge
|
|
8
8
|
module Runners
|
|
9
|
-
module Query
|
|
9
|
+
module Query # rubocop:disable Legion/Extension/RunnerIncludeHelpers
|
|
10
10
|
module_function
|
|
11
11
|
|
|
12
12
|
def query(question:, top_k: nil, synthesize: true)
|
|
@@ -74,7 +74,7 @@ module Legion
|
|
|
74
74
|
limit: top_k,
|
|
75
75
|
tags: ['document_chunk']
|
|
76
76
|
)
|
|
77
|
-
rescue StandardError
|
|
77
|
+
rescue StandardError => _e
|
|
78
78
|
[]
|
|
79
79
|
end
|
|
80
80
|
private_class_method :retrieve_chunks
|
|
@@ -90,7 +90,7 @@ module Legion
|
|
|
90
90
|
"Context:\n#{context_text}\n\nQuestion: #{question}\n\nAnswer:"
|
|
91
91
|
end
|
|
92
92
|
|
|
93
|
-
result =
|
|
93
|
+
result = llm_chat(message: prompt, caller: { extension: 'lex-knowledge' })
|
|
94
94
|
result.is_a?(Hash) ? result[:content] : result
|
|
95
95
|
rescue StandardError => e
|
|
96
96
|
"Error generating answer: #{e.message}"
|
|
@@ -159,7 +159,7 @@ module Legion
|
|
|
159
159
|
synthesized: synthesized,
|
|
160
160
|
rating: rating
|
|
161
161
|
})
|
|
162
|
-
rescue StandardError
|
|
162
|
+
rescue StandardError => _e
|
|
163
163
|
nil
|
|
164
164
|
end
|
|
165
165
|
private_class_method :emit_feedback_event
|
|
@@ -173,7 +173,7 @@ module Legion
|
|
|
173
173
|
return nil unless defined?(Legion::Settings)
|
|
174
174
|
|
|
175
175
|
Legion::Settings.dig(:knowledge, :query, :top_k)
|
|
176
|
-
rescue StandardError
|
|
176
|
+
rescue StandardError => _e
|
|
177
177
|
nil
|
|
178
178
|
end
|
|
179
179
|
private_class_method :settings_top_k
|
|
@@ -12,17 +12,17 @@ require_relative 'knowledge/runners/maintenance'
|
|
|
12
12
|
require_relative 'knowledge/runners/monitor'
|
|
13
13
|
require_relative 'knowledge/client'
|
|
14
14
|
|
|
15
|
-
if
|
|
15
|
+
if Legion.const_defined?(:Transport, false)
|
|
16
16
|
require_relative 'knowledge/transport/exchanges/knowledge'
|
|
17
17
|
require_relative 'knowledge/transport/queues/ingest'
|
|
18
18
|
require_relative 'knowledge/transport/messages/ingest_message'
|
|
19
19
|
require_relative 'knowledge/transport/messages/monitor_reload'
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
require_relative 'knowledge/actors/corpus_watcher'
|
|
23
|
-
require_relative 'knowledge/actors/maintenance_runner'
|
|
22
|
+
require_relative 'knowledge/actors/corpus_watcher'
|
|
23
|
+
require_relative 'knowledge/actors/maintenance_runner'
|
|
24
24
|
|
|
25
|
-
require_relative 'knowledge/actors/corpus_ingest'
|
|
25
|
+
require_relative 'knowledge/actors/corpus_ingest'
|
|
26
26
|
|
|
27
27
|
module Legion
|
|
28
28
|
module Extensions
|