lex-knowledge 0.6.10 → 0.6.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 319636e39b5a3f036355f455f03ae585a9902e7510028ae2d37ce88387d09ee9
-  data.tar.gz: 7a6c81ac4db6261d2388b3a16b8a6d28dc7b81571d4bff5e49c4cb623ca40576
+  metadata.gz: 21c773508b11dfb12b4d07bf2e37705483b82f4100382c29d84d883f3790ce5c
+  data.tar.gz: 96c6e3da83bb20eabb5db9f55b8ef4b7715ac08d3110ec4c6680ccf6b62210e4
 SHA512:
-  metadata.gz: 763094b4aad5b43d300823e8c74843cb8e7a199401a5a61cfe308f9d77a3461445d8175e3eb8e0b5599ad28c4c797d77bc37b038464f9ddb93f86918de6b40a9
-  data.tar.gz: a4d39884a98025e49509f281dfab9c94501a12c824f4ea08ec20795bea80d36b9bde7ec54c657b965f1b489f36a04f3e48bda6df85471246eb59d42a5bb10c27
+  metadata.gz: 5e2a5993ca4b4213c1dfb25eff797aaf7b89af3b5bc4c37072683d564bc49db49aad6f30702f47c19bd66c008bcdcc9b9a29b075920b66c03ab237dd4e9070e0
+  data.tar.gz: 04f3d33195e5c81c4b48c361170909b6f8a53e28660a0d30370584453a847f8d787443261ffa1b47a383c58aa387281616a29f1a2366c6c46e43308d82284808

data/lib/legion/extensions/knowledge/actors/corpus_ingest.rb

@@ -5,6 +5,9 @@ module Legion
     module Knowledge
       module Actor
         class CorpusIngest < Legion::Extensions::Actors::Subscription
+          include Legion::Logging::Helper
+          include Legion::Settings::Helper
+
           def runner_class    = 'Legion::Extensions::Knowledge::Runners::Ingest'
           def runner_function = 'ingest_file'
           def check_subtask?  = false
@@ -13,7 +16,8 @@ module Legion
           def enabled? # rubocop:disable Legion/Extension/ActorEnabledSideEffects
             Legion.const_defined?(:Transport, false) &&
               defined?(Legion::Extensions::Knowledge::Runners::Ingest)
-          rescue StandardError => _e
+          rescue StandardError => e
+            handle_exception(e, level: :warn, operation: 'knowledge.corpus_ingest.enabled')
             false
           end
         end

data/lib/legion/extensions/knowledge/actors/corpus_watcher.rb

@@ -5,26 +5,25 @@ module Legion
     module Knowledge
       module Actor
         class CorpusWatcher < Legion::Extensions::Actors::Every # rubocop:disable Legion/Extension/EveryActorRequiresTime
+          include Legion::Logging::Helper
+          include Legion::Settings::Helper
+
           def runner_class    = 'Legion::Extensions::Knowledge::Runners::Ingest'
           def runner_function = 'ingest_corpus'
           def check_subtask?  = false
           def generate_task?  = false
 
           def time
-            if defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
-              Legion::Settings.dig(:knowledge, :actors, :watcher_interval) || 300
-            else
-              300
-            end
+            settings[:actors][:watcher_interval]
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.corpus_watcher.time')
             300
           end
 
           def enabled? # rubocop:disable Legion/Extension/ActorEnabledSideEffects
             resolve_monitors.any?
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.corpus_watcher.enabled')
             false
           end
 
@@ -34,14 +33,10 @@ module Legion
 
           private
 
-          def log
-            Legion::Logging
-          end
-
           def resolve_monitors
             Runners::Monitor.resolve_monitors
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.corpus_watcher.resolve_monitors')
             []
           end
         end

data/lib/legion/extensions/knowledge/actors/maintenance_runner.rb

@@ -5,19 +5,18 @@ module Legion
     module Knowledge
       module Actor
         class MaintenanceRunner < Legion::Extensions::Actors::Every # rubocop:disable Legion/Extension/EveryActorRequiresTime
+          include Legion::Logging::Helper
+          include Legion::Settings::Helper
+
           def runner_class    = 'Legion::Extensions::Knowledge::Runners::Maintenance'
           def runner_function = 'health'
           def check_subtask?  = false
           def generate_task?  = false
 
           def time
-            if defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
-              Legion::Settings.dig(:knowledge, :actors, :maintenance_interval) || 21_600
-            else
-              21_600
-            end
+            settings[:actors][:maintenance_interval]
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.maintenance_runner.time')
             21_600
           end
 
@@ -26,7 +25,7 @@ module Legion
 
             true
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.maintenance_runner.enabled')
             false
           end
 
@@ -36,16 +35,10 @@ module Legion
 
           private
 
-          def log
-            Legion::Logging
-          end
-
           def corpus_path
-            return nil unless defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
-
-            Legion::Settings.dig(:knowledge, :corpus_path)
+            settings[:corpus_path]
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.maintenance_runner.corpus_path')
             nil
           end
         end

data/lib/legion/extensions/knowledge/helpers/apollo_models.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Helpers
+        module ApolloModels
+          class << self
+            def entry
+              namespaced_apollo_model(:Entry) || legacy_model(:ApolloEntry)
+            end
+
+            def access_log
+              namespaced_apollo_model(:AccessLog) || legacy_model(:ApolloAccessLog)
+            end
+
+            def entry_available?
+              !entry.nil?
+            end
+
+            def access_log_available?
+              !access_log.nil?
+            end
+
+            private
+
+            def namespaced_apollo_model(name)
+              return nil unless defined?(Legion::Data::Model::Apollo)
+              return nil unless Legion::Data::Model::Apollo.const_defined?(name, false)
+
+              Legion::Data::Model::Apollo.const_get(name, false)
+            end
+
+            def legacy_model(name)
+              return nil unless defined?(Legion::Data::Model)
+              return nil unless Legion::Data::Model.const_defined?(name, false)
+
+              Legion::Data::Model.const_get(name, false)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/helpers/chunker.rb

@@ -7,13 +7,16 @@ module Legion
     module Knowledge
       module Helpers
         module Chunker
+          extend Legion::Logging::Helper
+          extend Legion::Settings::Helper
+
           CHARS_PER_TOKEN = 4
 
           module_function
 
           def chunk(sections:, max_tokens: nil, overlap_tokens: nil)
-            resolved_max     = max_tokens     || settings_max_tokens     || 512
-            resolved_overlap = overlap_tokens || settings_overlap_tokens || 128
+            resolved_max     = max_tokens     || settings[:chunker][:max_tokens]
+            resolved_overlap = overlap_tokens || settings[:chunker][:overlap_tokens]
 
             max_chars     = resolved_max * CHARS_PER_TOKEN
             overlap_chars = resolved_overlap * CHARS_PER_TOKEN
@@ -89,24 +92,6 @@ module Legion
             end
           end
           private_class_method :apollo_compatible_content_hash
-
-          def settings_max_tokens
-            return nil unless defined?(Legion::Settings)
-
-            Legion::Settings.dig(:knowledge, :chunker, :max_tokens)
-          rescue StandardError => _e
-            nil
-          end
-          private_class_method :settings_max_tokens
-
-          def settings_overlap_tokens
-            return nil unless defined?(Legion::Settings)
-
-            Legion::Settings.dig(:knowledge, :chunker, :overlap_tokens)
-          rescue StandardError => _e
-            nil
-          end
-          private_class_method :settings_overlap_tokens
         end
       end
     end

data/lib/legion/extensions/knowledge/helpers/manifest.rb

@@ -7,6 +7,8 @@ module Legion
     module Knowledge
       module Helpers
         module Manifest
+          extend Legion::Logging::Helper
+
           module_function
 
           def scan(path:, extensions: %w[.md .txt .docx .pdf])
@@ -25,15 +27,10 @@ module Legion
               results << build_entry(entry)
             end
           rescue Errno::EPERM, Errno::EACCES, Errno::ELOOP, Errno::ENOENT => e
-            log.debug("[manifest] skipping unreadable #{entry}: #{e.class}: #{e.message}")
+            handle_exception(e, level: :warn, operation: 'knowledge.manifest.walk', entry: entry)
           end
           private_class_method :walk
 
-          def log
-            Legion::Logging
-          end
-          private_class_method :log
-
           def diff(current:, previous:)
             current_map  = current.to_h { |e| [e[:path], e[:sha256]] }
             previous_map = previous.to_h { |e| [e[:path], e[:sha256]] }

data/lib/legion/extensions/knowledge/helpers/manifest_store.rb

@@ -2,7 +2,7 @@
 
 require 'digest'
 require 'fileutils'
-require 'json'
+require 'legion/json'
 require 'tempfile'
 
 module Legion
@@ -10,6 +10,9 @@ module Legion
     module Knowledge
       module Helpers
         module ManifestStore
+          extend Legion::Logging::Helper
+          extend Legion::JSON::Helper
+
           module_function
 
           STORE_DIR = ::File.expand_path('~/.legionio/knowledge').freeze
@@ -19,8 +22,9 @@ module Legion
             return [] unless ::File.exist?(path)
 
             raw = ::File.read(path, encoding: 'utf-8')
-            ::JSON.parse(raw, symbolize_names: true)
-          rescue StandardError => _e
+            json_parse(raw)
+          rescue StandardError => e
+            handle_exception(e, level: :warn, operation: 'knowledge.manifest_store.load', corpus_path: corpus_path)
             []
           end
 
@@ -28,10 +32,11 @@ module Legion
             ::FileUtils.mkdir_p(STORE_DIR)
             path = store_path(corpus_path: corpus_path)
             tmp  = "#{path}.tmp"
-            ::File.write(tmp, ::JSON.generate(manifest.map { |e| serialize_entry(e) }))
+            ::File.write(tmp, json_generate(manifest.map { |e| serialize_entry(e) }))
             ::File.rename(tmp, path)
             true
-          rescue StandardError => _e
+          rescue StandardError => e
+            handle_exception(e, level: :warn, operation: 'knowledge.manifest_store.save', corpus_path: corpus_path)
             false
           end
 

data/lib/legion/extensions/knowledge/helpers/parser.rb

@@ -5,6 +5,8 @@ module Legion
     module Knowledge
       module Helpers
         module Parser
+          extend Legion::Logging::Helper
+
           module_function
 
           def parse(file_path:)
@@ -57,6 +59,7 @@ module Legion
             heading = ::File.basename(file_path, '.*')
             [{ heading: heading, section_path: [], content: result[:text].strip, source_file: file_path }]
           rescue StandardError => e
+            handle_exception(e, level: :warn, operation: 'knowledge.parser.extract_via_data', file_path: file_path)
             [{ error: 'extraction_failed', source_file: file_path, detail: e.message }]
           end
 

data/lib/legion/extensions/knowledge/runners/corpus.rb

@@ -5,6 +5,8 @@ module Legion
     module Knowledge
       module Runners
         module Corpus # rubocop:disable Legion/Extension/RunnerIncludeHelpers
+          extend Legion::Logging::Helper
+
           module_function
 
           def manifest_path(path:)
@@ -32,6 +34,7 @@ module Legion
               total_bytes:      entries.sum { |e| e[:size] }
             }
           rescue StandardError => e
+            handle_exception(e, level: :warn, operation: 'knowledge.corpus.corpus_stats', path: path)
             { success: false, error: e.message }
           end
         end

data/lib/legion/extensions/knowledge/runners/ingest.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../helpers/apollo_models'
+
 require 'securerandom'
 
 module Legion
@@ -7,12 +9,10 @@ module Legion
     module Knowledge
       module Runners
         module Ingest # rubocop:disable Legion/Extension/RunnerIncludeHelpers
-          module_function
+          extend Legion::Logging::Helper
+          extend Legion::Settings::Helper
 
-          def log
-            Legion::Logging
-          end
-          private_class_method :log
+          module_function
 
           def scan_corpus(path:, extensions: nil)
             opts = { path: path }
@@ -29,17 +29,27 @@ module Legion
             }
           end
 
-          def ingest_corpus(path: nil, monitors: nil, dry_run: false, force: false)
-            return ingest_monitors(monitors: monitors, dry_run: dry_run, force: force) if monitors&.any?
+          FILTER_SCHEMA = {
+            type:       'object',
+            properties: {
+              relevant:   { type: 'boolean' },
+              confidence: { type: 'number' },
+              reason:     { type: 'string' }
+            },
+            required:   %w[relevant confidence]
+          }.freeze
+
+          def ingest_corpus(path: nil, monitors: nil, dry_run: false, force: false, filter: true)
+            return ingest_monitors(monitors: monitors, dry_run: dry_run, force: force, filter: filter) if monitors&.any?
             raise ArgumentError, 'path is required when monitors is not provided' if path.nil?
 
-            ingest_corpus_path(path: path, dry_run: dry_run, force: force)
+            ingest_corpus_path(path: path, dry_run: dry_run, force: force, filter: filter)
           rescue ArgumentError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.ingest.ingest_corpus')
             { success: false, error: e.message }
           end
 
-          def ingest_corpus_path(path:, dry_run: false, force: false)
+          def ingest_corpus_path(path:, dry_run: false, force: false, filter: true)
             current  = Helpers::Manifest.scan(path: path)
             previous = force ? [] : Helpers::ManifestStore.load(corpus_path: path)
             delta    = Helpers::Manifest.diff(current: current, previous: previous)
@@ -50,7 +60,7 @@ module Legion
             chunks_updated = 0
 
             to_process.each do |file_path|
-              result = process_file(file_path, dry_run: dry_run, force: force)
+              result = process_file(file_path, dry_run: dry_run, force: force, filter: filter)
               chunks_created += result[:created]
               chunks_skipped += result[:skipped]
               chunks_updated += result[:updated]
@@ -71,16 +81,16 @@ module Legion
               chunks_updated: chunks_updated
             }
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.ingest.ingest_corpus_path', path: path)
             { success: false, error: e.message }
           end
           private_class_method :ingest_corpus_path
 
-          def ingest_monitors(monitors:, dry_run: false, force: false)
+          def ingest_monitors(monitors:, dry_run: false, force: false, filter: true)
             results = monitors.map do |monitor|
-              ingest_corpus(path: monitor[:path], dry_run: dry_run, force: force)
+              ingest_corpus(path: monitor[:path], dry_run: dry_run, force: force, filter: filter)
             rescue StandardError => e
-              log.warn(e.message)
+              handle_exception(e, level: :warn, operation: 'knowledge.ingest.ingest_monitor', path: monitor[:path])
               { success: false, path: monitor[:path], error: e.message }
             end
 
@@ -101,7 +111,7 @@ module Legion
 
             { success: true, monitors_processed: results.size, **total }
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.ingest.ingest_monitors')
             { success: false, error: e.message }
           end
           private_class_method :ingest_monitors
@@ -114,17 +124,17 @@ module Legion
               section_path: [source_type.to_s],
               source_file:  source_path
             }
-            chunks = Helpers::Chunker.chunk(sections: [section])
+            chunks = filter_chunks(Helpers::Chunker.chunk(sections: [section]), filter: true)
             paired = batch_embed_chunks(chunks, force: false)
             paired.each { |p| upsert_chunk_with_embedding(p[:chunk], p[:embedding], force: false, exists: p[:exists] || false) }
             { status: :ingested, chunks: chunks.size, source_type: source_type, metadata: metadata }
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.ingest.ingest_content', source_type: source_type)
             { status: :failed, error: e.message, source_type: source_type, metadata: metadata }
           end
 
-          def ingest_file(file_path:, force: false)
-            result = process_file(file_path, dry_run: false, force: force)
+          def ingest_file(file_path:, force: false, filter: true)
+            result = process_file(file_path, dry_run: false, force: force, filter: filter)
 
             {
               success:        true,
@@ -134,23 +144,24 @@ module Legion
               chunks_updated: result[:updated]
             }
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.ingest.ingest_file', file_path: file_path)
             { success: false, error: e.message }
           end
 
-          def process_file(file_path, dry_run: false, force: false)
+          def process_file(file_path, dry_run: false, force: false, filter: true)
             sections = Helpers::Parser.parse(file_path: file_path)
             return { created: 0, skipped: 0, updated: 0 } if sections.first&.key?(:error)
 
-            chunks  = Helpers::Chunker.chunk(sections: sections)
+            chunks = Helpers::Chunker.chunk(sections: sections)
+            filtered_chunks = filter_chunks(chunks, filter: filter)
             paired  = if dry_run
-                        chunks.map { |c| { chunk: c, embedding: nil } }
+                        filtered_chunks.map { |c| { chunk: c, embedding: nil } }
                       else
-                        batch_embed_chunks(chunks, force: force)
+                        batch_embed_chunks(filtered_chunks, force: force)
                       end
 
             created = 0
-            skipped = 0
+            skipped = chunks.size - filtered_chunks.size
             updated = 0
 
             paired.each do |p|
@@ -166,6 +177,49 @@ module Legion
           end
           private_class_method :process_file
 
+          def filter_chunks(chunks, filter:)
+            return chunks unless filter
+
+            prompt = settings[:ingest][:filter_prompt]
+            return chunks if prompt.to_s.strip.empty? || !llm_structured_available?
+
+            chunks.select { |chunk| chunk_allowed_by_filter?(chunk, prompt: prompt, threshold: settings[:ingest][:filter_threshold]) }
+          rescue StandardError => e
+            handle_exception(e, level: :warn, operation: 'knowledge.ingest.filter_chunks')
+            chunks
+          end
+          private_class_method :filter_chunks
+
+          def chunk_allowed_by_filter?(chunk, prompt:, threshold:)
+            hash = chunk[:content_hash] || Helpers::Chunker.send(:apollo_compatible_content_hash, chunk[:content].to_s)
+            return filter_cache[hash] if filter_cache.key?(hash)
+
+            result = Legion::LLM.structured( # rubocop:disable Legion/HelperMigration/DirectLlm
+              messages: [
+                { role: 'system', content: prompt },
+                { role: 'user', content: chunk[:content].to_s }
+              ],
+              schema:   FILTER_SCHEMA,
+              caller:   { extension: 'lex-knowledge', runner: 'ingest', operation: 'filter_chunk' }
+            )
+            data = result.is_a?(Hash) ? (result[:data] || result) : {}
+            filter_cache[hash] = data[:relevant] == true && data[:confidence].to_f >= threshold.to_f
+          rescue StandardError => e
+            handle_exception(e, level: :warn, operation: 'knowledge.ingest.filter_chunk', content_hash: hash)
+            filter_cache[hash] = true
+          end
+          private_class_method :chunk_allowed_by_filter?
+
+          def filter_cache
+            Thread.current[:lex_knowledge_filter_cache] ||= {}
+          end
+          private_class_method :filter_cache
+
+          def llm_structured_available?
+            defined?(Legion::LLM) && Legion::LLM.respond_to?(:structured)
+          end
+          private_class_method :llm_structured_available?
+
           def batch_embed_chunks(chunks, force:)
             exists_map = force ? {} : build_exists_map(chunks)
             return paired_without_embed(chunks, exists_map) unless llm_embed_available?
@@ -175,7 +229,7 @@ module Legion
 
             chunks.map { |c| { chunk: c, embedding: embed_map[c[:content_hash]], exists: exists_map.fetch(c[:content_hash], false) } }
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.ingest.batch_embed_chunks')
             paired_without_embed(chunks, {})
           end
           private_class_method :batch_embed_chunks
@@ -201,7 +255,7 @@ module Legion
               h[needs_embed[r[:index]][:content_hash]] = r[:vector] unless r[:error]
             end
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.ingest.build_embed_map')
             {}
           end
           private_class_method :build_embed_map
@@ -228,19 +282,19 @@ module Legion
             end
             force ? :updated : :created
           rescue StandardError => e
-            log.warn("[knowledge][upsert_chunk] unexpected error class=#{e.class} message=#{e.message} chunk_hash=#{chunk[:content_hash]&.slice(0, 12)}")
+            handle_exception(e, level: :warn, operation: 'knowledge.ingest.upsert_chunk', content_hash: chunk[:content_hash]&.slice(0, 12))
             :skipped
           end
           private_class_method :upsert_chunk_with_embedding
 
           def chunk_exists?(content_hash)
-            return false unless defined?(Legion::Data::Model::ApolloEntry)
+            return false unless Helpers::ApolloModels.entry_available?
 
-            Legion::Data::Model::ApolloEntry
-              .where(content_hash: content_hash)
-              .any?
+            Helpers::ApolloModels.entry
+                                 .where(content_hash: content_hash)
+                                 .any?
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.ingest.chunk_exists', content_hash: content_hash)
             false
           end
           private_class_method :chunk_exists?
@@ -248,18 +302,20 @@ module Legion
           def ingest_to_apollo(chunk, embedding)
             return unless defined?(Legion::Extensions::Apollo)
 
+            context = {
+              source_file:  chunk[:source_file],
+              heading:      chunk[:heading],
+              section_path: chunk[:section_path],
+              chunk_index:  chunk[:chunk_index],
+              token_count:  chunk[:token_count]
+            }
             payload = {
               content:      chunk[:content],
               content_type: 'document_chunk',
               content_hash: chunk[:content_hash],
               tags:         [chunk[:source_file], chunk[:heading], 'document_chunk'].compact.uniq,
-              metadata:     {
-                source_file:  chunk[:source_file],
-                heading:      chunk[:heading],
-                section_path: chunk[:section_path],
-                chunk_index:  chunk[:chunk_index],
-                token_count:  chunk[:token_count]
-              }
+              context:      context,
+              metadata:     context
             }
             payload[:embedding] = embedding if embedding
 
@@ -278,7 +334,7 @@ module Legion
               metadata:     { source_file: file_path, retired: true }
             )
           rescue StandardError => e
-            log.warn(e.message)
+            handle_exception(e, level: :warn, operation: 'knowledge.ingest.retire_file', file_path: file_path)
             nil
           end
           private_class_method :retire_file