lex-knowledge 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da332d52d481d4e45e8f7ee771a60fb13cebb74144a18ac54ff7b7ec4cb26f27
-  data.tar.gz: 3a858fbdae4511ec6e34573fccd85f330b33c7be818a2a78271af33cb0dcbfbd
+  metadata.gz: 8d512db91b31e6d8a9747a3987aa90dd406d04f8987400787f6de27b13eb10c5
+  data.tar.gz: 2d9df5e1289bb80f603dbd882863d1802bf942e8ff12940d259ba44e25b728f3
 SHA512:
-  metadata.gz: e7e15c2174b28ea518c3fee6ecc68723bc628ee3c6d9121a2292706ad32125d32f766da164aa581c56915cd178e54379014a67d8ad1d5cd765485a6eb88d0610
-  data.tar.gz: eb7f343423ea41ff46b6a478b3c32a22b0dd3ab3939ad858d0e8d540b5faaf8f01496a5839c79b5ccb1847529367ad9c55ffdc72bebf5959891ff4a7c06b72e4
+  metadata.gz: 2022e4654a3a815e8c5433daaad7c2e9767b039c51513e12d9cd6c71a2de47e3ee883c6e0624aa9ac8f6108439c19d4552260694482db4e2da9d58851fa093bd
+  data.tar.gz: d1afc8cb8fdcd0317ee9fa416c91827ca3536e40fed404791a355b4b4662360211e3c064dc09e8504d0ed5eb359f1dc906edaa57277f205b1aa9eafedcf261da

data/lib/legion/extensions/knowledge/actors/maintenance_runner.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Actor
+        class MaintenanceRunner < Legion::Extensions::Actors::Every
+          def runner_class    = 'Legion::Extensions::Knowledge::Runners::Maintenance'
+          def runner_function = 'health'
+          def check_subtask?  = false
+          def generate_task?  = false
+
+          def every_interval
+            if defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
+              Legion::Settings.dig(:knowledge, :actors, :maintenance_interval) || 21_600
+            else
+              21_600
+            end
+          rescue StandardError
+            21_600
+          end
+
+          def enabled?
+            return false unless corpus_path && !corpus_path.empty?
+
+            true
+          rescue StandardError
+            false
+          end
+
+          def args
+            { path: corpus_path }
+          end
+
+          private
+
+          def corpus_path
+            return nil unless defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
+
+            Legion::Settings.dig(:knowledge, :corpus_path)
+          rescue StandardError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/client.rb

@@ -7,6 +7,7 @@ module Legion
         include Runners::Ingest
         include Runners::Query
         include Runners::Corpus
+        include Runners::Maintenance
       end
     end
   end

data/lib/legion/extensions/knowledge/runners/maintenance.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Runners
+        module Maintenance
+          module_function
+
+          def detect_orphans(path:)
+            manifest_files = load_manifest_files(path)
+            apollo_files   = load_apollo_source_files
+
+            orphan_files = apollo_files - manifest_files
+
+            {
+              success:              true,
+              orphan_count:         orphan_files.size,
+              orphan_files:         orphan_files,
+              total_apollo_chunks:  count_apollo_chunks,
+              total_manifest_files: manifest_files.size
+            }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def cleanup_orphans(path:, dry_run: true)
+            detection = detect_orphans(path: path)
+            return detection unless detection[:success]
+            return detection.merge(archived: 0, files_cleaned: 0, dry_run: dry_run) if detection[:orphan_count].zero?
+            return detection.merge(archived: detection[:orphan_count], files_cleaned: detection[:orphan_files].size, dry_run: true) if dry_run
+
+            archived = archive_orphan_entries(detection[:orphan_files])
+
+            { success: true, archived: archived, files_cleaned: detection[:orphan_files].size, dry_run: false }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def reindex(path:)
+            store_path = Helpers::ManifestStore.store_path(corpus_path: path)
+            ::FileUtils.rm_f(store_path)
+
+            Runners::Ingest.ingest_corpus(path: path, force: true)
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def health(path:)
+            scan_entries  = Helpers::Manifest.scan(path: path)
+            store_path    = Helpers::ManifestStore.store_path(corpus_path: path)
+            manifest_file = ::File.exist?(store_path)
+            last_ingest   = manifest_file ? ::File.mtime(store_path).iso8601 : nil
+
+            {
+              success: true,
+              local:   build_local_stats(path, scan_entries, manifest_file, last_ingest),
+              apollo:  build_apollo_stats,
+              sync:    build_sync_stats(path, scan_entries)
+            }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def quality_report(limit: nil)
+            resolved_limit = limit || settings_quality_limit
+
+            {
+              success:        true,
+              hot_chunks:     hot_chunks(resolved_limit),
+              cold_chunks:    cold_chunks(resolved_limit),
+              low_confidence: low_confidence_chunks(resolved_limit),
+              poor_retrieval: [],
+              summary:        quality_summary
+            }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def build_local_stats(path, scan_entries, manifest_file, last_ingest)
+            {
+              corpus_path:     path,
+              file_count:      scan_entries.size,
+              total_bytes:     scan_entries.sum { |e| e[:size] },
+              manifest_exists: manifest_file,
+              last_ingest:     last_ingest
+            }
+          end
+          private_class_method :build_local_stats
+
+          def build_apollo_stats
+            return apollo_defaults unless defined?(Legion::Data::Model::ApolloEntry)
+
+            base  = Legion::Data::Model::ApolloEntry
+                    .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+                    .exclude(status: 'archived')
+            total = base.count
+            return apollo_defaults if total.zero?
+
+            rows = base.select(:confidence, :status, :access_count, :embedding, :created_at).all
+            apollo_stats_from_rows(base, rows, total)
+          rescue StandardError
+            apollo_defaults
+          end
+          private_class_method :build_apollo_stats
+
+          def apollo_stats_from_rows(base, rows, total)
+            confidences     = rows.map { |r| r[:confidence].to_f }
+            with_embeddings = rows.count { |r| !r[:embedding].nil? }
+            stale_threshold = settings_stale_threshold
+            timestamps      = rows.map { |r| r[:created_at] }
+
+            {
+              total_chunks:        total,
+              by_status:           base.group_and_count(:status).as_hash(:status, :count).transform_keys(&:to_sym),
+              embedding_coverage:  (with_embeddings.to_f / total).round(4),
+              avg_confidence:      confidences.sum / confidences.size.to_f,
+              confidence_range:    confidences.minmax,
+              stale_count:         confidences.count { |c| c < stale_threshold },
+              never_accessed:      rows.count { |r| r[:access_count].to_i.zero? },
+              unique_source_files: load_apollo_source_files.size,
+              oldest_chunk:        timestamps.min&.iso8601,
+              newest_chunk:        timestamps.max&.iso8601
+            }
+          end
+          private_class_method :apollo_stats_from_rows
+
+          def apollo_defaults
+            {
+              total_chunks:        0,
+              by_status:           {},
+              embedding_coverage:  0.0,
+              avg_confidence:      0.0,
+              confidence_range:    [0.0, 0.0],
+              stale_count:         0,
+              never_accessed:      0,
+              unique_source_files: 0,
+              oldest_chunk:        nil,
+              newest_chunk:        nil
+            }
+          end
+          private_class_method :apollo_defaults
+
+          def build_sync_stats(path, scan_entries)
+            manifest_paths = load_manifest_files(path)
+            apollo_paths   = load_apollo_source_files
+            scan_paths     = scan_entries.map { |e| e[:path] }
+
+            {
+              orphan_count:  (apollo_paths - manifest_paths).size,
+              missing_count: (scan_paths - apollo_paths).size
+            }
+          end
+          private_class_method :build_sync_stats
+
+          def load_manifest_files(path)
+            manifest = Helpers::ManifestStore.load(corpus_path: path)
+            manifest.map { |e| e[:path] }.compact.uniq
+          end
+          private_class_method :load_manifest_files
+
+          def load_apollo_source_files
+            return [] unless defined?(Legion::Data::Model::ApolloEntry)
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .exclude(status: 'archived')
+              .select_map(Sequel.lit("source_context->>'source_file'"))
+              .compact
+              .uniq
+          rescue StandardError
+            []
+          end
+          private_class_method :load_apollo_source_files
+
+          def count_apollo_chunks
+            return 0 unless defined?(Legion::Data::Model::ApolloEntry)
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .exclude(status: 'archived')
+              .count
+          rescue StandardError
+            0
+          end
+          private_class_method :count_apollo_chunks
+
+          def archive_orphan_entries(orphan_files)
+            return 0 unless defined?(Legion::Data::Model::ApolloEntry)
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .where(Sequel.lit("source_context->>'source_file' IN ?", orphan_files))
+              .exclude(status: 'archived')
+              .update(status: 'archived', updated_at: Time.now)
+          end
+          private_class_method :archive_orphan_entries
+
+          def hot_chunks(limit)
+            return [] unless defined?(Legion::Data::Model::ApolloEntry)
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .exclude(status: 'archived')
+              .where { access_count.positive? }
+              .order(Sequel.desc(:access_count))
+              .limit(limit)
+              .select_map([:id, :access_count, :confidence,
+                           Sequel.lit("source_context->>'source_file' AS source_file")])
+              .map { |r| { id: r[0], access_count: r[1], confidence: r[2], source_file: r[3] } }
+          rescue StandardError
+            []
+          end
+          private_class_method :hot_chunks
+
+          def cold_chunks(limit)
+            return [] unless defined?(Legion::Data::Model::ApolloEntry)
+
+            days   = settings_cold_chunk_days
+            cutoff = Time.now - (days * 86_400)
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .exclude(status: 'archived')
+              .where(access_count: 0)
+              .where { created_at < cutoff }
+              .order(:created_at)
+              .limit(limit)
+              .select_map([:id, :confidence, :created_at,
+                           Sequel.lit("source_context->>'source_file' AS source_file")])
+              .map { |r| { id: r[0], confidence: r[1], created_at: r[2]&.iso8601, source_file: r[3] } }
+          rescue StandardError
+            []
+          end
+          private_class_method :cold_chunks
+
+          def low_confidence_chunks(limit)
+            return [] unless defined?(Legion::Data::Model::ApolloEntry)
+
+            threshold = settings_stale_threshold
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .exclude(status: 'archived')
+              .where { confidence < threshold }
+              .order(:confidence)
+              .limit(limit)
+              .select_map([:id, :confidence, :access_count,
+                           Sequel.lit("source_context->>'source_file' AS source_file")])
+              .map { |r| { id: r[0], confidence: r[1], access_count: r[2], source_file: r[3] } }
+          rescue StandardError
+            []
+          end
+          private_class_method :low_confidence_chunks
+
+          def quality_summary
+            defaults = { total_queries: 0, avg_retrieval_score: nil, chunks_never_accessed: 0,
+                         chunks_below_threshold: 0 }
+            return defaults unless defined?(Legion::Data::Model::ApolloEntry)
+
+            base = Legion::Data::Model::ApolloEntry
+                   .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+                   .exclude(status: 'archived')
+
+            {
+              total_queries:          query_count,
+              avg_retrieval_score:    nil,
+              chunks_never_accessed:  base.where(access_count: 0).count,
+              chunks_below_threshold: base.where { confidence < settings_stale_threshold }.count
+            }
+          rescue StandardError
+            defaults
+          end
+          private_class_method :quality_summary
+
+          def query_count
+            return 0 unless defined?(Legion::Data::Model::ApolloAccessLog)
+
+            Legion::Data::Model::ApolloAccessLog.where(action: 'knowledge_query').count
+          rescue StandardError
+            0
+          end
+          private_class_method :query_count
+
+          def settings_stale_threshold
+            return 0.3 unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:knowledge, :maintenance, :stale_threshold) || 0.3
+          rescue StandardError
+            0.3
+          end
+          private_class_method :settings_stale_threshold
+
+          def settings_cold_chunk_days
+            return 7 unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:knowledge, :maintenance, :cold_chunk_days) || 7
+          rescue StandardError
+            7
+          end
+          private_class_method :settings_cold_chunk_days
+
+          def settings_quality_limit
+            return 10 unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:knowledge, :maintenance, :quality_report_limit) || 10
+          rescue StandardError
+            10
+          end
+          private_class_method :settings_quality_limit
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/runners/query.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'digest'
+
 module Legion
   module Extensions
     module Knowledge
@@ -17,15 +19,21 @@ module Legion
 
             latency_ms = ((::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - started) * 1000).round
 
+            score = average_score(chunks)
+            unless chunks.empty?
+              record_feedback(
+                question:        question,
+                chunk_ids:       chunks.filter_map { |c| c[:id] },
+                retrieval_score: score.to_f,
+                synthesized:     synthesize && llm_available?
+              )
+            end
+
             {
               success:  true,
               answer:   answer,
               sources:  chunks.map { |c| format_source(c) },
-              metadata: {
-                retrieval_score: average_score(chunks),
-                chunk_count:     chunks.size,
-                latency_ms:      latency_ms
-              }
+              metadata: build_metadata(chunks, score, latency_ms)
             }
           rescue StandardError => e
             { success: false, error: e.message }
@@ -38,14 +46,26 @@ module Legion
             {
               success:  true,
               sources:  chunks.map { |c| format_source(c) },
-              metadata: {
-                chunk_count: chunks.size
-              }
+              metadata: build_metadata(chunks, average_score(chunks))
             }
           rescue StandardError => e
             { success: false, error: e.message }
           end
 
+          def record_feedback(question:, chunk_ids:, retrieval_score:, synthesized: true, rating: nil)
+            question_hash = ::Digest::SHA256.hexdigest(question.to_s)[0, 16]
+            emit_feedback_event(
+              question_hash:   question_hash,
+              chunk_ids:       chunk_ids,
+              retrieval_score: retrieval_score,
+              synthesized:     synthesized,
+              rating:          rating
+            )
+            { success: true, question_hash: question_hash, rating: rating }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
           def retrieve_chunks(question, top_k)
             return [] unless defined?(Legion::Extensions::Apollo)
 
@@ -97,6 +117,53 @@ module Legion
           end
           private_class_method :average_score
 
+          def build_metadata(chunks, score, latency_ms = nil)
+            confidences = chunks.filter_map { |c| c[:confidence] }
+            distances   = chunks.filter_map { |c| c[:distance] }
+            source_names = chunks.filter_map do |c|
+              c.dig(:metadata, :source_file) || c[:source_file]
+            end.uniq
+            statuses = chunks.group_by { |c| c[:status] }.transform_values(&:size)
+
+            meta = {
+              retrieval_score:   score,
+              chunk_count:       chunks.size,
+              confidence_avg:    confidences.empty? ? nil : (confidences.sum.to_f / confidences.size).round(4),
+              confidence_range:  confidences.empty? ? nil : confidences.minmax,
+              distance_range:    distances.empty?   ? nil : distances.minmax,
+              source_files:      source_names,
+              source_file_count: source_names.size,
+              all_embedded:      chunks.none? { |c| zero_embedding?(c) },
+              statuses:          statuses
+            }
+            meta[:latency_ms] = latency_ms unless latency_ms.nil?
+            meta
+          end
+          private_class_method :build_metadata
+
+          def zero_embedding?(chunk)
+            emb = chunk[:embedding]
+            return true if emb.nil?
+
+            emb.is_a?(Array) && (emb.empty? || emb.all?(&:zero?))
+          end
+          private_class_method :zero_embedding?
+
+          def emit_feedback_event(question_hash:, chunk_ids:, retrieval_score:, synthesized:, rating:)
+            return unless defined?(Legion::Events)
+
+            Legion::Events.emit('knowledge.query_feedback', {
+                                  question_hash:   question_hash,
+                                  chunk_ids:       chunk_ids,
+                                  retrieval_score: retrieval_score,
+                                  synthesized:     synthesized,
+                                  rating:          rating
+                                })
+          rescue StandardError
+            nil
+          end
+          private_class_method :emit_feedback_event
+
           def llm_available?
             defined?(Legion::LLM)
           end

data/lib/legion/extensions/knowledge/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Knowledge
-      VERSION = '0.4.0'
+      VERSION = '0.5.0'
     end
   end
 end

data/lib/legion/extensions/knowledge.rb

@@ -8,6 +8,7 @@ require_relative 'knowledge/helpers/chunker'
 require_relative 'knowledge/runners/ingest'
 require_relative 'knowledge/runners/query'
 require_relative 'knowledge/runners/corpus'
+require_relative 'knowledge/runners/maintenance'
 require_relative 'knowledge/client'
 
 if defined?(Legion::Transport)
@@ -17,6 +18,7 @@ if defined?(Legion::Transport)
 end
 
 require_relative 'knowledge/actors/corpus_watcher' if defined?(Legion::Extensions::Actors::Every)
+require_relative 'knowledge/actors/maintenance_runner' if defined?(Legion::Extensions::Actors::Every)
 
 require_relative 'knowledge/actors/corpus_ingest' if defined?(Legion::Extensions::Actors::Subscription)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-knowledge
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Matthew Iverson
@@ -118,6 +118,7 @@ files:
 - lib/legion/extensions/knowledge.rb
 - lib/legion/extensions/knowledge/actors/corpus_ingest.rb
 - lib/legion/extensions/knowledge/actors/corpus_watcher.rb
+- lib/legion/extensions/knowledge/actors/maintenance_runner.rb
 - lib/legion/extensions/knowledge/client.rb
 - lib/legion/extensions/knowledge/helpers/chunker.rb
 - lib/legion/extensions/knowledge/helpers/manifest.rb
@@ -125,6 +126,7 @@ files:
 - lib/legion/extensions/knowledge/helpers/parser.rb
 - lib/legion/extensions/knowledge/runners/corpus.rb
 - lib/legion/extensions/knowledge/runners/ingest.rb
+- lib/legion/extensions/knowledge/runners/maintenance.rb
 - lib/legion/extensions/knowledge/runners/query.rb
 - lib/legion/extensions/knowledge/transport/exchanges/knowledge.rb
 - lib/legion/extensions/knowledge/transport/messages/ingest_message.rb