lex-knowledge 0.2.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a5307a5b8c19abaedd5f7d6be95d0fc3d068fa0b6d7366293c70da0edec7825
-  data.tar.gz: 756114f38b345f356a826e09c50714d8b14ff2a4baa6ef0087ddab9f2d75ff78
+  metadata.gz: 8d512db91b31e6d8a9747a3987aa90dd406d04f8987400787f6de27b13eb10c5
+  data.tar.gz: 2d9df5e1289bb80f603dbd882863d1802bf942e8ff12940d259ba44e25b728f3
 SHA512:
-  metadata.gz: 6d77ae8947c2ac53af380a35935c33cb075f4ae1a7b8c2b495389701fb15652a7baaefeeddd32606bc81d9e4c6f8f319562fe869906523b34ca4563ce0e245c4
-  data.tar.gz: a30c57db2c8cb0da0d54ac73afc178ef55892123e4a6878aa1bf1ca62014663c508c699857970f5582a0e0361a5fd3c29349dbe913a6d1ae8a19aed78515fcc0
+  metadata.gz: 2022e4654a3a815e8c5433daaad7c2e9767b039c51513e12d9cd6c71a2de47e3ee883c6e0624aa9ac8f6108439c19d4552260694482db4e2da9d58851fa093bd
+  data.tar.gz: d1afc8cb8fdcd0317ee9fa416c91827ca3536e40fed404791a355b4b4662360211e3c064dc09e8504d0ed5eb359f1dc906edaa57277f205b1aa9eafedcf261da

data/lib/legion/extensions/knowledge/actors/maintenance_runner.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Actor
+        class MaintenanceRunner < Legion::Extensions::Actors::Every
+          def runner_class    = 'Legion::Extensions::Knowledge::Runners::Maintenance'
+          def runner_function = 'health'
+          def check_subtask?  = false
+          def generate_task?  = false
+
+          def every_interval
+            if defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
+              Legion::Settings.dig(:knowledge, :actors, :maintenance_interval) || 21_600
+            else
+              21_600
+            end
+          rescue StandardError
+            21_600
+          end
+
+          def enabled?
+            return false unless corpus_path && !corpus_path.empty?
+
+            true
+          rescue StandardError
+            false
+          end
+
+          def args
+            { path: corpus_path }
+          end
+
+          private
+
+          def corpus_path
+            return nil unless defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
+
+            Legion::Settings.dig(:knowledge, :corpus_path)
+          rescue StandardError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/client.rb

@@ -7,6 +7,7 @@ module Legion
         include Runners::Ingest
         include Runners::Query
         include Runners::Corpus
+        include Runners::Maintenance
       end
     end
   end

data/lib/legion/extensions/knowledge/helpers/manifest_store.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'fileutils'
+require 'json'
+require 'tempfile'
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Helpers
+        module ManifestStore
+          module_function
+
+          STORE_DIR = ::File.expand_path('~/.legionio/knowledge').freeze
+
+          def load(corpus_path:)
+            path = store_path(corpus_path: corpus_path)
+            return [] unless ::File.exist?(path)
+
+            raw = ::File.read(path, encoding: 'utf-8')
+            ::JSON.parse(raw, symbolize_names: true)
+          rescue StandardError
+            []
+          end
+
+          def save(corpus_path:, manifest:)
+            ::FileUtils.mkdir_p(STORE_DIR)
+            path = store_path(corpus_path: corpus_path)
+            tmp  = "#{path}.tmp"
+            ::File.write(tmp, ::JSON.generate(manifest.map { |e| serialize_entry(e) }))
+            ::File.rename(tmp, path)
+            true
+          rescue StandardError
+            false
+          end
+
+          def store_path(corpus_path:)
+            hash = ::Digest::SHA256.hexdigest(corpus_path.to_s)[0, 16]
+            ::File.join(STORE_DIR, "#{hash}.manifest.json")
+          end
+
+          def serialize_entry(entry)
+            entry.merge(mtime: entry[:mtime].to_s)
+          end
+          private_class_method :serialize_entry
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/helpers/parser.rb

@@ -15,6 +15,8 @@ module Legion
               parse_markdown(file_path: file_path)
             when '.txt'
               parse_text(file_path: file_path)
+            when '.pdf', '.docx'
+              extract_via_data(file_path: file_path)
             else
               [{ error: 'unsupported format', source_file: file_path }]
             end
@@ -22,32 +24,42 @@ module Legion
 
           def parse_markdown(file_path:)
             content = ::File.read(file_path, encoding: 'utf-8')
-            sections = []
+            sections        = []
             current_heading = ::File.basename(file_path, '.*')
             current_lines   = []
-            section_path    = []
+            heading_stack   = {}
 
             content.each_line do |line|
-              if line.start_with?('# ')
-                flush_section(sections, current_heading, section_path, current_lines, file_path) unless current_lines.empty?
-                current_heading = line.sub(/^#+\s*/, '').chomp
-                section_path    = [current_heading]
-                current_lines   = []
-              elsif line.start_with?('## ')
-                flush_section(sections, current_heading, section_path, current_lines, file_path) unless current_lines.empty?
-                current_heading = line.sub(/^#+\s*/, '').chomp
-                section_path    = section_path.first(1) + [current_heading]
+              level = heading_level(line)
+              if level
+                flush_section(sections, current_heading, build_section_path(heading_stack), current_lines, file_path)
+                title = line.sub(/^#+\s*/, '').chomp
+                heading_stack.delete_if { |d, _| d >= level }
+                heading_stack[level] = title
+                current_heading = title
                 current_lines   = []
               else
                 current_lines << line
               end
             end
 
-            flush_section(sections, current_heading, section_path, current_lines, file_path) unless current_lines.empty?
+            flush_section(sections, current_heading, build_section_path(heading_stack), current_lines, file_path)
 
             sections.empty? ? [{ heading: ::File.basename(file_path, '.*'), section_path: [], content: content.strip, source_file: file_path }] : sections
           end
 
+          def extract_via_data(file_path:)
+            return [{ error: 'unsupported format', source_file: file_path }] unless defined?(::Legion::Data::Extract)
+
+            result = ::Legion::Data::Extract.extract(file_path, type: :auto)
+            return [{ error: 'extraction_failed', source_file: file_path, detail: result }] unless result.is_a?(Hash) && result[:text]
+
+            heading = ::File.basename(file_path, '.*')
+            [{ heading: heading, section_path: [], content: result[:text].strip, source_file: file_path }]
+          rescue StandardError => e
+            [{ error: 'extraction_failed', source_file: file_path, detail: e.message }]
+          end
+
           def parse_text(file_path:)
             content = ::File.read(file_path, encoding: 'utf-8')
             heading = ::File.basename(file_path, '.*')
@@ -67,6 +79,17 @@ module Legion
             }
           end
           private_class_method :flush_section
+
+          def heading_level(line)
+            m = line.match(/^(\#{1,6})\s/)
+            m ? m[1].length : nil
+          end
+          private_class_method :heading_level
+
+          def build_section_path(stack)
+            stack.sort.map { |_, title| title }
+          end
+          private_class_method :build_section_path
         end
       end
     end

data/lib/legion/extensions/knowledge/runners/corpus.rb

@@ -7,6 +7,10 @@ module Legion
         module Corpus
           module_function
 
+          def manifest_path(path:)
+            Helpers::ManifestStore.store_path(corpus_path: path)
+          end
+
           def corpus_stats(path:, extensions: nil)
             return { success: false, error: 'path does not exist' } unless ::File.exist?(path)
 

data/lib/legion/extensions/knowledge/runners/ingest.rb

@@ -23,23 +23,32 @@ module Legion
           end
 
           def ingest_corpus(path:, dry_run: false, force: false)
-            entries = Helpers::Manifest.scan(path: path)
-
-            files_scanned   = entries.size
-            chunks_created  = 0
-            chunks_skipped  = 0
-            chunks_updated  = 0
-
-            entries.each do |entry|
-              result = process_file(entry[:path], dry_run: dry_run, force: force)
-              chunks_created  += result[:created]
-              chunks_skipped  += result[:skipped]
-              chunks_updated  += result[:updated]
+            current  = Helpers::Manifest.scan(path: path)
+            previous = force ? [] : Helpers::ManifestStore.load(corpus_path: path)
+            delta    = Helpers::Manifest.diff(current: current, previous: previous)
+
+            to_process     = delta[:added] + delta[:changed]
+            chunks_created = 0
+            chunks_skipped = 0
+            chunks_updated = 0
+
+            to_process.each do |file_path|
+              result = process_file(file_path, dry_run: dry_run, force: force)
+              chunks_created += result[:created]
+              chunks_skipped += result[:skipped]
+              chunks_updated += result[:updated]
             end
 
+            delta[:removed].each { |file_path| retire_file(file_path: file_path) } unless dry_run
+
+            Helpers::ManifestStore.save(corpus_path: path, manifest: current) unless dry_run
+
             {
               success:        true,
-              files_scanned:  files_scanned,
+              files_scanned:  current.size,
+              files_added:    delta[:added].size,
+              files_changed:  delta[:changed].size,
+              files_removed:  delta[:removed].size,
               chunks_created: chunks_created,
               chunks_skipped: chunks_skipped,
               chunks_updated: chunks_updated
@@ -67,12 +76,18 @@ module Legion
             return { created: 0, skipped: 0, updated: 0 } if sections.first&.key?(:error)
 
             chunks  = Helpers::Chunker.chunk(sections: sections)
+            paired  = if dry_run
+                        chunks.map { |c| { chunk: c, embedding: nil } }
+                      else
+                        batch_embed_chunks(chunks, force: force)
+                      end
+
             created = 0
             skipped = 0
             updated = 0
 
-            chunks.each do |chunk|
-              outcome = upsert_chunk(chunk, dry_run: dry_run, force: force)
+            paired.each do |p|
+              outcome = upsert_chunk_with_embedding(p[:chunk], p[:embedding], dry_run: dry_run, force: force, exists: p[:exists] || false)
               case outcome
               when :created then created += 1
               when :skipped then skipped += 1
@@ -84,21 +99,55 @@ module Legion
           end
           private_class_method :process_file
 
-          def upsert_chunk(chunk, dry_run: false, force: false)
-            return :created if dry_run
+          def batch_embed_chunks(chunks, force:)
+            exists_map = force ? {} : build_exists_map(chunks)
+            return paired_without_embed(chunks, exists_map) unless llm_embed_available?
 
-            return :created unless defined?(Legion::Extensions::Apollo)
+            needs_embed = force ? chunks : chunks.reject { |c| exists_map[c[:content_hash]] }
+            embed_map   = needs_embed.empty? ? {} : build_embed_map(needs_embed)
 
-            return :skipped if !force && chunk_exists?(chunk[:content_hash])
+            chunks.map { |c| { chunk: c, embedding: embed_map[c[:content_hash]], exists: exists_map.fetch(c[:content_hash], false) } }
+          rescue StandardError
+            paired_without_embed(chunks, {})
+          end
+          private_class_method :batch_embed_chunks
 
-            embedding = generate_embedding(chunk[:content])
-            ingest_to_apollo(chunk, embedding)
+          def build_exists_map(chunks)
+            chunks.to_h { |c| [c[:content_hash], chunk_exists?(c[:content_hash])] }
+          end
+          private_class_method :build_exists_map
+
+          def llm_embed_available?
+            defined?(Legion::LLM) && Legion::LLM.respond_to?(:embed_batch)
+          end
+          private_class_method :llm_embed_available?
+
+          def paired_without_embed(chunks, exists_map)
+            chunks.map { |c| { chunk: c, embedding: nil, exists: exists_map.fetch(c[:content_hash], false) } }
+          end
+          private_class_method :paired_without_embed
 
+          def build_embed_map(needs_embed)
+            results = Legion::LLM.embed_batch(needs_embed.map { |c| c[:content] })
+            results.each_with_object({}) do |r, h|
+              h[needs_embed[r[:index]][:content_hash]] = r[:vector] unless r[:error]
+            end
+          rescue StandardError
+            {}
+          end
+          private_class_method :build_embed_map
+
+          def upsert_chunk_with_embedding(chunk, embedding, dry_run: false, force: false, exists: false)
+            return :created if dry_run
+            return :created unless defined?(Legion::Extensions::Apollo)
+            return :skipped if !force && exists
+
+            ingest_to_apollo(chunk, embedding)
             force ? :updated : :created
           rescue StandardError
             :skipped
           end
-          private_class_method :upsert_chunk
+          private_class_method :upsert_chunk_with_embedding
 
           def chunk_exists?(content_hash)
             return false unless defined?(Legion::Data::Model::ApolloEntry)
@@ -112,16 +161,6 @@ module Legion
           end
           private_class_method :chunk_exists?
 
-          def generate_embedding(content)
-            return nil unless defined?(Legion::LLM) && Legion::LLM.respond_to?(:embed)
-
-            result = Legion::LLM.embed(content)
-            result.is_a?(Hash) ? result[:vector] : nil
-          rescue StandardError
-            nil
-          end
-          private_class_method :generate_embedding
-
           def ingest_to_apollo(chunk, embedding)
             return unless defined?(Legion::Extensions::Apollo)
 
@@ -143,6 +182,21 @@ module Legion
             Legion::Extensions::Apollo::Runners::Knowledge.handle_ingest(**payload)
           end
           private_class_method :ingest_to_apollo
+
+          def retire_file(file_path:)
+            return unless defined?(Legion::Apollo)
+            return unless Legion::Apollo.respond_to?(:ingest) && Legion::Apollo.started?
+
+            Legion::Apollo.ingest(
+              content:      file_path,
+              content_type: 'document_retired',
+              tags:         [file_path, 'retired', 'document_chunk'].uniq,
+              metadata:     { source_file: file_path, retired: true }
+            )
+          rescue StandardError
+            nil
+          end
+          private_class_method :retire_file
         end
       end
     end

data/lib/legion/extensions/knowledge/runners/maintenance.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Runners
+        module Maintenance
+          module_function
+
+          def detect_orphans(path:)
+            manifest_files = load_manifest_files(path)
+            apollo_files   = load_apollo_source_files
+
+            orphan_files = apollo_files - manifest_files
+
+            {
+              success:              true,
+              orphan_count:         orphan_files.size,
+              orphan_files:         orphan_files,
+              total_apollo_chunks:  count_apollo_chunks,
+              total_manifest_files: manifest_files.size
+            }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def cleanup_orphans(path:, dry_run: true)
+            detection = detect_orphans(path: path)
+            return detection unless detection[:success]
+            return detection.merge(archived: 0, files_cleaned: 0, dry_run: dry_run) if detection[:orphan_count].zero?
+            return detection.merge(archived: detection[:orphan_count], files_cleaned: detection[:orphan_files].size, dry_run: true) if dry_run
+
+            archived = archive_orphan_entries(detection[:orphan_files])
+
+            { success: true, archived: archived, files_cleaned: detection[:orphan_files].size, dry_run: false }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def reindex(path:)
+            store_path = Helpers::ManifestStore.store_path(corpus_path: path)
+            ::FileUtils.rm_f(store_path)
+
+            Runners::Ingest.ingest_corpus(path: path, force: true)
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def health(path:)
+            scan_entries  = Helpers::Manifest.scan(path: path)
+            store_path    = Helpers::ManifestStore.store_path(corpus_path: path)
+            manifest_file = ::File.exist?(store_path)
+            last_ingest   = manifest_file ? ::File.mtime(store_path).iso8601 : nil
+
+            {
+              success: true,
+              local:   build_local_stats(path, scan_entries, manifest_file, last_ingest),
+              apollo:  build_apollo_stats,
+              sync:    build_sync_stats(path, scan_entries)
+            }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def quality_report(limit: nil)
+            resolved_limit = limit || settings_quality_limit
+
+            {
+              success:        true,
+              hot_chunks:     hot_chunks(resolved_limit),
+              cold_chunks:    cold_chunks(resolved_limit),
+              low_confidence: low_confidence_chunks(resolved_limit),
+              poor_retrieval: [],
+              summary:        quality_summary
+            }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def build_local_stats(path, scan_entries, manifest_file, last_ingest)
+            {
+              corpus_path:     path,
+              file_count:      scan_entries.size,
+              total_bytes:     scan_entries.sum { |e| e[:size] },
+              manifest_exists: manifest_file,
+              last_ingest:     last_ingest
+            }
+          end
+          private_class_method :build_local_stats
+
+          def build_apollo_stats
+            return apollo_defaults unless defined?(Legion::Data::Model::ApolloEntry)
+
+            base  = Legion::Data::Model::ApolloEntry
+                    .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+                    .exclude(status: 'archived')
+            total = base.count
+            return apollo_defaults if total.zero?
+
+            rows = base.select(:confidence, :status, :access_count, :embedding, :created_at).all
+            apollo_stats_from_rows(base, rows, total)
+          rescue StandardError
+            apollo_defaults
+          end
+          private_class_method :build_apollo_stats
+
+          def apollo_stats_from_rows(base, rows, total)
+            confidences     = rows.map { |r| r[:confidence].to_f }
+            with_embeddings = rows.count { |r| !r[:embedding].nil? }
+            stale_threshold = settings_stale_threshold
+            timestamps      = rows.map { |r| r[:created_at] }
+
+            {
+              total_chunks:        total,
+              by_status:           base.group_and_count(:status).as_hash(:status, :count).transform_keys(&:to_sym),
+              embedding_coverage:  (with_embeddings.to_f / total).round(4),
+              avg_confidence:      confidences.sum / confidences.size.to_f,
+              confidence_range:    confidences.minmax,
+              stale_count:         confidences.count { |c| c < stale_threshold },
+              never_accessed:      rows.count { |r| r[:access_count].to_i.zero? },
+              unique_source_files: load_apollo_source_files.size,
+              oldest_chunk:        timestamps.min&.iso8601,
+              newest_chunk:        timestamps.max&.iso8601
+            }
+          end
+          private_class_method :apollo_stats_from_rows
+
+          def apollo_defaults
+            {
+              total_chunks:        0,
+              by_status:           {},
+              embedding_coverage:  0.0,
+              avg_confidence:      0.0,
+              confidence_range:    [0.0, 0.0],
+              stale_count:         0,
+              never_accessed:      0,
+              unique_source_files: 0,
+              oldest_chunk:        nil,
+              newest_chunk:        nil
+            }
+          end
+          private_class_method :apollo_defaults
+
+          def build_sync_stats(path, scan_entries)
+            manifest_paths = load_manifest_files(path)
+            apollo_paths   = load_apollo_source_files
+            scan_paths     = scan_entries.map { |e| e[:path] }
+
+            {
+              orphan_count:  (apollo_paths - manifest_paths).size,
+              missing_count: (scan_paths - apollo_paths).size
+            }
+          end
+          private_class_method :build_sync_stats
+
+          def load_manifest_files(path)
+            manifest = Helpers::ManifestStore.load(corpus_path: path)
+            manifest.map { |e| e[:path] }.compact.uniq
+          end
+          private_class_method :load_manifest_files
+
+          def load_apollo_source_files
+            return [] unless defined?(Legion::Data::Model::ApolloEntry)
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .exclude(status: 'archived')
+              .select_map(Sequel.lit("source_context->>'source_file'"))
+              .compact
+              .uniq
+          rescue StandardError
+            []
+          end
+          private_class_method :load_apollo_source_files
+
+          def count_apollo_chunks
+            return 0 unless defined?(Legion::Data::Model::ApolloEntry)
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .exclude(status: 'archived')
+              .count
+          rescue StandardError
+            0
+          end
+          private_class_method :count_apollo_chunks
+
+          def archive_orphan_entries(orphan_files)
+            return 0 unless defined?(Legion::Data::Model::ApolloEntry)
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .where(Sequel.lit("source_context->>'source_file' IN ?", orphan_files))
+              .exclude(status: 'archived')
+              .update(status: 'archived', updated_at: Time.now)
+          end
+          private_class_method :archive_orphan_entries
+
+          def hot_chunks(limit)
+            return [] unless defined?(Legion::Data::Model::ApolloEntry)
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .exclude(status: 'archived')
+              .where { access_count.positive? }
+              .order(Sequel.desc(:access_count))
+              .limit(limit)
+              .select_map([:id, :access_count, :confidence,
+                           Sequel.lit("source_context->>'source_file' AS source_file")])
+              .map { |r| { id: r[0], access_count: r[1], confidence: r[2], source_file: r[3] } }
+          rescue StandardError
+            []
+          end
+          private_class_method :hot_chunks
+
+          def cold_chunks(limit)
+            return [] unless defined?(Legion::Data::Model::ApolloEntry)
+
+            days   = settings_cold_chunk_days
+            cutoff = Time.now - (days * 86_400)
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .exclude(status: 'archived')
+              .where(access_count: 0)
+              .where { created_at < cutoff }
+              .order(:created_at)
+              .limit(limit)
+              .select_map([:id, :confidence, :created_at,
+                           Sequel.lit("source_context->>'source_file' AS source_file")])
+              .map { |r| { id: r[0], confidence: r[1], created_at: r[2]&.iso8601, source_file: r[3] } }
+          rescue StandardError
+            []
+          end
+          private_class_method :cold_chunks
+
+          def low_confidence_chunks(limit)
+            return [] unless defined?(Legion::Data::Model::ApolloEntry)
+
+            threshold = settings_stale_threshold
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .exclude(status: 'archived')
+              .where { confidence < threshold }
+              .order(:confidence)
+              .limit(limit)
+              .select_map([:id, :confidence, :access_count,
+                           Sequel.lit("source_context->>'source_file' AS source_file")])
+              .map { |r| { id: r[0], confidence: r[1], access_count: r[2], source_file: r[3] } }
+          rescue StandardError
+            []
+          end
+          private_class_method :low_confidence_chunks
+
+          def quality_summary
+            defaults = { total_queries: 0, avg_retrieval_score: nil, chunks_never_accessed: 0,
+                         chunks_below_threshold: 0 }
+            return defaults unless defined?(Legion::Data::Model::ApolloEntry)
+
+            base = Legion::Data::Model::ApolloEntry
+                   .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+                   .exclude(status: 'archived')
+
+            {
+              total_queries:          query_count,
+              avg_retrieval_score:    nil,
+              chunks_never_accessed:  base.where(access_count: 0).count,
+              chunks_below_threshold: base.where { confidence < settings_stale_threshold }.count
+            }
+          rescue StandardError
+            defaults
+          end
+          private_class_method :quality_summary
+
+          def query_count
+            return 0 unless defined?(Legion::Data::Model::ApolloAccessLog)
+
+            Legion::Data::Model::ApolloAccessLog.where(action: 'knowledge_query').count
+          rescue StandardError
+            0
+          end
+          private_class_method :query_count
+
+          def settings_stale_threshold
+            return 0.3 unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:knowledge, :maintenance, :stale_threshold) || 0.3
+          rescue StandardError
+            0.3
+          end
+          private_class_method :settings_stale_threshold
+
+          def settings_cold_chunk_days
+            return 7 unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:knowledge, :maintenance, :cold_chunk_days) || 7
+          rescue StandardError
+            7
+          end
+          private_class_method :settings_cold_chunk_days
+
+          def settings_quality_limit
+            return 10 unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:knowledge, :maintenance, :quality_report_limit) || 10
+          rescue StandardError
+            10
+          end
+          private_class_method :settings_quality_limit
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/runners/query.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'digest'
+
 module Legion
   module Extensions
     module Knowledge
@@ -17,15 +19,21 @@ module Legion
 
             latency_ms = ((::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - started) * 1000).round
 
+            score = average_score(chunks)
+            unless chunks.empty?
+              record_feedback(
+                question:        question,
+                chunk_ids:       chunks.filter_map { |c| c[:id] },
+                retrieval_score: score.to_f,
+                synthesized:     synthesize && llm_available?
+              )
+            end
+
             {
               success:  true,
               answer:   answer,
               sources:  chunks.map { |c| format_source(c) },
-              metadata: {
-                retrieval_score: average_score(chunks),
-                chunk_count:     chunks.size,
-                latency_ms:      latency_ms
-              }
+              metadata: build_metadata(chunks, score, latency_ms)
             }
           rescue StandardError => e
             { success: false, error: e.message }
@@ -38,14 +46,26 @@ module Legion
             {
               success:  true,
               sources:  chunks.map { |c| format_source(c) },
-              metadata: {
-                chunk_count: chunks.size
-              }
+              metadata: build_metadata(chunks, average_score(chunks))
             }
           rescue StandardError => e
             { success: false, error: e.message }
           end
 
+          def record_feedback(question:, chunk_ids:, retrieval_score:, synthesized: true, rating: nil)
+            question_hash = ::Digest::SHA256.hexdigest(question.to_s)[0, 16]
+            emit_feedback_event(
+              question_hash:   question_hash,
+              chunk_ids:       chunk_ids,
+              retrieval_score: retrieval_score,
+              synthesized:     synthesized,
+              rating:          rating
+            )
+            { success: true, question_hash: question_hash, rating: rating }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
           def retrieve_chunks(question, top_k)
             return [] unless defined?(Legion::Extensions::Apollo)
 
@@ -97,6 +117,53 @@ module Legion
           end
           private_class_method :average_score
 
+          def build_metadata(chunks, score, latency_ms = nil)
+            confidences = chunks.filter_map { |c| c[:confidence] }
+            distances   = chunks.filter_map { |c| c[:distance] }
+            source_names = chunks.filter_map do |c|
+              c.dig(:metadata, :source_file) || c[:source_file]
+            end.uniq
+            statuses = chunks.group_by { |c| c[:status] }.transform_values(&:size)
+
+            meta = {
+              retrieval_score:   score,
+              chunk_count:       chunks.size,
+              confidence_avg:    confidences.empty? ? nil : (confidences.sum.to_f / confidences.size).round(4),
+              confidence_range:  confidences.empty? ? nil : confidences.minmax,
+              distance_range:    distances.empty?   ? nil : distances.minmax,
+              source_files:      source_names,
+              source_file_count: source_names.size,
+              all_embedded:      chunks.none? { |c| zero_embedding?(c) },
+              statuses:          statuses
+            }
+            meta[:latency_ms] = latency_ms unless latency_ms.nil?
+            meta
+          end
+          private_class_method :build_metadata
+
+          def zero_embedding?(chunk)
+            emb = chunk[:embedding]
+            return true if emb.nil?
+
+            emb.is_a?(Array) && (emb.empty? || emb.all?(&:zero?))
+          end
+          private_class_method :zero_embedding?
+
+          def emit_feedback_event(question_hash:, chunk_ids:, retrieval_score:, synthesized:, rating:)
+            return unless defined?(Legion::Events)
+
+            Legion::Events.emit('knowledge.query_feedback', {
+                                  question_hash:   question_hash,
+                                  chunk_ids:       chunk_ids,
+                                  retrieval_score: retrieval_score,
+                                  synthesized:     synthesized,
+                                  rating:          rating
+                                })
+          rescue StandardError
+            nil
+          end
+          private_class_method :emit_feedback_event
+
           def llm_available?
             defined?(Legion::LLM)
           end

data/lib/legion/extensions/knowledge/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Knowledge
-      VERSION = '0.2.0'
+      VERSION = '0.5.0'
     end
   end
 end

data/lib/legion/extensions/knowledge.rb

@@ -2,11 +2,13 @@
 
 require_relative 'knowledge/version'
 require_relative 'knowledge/helpers/manifest'
+require_relative 'knowledge/helpers/manifest_store'
 require_relative 'knowledge/helpers/parser'
 require_relative 'knowledge/helpers/chunker'
 require_relative 'knowledge/runners/ingest'
 require_relative 'knowledge/runners/query'
 require_relative 'knowledge/runners/corpus'
+require_relative 'knowledge/runners/maintenance'
 require_relative 'knowledge/client'
 
 if defined?(Legion::Transport)
@@ -16,6 +18,7 @@ if defined?(Legion::Transport)
 end
 
 require_relative 'knowledge/actors/corpus_watcher' if defined?(Legion::Extensions::Actors::Every)
+require_relative 'knowledge/actors/maintenance_runner' if defined?(Legion::Extensions::Actors::Every)
 
 require_relative 'knowledge/actors/corpus_ingest' if defined?(Legion::Extensions::Actors::Subscription)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-knowledge
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Matthew Iverson
@@ -118,12 +118,15 @@ files:
 - lib/legion/extensions/knowledge.rb
 - lib/legion/extensions/knowledge/actors/corpus_ingest.rb
 - lib/legion/extensions/knowledge/actors/corpus_watcher.rb
+- lib/legion/extensions/knowledge/actors/maintenance_runner.rb
 - lib/legion/extensions/knowledge/client.rb
 - lib/legion/extensions/knowledge/helpers/chunker.rb
 - lib/legion/extensions/knowledge/helpers/manifest.rb
+- lib/legion/extensions/knowledge/helpers/manifest_store.rb
 - lib/legion/extensions/knowledge/helpers/parser.rb
 - lib/legion/extensions/knowledge/runners/corpus.rb
 - lib/legion/extensions/knowledge/runners/ingest.rb
+- lib/legion/extensions/knowledge/runners/maintenance.rb
 - lib/legion/extensions/knowledge/runners/query.rb
 - lib/legion/extensions/knowledge/transport/exchanges/knowledge.rb
 - lib/legion/extensions/knowledge/transport/messages/ingest_message.rb