lex-knowledge 0.5.0 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d512db91b31e6d8a9747a3987aa90dd406d04f8987400787f6de27b13eb10c5
-  data.tar.gz: 2d9df5e1289bb80f603dbd882863d1802bf942e8ff12940d259ba44e25b728f3
+  metadata.gz: 40412f676840f9927c0e66d7603678bbde1489dcb9a2faba958b1a684fd47b3d
+  data.tar.gz: f9247b163289a8e3e07deaf63efbab11703e153150e91d77745eca010cb49b21
 SHA512:
-  metadata.gz: 2022e4654a3a815e8c5433daaad7c2e9767b039c51513e12d9cd6c71a2de47e3ee883c6e0624aa9ac8f6108439c19d4552260694482db4e2da9d58851fa093bd
-  data.tar.gz: d1afc8cb8fdcd0317ee9fa416c91827ca3536e40fed404791a355b4b4662360211e3c064dc09e8504d0ed5eb359f1dc906edaa57277f205b1aa9eafedcf261da
+  metadata.gz: 83cdc62303978876ae5f04c50d3bcf5fcd08faddaa3ee49dc8ac092c49df9665419fb2c52452d7247357b5e424d6122111d82b9e772879bd2afd0068dc780918
+  data.tar.gz: 24f0a1208fc396058ae014c497ef5bc49aa96dfe185ad32d0628a19b8fee4ab1c8754048d77d06a73a4a3d42f7c2c1537e6653fecae2b2b280c4b38e6f169745

data/lib/legion/extensions/knowledge/actors/corpus_watcher.rb

@@ -10,34 +10,39 @@ module Legion
           def check_subtask?  = false
           def generate_task?  = false
 
-          def every_interval
+          def time
             if defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
               Legion::Settings.dig(:knowledge, :actors, :watcher_interval) || 300
             else
               300
             end
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             300
           end
 
           def enabled?
-            corpus_path && !corpus_path.empty?
-          rescue StandardError
+            resolve_monitors.any?
+          rescue StandardError => e
+            log.warn(e.message)
             false
           end
 
           def args
-            { path: corpus_path }
+            { monitors: resolve_monitors }
           end
 
           private
 
-          def corpus_path
-            return nil unless defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
+          def log
+            Legion::Logging
+          end
 
-            Legion::Settings.dig(:knowledge, :corpus_path)
-          rescue StandardError
-            nil
+          def resolve_monitors
+            Runners::Monitor.resolve_monitors
+          rescue StandardError => e
+            log.warn(e.message)
+            []
           end
         end
       end

data/lib/legion/extensions/knowledge/actors/maintenance_runner.rb

@@ -10,13 +10,14 @@ module Legion
           def check_subtask?  = false
           def generate_task?  = false
 
-          def every_interval
+          def time
             if defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
               Legion::Settings.dig(:knowledge, :actors, :maintenance_interval) || 21_600
             else
               21_600
             end
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             21_600
           end
 
@@ -24,7 +25,8 @@ module Legion
             return false unless corpus_path && !corpus_path.empty?
 
             true
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             false
           end
 
@@ -34,11 +36,16 @@ module Legion
 
           private
 
+          def log
+            Legion::Logging
+          end
+
           def corpus_path
             return nil unless defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
 
             Legion::Settings.dig(:knowledge, :corpus_path)
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             nil
           end
         end

data/lib/legion/extensions/knowledge/client.rb

@@ -8,6 +8,7 @@ module Legion
         include Runners::Query
         include Runners::Corpus
         include Runners::Maintenance
+        include Runners::Monitor
       end
     end
   end

data/lib/legion/extensions/knowledge/runners/ingest.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'securerandom'
+
 module Legion
   module Extensions
     module Knowledge
@@ -7,6 +9,11 @@ module Legion
         module Ingest
           module_function
 
+          def log
+            Legion::Logging
+          end
+          private_class_method :log
+
           def scan_corpus(path:, extensions: nil)
             opts = { path: path }
             opts[:extensions] = extensions if extensions
@@ -22,7 +29,17 @@ module Legion
             }
           end
 
-          def ingest_corpus(path:, dry_run: false, force: false)
+          def ingest_corpus(path: nil, monitors: nil, dry_run: false, force: false)
+            return ingest_monitors(monitors: monitors, dry_run: dry_run, force: force) if monitors&.any?
+            raise ArgumentError, 'path is required when monitors is not provided' if path.nil?
+
+            ingest_corpus_path(path: path, dry_run: dry_run, force: force)
+          rescue ArgumentError => e
+            log.warn(e.message)
+            { success: false, error: e.message }
+          end
+
+          def ingest_corpus_path(path:, dry_run: false, force: false)
             current  = Helpers::Manifest.scan(path: path)
             previous = force ? [] : Helpers::ManifestStore.load(corpus_path: path)
             delta    = Helpers::Manifest.diff(current: current, previous: previous)
@@ -54,8 +71,57 @@ module Legion
               chunks_updated: chunks_updated
             }
           rescue StandardError => e
+            log.warn(e.message)
+            { success: false, error: e.message }
+          end
+          private_class_method :ingest_corpus_path
+
+          def ingest_monitors(monitors:, dry_run: false, force: false)
+            results = monitors.map do |monitor|
+              ingest_corpus(path: monitor[:path], dry_run: dry_run, force: force)
+            rescue StandardError => e
+              log.warn(e.message)
+              { success: false, path: monitor[:path], error: e.message }
+            end
+
+            total = {
+              files_scanned:  0,
+              files_added:    0,
+              files_changed:  0,
+              files_removed:  0,
+              chunks_created: 0,
+              chunks_skipped: 0,
+              chunks_updated: 0
+            }
+            results.each do |r|
+              next unless r[:success]
+
+              total.each_key { |k| total[k] += r[k].to_i }
+            end
+
+            { success: true, monitors_processed: results.size, **total }
+          rescue StandardError => e
+            log.warn(e.message)
             { success: false, error: e.message }
           end
+          private_class_method :ingest_monitors
+
+          def ingest_content(content:, source_type: :text, metadata: {})
+            source_path = "content://#{source_type}/#{SecureRandom.uuid}"
+            section = {
+              content:      content,
+              heading:      source_type.to_s,
+              section_path: [source_type.to_s],
+              source_file:  source_path
+            }
+            chunks = Helpers::Chunker.chunk(sections: [section])
+            paired = batch_embed_chunks(chunks, force: false)
+            paired.each { |p| upsert_chunk_with_embedding(p[:chunk], p[:embedding], force: false, exists: p[:exists] || false) }
+            { status: :ingested, chunks: chunks.size, source_type: source_type, metadata: metadata }
+          rescue StandardError => e
+            log.warn(e.message)
+            { status: :failed, error: e.message, source_type: source_type, metadata: metadata }
+          end
 
           def ingest_file(file_path:, force: false)
             result = process_file(file_path, dry_run: false, force: force)
@@ -68,6 +134,7 @@ module Legion
               chunks_updated: result[:updated]
             }
           rescue StandardError => e
+            log.warn(e.message)
             { success: false, error: e.message }
           end
 
@@ -107,7 +174,8 @@ module Legion
             embed_map   = needs_embed.empty? ? {} : build_embed_map(needs_embed)
 
             chunks.map { |c| { chunk: c, embedding: embed_map[c[:content_hash]], exists: exists_map.fetch(c[:content_hash], false) } }
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             paired_without_embed(chunks, {})
           end
           private_class_method :batch_embed_chunks
@@ -132,7 +200,8 @@ module Legion
             results.each_with_object({}) do |r, h|
               h[needs_embed[r[:index]][:content_hash]] = r[:vector] unless r[:error]
             end
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             {}
           end
           private_class_method :build_embed_map
@@ -144,7 +213,8 @@ module Legion
 
             ingest_to_apollo(chunk, embedding)
             force ? :updated : :created
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             :skipped
           end
           private_class_method :upsert_chunk_with_embedding
@@ -156,7 +226,8 @@ module Legion
               .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
               .where(Sequel.like(:content, "%#{content_hash}%"))
               .any?
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             false
           end
           private_class_method :chunk_exists?
@@ -193,7 +264,8 @@ module Legion
               tags:         [file_path, 'retired', 'document_chunk'].uniq,
               metadata:     { source_file: file_path, retired: true }
             )
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             nil
           end
           private_class_method :retire_file

data/lib/legion/extensions/knowledge/runners/monitor.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Runners
+        module Monitor
+          module_function
+
+          DEFAULT_EXTENSIONS = %w[.md .txt].freeze
+
+          def resolve_monitors
+            monitors = Array(read_monitors_setting)
+            legacy   = read_legacy_corpus_path
+
+            if legacy && !legacy.empty? && monitors.none? { |m| m[:path] == legacy }
+              monitors << { path: legacy, extensions: %w[.md .txt .docx .pdf], label: 'legacy' }
+            end
+
+            monitors
+          rescue StandardError
+            []
+          end
+
+          def add_monitor(path:, extensions: nil, label: nil)
+            abs_path = File.expand_path(path)
+            return { success: false, error: "Path #{abs_path} does not exist or is not a directory" } unless File.directory?(abs_path)
+
+            existing = Array(read_monitors_setting)
+            return { success: false, error: "Path #{abs_path} is already registered" } if existing.any? { |m| m[:path] == abs_path }
+
+            entry = {
+              path:       abs_path,
+              extensions: extensions || DEFAULT_EXTENSIONS.dup,
+              label:      label || File.basename(abs_path),
+              added_at:   Time.now.utc.iso8601
+            }
+
+            existing << entry
+            persist_monitors(existing)
+
+            { success: true, monitor: entry }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def remove_monitor(identifier:)
+            existing = Array(read_monitors_setting)
+            found = existing.find { |m| m[:path] == identifier || m[:label] == identifier }
+            return { success: false, error: "Monitor '#{identifier}' not found" } unless found
+
+            existing.delete(found)
+            persist_monitors(existing)
+
+            { success: true, removed: found }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def list_monitors
+            { success: true, monitors: resolve_monitors }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def monitor_status
+            monitors    = resolve_monitors
+            total_files = 0
+
+            monitors.each do |m|
+              scan = Helpers::Manifest.scan(path: m[:path], extensions: m[:extensions])
+              total_files += scan.size
+            rescue StandardError
+              next
+            end
+
+            { success: true, total_monitors: monitors.size, total_files: total_files }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          # --- private helpers ---
+
+          def read_monitors_setting
+            return nil unless defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
+
+            Legion::Settings.dig(:knowledge, :monitors)
+          rescue StandardError
+            nil
+          end
+          private_class_method :read_monitors_setting
+
+          def read_legacy_corpus_path
+            return nil unless defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
+
+            Legion::Settings.dig(:knowledge, :corpus_path)
+          rescue StandardError
+            nil
+          end
+          private_class_method :read_legacy_corpus_path
+
+          def persist_monitors(monitors)
+            return false unless defined?(Legion::Settings)
+
+            loader = Legion::Settings.loader
+            knowledge = loader.settings[:knowledge] || {}
+            knowledge[:monitors] = monitors
+            loader.settings[:knowledge] = knowledge
+            true
+          rescue StandardError
+            false
+          end
+          private_class_method :persist_monitors
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/transport/messages/monitor_reload.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Transport
+        module Messages
+          class MonitorReload < Legion::Transport::Message
+            def exchange_name = 'knowledge'
+            def routing_key = 'knowledge.monitor.reload'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Knowledge
-      VERSION = '0.5.0'
+      VERSION = '0.6.3'
     end
   end
 end

data/lib/legion/extensions/knowledge.rb

@@ -9,12 +9,14 @@ require_relative 'knowledge/runners/ingest'
 require_relative 'knowledge/runners/query'
 require_relative 'knowledge/runners/corpus'
 require_relative 'knowledge/runners/maintenance'
+require_relative 'knowledge/runners/monitor'
 require_relative 'knowledge/client'
 
 if defined?(Legion::Transport)
   require_relative 'knowledge/transport/exchanges/knowledge'
   require_relative 'knowledge/transport/queues/ingest'
   require_relative 'knowledge/transport/messages/ingest_message'
+  require_relative 'knowledge/transport/messages/monitor_reload'
 end
 
 require_relative 'knowledge/actors/corpus_watcher' if defined?(Legion::Extensions::Actors::Every)
@@ -26,6 +28,10 @@ module Legion
   module Extensions
     module Knowledge
       extend Legion::Extensions::Core if defined?(Legion::Extensions::Core)
+
+      def self.remote_invocable?
+        false
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-knowledge
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.3
 platform: ruby
 authors:
 - Matthew Iverson
@@ -127,9 +127,11 @@ files:
 - lib/legion/extensions/knowledge/runners/corpus.rb
 - lib/legion/extensions/knowledge/runners/ingest.rb
 - lib/legion/extensions/knowledge/runners/maintenance.rb
+- lib/legion/extensions/knowledge/runners/monitor.rb
 - lib/legion/extensions/knowledge/runners/query.rb
 - lib/legion/extensions/knowledge/transport/exchanges/knowledge.rb
 - lib/legion/extensions/knowledge/transport/messages/ingest_message.rb
+- lib/legion/extensions/knowledge/transport/messages/monitor_reload.rb
 - lib/legion/extensions/knowledge/transport/queues/ingest.rb
 - lib/legion/extensions/knowledge/version.rb
 homepage: https://github.com/LegionIO/lex-knowledge