lex-knowledge 0.6.0 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 93e36e27a559476697a65f659e5c5a21db2e061b40877db6ea467875642d3232
-  data.tar.gz: dfae368bcc8db4eba0c6b41da9bb79537ed3a9b4fd898b189905fb1ca58a6bc7
+  metadata.gz: 40412f676840f9927c0e66d7603678bbde1489dcb9a2faba958b1a684fd47b3d
+  data.tar.gz: f9247b163289a8e3e07deaf63efbab11703e153150e91d77745eca010cb49b21
 SHA512:
-  metadata.gz: c9d11bdbcc27f14431c6e900c703a02cea7389bf2220c6e8c2fb1c920a97d7a86993d11e207453ef79ab928da68629a98aa711d51d009dd04d1f5215337dc4a6
-  data.tar.gz: 9fe9dc96f91a19aca0d58b4ad49abbcf0876797bba5ddea6984cff0d51b7a577ce0f165fa8d54d9271b185da84d6a267d7c9883903e99937f7b084454b1affcc
+  metadata.gz: 83cdc62303978876ae5f04c50d3bcf5fcd08faddaa3ee49dc8ac092c49df9665419fb2c52452d7247357b5e424d6122111d82b9e772879bd2afd0068dc780918
+  data.tar.gz: 24f0a1208fc396058ae014c497ef5bc49aa96dfe185ad32d0628a19b8fee4ab1c8754048d77d06a73a4a3d42f7c2c1537e6653fecae2b2b280c4b38e6f169745

data/lib/legion/extensions/knowledge/actors/corpus_watcher.rb

@@ -10,19 +10,21 @@ module Legion
           def check_subtask?  = false
           def generate_task?  = false
 
-          def every_interval
+          def time
             if defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
               Legion::Settings.dig(:knowledge, :actors, :watcher_interval) || 300
             else
               300
             end
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             300
           end
 
           def enabled?
             resolve_monitors.any?
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             false
           end
 
@@ -32,9 +34,14 @@ module Legion
 
           private
 
+          def log
+            Legion::Logging
+          end
+
           def resolve_monitors
             Runners::Monitor.resolve_monitors
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             []
           end
         end

data/lib/legion/extensions/knowledge/actors/maintenance_runner.rb

@@ -10,13 +10,14 @@ module Legion
           def check_subtask?  = false
           def generate_task?  = false
 
-          def every_interval
+          def time
             if defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
               Legion::Settings.dig(:knowledge, :actors, :maintenance_interval) || 21_600
             else
               21_600
             end
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             21_600
           end
 
@@ -24,7 +25,8 @@ module Legion
             return false unless corpus_path && !corpus_path.empty?
 
             true
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             false
           end
 
@@ -34,11 +36,16 @@ module Legion
 
           private
 
+          def log
+            Legion::Logging
+          end
+
           def corpus_path
             return nil unless defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
 
             Legion::Settings.dig(:knowledge, :corpus_path)
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             nil
           end
         end

data/lib/legion/extensions/knowledge/runners/ingest.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'securerandom'
+
 module Legion
   module Extensions
     module Knowledge
@@ -7,6 +9,11 @@ module Legion
         module Ingest
           module_function
 
+          def log
+            Legion::Logging
+          end
+          private_class_method :log
+
           def scan_corpus(path:, extensions: nil)
             opts = { path: path }
             opts[:extensions] = extensions if extensions
@@ -28,6 +35,7 @@ module Legion
 
             ingest_corpus_path(path: path, dry_run: dry_run, force: force)
           rescue ArgumentError => e
+            log.warn(e.message)
             { success: false, error: e.message }
           end
 
@@ -63,6 +71,7 @@ module Legion
               chunks_updated: chunks_updated
             }
           rescue StandardError => e
+            log.warn(e.message)
             { success: false, error: e.message }
           end
           private_class_method :ingest_corpus_path
@@ -71,6 +80,7 @@ module Legion
             results = monitors.map do |monitor|
               ingest_corpus(path: monitor[:path], dry_run: dry_run, force: force)
             rescue StandardError => e
+              log.warn(e.message)
               { success: false, path: monitor[:path], error: e.message }
             end
 
@@ -91,10 +101,28 @@ module Legion
 
             { success: true, monitors_processed: results.size, **total }
           rescue StandardError => e
+            log.warn(e.message)
             { success: false, error: e.message }
           end
           private_class_method :ingest_monitors
 
+          def ingest_content(content:, source_type: :text, metadata: {})
+            source_path = "content://#{source_type}/#{SecureRandom.uuid}"
+            section = {
+              content:      content,
+              heading:      source_type.to_s,
+              section_path: [source_type.to_s],
+              source_file:  source_path
+            }
+            chunks = Helpers::Chunker.chunk(sections: [section])
+            paired = batch_embed_chunks(chunks, force: false)
+            paired.each { |p| upsert_chunk_with_embedding(p[:chunk], p[:embedding], force: false, exists: p[:exists] || false) }
+            { status: :ingested, chunks: chunks.size, source_type: source_type, metadata: metadata }
+          rescue StandardError => e
+            log.warn(e.message)
+            { status: :failed, error: e.message, source_type: source_type, metadata: metadata }
+          end
+
           def ingest_file(file_path:, force: false)
             result = process_file(file_path, dry_run: false, force: force)
 
@@ -106,6 +134,7 @@ module Legion
               chunks_updated: result[:updated]
             }
           rescue StandardError => e
+            log.warn(e.message)
             { success: false, error: e.message }
           end
 
@@ -145,7 +174,8 @@ module Legion
             embed_map   = needs_embed.empty? ? {} : build_embed_map(needs_embed)
 
             chunks.map { |c| { chunk: c, embedding: embed_map[c[:content_hash]], exists: exists_map.fetch(c[:content_hash], false) } }
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             paired_without_embed(chunks, {})
           end
           private_class_method :batch_embed_chunks
@@ -170,7 +200,8 @@ module Legion
             results.each_with_object({}) do |r, h|
               h[needs_embed[r[:index]][:content_hash]] = r[:vector] unless r[:error]
             end
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             {}
           end
           private_class_method :build_embed_map
@@ -182,7 +213,8 @@ module Legion
 
             ingest_to_apollo(chunk, embedding)
             force ? :updated : :created
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             :skipped
           end
           private_class_method :upsert_chunk_with_embedding
@@ -194,7 +226,8 @@ module Legion
               .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
               .where(Sequel.like(:content, "%#{content_hash}%"))
               .any?
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             false
           end
           private_class_method :chunk_exists?
@@ -231,7 +264,8 @@ module Legion
               tags:         [file_path, 'retired', 'document_chunk'].uniq,
               metadata:     { source_file: file_path, retired: true }
             )
-          rescue StandardError
+          rescue StandardError => e
+            log.warn(e.message)
             nil
           end
           private_class_method :retire_file

data/lib/legion/extensions/knowledge/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Knowledge
-      VERSION = '0.6.0'
+      VERSION = '0.6.3'
     end
   end
 end

data/lib/legion/extensions/knowledge.rb

@@ -28,6 +28,10 @@ module Legion
   module Extensions
     module Knowledge
       extend Legion::Extensions::Core if defined?(Legion::Extensions::Core)
+
+      def self.remote_invocable?
+        false
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-knowledge
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.3
 platform: ruby
 authors:
 - Matthew Iverson