lex-knowledge 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea1661f154fc5184b961caac74654775240dcc49fd7d8c6be469f6cd5520429c
-  data.tar.gz: 3499bcb535edb1f56e896ed9c462a2cfabb9d7d1804dc55ecd559fc3b1941ae9
+  metadata.gz: 2a5307a5b8c19abaedd5f7d6be95d0fc3d068fa0b6d7366293c70da0edec7825
+  data.tar.gz: 756114f38b345f356a826e09c50714d8b14ff2a4baa6ef0087ddab9f2d75ff78
 SHA512:
-  metadata.gz: 8b50acc322c82ae7629e5152e31c84ce9b348fdd1226e96aa7bc3c1f9f67cd330b809212f49c19636d93a45a199ce4e936a3d172cf71997ffad5aa69eb9712ef
-  data.tar.gz: c1feac59942cf706beb303cbac9989e569937048d2f666af3e762ef50d69659c46de6bb350f69cc16a5103581d54682c3f68886f84a4a840b16b1f0e8cfb82e6
+  metadata.gz: 6d77ae8947c2ac53af380a35935c33cb075f4ae1a7b8c2b495389701fb15652a7baaefeeddd32606bc81d9e4c6f8f319562fe869906523b34ca4563ce0e245c4
+  data.tar.gz: a30c57db2c8cb0da0d54ac73afc178ef55892123e4a6878aa1bf1ca62014663c508c699857970f5582a0e0361a5fd3c29349dbe913a6d1ae8a19aed78515fcc0

data/lib/legion/extensions/knowledge/actors/corpus_ingest.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Actor
+        class CorpusIngest < Legion::Extensions::Actors::Subscription
+          def runner_class    = 'Legion::Extensions::Knowledge::Runners::Ingest'
+          def runner_function = 'ingest_file'
+          def check_subtask?  = false
+          def generate_task?  = false
+
+          def enabled?
+            defined?(Legion::Transport) &&
+              defined?(Legion::Extensions::Knowledge::Runners::Ingest)
+          rescue StandardError
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/actors/corpus_watcher.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Actor
+        class CorpusWatcher < Legion::Extensions::Actors::Every
+          def runner_class    = 'Legion::Extensions::Knowledge::Runners::Ingest'
+          def runner_function = 'ingest_corpus'
+          def check_subtask?  = false
+          def generate_task?  = false
+
+          def every_interval
+            if defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
+              Legion::Settings.dig(:knowledge, :actors, :watcher_interval) || 300
+            else
+              300
+            end
+          rescue StandardError
+            300
+          end
+
+          def enabled?
+            corpus_path && !corpus_path.empty?
+          rescue StandardError
+            false
+          end
+
+          def args
+            { path: corpus_path }
+          end
+
+          private
+
+          def corpus_path
+            return nil unless defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
+
+            Legion::Settings.dig(:knowledge, :corpus_path)
+          rescue StandardError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/runners/corpus.rb

@@ -7,8 +7,28 @@ module Legion
         module Corpus
           module_function
 
-          def corpus_stats
-            { success: true, info: 'not yet implemented' }
+          def corpus_stats(path:, extensions: nil)
+            return { success: false, error: 'path does not exist' } unless ::File.exist?(path)
+
+            opts = { path: path }
+            opts[:extensions] = extensions if extensions
+            entries = Helpers::Manifest.scan(**opts)
+            chunk_count = entries.sum do |entry|
+              sections = Helpers::Parser.parse(file_path: entry[:path])
+              next 0 if sections.first&.key?(:error)
+
+              Helpers::Chunker.chunk(sections: sections).size
+            end
+
+            {
+              success:          true,
+              path:             path,
+              file_count:       entries.size,
+              estimated_chunks: chunk_count,
+              total_bytes:      entries.sum { |e| e[:size] }
+            }
+          rescue StandardError => e
+            { success: false, error: e.message }
           end
         end
       end

data/lib/legion/extensions/knowledge/runners/ingest.rb

@@ -101,24 +101,22 @@ module Legion
           private_class_method :upsert_chunk
 
           def chunk_exists?(content_hash)
-            return false unless defined?(Legion::Extensions::Apollo)
-
-            Legion::Extensions::Apollo::Runners::Knowledge.retrieve_relevant(
-              query:  content_hash,
-              limit:  1,
-              tags:   ['document_chunk'],
-              filter: { content_hash: content_hash }
-            ).any?
+            return false unless defined?(Legion::Data::Model::ApolloEntry)
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .where(Sequel.like(:content, "%#{content_hash}%"))
+              .any?
           rescue StandardError
             false
           end
           private_class_method :chunk_exists?
 
           def generate_embedding(content)
-            return nil unless defined?(Legion::Extensions::Apollo)
-            return nil unless defined?(Legion::Extensions::Apollo::Helpers::Embedding)
+            return nil unless defined?(Legion::LLM) && Legion::LLM.respond_to?(:embed)
 
-            Legion::Extensions::Apollo::Helpers::Embedding.generate(content)
+            result = Legion::LLM.embed(content)
+            result.is_a?(Hash) ? result[:vector] : nil
           rescue StandardError
             nil
           end

data/lib/legion/extensions/knowledge/transport/exchanges/knowledge.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Transport
+        module Exchanges
+          class Knowledge < Legion::Transport::Exchange
+            def exchange_name = 'knowledge'
+            def type = 'topic'
+            def durable = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/transport/messages/ingest_message.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Transport
+        module Messages
+          class IngestMessage < Legion::Transport::Message
+            def exchange_name = 'knowledge'
+            def routing_key = 'knowledge.ingest'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/transport/queues/ingest.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Transport
+        module Queues
+          class Ingest < Legion::Transport::Queue
+            def queue_name = 'knowledge.ingest'
+            def exchange_name = 'knowledge'
+            def routing_key = 'knowledge.ingest'
+            def durable = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Knowledge
-      VERSION = '0.1.2'
+      VERSION = '0.2.0'
     end
   end
 end

data/lib/legion/extensions/knowledge.rb

@@ -9,6 +9,16 @@ require_relative 'knowledge/runners/query'
 require_relative 'knowledge/runners/corpus'
 require_relative 'knowledge/client'
 
+if defined?(Legion::Transport)
+  require_relative 'knowledge/transport/exchanges/knowledge'
+  require_relative 'knowledge/transport/queues/ingest'
+  require_relative 'knowledge/transport/messages/ingest_message'
+end
+
+require_relative 'knowledge/actors/corpus_watcher' if defined?(Legion::Extensions::Actors::Every)
+
+require_relative 'knowledge/actors/corpus_ingest' if defined?(Legion::Extensions::Actors::Subscription)
+
 module Legion
   module Extensions
     module Knowledge

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-knowledge
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Matthew Iverson
@@ -116,6 +116,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/legion/extensions/knowledge.rb
+- lib/legion/extensions/knowledge/actors/corpus_ingest.rb
+- lib/legion/extensions/knowledge/actors/corpus_watcher.rb
 - lib/legion/extensions/knowledge/client.rb
 - lib/legion/extensions/knowledge/helpers/chunker.rb
 - lib/legion/extensions/knowledge/helpers/manifest.rb
@@ -123,6 +125,9 @@ files:
 - lib/legion/extensions/knowledge/runners/corpus.rb
 - lib/legion/extensions/knowledge/runners/ingest.rb
 - lib/legion/extensions/knowledge/runners/query.rb
+- lib/legion/extensions/knowledge/transport/exchanges/knowledge.rb
+- lib/legion/extensions/knowledge/transport/messages/ingest_message.rb
+- lib/legion/extensions/knowledge/transport/queues/ingest.rb
 - lib/legion/extensions/knowledge/version.rb
 homepage: https://github.com/LegionIO/lex-knowledge
 licenses: