lex-knowledge 0.1.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea1661f154fc5184b961caac74654775240dcc49fd7d8c6be469f6cd5520429c
-  data.tar.gz: 3499bcb535edb1f56e896ed9c462a2cfabb9d7d1804dc55ecd559fc3b1941ae9
+  metadata.gz: da332d52d481d4e45e8f7ee771a60fb13cebb74144a18ac54ff7b7ec4cb26f27
+  data.tar.gz: 3a858fbdae4511ec6e34573fccd85f330b33c7be818a2a78271af33cb0dcbfbd
 SHA512:
-  metadata.gz: 8b50acc322c82ae7629e5152e31c84ce9b348fdd1226e96aa7bc3c1f9f67cd330b809212f49c19636d93a45a199ce4e936a3d172cf71997ffad5aa69eb9712ef
-  data.tar.gz: c1feac59942cf706beb303cbac9989e569937048d2f666af3e762ef50d69659c46de6bb350f69cc16a5103581d54682c3f68886f84a4a840b16b1f0e8cfb82e6
+  metadata.gz: e7e15c2174b28ea518c3fee6ecc68723bc628ee3c6d9121a2292706ad32125d32f766da164aa581c56915cd178e54379014a67d8ad1d5cd765485a6eb88d0610
+  data.tar.gz: eb7f343423ea41ff46b6a478b3c32a22b0dd3ab3939ad858d0e8d540b5faaf8f01496a5839c79b5ccb1847529367ad9c55ffdc72bebf5959891ff4a7c06b72e4

data/lib/legion/extensions/knowledge/actors/corpus_ingest.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Actor
+        class CorpusIngest < Legion::Extensions::Actors::Subscription
+          def runner_class    = 'Legion::Extensions::Knowledge::Runners::Ingest'
+          def runner_function = 'ingest_file'
+          def check_subtask?  = false
+          def generate_task?  = false
+
+          def enabled?
+            defined?(Legion::Transport) &&
+              defined?(Legion::Extensions::Knowledge::Runners::Ingest)
+          rescue StandardError
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/actors/corpus_watcher.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Actor
+        class CorpusWatcher < Legion::Extensions::Actors::Every
+          def runner_class    = 'Legion::Extensions::Knowledge::Runners::Ingest'
+          def runner_function = 'ingest_corpus'
+          def check_subtask?  = false
+          def generate_task?  = false
+
+          def every_interval
+            if defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
+              Legion::Settings.dig(:knowledge, :actors, :watcher_interval) || 300
+            else
+              300
+            end
+          rescue StandardError
+            300
+          end
+
+          def enabled?
+            corpus_path && !corpus_path.empty?
+          rescue StandardError
+            false
+          end
+
+          def args
+            { path: corpus_path }
+          end
+
+          private
+
+          def corpus_path
+            return nil unless defined?(Legion::Settings) && !Legion::Settings[:knowledge].nil?
+
+            Legion::Settings.dig(:knowledge, :corpus_path)
+          rescue StandardError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/helpers/manifest_store.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'fileutils'
+require 'json'
+require 'tempfile'
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Helpers
+        module ManifestStore
+          module_function
+
+          STORE_DIR = ::File.expand_path('~/.legionio/knowledge').freeze
+
+          def load(corpus_path:)
+            path = store_path(corpus_path: corpus_path)
+            return [] unless ::File.exist?(path)
+
+            raw = ::File.read(path, encoding: 'utf-8')
+            ::JSON.parse(raw, symbolize_names: true)
+          rescue StandardError
+            []
+          end
+
+          def save(corpus_path:, manifest:)
+            ::FileUtils.mkdir_p(STORE_DIR)
+            path = store_path(corpus_path: corpus_path)
+            tmp  = "#{path}.tmp"
+            ::File.write(tmp, ::JSON.generate(manifest.map { |e| serialize_entry(e) }))
+            ::File.rename(tmp, path)
+            true
+          rescue StandardError
+            false
+          end
+
+          def store_path(corpus_path:)
+            hash = ::Digest::SHA256.hexdigest(corpus_path.to_s)[0, 16]
+            ::File.join(STORE_DIR, "#{hash}.manifest.json")
+          end
+
+          def serialize_entry(entry)
+            entry.merge(mtime: entry[:mtime].to_s)
+          end
+          private_class_method :serialize_entry
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/helpers/parser.rb

@@ -15,6 +15,8 @@ module Legion
               parse_markdown(file_path: file_path)
             when '.txt'
               parse_text(file_path: file_path)
+            when '.pdf', '.docx'
+              extract_via_data(file_path: file_path)
             else
               [{ error: 'unsupported format', source_file: file_path }]
             end
@@ -22,32 +24,42 @@ module Legion
 
           def parse_markdown(file_path:)
             content = ::File.read(file_path, encoding: 'utf-8')
-            sections = []
+            sections        = []
             current_heading = ::File.basename(file_path, '.*')
             current_lines   = []
-            section_path    = []
+            heading_stack   = {}
 
             content.each_line do |line|
-              if line.start_with?('# ')
-                flush_section(sections, current_heading, section_path, current_lines, file_path) unless current_lines.empty?
-                current_heading = line.sub(/^#+\s*/, '').chomp
-                section_path    = [current_heading]
-                current_lines   = []
-              elsif line.start_with?('## ')
-                flush_section(sections, current_heading, section_path, current_lines, file_path) unless current_lines.empty?
-                current_heading = line.sub(/^#+\s*/, '').chomp
-                section_path    = section_path.first(1) + [current_heading]
+              level = heading_level(line)
+              if level
+                flush_section(sections, current_heading, build_section_path(heading_stack), current_lines, file_path)
+                title = line.sub(/^#+\s*/, '').chomp
+                heading_stack.delete_if { |d, _| d >= level }
+                heading_stack[level] = title
+                current_heading = title
                 current_lines   = []
               else
                 current_lines << line
               end
             end
 
-            flush_section(sections, current_heading, section_path, current_lines, file_path) unless current_lines.empty?
+            flush_section(sections, current_heading, build_section_path(heading_stack), current_lines, file_path)
 
             sections.empty? ? [{ heading: ::File.basename(file_path, '.*'), section_path: [], content: content.strip, source_file: file_path }] : sections
           end
 
+          def extract_via_data(file_path:)
+            return [{ error: 'unsupported format', source_file: file_path }] unless defined?(::Legion::Data::Extract)
+
+            result = ::Legion::Data::Extract.extract(file_path, type: :auto)
+            return [{ error: 'extraction_failed', source_file: file_path, detail: result }] unless result.is_a?(Hash) && result[:text]
+
+            heading = ::File.basename(file_path, '.*')
+            [{ heading: heading, section_path: [], content: result[:text].strip, source_file: file_path }]
+          rescue StandardError => e
+            [{ error: 'extraction_failed', source_file: file_path, detail: e.message }]
+          end
+
           def parse_text(file_path:)
             content = ::File.read(file_path, encoding: 'utf-8')
             heading = ::File.basename(file_path, '.*')
@@ -67,6 +79,17 @@ module Legion
             }
           end
           private_class_method :flush_section
+
+          def heading_level(line)
+            m = line.match(/^(\#{1,6})\s/)
+            m ? m[1].length : nil
+          end
+          private_class_method :heading_level
+
+          def build_section_path(stack)
+            stack.sort.map { |_, title| title }
+          end
+          private_class_method :build_section_path
         end
       end
     end

data/lib/legion/extensions/knowledge/runners/corpus.rb

@@ -7,8 +7,32 @@ module Legion
         module Corpus
           module_function
 
-          def corpus_stats
-            { success: true, info: 'not yet implemented' }
+          def manifest_path(path:)
+            Helpers::ManifestStore.store_path(corpus_path: path)
+          end
+
+          def corpus_stats(path:, extensions: nil)
+            return { success: false, error: 'path does not exist' } unless ::File.exist?(path)
+
+            opts = { path: path }
+            opts[:extensions] = extensions if extensions
+            entries = Helpers::Manifest.scan(**opts)
+            chunk_count = entries.sum do |entry|
+              sections = Helpers::Parser.parse(file_path: entry[:path])
+              next 0 if sections.first&.key?(:error)
+
+              Helpers::Chunker.chunk(sections: sections).size
+            end
+
+            {
+              success:          true,
+              path:             path,
+              file_count:       entries.size,
+              estimated_chunks: chunk_count,
+              total_bytes:      entries.sum { |e| e[:size] }
+            }
+          rescue StandardError => e
+            { success: false, error: e.message }
           end
         end
       end

data/lib/legion/extensions/knowledge/runners/ingest.rb

@@ -23,23 +23,32 @@ module Legion
           end
 
           def ingest_corpus(path:, dry_run: false, force: false)
-            entries = Helpers::Manifest.scan(path: path)
-
-            files_scanned   = entries.size
-            chunks_created  = 0
-            chunks_skipped  = 0
-            chunks_updated  = 0
-
-            entries.each do |entry|
-              result = process_file(entry[:path], dry_run: dry_run, force: force)
-              chunks_created  += result[:created]
-              chunks_skipped  += result[:skipped]
-              chunks_updated  += result[:updated]
+            current  = Helpers::Manifest.scan(path: path)
+            previous = force ? [] : Helpers::ManifestStore.load(corpus_path: path)
+            delta    = Helpers::Manifest.diff(current: current, previous: previous)
+
+            to_process     = delta[:added] + delta[:changed]
+            chunks_created = 0
+            chunks_skipped = 0
+            chunks_updated = 0
+
+            to_process.each do |file_path|
+              result = process_file(file_path, dry_run: dry_run, force: force)
+              chunks_created += result[:created]
+              chunks_skipped += result[:skipped]
+              chunks_updated += result[:updated]
             end
 
+            delta[:removed].each { |file_path| retire_file(file_path: file_path) } unless dry_run
+
+            Helpers::ManifestStore.save(corpus_path: path, manifest: current) unless dry_run
+
             {
               success:        true,
-              files_scanned:  files_scanned,
+              files_scanned:  current.size,
+              files_added:    delta[:added].size,
+              files_changed:  delta[:changed].size,
+              files_removed:  delta[:removed].size,
               chunks_created: chunks_created,
               chunks_skipped: chunks_skipped,
               chunks_updated: chunks_updated
@@ -67,12 +76,18 @@ module Legion
             return { created: 0, skipped: 0, updated: 0 } if sections.first&.key?(:error)
 
             chunks  = Helpers::Chunker.chunk(sections: sections)
+            paired  = if dry_run
+                        chunks.map { |c| { chunk: c, embedding: nil } }
+                      else
+                        batch_embed_chunks(chunks, force: force)
+                      end
+
             created = 0
             skipped = 0
             updated = 0
 
-            chunks.each do |chunk|
-              outcome = upsert_chunk(chunk, dry_run: dry_run, force: force)
+            paired.each do |p|
+              outcome = upsert_chunk_with_embedding(p[:chunk], p[:embedding], dry_run: dry_run, force: force, exists: p[:exists] || false)
               case outcome
               when :created then created += 1
               when :skipped then skipped += 1
@@ -84,46 +99,68 @@ module Legion
           end
           private_class_method :process_file
 
-          def upsert_chunk(chunk, dry_run: false, force: false)
-            return :created if dry_run
+          def batch_embed_chunks(chunks, force:)
+            exists_map = force ? {} : build_exists_map(chunks)
+            return paired_without_embed(chunks, exists_map) unless llm_embed_available?
 
-            return :created unless defined?(Legion::Extensions::Apollo)
+            needs_embed = force ? chunks : chunks.reject { |c| exists_map[c[:content_hash]] }
+            embed_map   = needs_embed.empty? ? {} : build_embed_map(needs_embed)
 
-            return :skipped if !force && chunk_exists?(chunk[:content_hash])
+            chunks.map { |c| { chunk: c, embedding: embed_map[c[:content_hash]], exists: exists_map.fetch(c[:content_hash], false) } }
+          rescue StandardError
+            paired_without_embed(chunks, {})
+          end
+          private_class_method :batch_embed_chunks
 
-            embedding = generate_embedding(chunk[:content])
-            ingest_to_apollo(chunk, embedding)
+          def build_exists_map(chunks)
+            chunks.to_h { |c| [c[:content_hash], chunk_exists?(c[:content_hash])] }
+          end
+          private_class_method :build_exists_map
+
+          def llm_embed_available?
+            defined?(Legion::LLM) && Legion::LLM.respond_to?(:embed_batch)
+          end
+          private_class_method :llm_embed_available?
+
+          def paired_without_embed(chunks, exists_map)
+            chunks.map { |c| { chunk: c, embedding: nil, exists: exists_map.fetch(c[:content_hash], false) } }
+          end
+          private_class_method :paired_without_embed
+
+          def build_embed_map(needs_embed)
+            results = Legion::LLM.embed_batch(needs_embed.map { |c| c[:content] })
+            results.each_with_object({}) do |r, h|
+              h[needs_embed[r[:index]][:content_hash]] = r[:vector] unless r[:error]
+            end
+          rescue StandardError
+            {}
+          end
+          private_class_method :build_embed_map
+
+          def upsert_chunk_with_embedding(chunk, embedding, dry_run: false, force: false, exists: false)
+            return :created if dry_run
+            return :created unless defined?(Legion::Extensions::Apollo)
+            return :skipped if !force && exists
 
+            ingest_to_apollo(chunk, embedding)
             force ? :updated : :created
           rescue StandardError
             :skipped
           end
-          private_class_method :upsert_chunk
+          private_class_method :upsert_chunk_with_embedding
 
           def chunk_exists?(content_hash)
-            return false unless defined?(Legion::Extensions::Apollo)
-
-            Legion::Extensions::Apollo::Runners::Knowledge.retrieve_relevant(
-              query:  content_hash,
-              limit:  1,
-              tags:   ['document_chunk'],
-              filter: { content_hash: content_hash }
-            ).any?
+            return false unless defined?(Legion::Data::Model::ApolloEntry)
+
+            Legion::Data::Model::ApolloEntry
+              .where(Sequel.pg_array_op(:tags).contains(Sequel.pg_array(['document_chunk'])))
+              .where(Sequel.like(:content, "%#{content_hash}%"))
+              .any?
           rescue StandardError
             false
           end
           private_class_method :chunk_exists?
 
-          def generate_embedding(content)
-            return nil unless defined?(Legion::Extensions::Apollo)
-            return nil unless defined?(Legion::Extensions::Apollo::Helpers::Embedding)
-
-            Legion::Extensions::Apollo::Helpers::Embedding.generate(content)
-          rescue StandardError
-            nil
-          end
-          private_class_method :generate_embedding
-
           def ingest_to_apollo(chunk, embedding)
             return unless defined?(Legion::Extensions::Apollo)
 
@@ -145,6 +182,21 @@ module Legion
             Legion::Extensions::Apollo::Runners::Knowledge.handle_ingest(**payload)
           end
           private_class_method :ingest_to_apollo
+
+          def retire_file(file_path:)
+            return unless defined?(Legion::Apollo)
+            return unless Legion::Apollo.respond_to?(:ingest) && Legion::Apollo.started?
+
+            Legion::Apollo.ingest(
+              content:      file_path,
+              content_type: 'document_retired',
+              tags:         [file_path, 'retired', 'document_chunk'].uniq,
+              metadata:     { source_file: file_path, retired: true }
+            )
+          rescue StandardError
+            nil
+          end
+          private_class_method :retire_file
         end
       end
     end

data/lib/legion/extensions/knowledge/transport/exchanges/knowledge.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Transport
+        module Exchanges
+          class Knowledge < Legion::Transport::Exchange
+            def exchange_name = 'knowledge'
+            def type = 'topic'
+            def durable = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/transport/messages/ingest_message.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Transport
+        module Messages
+          class IngestMessage < Legion::Transport::Message
+            def exchange_name = 'knowledge'
+            def routing_key = 'knowledge.ingest'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/transport/queues/ingest.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Transport
+        module Queues
+          class Ingest < Legion::Transport::Queue
+            def queue_name = 'knowledge.ingest'
+            def exchange_name = 'knowledge'
+            def routing_key = 'knowledge.ingest'
+            def durable = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Knowledge
-      VERSION = '0.1.2'
+      VERSION = '0.4.0'
     end
   end
 end

data/lib/legion/extensions/knowledge.rb

@@ -2,6 +2,7 @@
 
 require_relative 'knowledge/version'
 require_relative 'knowledge/helpers/manifest'
+require_relative 'knowledge/helpers/manifest_store'
 require_relative 'knowledge/helpers/parser'
 require_relative 'knowledge/helpers/chunker'
 require_relative 'knowledge/runners/ingest'
@@ -9,6 +10,16 @@ require_relative 'knowledge/runners/query'
 require_relative 'knowledge/runners/corpus'
 require_relative 'knowledge/client'
 
+if defined?(Legion::Transport)
+  require_relative 'knowledge/transport/exchanges/knowledge'
+  require_relative 'knowledge/transport/queues/ingest'
+  require_relative 'knowledge/transport/messages/ingest_message'
+end
+
+require_relative 'knowledge/actors/corpus_watcher' if defined?(Legion::Extensions::Actors::Every)
+
+require_relative 'knowledge/actors/corpus_ingest' if defined?(Legion::Extensions::Actors::Subscription)
+
 module Legion
   module Extensions
     module Knowledge

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-knowledge
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Matthew Iverson
@@ -116,13 +116,19 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/legion/extensions/knowledge.rb
+- lib/legion/extensions/knowledge/actors/corpus_ingest.rb
+- lib/legion/extensions/knowledge/actors/corpus_watcher.rb
 - lib/legion/extensions/knowledge/client.rb
 - lib/legion/extensions/knowledge/helpers/chunker.rb
 - lib/legion/extensions/knowledge/helpers/manifest.rb
+- lib/legion/extensions/knowledge/helpers/manifest_store.rb
 - lib/legion/extensions/knowledge/helpers/parser.rb
 - lib/legion/extensions/knowledge/runners/corpus.rb
 - lib/legion/extensions/knowledge/runners/ingest.rb
 - lib/legion/extensions/knowledge/runners/query.rb
+- lib/legion/extensions/knowledge/transport/exchanges/knowledge.rb
+- lib/legion/extensions/knowledge/transport/messages/ingest_message.rb
+- lib/legion/extensions/knowledge/transport/queues/ingest.rb
 - lib/legion/extensions/knowledge/version.rb
 homepage: https://github.com/LegionIO/lex-knowledge
 licenses: