lex-knowledge 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a5307a5b8c19abaedd5f7d6be95d0fc3d068fa0b6d7366293c70da0edec7825
-  data.tar.gz: 756114f38b345f356a826e09c50714d8b14ff2a4baa6ef0087ddab9f2d75ff78
+  metadata.gz: da332d52d481d4e45e8f7ee771a60fb13cebb74144a18ac54ff7b7ec4cb26f27
+  data.tar.gz: 3a858fbdae4511ec6e34573fccd85f330b33c7be818a2a78271af33cb0dcbfbd
 SHA512:
-  metadata.gz: 6d77ae8947c2ac53af380a35935c33cb075f4ae1a7b8c2b495389701fb15652a7baaefeeddd32606bc81d9e4c6f8f319562fe869906523b34ca4563ce0e245c4
-  data.tar.gz: a30c57db2c8cb0da0d54ac73afc178ef55892123e4a6878aa1bf1ca62014663c508c699857970f5582a0e0361a5fd3c29349dbe913a6d1ae8a19aed78515fcc0
+  metadata.gz: e7e15c2174b28ea518c3fee6ecc68723bc628ee3c6d9121a2292706ad32125d32f766da164aa581c56915cd178e54379014a67d8ad1d5cd765485a6eb88d0610
+  data.tar.gz: eb7f343423ea41ff46b6a478b3c32a22b0dd3ab3939ad858d0e8d540b5faaf8f01496a5839c79b5ccb1847529367ad9c55ffdc72bebf5959891ff4a7c06b72e4

data/lib/legion/extensions/knowledge/helpers/manifest_store.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'fileutils'
+require 'json'
+require 'tempfile'
+
+module Legion
+  module Extensions
+    module Knowledge
+      module Helpers
+        module ManifestStore
+          module_function
+
+          STORE_DIR = ::File.expand_path('~/.legionio/knowledge').freeze
+
+          def load(corpus_path:)
+            path = store_path(corpus_path: corpus_path)
+            return [] unless ::File.exist?(path)
+
+            raw = ::File.read(path, encoding: 'utf-8')
+            ::JSON.parse(raw, symbolize_names: true)
+          rescue StandardError
+            []
+          end
+
+          def save(corpus_path:, manifest:)
+            ::FileUtils.mkdir_p(STORE_DIR)
+            path = store_path(corpus_path: corpus_path)
+            tmp  = "#{path}.tmp"
+            ::File.write(tmp, ::JSON.generate(manifest.map { |e| serialize_entry(e) }))
+            ::File.rename(tmp, path)
+            true
+          rescue StandardError
+            false
+          end
+
+          def store_path(corpus_path:)
+            hash = ::Digest::SHA256.hexdigest(corpus_path.to_s)[0, 16]
+            ::File.join(STORE_DIR, "#{hash}.manifest.json")
+          end
+
+          def serialize_entry(entry)
+            entry.merge(mtime: entry[:mtime].to_s)
+          end
+          private_class_method :serialize_entry
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/knowledge/helpers/parser.rb

@@ -15,6 +15,8 @@ module Legion
               parse_markdown(file_path: file_path)
             when '.txt'
               parse_text(file_path: file_path)
+            when '.pdf', '.docx'
+              extract_via_data(file_path: file_path)
             else
               [{ error: 'unsupported format', source_file: file_path }]
             end
@@ -22,32 +24,42 @@ module Legion
 
           def parse_markdown(file_path:)
             content = ::File.read(file_path, encoding: 'utf-8')
-            sections = []
+            sections        = []
             current_heading = ::File.basename(file_path, '.*')
             current_lines   = []
-            section_path    = []
+            heading_stack   = {}
 
             content.each_line do |line|
-              if line.start_with?('# ')
-                flush_section(sections, current_heading, section_path, current_lines, file_path) unless current_lines.empty?
-                current_heading = line.sub(/^#+\s*/, '').chomp
-                section_path    = [current_heading]
-                current_lines   = []
-              elsif line.start_with?('## ')
-                flush_section(sections, current_heading, section_path, current_lines, file_path) unless current_lines.empty?
-                current_heading = line.sub(/^#+\s*/, '').chomp
-                section_path    = section_path.first(1) + [current_heading]
+              level = heading_level(line)
+              if level
+                flush_section(sections, current_heading, build_section_path(heading_stack), current_lines, file_path)
+                title = line.sub(/^#+\s*/, '').chomp
+                heading_stack.delete_if { |d, _| d >= level }
+                heading_stack[level] = title
+                current_heading = title
                 current_lines   = []
               else
                 current_lines << line
               end
             end
 
-            flush_section(sections, current_heading, section_path, current_lines, file_path) unless current_lines.empty?
+            flush_section(sections, current_heading, build_section_path(heading_stack), current_lines, file_path)
 
             sections.empty? ? [{ heading: ::File.basename(file_path, '.*'), section_path: [], content: content.strip, source_file: file_path }] : sections
           end
 
+          def extract_via_data(file_path:)
+            return [{ error: 'unsupported format', source_file: file_path }] unless defined?(::Legion::Data::Extract)
+
+            result = ::Legion::Data::Extract.extract(file_path, type: :auto)
+            return [{ error: 'extraction_failed', source_file: file_path, detail: result }] unless result.is_a?(Hash) && result[:text]
+
+            heading = ::File.basename(file_path, '.*')
+            [{ heading: heading, section_path: [], content: result[:text].strip, source_file: file_path }]
+          rescue StandardError => e
+            [{ error: 'extraction_failed', source_file: file_path, detail: e.message }]
+          end
+
           def parse_text(file_path:)
             content = ::File.read(file_path, encoding: 'utf-8')
             heading = ::File.basename(file_path, '.*')
@@ -67,6 +79,17 @@ module Legion
             }
           end
           private_class_method :flush_section
+
+          def heading_level(line)
+            m = line.match(/^(\#{1,6})\s/)
+            m ? m[1].length : nil
+          end
+          private_class_method :heading_level
+
+          def build_section_path(stack)
+            stack.sort.map { |_, title| title }
+          end
+          private_class_method :build_section_path
         end
       end
     end

data/lib/legion/extensions/knowledge/runners/corpus.rb

@@ -7,6 +7,10 @@ module Legion
         module Corpus
           module_function
 
+          def manifest_path(path:)
+            Helpers::ManifestStore.store_path(corpus_path: path)
+          end
+
           def corpus_stats(path:, extensions: nil)
             return { success: false, error: 'path does not exist' } unless ::File.exist?(path)
 

data/lib/legion/extensions/knowledge/runners/ingest.rb

@@ -23,23 +23,32 @@ module Legion
           end
 
           def ingest_corpus(path:, dry_run: false, force: false)
-            entries = Helpers::Manifest.scan(path: path)
-
-            files_scanned   = entries.size
-            chunks_created  = 0
-            chunks_skipped  = 0
-            chunks_updated  = 0
-
-            entries.each do |entry|
-              result = process_file(entry[:path], dry_run: dry_run, force: force)
-              chunks_created  += result[:created]
-              chunks_skipped  += result[:skipped]
-              chunks_updated  += result[:updated]
+            current  = Helpers::Manifest.scan(path: path)
+            previous = force ? [] : Helpers::ManifestStore.load(corpus_path: path)
+            delta    = Helpers::Manifest.diff(current: current, previous: previous)
+
+            to_process     = delta[:added] + delta[:changed]
+            chunks_created = 0
+            chunks_skipped = 0
+            chunks_updated = 0
+
+            to_process.each do |file_path|
+              result = process_file(file_path, dry_run: dry_run, force: force)
+              chunks_created += result[:created]
+              chunks_skipped += result[:skipped]
+              chunks_updated += result[:updated]
             end
 
+            delta[:removed].each { |file_path| retire_file(file_path: file_path) } unless dry_run
+
+            Helpers::ManifestStore.save(corpus_path: path, manifest: current) unless dry_run
+
             {
               success:        true,
-              files_scanned:  files_scanned,
+              files_scanned:  current.size,
+              files_added:    delta[:added].size,
+              files_changed:  delta[:changed].size,
+              files_removed:  delta[:removed].size,
               chunks_created: chunks_created,
               chunks_skipped: chunks_skipped,
               chunks_updated: chunks_updated
@@ -67,12 +76,18 @@ module Legion
             return { created: 0, skipped: 0, updated: 0 } if sections.first&.key?(:error)
 
             chunks  = Helpers::Chunker.chunk(sections: sections)
+            paired  = if dry_run
+                        chunks.map { |c| { chunk: c, embedding: nil } }
+                      else
+                        batch_embed_chunks(chunks, force: force)
+                      end
+
             created = 0
             skipped = 0
             updated = 0
 
-            chunks.each do |chunk|
-              outcome = upsert_chunk(chunk, dry_run: dry_run, force: force)
+            paired.each do |p|
+              outcome = upsert_chunk_with_embedding(p[:chunk], p[:embedding], dry_run: dry_run, force: force, exists: p[:exists] || false)
               case outcome
               when :created then created += 1
               when :skipped then skipped += 1
@@ -84,21 +99,55 @@ module Legion
           end
           private_class_method :process_file
 
-          def upsert_chunk(chunk, dry_run: false, force: false)
-            return :created if dry_run
+          def batch_embed_chunks(chunks, force:)
+            exists_map = force ? {} : build_exists_map(chunks)
+            return paired_without_embed(chunks, exists_map) unless llm_embed_available?
 
-            return :created unless defined?(Legion::Extensions::Apollo)
+            needs_embed = force ? chunks : chunks.reject { |c| exists_map[c[:content_hash]] }
+            embed_map   = needs_embed.empty? ? {} : build_embed_map(needs_embed)
 
-            return :skipped if !force && chunk_exists?(chunk[:content_hash])
+            chunks.map { |c| { chunk: c, embedding: embed_map[c[:content_hash]], exists: exists_map.fetch(c[:content_hash], false) } }
+          rescue StandardError
+            paired_without_embed(chunks, {})
+          end
+          private_class_method :batch_embed_chunks
 
-            embedding = generate_embedding(chunk[:content])
-            ingest_to_apollo(chunk, embedding)
+          def build_exists_map(chunks)
+            chunks.to_h { |c| [c[:content_hash], chunk_exists?(c[:content_hash])] }
+          end
+          private_class_method :build_exists_map
+
+          def llm_embed_available?
+            defined?(Legion::LLM) && Legion::LLM.respond_to?(:embed_batch)
+          end
+          private_class_method :llm_embed_available?
+
+          def paired_without_embed(chunks, exists_map)
+            chunks.map { |c| { chunk: c, embedding: nil, exists: exists_map.fetch(c[:content_hash], false) } }
+          end
+          private_class_method :paired_without_embed
 
+          def build_embed_map(needs_embed)
+            results = Legion::LLM.embed_batch(needs_embed.map { |c| c[:content] })
+            results.each_with_object({}) do |r, h|
+              h[needs_embed[r[:index]][:content_hash]] = r[:vector] unless r[:error]
+            end
+          rescue StandardError
+            {}
+          end
+          private_class_method :build_embed_map
+
+          def upsert_chunk_with_embedding(chunk, embedding, dry_run: false, force: false, exists: false)
+            return :created if dry_run
+            return :created unless defined?(Legion::Extensions::Apollo)
+            return :skipped if !force && exists
+
+            ingest_to_apollo(chunk, embedding)
             force ? :updated : :created
           rescue StandardError
             :skipped
           end
-          private_class_method :upsert_chunk
+          private_class_method :upsert_chunk_with_embedding
 
           def chunk_exists?(content_hash)
             return false unless defined?(Legion::Data::Model::ApolloEntry)
@@ -112,16 +161,6 @@ module Legion
           end
           private_class_method :chunk_exists?
 
-          def generate_embedding(content)
-            return nil unless defined?(Legion::LLM) && Legion::LLM.respond_to?(:embed)
-
-            result = Legion::LLM.embed(content)
-            result.is_a?(Hash) ? result[:vector] : nil
-          rescue StandardError
-            nil
-          end
-          private_class_method :generate_embedding
-
           def ingest_to_apollo(chunk, embedding)
             return unless defined?(Legion::Extensions::Apollo)
 
@@ -143,6 +182,21 @@ module Legion
             Legion::Extensions::Apollo::Runners::Knowledge.handle_ingest(**payload)
           end
           private_class_method :ingest_to_apollo
+
+          def retire_file(file_path:)
+            return unless defined?(Legion::Apollo)
+            return unless Legion::Apollo.respond_to?(:ingest) && Legion::Apollo.started?
+
+            Legion::Apollo.ingest(
+              content:      file_path,
+              content_type: 'document_retired',
+              tags:         [file_path, 'retired', 'document_chunk'].uniq,
+              metadata:     { source_file: file_path, retired: true }
+            )
+          rescue StandardError
+            nil
+          end
+          private_class_method :retire_file
         end
       end
     end

data/lib/legion/extensions/knowledge/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Knowledge
-      VERSION = '0.2.0'
+      VERSION = '0.4.0'
     end
   end
 end

data/lib/legion/extensions/knowledge.rb

@@ -2,6 +2,7 @@
 
 require_relative 'knowledge/version'
 require_relative 'knowledge/helpers/manifest'
+require_relative 'knowledge/helpers/manifest_store'
 require_relative 'knowledge/helpers/parser'
 require_relative 'knowledge/helpers/chunker'
 require_relative 'knowledge/runners/ingest'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-knowledge
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Matthew Iverson
@@ -121,6 +121,7 @@ files:
 - lib/legion/extensions/knowledge/client.rb
 - lib/legion/extensions/knowledge/helpers/chunker.rb
 - lib/legion/extensions/knowledge/helpers/manifest.rb
+- lib/legion/extensions/knowledge/helpers/manifest_store.rb
 - lib/legion/extensions/knowledge/helpers/parser.rb
 - lib/legion/extensions/knowledge/runners/corpus.rb
 - lib/legion/extensions/knowledge/runners/ingest.rb