lex-apollo 0.4.6 → 0.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9a1c9042626e5d9e09a783b8c703b11f261573fe1658498a4fa48ad023306baf
-  data.tar.gz: 2f3e561b6851825d9d5c67325007802eafea63c10c93a00e37a40170202a0bdc
+  metadata.gz: ef4d63cf3e8b7a9ce86f11af1150b851a911fa7383ca2233d753bc328be31cb5
+  data.tar.gz: 8d4e3f73d3d7d04c1dff338eb226a3385d0977d57d77c2378d41723b3fff80e6
 SHA512:
-  metadata.gz: d2dae8784e591deeb1608671dbabbd313813a6f94214b77de0539d2316d6ef3a34e21d5002eb28a262e98c4c4d57773793be4ba006d185e0fd3afcf6816902c5
-  data.tar.gz: 41e11ae62a001fab8e79b514d30c3c28374d38d1d4379926e00752df74e35ccfa79c61b3ac75e7e60341de0420287635413b265eca5fa0cd7fe4e7f02a5fad0f
+  metadata.gz: fae193470f3da5dedcdb5ee91310ac824064aa3ebd60de9f21e9d9a6c5eb770db23ad21603cb6c8d7337fb7fb84b1f85f9c3155f1ff38cac2bd426a3270eaabf
+  data.tar.gz: 6d6827a8defe1e41724a3033193f774034830cddf28bd34096622410f736d233a79853567f71b5e2f7547e48194ff3d23ce685633b55f832747ad37026c54df5

data/CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## [0.4.8] - 2026-03-25
+
+### Changed
+- Removed `Helpers::Embedding` direct Ollama bypass; all embedding generation now routes through `Legion::LLM::Embeddings.generate`
+- `Runners::Knowledge`, `Helpers::Writeback`, and `Actor::WritebackVectorize` all use `Legion::LLM::Embeddings.generate` and extract `[:vector]` from the result hash
+- Zero-vector fallback (`Array.new(1024, 0.0)`) preserved when LLM is unavailable or returns nil vector
+
+### Removed
+- `lib/legion/extensions/apollo/helpers/embedding.rb` — direct Faraday/Ollama embedding bypass (superseded by `Legion::LLM` provider routing)
+- `spec/legion/extensions/apollo/helpers/embedding_spec.rb` — tests for removed helper
+
+## [0.4.7] - 2026-03-25
+
+### Added
+- Knowledge capture writeback system: `Helpers::Writeback` evaluates LLM responses for Apollo ingest with echo chamber prevention
+- `Helpers::TagNormalizer` for mechanical tag normalization (aliases, special chars, max 5)
+- `Helpers::Capability` for detecting embed/write capabilities per node
+- Writeback transport layer: `Messages::Writeback`, `Queues::WritebackStore`, `Queues::WritebackVectorize`
+- Writeback subscription actors: `Actor::WritebackStore` (pre-embedded), `Actor::WritebackVectorize` (needs embedding)
+- Content hash dedup in `Runners::Knowledge#handle_ingest` — collision boosts confidence instead of inserting
+- Identity tracking: `submitted_by`, `submitted_from`, `content_hash` fields on knowledge entries
+
 ## [0.4.6] - 2026-03-25
 
 ### Added

data/lib/legion/extensions/apollo/actors/writeback_store.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/actors/subscription' if defined?(Legion::Extensions::Actors::Subscription)
+
+module Legion
+  module Extensions
+    module Apollo
+      module Actor
+        class WritebackStore < Legion::Extensions::Actors::Subscription
+          def runner_class    = 'Legion::Extensions::Apollo::Runners::Knowledge'
+          def runner_function = 'handle_ingest'
+          def check_subtask?  = false
+          def generate_task?  = false
+
+          def enabled?
+            defined?(Legion::Extensions::Apollo::Runners::Knowledge) &&
+              defined?(Legion::Transport) &&
+              Helpers::Capability.apollo_write_enabled?
+          rescue StandardError
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/apollo/actors/writeback_vectorize.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/actors/subscription' if defined?(Legion::Extensions::Actors::Subscription)
+
+module Legion
+  module Extensions
+    module Apollo
+      module Actor
+        class WritebackVectorize < Legion::Extensions::Actors::Subscription
+          def runner_class    = self.class
+          def runner_function = 'handle_vectorize'
+          def check_subtask?  = false
+          def generate_task?  = false
+
+          def handle_vectorize(payload)
+            payload = symbolize(payload)
+            result = Legion::LLM::Embeddings.generate(text: payload[:content])
+            vector = result.is_a?(Hash) ? result[:vector] : result
+            embedding = vector.is_a?(Array) && vector.any? ? vector : Array.new(1024, 0.0)
+            enriched = payload.merge(embedding: embedding)
+
+            if Helpers::Capability.can_write?
+              Runners::Knowledge.handle_ingest(**enriched)
+            else
+              Transport::Messages::Writeback.new(
+                **enriched, has_embedding: true
+              ).publish
+            end
+
+            { success: true, action: :vectorized }
+          rescue StandardError => e
+            { success: false, error: e.message }
+          end
+
+          def enabled?
+            defined?(Legion::Transport) && Helpers::Capability.can_embed?
+          rescue StandardError
+            false
+          end
+
+          private
+
+          def symbolize(hash)
+            hash.transform_keys(&:to_sym)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/apollo/helpers/capability.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Apollo
+      module Helpers
+        module Capability
+          EMBEDDING_MODELS = %w[mxbai-embed-large bge-large snowflake-arctic-embed].freeze
+
+          module_function
+
+          def can_embed?
+            return false unless defined?(Legion::LLM) && Legion::LLM.started?
+
+            ollama_embedding_available? || cloud_embedding_configured?
+          rescue StandardError
+            false
+          end
+
+          def can_write?
+            return false unless apollo_write_enabled?
+            return false unless defined?(Legion::Data) && Legion::Data.connected?
+
+            check_db_write_privilege
+          rescue StandardError
+            false
+          end
+
+          def apollo_write_enabled?
+            Legion::Settings.dig(:data, :apollo_write) == true
+          rescue StandardError
+            false
+          end
+
+          def ollama_embedding_available?
+            return false unless defined?(Legion::LLM::Discovery::Ollama)
+
+            EMBEDDING_MODELS.any? { |m| Legion::LLM::Discovery::Ollama.model_available?(m) }
+          rescue StandardError
+            false
+          end
+
+          def cloud_embedding_configured?
+            provider = Legion::Settings.dig(:apollo, :embedding, :provider)
+            model = Legion::Settings.dig(:apollo, :embedding, :model)
+            !provider.nil? && !model.nil?
+          rescue StandardError
+            false
+          end
+
+          def check_db_write_privilege
+            return @apollo_write_privilege unless @apollo_write_privilege.nil?
+
+            @apollo_write_privilege = Legion::Data.connection
+                                                  .fetch("SELECT has_table_privilege(current_user, 'apollo_entries', 'INSERT') AS can_insert")
+                                                  .first[:can_insert] == true
+          rescue StandardError
+            @apollo_write_privilege = false
+          end
+
+          def reset!
+            @apollo_write_privilege = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/apollo/helpers/tag_normalizer.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Apollo
+      module Helpers
+        module TagNormalizer
+          ALIASES = {
+            'c#' => 'csharp', '.net' => 'dotnet', 'c++' => 'cplusplus',
+            'node.js' => 'nodejs', 'vue.js' => 'vuejs', 'react.js' => 'reactjs'
+          }.freeze
+
+          module_function
+
+          def normalize(raw)
+            tag = raw.to_s.strip.downcase
+            tag = ALIASES[tag] if ALIASES.key?(tag)
+            tag = tag.gsub(/[^a-z0-9\- ]/, '')
+                     .gsub(/\s+/, '-')
+                     .gsub(/-+/, '-')
+                     .sub(/^-/, '')
+                     .sub(/-$/, '')
+            tag.empty? ? nil : tag
+          end
+
+          def normalize_all(tags, max: 5)
+            Array(tags)
+              .filter_map { |t| normalize(t) }
+              .uniq
+              .first(max)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/apollo/helpers/writeback.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'socket'
+
+module Legion
+  module Extensions
+    module Apollo
+      module Helpers
+        module Writeback
+          RESEARCH_TOOLS = %w[read_file search_files search_content run_command].freeze
+          MAX_CONTENT_LENGTH = 4000
+          MIN_CONTENT_LENGTH = 50
+
+          module_function
+
+          def evaluate_and_route(request:, response:, enrichments: {})
+            return unless writeback_enabled?
+            return unless should_capture?(request, response, enrichments)
+
+            payload = build_payload(request: request, response: response)
+            route_payload(payload)
+          rescue StandardError => e
+            Legion::Logging.warn("apollo writeback failed: #{e.message}") if defined?(Legion::Logging)
+          end
+
+          def should_capture?(_request, response, enrichments)
+            content = response_content(response)
+            return false if content.nil? || content.length < min_content_length
+
+            tool_calls = extract_tool_calls(response, enrichments)
+            research_calls = tool_calls.select { |tc| RESEARCH_TOOLS.include?(tc[:name] || tc['name']) }
+
+            return false if research_calls.empty?
+
+            apollo_results = enrichments['rag_context:apollo_results']
+            return true if apollo_results.nil? || (apollo_results[:count] || 0).zero?
+
+            # Apollo had results — only capture if LLM also did additional research
+            research_calls.any?
+          end
+
+          def build_payload(request:, response:, source_channel: nil)
+            content = response_content(response)
+            caller_identity = extract_identity(request)
+            user_query = extract_user_query(request)
+            tags = derive_tags(user_query)
+
+            {
+              content:          content[0...MAX_CONTENT_LENGTH],
+              content_type:     'observation',
+              tags:             Helpers::TagNormalizer.normalize_all(tags),
+              source_agent:     response.respond_to?(:model) ? response.model : 'unknown',
+              source_channel:   "#{source_channel || 'pipeline'}_synthesis",
+              submitted_by:     caller_identity,
+              submitted_from:   Socket.gethostname,
+              knowledge_domain: nil,
+              content_hash:     content_hash(content)
+            }
+          end
+
+          def route_payload(payload)
+            can_embed = Helpers::Capability.can_embed?
+            can_write = Helpers::Capability.can_write?
+
+            if can_embed
+              result = Legion::LLM::Embeddings.generate(text: payload[:content])
+              vector = result.is_a?(Hash) ? result[:vector] : result
+              payload[:embedding] = vector.is_a?(Array) && vector.any? ? vector : Array.new(1024, 0.0)
+            end
+
+            if can_write && can_embed
+              write_directly(payload)
+            else
+              publish_to_transport(payload, has_embedding: can_embed)
+            end
+          end
+
+          def write_directly(payload)
+            Runners::Knowledge.handle_ingest(**payload)
+          rescue StandardError => e
+            Legion::Logging.warn("apollo direct write failed, falling back to transport: #{e.message}") if defined?(Legion::Logging)
+            publish_to_transport(payload, has_embedding: !payload[:embedding].nil?)
+          end
+
+          def publish_to_transport(payload, has_embedding: false)
+            return unless defined?(Legion::Transport)
+
+            Transport::Messages::Writeback.new(
+              **payload, has_embedding: has_embedding
+            ).publish
+          rescue StandardError => e
+            Legion::Logging.warn("apollo writeback publish failed: #{e.message}") if defined?(Legion::Logging)
+          end
+
+          def writeback_enabled?
+            Legion::Settings.dig(:apollo, :writeback, :enabled) != false
+          rescue StandardError
+            true
+          end
+
+          def min_content_length
+            Legion::Settings.dig(:apollo, :writeback, :min_content_length) || MIN_CONTENT_LENGTH
+          rescue StandardError
+            MIN_CONTENT_LENGTH
+          end
+
+          def content_hash(content)
+            normalized = content.to_s.strip.downcase.gsub(/\s+/, ' ')
+            Digest::MD5.hexdigest(normalized)
+          end
+
+          def response_content(response)
+            msg = response.respond_to?(:message) ? response.message : nil
+            return nil unless msg.is_a?(Hash)
+
+            msg[:content] || msg['content']
+          end
+
+          def extract_identity(request)
+            return 'unknown' unless request.respond_to?(:caller) && request.caller.is_a?(Hash)
+
+            request.caller.dig(:requested_by, :identity) || 'unknown'
+          rescue StandardError
+            'unknown'
+          end
+
+          def extract_user_query(request)
+            return '' unless request.respond_to?(:messages)
+
+            user_msgs = Array(request.messages).select { |m| m[:role] == 'user' || m['role'] == 'user' }
+            (user_msgs.last || {})[:content] || ''
+          rescue StandardError
+            ''
+          end
+
+          def extract_tool_calls(response, enrichments)
+            calls = []
+            calls += Array(response.tool_calls) if response.respond_to?(:tool_calls)
+            calls += Array(enrichments['tool_calls']) if enrichments['tool_calls']
+            calls.uniq { |tc| tc[:name] || tc['name'] }
+          end
+
+          def derive_tags(query)
+            stop_words = %w[a an the is are was were be been being have has had do does did will would shall
+                            should may might can could of in to for on with at by from as into about between
+                            how what when where why who which this that these those it its and or but not]
+            words = query.to_s.downcase.gsub(/[^a-z0-9\s]/, '').split
+            words.reject { |w| stop_words.include?(w) || w.length < 3 }
+                 .uniq
+                 .first(5)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/apollo/runners/knowledge.rb

@@ -2,7 +2,6 @@
 
 require 'json'
 require_relative '../helpers/confidence'
-require_relative '../helpers/embedding'
 
 module Legion
   module Extensions
@@ -59,12 +58,25 @@ module Legion
             }
           end
 
-          def handle_ingest(content:, content_type:, tags: [], source_agent: 'unknown', source_provider: nil, source_channel: nil, knowledge_domain: nil, context: {}, **) # rubocop:disable Metrics/ParameterLists, Layout/LineLength
+          def handle_ingest(content:, content_type:, tags: [], source_agent: 'unknown', source_provider: nil, source_channel: nil, knowledge_domain: nil, submitted_by: nil, submitted_from: nil, content_hash: nil, context: {}, **) # rubocop:disable Metrics/ParameterLists, Layout/LineLength
             return { success: false, error: 'apollo_data_not_available' } unless defined?(Legion::Data::Model::ApolloEntry)
 
-            embedding = Helpers::Embedding.generate(text: content)
+            # Content hash dedup
+            hash = content_hash || (defined?(Helpers::Writeback) ? Helpers::Writeback.content_hash(content) : nil)
+            if hash
+              existing = Legion::Data::Model::ApolloEntry
+                         .where(content_hash: hash)
+                         .exclude(status: 'archived')
+                         .first
+              if existing
+                existing.update(confidence: [existing.confidence + Helpers::Confidence.retrieval_boost, 1.0].min)
+                return { success: true, entry_id: existing.id, deduped: true }
+              end
+            end
+
+            embedding = embed_text(content)
             content_type_sym = content_type.to_s
-            tag_array = Array(tags)
+            tag_array = defined?(Helpers::TagNormalizer) ? Helpers::TagNormalizer.normalize_all(tags) : Array(tags)
             domain = knowledge_domain || tag_array.first || 'general'
 
             corroborated, existing_id = find_corroboration(embedding, content_type_sym, source_agent, source_channel)
@@ -81,6 +93,9 @@ module Legion
                 tags:             Sequel.pg_array(tag_array),
                 status:           'candidate',
                 knowledge_domain: domain,
+                submitted_by:     submitted_by,
+                submitted_from:   submitted_from,
+                content_hash:     hash,
                 embedding:        Sequel.lit("'[#{embedding.join(',')}]'::vector")
               )
               existing_id = new_entry.id
@@ -103,7 +118,7 @@ module Legion
           def handle_query(query:, limit: Helpers::GraphQuery.default_query_limit, min_confidence: Helpers::GraphQuery.default_query_min_confidence, status: [:confirmed], tags: nil, domain: nil, agent_id: 'unknown', **) # rubocop:disable Metrics/ParameterLists, Layout/LineLength
             return { success: false, error: 'apollo_data_not_available' } unless defined?(Legion::Data::Model::ApolloEntry)
 
-            embedding = Helpers::Embedding.generate(text: query)
+            embedding = embed_text(query)
             sql = Helpers::GraphQuery.build_semantic_search_sql(
               limit: limit, min_confidence: min_confidence,
               statuses: Array(status).map(&:to_s), tags: tags, domain: domain
@@ -212,7 +227,7 @@ module Legion
 
             return { success: true, entries: [], count: 0 } if query.nil? || query.to_s.strip.empty?
 
-            embedding = Helpers::Embedding.generate(text: query.to_s)
+            embedding = embed_text(query.to_s)
             sql = Helpers::GraphQuery.build_semantic_search_sql(
               limit: limit, min_confidence: min_confidence,
               statuses: ['confirmed'], tags: tags, domain: domain
@@ -294,6 +309,14 @@ module Legion
 
           private
 
+          def embed_text(text)
+            result = Legion::LLM::Embeddings.generate(text: text)
+            vector = result.is_a?(Hash) ? result[:vector] : result
+            vector.is_a?(Array) && vector.any? ? vector : Array.new(1024, 0.0)
+          rescue StandardError
+            Array.new(1024, 0.0)
+          end
+
           def allowed_domains_for(target_domain)
             rules = if defined?(Legion::Settings) && Legion::Settings.dig(:apollo, :domain_isolation)
                       Legion::Settings.dig(:apollo, :domain_isolation)

data/lib/legion/extensions/apollo/runners/maintenance.rb

@@ -2,7 +2,6 @@
 
 require_relative '../helpers/confidence'
 require_relative '../helpers/similarity'
-require_relative '../helpers/embedding'
 
 module Legion
   module Extensions

data/lib/legion/extensions/apollo/transport/messages/writeback.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'legion/transport/message' if defined?(Legion::Transport)
+
+module Legion
+  module Extensions
+    module Apollo
+      module Transport
+        module Messages
+          class Writeback < Legion::Transport::Message
+            def exchange
+              Exchanges::Apollo
+            end
+
+            def routing_key
+              @options[:has_embedding] ? 'apollo.writeback.store' : 'apollo.writeback.vectorize'
+            end
+
+            def type
+              'apollo_writeback'
+            end
+
+            def message
+              {
+                content:          @options[:content],
+                content_type:     @options[:content_type],
+                tags:             @options[:tags],
+                source_agent:     @options[:source_agent],
+                source_channel:   @options[:source_channel],
+                submitted_by:     @options[:submitted_by],
+                submitted_from:   @options[:submitted_from],
+                embedding:        @options[:embedding],
+                knowledge_domain: @options[:knowledge_domain],
+                context:          @options[:context] || {}
+              }.compact
+            end
+
+            def validate
+              raise TypeError, 'content is required' unless @options[:content].is_a?(String)
+
+              @valid = true
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/apollo/transport/queues/writeback_store.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'legion/transport/queue' if defined?(Legion::Transport)
+
+module Legion
+  module Extensions
+    module Apollo
+      module Transport
+        module Queues
+          class WritebackStore < Legion::Transport::Queue
+            def queue_name
+              'apollo.writeback.store'
+            end
+
+            def queue_options
+              { manual_ack: true, durable: true, arguments: { 'x-dead-letter-exchange': 'apollo.dlx' } }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/apollo/transport/queues/writeback_vectorize.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'legion/transport/queue' if defined?(Legion::Transport)
+
+module Legion
+  module Extensions
+    module Apollo
+      module Transport
+        module Queues
+          class WritebackVectorize < Legion::Transport::Queue
+            def queue_name
+              'apollo.writeback.vectorize'
+            end
+
+            def queue_options
+              { manual_ack: true, durable: true, arguments: { 'x-dead-letter-exchange': 'apollo.dlx' } }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/apollo/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Apollo
-      VERSION = '0.4.6'
+      VERSION = '0.4.8'
     end
   end
 end

data/lib/legion/extensions/apollo.rb

@@ -4,6 +4,9 @@ require 'legion/extensions/apollo/version'
 require 'legion/extensions/apollo/helpers/confidence'
 require 'legion/extensions/apollo/helpers/similarity'
 require 'legion/extensions/apollo/helpers/graph_query'
+require 'legion/extensions/apollo/helpers/tag_normalizer'
+require 'legion/extensions/apollo/helpers/capability'
+require 'legion/extensions/apollo/helpers/writeback'
 require 'legion/extensions/apollo/runners/knowledge'
 require 'legion/extensions/apollo/runners/expertise'
 require 'legion/extensions/apollo/runners/maintenance'
@@ -21,6 +24,9 @@ if defined?(Legion::Transport)
   require 'legion/extensions/apollo/transport/queues/gas'
   require 'legion/extensions/apollo/transport/messages/ingest'
   require 'legion/extensions/apollo/transport/messages/query'
+  require 'legion/extensions/apollo/transport/messages/writeback'
+  require 'legion/extensions/apollo/transport/queues/writeback_store'
+  require 'legion/extensions/apollo/transport/queues/writeback_vectorize'
 end
 
 module Legion

data/spec/legion/extensions/apollo/actors/writeback_store_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+unless defined?(Legion::Extensions::Actors::Subscription)
+  module Legion
+    module Extensions
+      module Actors
+        class Subscription; end # rubocop:disable Lint/EmptyClass
+      end
+    end
+  end
+end
+$LOADED_FEATURES << 'legion/extensions/actors/subscription' unless $LOADED_FEATURES.include?('legion/extensions/actors/subscription')
+
+require 'legion/extensions/apollo/helpers/capability'
+require 'legion/extensions/apollo/runners/knowledge'
+require 'legion/extensions/apollo/actors/writeback_store'
+
+RSpec.describe Legion::Extensions::Apollo::Actor::WritebackStore do
+  subject(:actor) { described_class.new }
+
+  describe '#runner_class' do
+    it 'returns Knowledge runner string' do
+      expect(actor.runner_class).to eq('Legion::Extensions::Apollo::Runners::Knowledge')
+    end
+  end
+
+  describe '#runner_function' do
+    it 'returns handle_ingest' do
+      expect(actor.runner_function).to eq('handle_ingest')
+    end
+  end
+
+  describe '#check_subtask?' do
+    it { expect(actor.check_subtask?).to be false }
+  end
+
+  describe '#generate_task?' do
+    it { expect(actor.generate_task?).to be false }
+  end
+end