lex-pilot-knowledge-assist 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f398e78d2e02cceb805989cab08ce330e62fcaa458afa2c07d864553789b708
-  data.tar.gz: 1a677c86d87bbe95617c95d55dfe20ce20aeec0452f9adef8a723200576e77d7
+  metadata.gz: e9cd5f20f185374b02c185d07efc68ac953eb96b51dc742dde00060a17dc7f25
+  data.tar.gz: b3340c3aaad049daef0d6c102518398b6e10e834a88f8a87849c08a86ba5bdb5
 SHA512:
-  metadata.gz: c328df777ead50e852bc92486d54193d56eea53526cc03ad9fec78556c75aad8bbd11deb0b37fc4b25fd0955f025cf170a414f0fbf9e5b132e8bec5355d51dbe
-  data.tar.gz: 45acdab5dc6c139132499fa24c5ea48e10be2fa93b1bef3972175f8badc07d4a6115e38bd05b502a5771aac253d32f4f8338f9b7d23b107a12ca241f9ef6e859
+  metadata.gz: a2d00b05083629bc9b4cf0551e1bbf6783e3aa5695cfb69cba4ad3dfe1b6b8054ea2272199467340ff2ff431064e326918051b502569a39056bd61fbadad6f84
+  data.tar.gz: 2567071743f3f751c69f7afdea3f8379b6b0c31823cee33f7a71987d15c1f85079bfef1b5f0359c18d4b9bf86289032712da5c757ccdcb2d530760d32806bb60

data/lib/legion/extensions/pilot_knowledge_assist/actors/question_subscriber.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module PilotKnowledgeAssist
+      module Actors
+        class QuestionSubscriber < Legion::Extensions::Actors::Subscription
+          def runner_class
+            'Legion::Extensions::PilotKnowledgeAssist::Runners::Assistant'
+          end
+
+          def runner_function
+            'answer_question'
+          end
+
+          def check_subtask?
+            false
+          end
+
+          def generate_task?
+            false
+          end
+
+          def use_runner?
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/pilot_knowledge_assist/runners/assistant.rb

@@ -5,20 +5,51 @@ module Legion
     module PilotKnowledgeAssist
       module Runners
         module Assistant
+          include Classifier
+
+          DISCLAIMER_THRESHOLD = 0.5
+          ESCALATION_THRESHOLD = 0.2
+          DISCLAIMER_PREFIX = "I'm not fully certain about this answer -- please verify:\n\n"
+          INTENT_ANSWERS = {
+            greeting: 'Hello! I can help with Grid documentation and support questions.',
+            out_of_scope: 'I can only help with Grid infrastructure and documentation questions.',
+            ops_request: 'I can answer documentation questions but cannot perform operations. ' \
+                         'Please contact the Grid team.'
+          }.freeze
+
           def answer_question(question:, agent_id: 'knowledge-assist')
-            context = retrieve_context(question, agent_id)
-            answer = generate_answer(question, context)
-
-            {
-              question: question,
-              answer: answer,
-              sources: context.map { |c| c[:id] },
-              confidence: derive_confidence(context)
-            }
+            intent_response = classify_and_route(question: question)
+            return intent_response if intent_response
+
+            context_entries = retrieve_context(question: question, agent_id: agent_id)
+            confidence = derive_confidence(context_entries)
+
+            if confidence < ESCALATION_THRESHOLD
+              escalate(question: question, confidence: confidence)
+              return { question: question, answer: 'This question has been escalated to the support team.',
+                       sources: [], confidence: confidence, escalated: true, flagged: true }
+            end
+
+            answer = generate_answer(question: question, context: context_entries)
+            flagged = confidence < DISCLAIMER_THRESHOLD
+            final_answer = flagged ? "#{DISCLAIMER_PREFIX}#{answer}" : answer
+
+            { question: question, answer: final_answer, sources: context_entries.map { |e| e[:id] },
+              confidence: confidence, flagged: flagged, escalated: false }
           end
 
           private
 
+          def classify_and_route(question:)
+            classification = classify_intent(message: question)
+            intent = classification[:intent]
+            answer = INTENT_ANSWERS[intent]
+            return unless answer
+
+            { question: question, answer: answer, intent: intent,
+              confidence: 1.0, sources: [], flagged: false, escalated: false }
+          end
+
           def derive_confidence(context)
             return 0.3 if context.empty?
 
@@ -28,7 +59,7 @@ module Legion
             scores.max.clamp(0.1, 1.0)
           end
 
-          def retrieve_context(question, agent_id)
+          def retrieve_context(question:, agent_id:)
             return [] unless defined?(Legion::Extensions::Apollo::Client)
 
             client = Legion::Extensions::Apollo::Client.new(agent_id: agent_id)
@@ -37,7 +68,7 @@ module Legion
             []
           end
 
-          def generate_answer(question, context)
+          def generate_answer(question:, context:)
             return 'LLM unavailable' unless defined?(Legion::LLM)
 
             context_text = context.map { |c| c[:content] }.join("\n\n")
@@ -48,10 +79,29 @@ module Legion
                      end
 
             result = Legion::LLM.chat(message: prompt, caller: { extension: 'lex-pilot-knowledge-assist' })
-            result[:content]
+            result.is_a?(Hash) ? result[:content] : result.to_s
           rescue StandardError => e
             "Error: #{e.message}"
           end
+
+          def escalate(question:, confidence:)
+            return unless defined?(Legion::Extensions::Slack::Client)
+
+            webhook = escalation_webhook
+            return unless webhook
+
+            message = "Knowledge assist escalation:\n*Question:* #{question}\n*Confidence:* #{confidence}"
+            Legion::Extensions::Slack::Client.new.send_webhook(message: message, webhook: webhook)
+          rescue StandardError => e
+            Legion::Logging::Logger.warn("Escalation failed: #{e.message}") if defined?(Legion::Logging)
+          end
+
+          def escalation_webhook
+            return nil unless defined?(Legion::Settings)
+
+            config = Legion::Settings[:pilot_knowledge_assist] || {}
+            config[:escalation_webhook]
+          end
         end
       end
     end

data/lib/legion/extensions/pilot_knowledge_assist/runners/classifier.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module PilotKnowledgeAssist
+      module Runners
+        module Classifier
+          VALID_INTENTS = %i[doc_question ops_request greeting out_of_scope].freeze
+
+          CLASSIFY_PROMPT = <<~PROMPT
+            Classify this message into exactly one category. Reply with ONLY the category name:
+            - doc_question (asking about documentation, how-to, configuration, architecture)
+            - ops_request (asking to perform an action, run a command, make a change)
+            - greeting (hello, hi, thanks, goodbye)
+            - out_of_scope (weather, sports, personal questions, unrelated topics)
+
+            Message: %<message>s
+          PROMPT
+
+          def classify_intent(message:)
+            return { intent: :doc_question, method: :default } unless llm_available?
+
+            prompt = format(CLASSIFY_PROMPT, message: message)
+            response = Legion::LLM.chat(
+              message: prompt,
+              caller: { extension: 'lex-pilot-knowledge-assist', function: 'classify_intent' }
+            )
+            intent = response.to_s.strip.downcase.to_sym
+            intent = :doc_question unless VALID_INTENTS.include?(intent)
+
+            { intent: intent, method: :llm }
+          rescue StandardError
+            { intent: :doc_question, method: :fallback }
+          end
+
+          private
+
+          def llm_available?
+            defined?(Legion::LLM) && Legion::LLM.respond_to?(:started?) && Legion::LLM.started?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/pilot_knowledge_assist/runners/feedback.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module PilotKnowledgeAssist
+      module Runners
+        module Feedback
+          VALID_RATINGS = %i[positive negative].freeze
+
+          def record_feedback(question:, answer:, rating:, user_id: nil)
+            rating = rating.to_sym
+            return { success: false, reason: :invalid_rating } unless VALID_RATINGS.include?(rating)
+
+            entry = { question: question, answer: answer, rating: rating,
+                      user_id: user_id, timestamp: Time.now }
+            feedback_store << entry
+
+            store_to_apollo(question: question, answer: answer) if rating == :positive
+
+            { success: true, rating: rating }
+          end
+
+          def feedback_stats
+            total = feedback_store.size
+            positive = feedback_store.count { |e| e[:rating] == :positive }
+            negative = total - positive
+
+            { total: total, positive: positive, negative: negative,
+              accuracy: total.positive? ? positive.to_f / total : 0.0 }
+          end
+
+          private
+
+          def feedback_store
+            @feedback_store ||= []
+          end
+
+          def store_to_apollo(question:, answer:)
+            return unless defined?(Legion::Extensions::Apollo::Client)
+
+            Legion::Extensions::Apollo::Client.new.handle_ingest(
+              content: "Q: #{question}\nA: #{answer}",
+              content_type: 'qa_pair',
+              confidence: 0.5,
+              tags: %w[knowledge_assist qa_pair]
+            )
+          rescue StandardError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/pilot_knowledge_assist/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module PilotKnowledgeAssist
-      VERSION = '0.1.3'
+      VERSION = '0.2.0'
     end
   end
 end

data/lib/legion/extensions/pilot_knowledge_assist.rb

@@ -1,7 +1,11 @@
 # frozen_string_literal: true
 
 require_relative 'pilot_knowledge_assist/version'
+require_relative 'pilot_knowledge_assist/runners/classifier'
 require_relative 'pilot_knowledge_assist/runners/assistant'
+require_relative 'pilot_knowledge_assist/runners/feedback'
+
+require_relative 'pilot_knowledge_assist/actors/question_subscriber' if defined?(Legion::Extensions::Actors::Subscription)
 
 module Legion
   module Extensions

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-pilot-knowledge-assist
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Esity
@@ -115,7 +115,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/legion/extensions/pilot_knowledge_assist.rb
+- lib/legion/extensions/pilot_knowledge_assist/actors/question_subscriber.rb
 - lib/legion/extensions/pilot_knowledge_assist/runners/assistant.rb
+- lib/legion/extensions/pilot_knowledge_assist/runners/classifier.rb
+- lib/legion/extensions/pilot_knowledge_assist/runners/feedback.rb
 - lib/legion/extensions/pilot_knowledge_assist/version.rb
 homepage: https://github.com/LegionIO
 licenses: