legion-llm 0.8.21 → 0.8.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5fdd5beb0c0e3a464cd06848369ec4cf9c07beab7a9c1560cfc0d13e1a6dc721
-  data.tar.gz: 2539e7e1a1cebcd9ba363ac66f79c08f9314d7477d482f1d17a7bf6185e1fd0b
+  metadata.gz: bea0deb0330e257b0a513675970bd988c5b157170e1bf46569482e9203578681
+  data.tar.gz: 9af8c0c5e9d6911f95f738bfd840c3dd1989e2503da7042357332e1c394fe930
 SHA512:
-  metadata.gz: 377c8498427bbe07d0dd171ce8fba340683ac8fb0286b781092caa24eb19c038fc9e621115bfcbfe12aee25c1f84aa267b4c289b03cd9424a2beee0d31ccf128
-  data.tar.gz: 611ab93b2732f10aaac6015b2fc042af6de8963b23012c73a00a6c6a36477639a61763ace1be0e65eb2f38c815cd0418c9d118a644090324a6de53b98cc2f5a6
+  metadata.gz: ff80717d479fb79c9c2ea60123828b50c218ce549d90c7d6a9605885c8791c1a078c48a232d4b5437213c904206c326d7b832348eabf694caef8e7cb30abdfcd
+  data.tar.gz: d81969d08b0dd13e6447a662aaeb4c4a0c43fe07cfe3e4a2af328bd30ab9d09df0e817a6ffaa8b296e44ebdb00d4a4f2b70f55a3f86c59e1f8b18c0207fb4da2

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Legion LLM Changelog
 
+## [0.8.23] - 2026-04-23
+
+### Fixed
+- `Call::StructuredOutput` prompt-fallback path passed `messages:` (plural) to `chat_single` which only accepts `message:` (singular), leaking the unknown kwarg into `RubyLLM::Chat.new`. Visible as repeated "unknown keyword: :messages" warnings during dream cycle contradiction detection. Flattened instruction + messages into a single string via `extract_user_content`.
+
+## [0.8.22] - 2026-04-22
+
+### Fixed
+- Error paths in `Executor#run_provider_call_single` and `#step_provider_call_stream` now emit audit events (`Audit.emit_prompt`) before re-raising `RateLimitError`, `ProviderError`, and `ProviderDown`. Previously these errors produced no audit trail.
+- Escalation exhaustion (`EscalationExhausted`) in the pipeline executor now emits an audit event with `status: 'escalation_exhausted'` before raising.
+- `assert_external_allowed!` in the Inference module now emits an audit event with `status: 'privacy_blocked'` before raising `PrivacyModeError`, so enterprise privacy blocks are observable in the audit trail.
+- `step_metering` in `Executor` now passes `request_id:` and `caller:` to `Steps::Metering.build_event` so every metering event carries caller identity and request correlation.
+- `Steps::Metering.identity_fields` updated to include `request_id` and `caller` fields in the emitted metering event payload.
+- `Call::Embeddings.generate` now emits a metering event via `Metering.emit` after each successful `RubyLLM.embed` call, covering the previously unmetered embedding path.
+- `chat_single` in Inference now calls `emit_non_pipeline_metering` after a direct (non-pipeline) `session.ask` so token usage is recorded when the pipeline is disabled.
+- `Call::StructuredOutput.generate` now logs `info` on successful parse and `warn` on `JSON::ParserError` for observability.
+
 ## [0.8.21] - 2026-04-22
 
 ### Fixed

data/lib/legion/llm/call/embeddings.rb

@@ -25,8 +25,9 @@ module Legion
             return { vector: nil, model: model, provider: provider, error: "provider #{provider} does not support embeddings" } \
               if provider && !provider_supports_embeddings?(provider)
 
-            response   = RubyLLM.embed(text, **build_opts(model, provider, dimensions))
-            vector     = apply_dimension_enforcement(response.vectors.first, provider)
+            response = RubyLLM.embed(text, **build_opts(model, provider, dimensions))
+            emit_embedding_metering(provider: provider, model: model, tokens: response.input_tokens)
+            vector = apply_dimension_enforcement(response.vectors.first, provider)
             return dimension_error(model, provider, vector) if vector.is_a?(String)
 
             { vector: vector, model: model, provider: provider, dimensions: vector&.size || 0, tokens: response.input_tokens }
@@ -459,6 +460,15 @@ module Legion
 
             []
           end
+
+          def emit_embedding_metering(provider:, model:, tokens:)
+            Legion::LLM::Metering.emit(
+              provider: provider, model_id: model, request_type: 'embed',
+              tier: 'cloud', input_tokens: tokens.to_i, output_tokens: 0, total_tokens: tokens.to_i
+            )
+          rescue StandardError => e
+            handle_exception(e, level: :warn, operation: 'llm.embeddings.metering')
+          end
         end
       end
     end

data/lib/legion/llm/call/structured_output.rb

@@ -13,10 +13,12 @@ module Legion
           def generate(messages:, schema:, model: nil, provider: nil, **)
             model ||= Legion::LLM.settings[:default_model]
             result = call_with_schema(messages, schema, model, provider: provider, **)
+            log.info "[llm][structured_output] model=#{model} provider=#{provider} valid=true"
 
             parsed = Legion::JSON.load(result[:content])
             { data: parsed, raw: result[:content], model: result[:model], valid: true }
           rescue ::JSON::ParserError => e
+            log.warn "[llm][structured_output] model=#{model} provider=#{provider} parse_error=#{e.message}"
             handle_parse_error(e, messages, schema, model, provider, result, **)
           end
 
@@ -34,10 +36,10 @@ module Legion
               instruction = "You MUST respond with valid JSON matching this schema:\n" \
                             "```json\n#{Legion::JSON.dump(schema)}\n```\n" \
                             'Respond with ONLY the JSON object, no other text.'
-              augmented = [{ role: 'system', content: instruction }] + Array(messages)
+              user_content = extract_user_content(messages, instruction)
               Legion::LLM::Inference.send(:chat_single,
                                           model: model, provider: provider, intent: nil, tier: nil,
-                                          messages: augmented, **opts.except(:attempt))
+                                          message: user_content, **opts.except(:attempt))
             end
           end
 
@@ -53,10 +55,10 @@ module Legion
 
           def retry_with_instruction(messages, schema, model, provider: nil, **opts)
             instruction = "Your previous response was not valid JSON. Respond with ONLY a valid JSON object matching this schema:\n#{Legion::JSON.dump(schema)}"
-            augmented = Array(messages) + [{ role: 'user', content: instruction }]
+            user_content = extract_user_content(messages, instruction)
             result = Legion::LLM::Inference.send(:chat_single,
                                                  model: model, provider: provider, intent: nil, tier: nil,
-                                                 messages: augmented, **opts.except(:attempt))
+                                                 message: user_content, **opts.except(:attempt))
 
             parsed = Legion::JSON.load(result[:content])
             { data: parsed, raw: result[:content], model: result[:model], valid: true, retried: true }
@@ -65,6 +67,15 @@ module Legion
             { data: nil, error: e.message, valid: false }
           end
 
+          def extract_user_content(messages, instruction)
+            parts = [instruction]
+            Array(messages).each do |msg|
+              content = msg[:content] || msg['content']
+              parts << content.to_s unless content.to_s.empty?
+            end
+            parts.join("\n\n")
+          end
+
           def supports_response_format?(model)
             SCHEMA_CAPABLE_MODELS.any? { |m| model.to_s.include?(m) }
           end

data/lib/legion/llm/inference/executor.rb

@@ -371,19 +371,23 @@ module Legion
           rescue RubyLLM::RateLimitError => e
             handle_exception(e, level: :warn, operation: 'llm.pipeline.provider_call.rate_limit',
                               provider: @resolved_provider, model: @resolved_model)
+            emit_error_audit(e, status: 'rate_limited')
             raise Legion::LLM::RateLimitError, e.message
           rescue RubyLLM::ServerError, RubyLLM::ServiceUnavailableError, RubyLLM::OverloadedError,
                  Faraday::ServerError => e
             handle_exception(e, level: :warn, operation: 'llm.pipeline.provider_call.provider_error',
                               provider: @resolved_provider, model: @resolved_model)
+            emit_error_audit(e, status: 'provider_error')
             raise Legion::LLM::ProviderError, e.message
           rescue Faraday::TooManyRequestsError => e
             handle_exception(e, level: :warn, operation: 'llm.pipeline.provider_call.http_rate_limit',
                               provider: @resolved_provider, model: @resolved_model)
+            emit_error_audit(e, status: 'rate_limited')
             raise Legion::LLM::RateLimitError.new(e.message, retry_after: extract_retry_after(e))
           rescue Faraday::ConnectionFailed, Faraday::TimeoutError => e
             handle_exception(e, level: :warn, operation: 'llm.pipeline.provider_call.provider_down',
                               provider: @resolved_provider, model: @resolved_model)
+            emit_error_audit(e, status: 'provider_down')
             raise Legion::LLM::ProviderDown, e.message
           end
         end
@@ -412,7 +416,13 @@ module Legion
             record_escalation_failure(e, resolution, start_time, outcome:   :error,
                                                                  operation: 'llm.pipeline.escalation_attempt')
           end
-          raise EscalationExhausted, "All #{@escalation_history.size} escalation attempts failed" unless succeeded
+          return if succeeded
+
+          emit_error_audit(
+            EscalationExhausted.new("All #{@escalation_history.size} attempts failed"),
+            status: 'escalation_exhausted'
+          )
+          raise EscalationExhausted, "All #{@escalation_history.size} escalation attempts failed"
         end
 
         def attempt_escalation(resolution, threshold, quality_check, start_time)
@@ -567,6 +577,23 @@ module Legion
           error.response[:headers]&.fetch('retry-after', nil)&.to_i
         end
 
+        def emit_error_audit(error, status:, provider: @resolved_provider, model: @resolved_model)
+          Legion::LLM::Audit.emit_prompt(
+            request_id:      @request.id,
+            conversation_id: @request.conversation_id,
+            caller:          @request.caller,
+            routing:         { provider: provider, model: model },
+            tokens:          {},
+            status:          status,
+            error:           { class: error.class.name, message: error.message },
+            tracing:         @tracing,
+            timestamp:       Time.now,
+            request_type:    'chat'
+          )
+        rescue StandardError => e
+          handle_exception(e, level: :warn, operation: 'llm.pipeline.emit_error_audit')
+        end
+
         def execute_pre_provider_steps
           PRE_PROVIDER_STEPS.each do |step|
             next if Profile.skip?(@profile, step)
@@ -645,19 +672,23 @@ module Legion
           rescue RubyLLM::RateLimitError => e
             handle_exception(e, level: :warn, operation: 'llm.pipeline.provider_call_stream.rate_limit',
                               provider: @resolved_provider, model: @resolved_model)
+            emit_error_audit(e, status: 'rate_limited')
             raise Legion::LLM::RateLimitError, e.message
           rescue RubyLLM::ServerError, RubyLLM::ServiceUnavailableError, RubyLLM::OverloadedError,
                  Faraday::ServerError => e
             handle_exception(e, level: :warn, operation: 'llm.pipeline.provider_call_stream.provider_error',
                               provider: @resolved_provider, model: @resolved_model)
+            emit_error_audit(e, status: 'provider_error')
             raise Legion::LLM::ProviderError, e.message
           rescue Faraday::TooManyRequestsError => e
             handle_exception(e, level: :warn, operation: 'llm.pipeline.provider_call_stream.http_rate_limit',
                               provider: @resolved_provider, model: @resolved_model)
+            emit_error_audit(e, status: 'rate_limited')
             raise Legion::LLM::RateLimitError.new(e.message, retry_after: extract_retry_after(e))
           rescue Faraday::ConnectionFailed, Faraday::TimeoutError => e
             handle_exception(e, level: :warn, operation: 'llm.pipeline.provider_call_stream.provider_down',
                               provider: @resolved_provider, model: @resolved_model)
+            emit_error_audit(e, status: 'provider_down')
             raise Legion::LLM::ProviderDown, e.message
           end
         end
@@ -1034,7 +1065,9 @@ module Legion
             request_type:  'chat',
             input_tokens:  input_tokens,
             output_tokens: output_tokens,
-            latency_ms:    latency_ms
+            latency_ms:    latency_ms,
+            request_id:    @request.id,
+            caller:        @request.caller
           )
           Steps::Metering.publish_or_spool(event)
         rescue StandardError => e

data/lib/legion/llm/inference/steps/metering.rb

@@ -29,6 +29,8 @@ module Legion
               node_id:      opts[:node_id],
               worker_id:    opts[:worker_id],
               agent_id:     opts[:agent_id],
+              request_id:   opts[:request_id],
+              caller:       opts[:caller],
               request_type: opts[:request_type],
               tier:         opts[:tier],
               provider:     opts[:provider],

data/lib/legion/llm/inference.rb

@@ -514,6 +514,7 @@ module Legion
         log.debug '[llm][inference] chat_single asking session'
         response = block ? session.ask(message, &block) : session.ask(message)
         log.debug "[llm][inference] chat_single response_class=#{response.class} response_nil=#{response.nil?}"
+        emit_non_pipeline_metering(response, model: opts[:model], provider: opts[:provider])
 
         if response && !block && defined?(Quality::ShadowEval) && Quality::ShadowEval.enabled?
           msgs = session.respond_to?(:messages) ? session.messages : nil
@@ -711,6 +712,19 @@ module Legion
         esc.fetch(:quality_threshold, 50)
       end
 
+      def emit_non_pipeline_metering(response, model:, provider:)
+        return unless response
+
+        input  = response.respond_to?(:input_tokens)  ? response.input_tokens.to_i  : 0
+        output = response.respond_to?(:output_tokens) ? response.output_tokens.to_i : 0
+        Legion::LLM::Metering.emit(
+          provider: provider, model_id: model, request_type: 'chat',
+          tier: 'direct', input_tokens: input, output_tokens: output, total_tokens: input + output
+        )
+      rescue StandardError => e
+        handle_exception(e, level: :warn, operation: 'llm.inference.non_pipeline_metering')
+      end
+
       def enterprise_privacy?
         if Legion.const_defined?('Settings', false) && Legion::Settings.respond_to?(:enterprise_privacy?)
           Legion::Settings.enterprise_privacy?
@@ -719,9 +733,21 @@ module Legion
         end
       end
 
+      def emit_privacy_blocked_audit
+        Legion::LLM::Audit.emit_prompt(
+          request_id: nil, conversation_id: nil, caller: nil,
+          routing: {}, tokens: {}, status: 'privacy_blocked',
+          error: { class: 'PrivacyModeError', message: 'External tiers blocked by enterprise privacy' },
+          timestamp: Time.now, request_type: 'chat'
+        )
+      rescue StandardError => e
+        handle_exception(e, level: :warn, operation: 'llm.inference.emit_privacy_blocked_audit')
+      end
+
       def assert_external_allowed!
         return unless enterprise_privacy?
 
+        emit_privacy_blocked_audit
         raise Legion::LLM::PrivacyModeError,
               'External LLM tiers are disabled: enterprise_data_privacy is enabled. ' \
               'Only local and fleet tiers are permitted.'

data/lib/legion/llm/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module LLM
-    VERSION = '0.8.21'
+    VERSION = '0.8.23'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-llm
 version: !ruby/object:Gem::Version
-  version: 0.8.21
+  version: 0.8.23
 platform: ruby
 authors:
 - Esity