nitro_intelligence 2.2.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1181b17cc855ffa2f039b1401fd5dd2040c262d2878e6d265e65f9a581f6106e
-  data.tar.gz: 28b3a022b8d6d7e8e8472930fcf089fa5a3ead8a6c0d917e7587a73b69b8fba5
+  metadata.gz: 2c196d3b804d5a9137a2af97b45df17a137382114b2d592d835d3477af0541e6
+  data.tar.gz: f0fb2d024492cd2f04b52d12f41930fc0ac9ed323aca3681a59016eccc3db7fb
 SHA512:
-  metadata.gz: e5fd5911aa6740d71f9dc5564766fe80137d7c2da6e7195ee406649387a22a8387bc02db78671e643d90445c6b8578e2b7b47d7573d4abdf677b35edcc41c6a3
-  data.tar.gz: d11585c4f9731911910660e67eb8702634475fd7bbf523b7313c81910d2775852f918a789a669719840196d64c474c1452842f6948e7618563ee1d5b2a1cd5e1
+  metadata.gz: c41e6aa4592a56a75a7a45715ffc2a313f0a1ecf39c227a780334d60b204292790bb715da83783763931da5af95d1e845dffb3addc208329c9dbd01ed2f75d0b
+  data.tar.gz: 7eb926cd024f0b73c2b372e75d65a43cf51a82f7afd487cd589b61be0f4130549d3fe08c3f9a01965668e092f092a5d431423af925533997f863c88f70c7534c

data/docs/README.md

@@ -36,28 +36,28 @@ NitroIntelligence.configure do |config|
 
   # Model configuration
   config.model_config = {
-    "default_audio_transcription_model" => "gpt-4o-transcribe",
-    "default_text_model" => "gpt-4o-mini",
-    "default_image_model" => "nano-banana-2",
-    "default_text_to_speech_model" => "gpt-4o-mini-tts",
+    "default_audio_transcription_model" => "Qwen/Qwen3-ASR-1.7B",
+    "default_text_model" => "Qwen/Qwen3.6-35B-A3B",
+    "default_image_model" => "google/gemini-3.1-flash-image",
+    "default_text_to_speech_model" => "openai/gpt-4o-mini-tts",
     "models" => [
       {
-        "name" => "gpt-4o-mini",
+        "name" => "Qwen/Qwen3.6-35B-A3B",
         "type" => "text"
       },
       {
-        "name" => "gpt-4o-transcribe",
+        "name" => "Qwen/Qwen3-ASR-1.7B",
         "type" => "audio_transcription"
       },
       {
-        "name" => "nano-banana-2",
+        "name" => "google/gemini-3.1-flash-image",
         "type" => "image",
-        "aspect_ratios" => ["1:1", "2:3", "3:2", "3:4", "4:3"],
+        "aspect_ratios" => ["1:1", "2:3", "3:2", "3:4", "4:3", "9:16", "16:9"],
         "resolutions" => ["512", "1K", "2K"],
         "omit_output_fields" => ["provider_specific_fields.thought_signatures"]
       },
       {
-        "name" => "gpt-4o-mini-tts",
+        "name" => "openai/gpt-4o-mini-tts",
         "type" => "text_to_speech",
         "default_voice" => "marin",
         "default_response_format" => "mp3",
@@ -101,7 +101,7 @@ You may also use [`openai-ruby`](https://github.com/openai/openai-ruby) compatib
 
 ```ruby
 client = NitroIntelligence::Client.new
-client.chat(parameters: { model: "meta-llama/Llama-3.1-8B-Instruct", messages: [{ role: "user", content: "Why is the sky blue?" }]})
+client.chat(parameters: { model: "Qwen/Qwen3.8-27B", messages: [{ role: "user", content: "Why is the sky blue?" }]})
 ```
 
 #### Providing Parameters
@@ -110,7 +110,7 @@ Parameters such as 'max_tokens' and 'temperature' can be passed in under the `pa
 
 ```ruby
 client = NitroIntelligence::Client.new
-client.chat(parameters: { model: "meta-llama/Llama-3.1-8B-Instruct", max_tokens: 1000, temperature: 0.7, messages: [{ role: "user", content: "Why is the sky blue?" }]})
+client.chat(parameters: { model: "Qwen/Qwen3.8-27B", max_tokens: 1000, temperature: 0.7, messages: [{ role: "user", content: "Why is the sky blue?" }]})
 ```
 
 For a full list of supported parameters, see the [API reference here](https://developers.openai.com/api/reference/resources/completions/methods/create).
@@ -330,6 +330,72 @@ client.chat(
 )
 ```
 
+### Grouping Traces: Sessions and Tags
+
+`session_id` groups related traces in the observability platform, and `tags` label
+them for filtering. Both are optional and are only sent when you set them.
+
+```ruby
+client = NitroIntelligence::Client.new(observability_project_slug: "fake-feature-project")
+client.chat(
+  message: "why did this deploy fail?",
+  parameters: {
+    session_id: "deploy-9f2c1ab",
+    tags: ["deployment-failure-analyzer"],
+  }
+)
+```
+
+### Correlating With Application Logs and the Inference Gateway
+
+Every observed request is correlated across three systems automatically:
+
+* **Observability platform → inference gateway.** The trace ID is sent to the
+  gateway as `x-litellm-trace-id`, and `metadata` is sent as
+  `x-litellm-spend-logs-metadata` (dropped if it exceeds 4KB).
+
+  The two are independent. Metadata is sent whenever you set it, observed or not,
+  so gateway spend can be attributed even without observability. The trace ID is
+  sent only on the observed path - it comes from the observation being recorded,
+  never from whatever span happens to be active, so a client built without an
+  `observability_project_slug` sends none even inside a host application with its
+  own OpenTelemetry instrumentation.
+* **Inference gateway → observability platform.** When a request fails, the
+  gateway's own request identifier is read from the error response and recorded on
+  the observation as `litellm_call_id` metadata.
+* **Application logs → observability platform.** Put whatever your logs are keyed
+  by into `metadata` — it lands on the trace *and* in the gateway's spend logs.
+
+```ruby
+client.chat(
+  message: "why did this deploy fail?",
+  parameters: {
+    trace_seed: deploy_url,
+    metadata: {
+      source: self.class,
+      rails_request_id: request_id,
+      job_id: job_id,
+    },
+  }
+)
+```
+
+To log the trace ID from application code, derive it from the same seed with
+`NitroIntelligence::Trace.create_id(seed:)`.
+
+#### Failed Requests
+
+A request that raises still produces a usable trace. The observation records:
+
+* the `input` that was sent (recorded before the request runs)
+* `level: "ERROR"` and a `status_message` carrying the exception class and message
+* `litellm_call_id` metadata when the gateway returned one
+
+The exception is then re-raised, so this changes what is observed, not how callers
+handle failures. Image generation is the one exception to input recording: its
+input carries base64 payloads that are replaced with media references only on
+success, so it records no input up front.
+
 ### Scoring
 
 You can use `NitroIntelligence::Reporter` to evaluate existing traces. Calling `NitroIntelligence::Reporter#score` lets you attach metrics to a trace in the observability platform.
@@ -380,18 +446,18 @@ Consider this prompt config:
 
 ```json
 {
-  "model": "gpt-4o-mini"
+  "model": "Qwen/Qwen3.6-35B-A3B"
 }
 ```
 
-Invoking this request would result in "gpt-4o-mini" being used as the model, even if supplied manually:
+Invoking this request would result in "Qwen/Qwen3.6-35B-A3B" being used as the model, even if supplied manually:
 
 ```ruby
 client = NitroIntelligence::Client.new(observability_project_slug: "fake-feature-project")
 client.chat(
   message: "Where is the appointment?",
   parameters: {
-    model: "meta-llama/Llama-3.1-8B-Instruct", # Will not be used, will be overridden by config "gpt-4o-mini"
+    model: "Qwen/Qwen3.8-27B", # Will not be used, will be overridden by config "Qwen/Qwen3.6-35B-A3B"
     prompt_name: "My Prompt With Variables",
     prompt_variables: {
       appointment_id: "1234",
@@ -410,7 +476,7 @@ client = NitroIntelligence::Client.new(observability_project_slug: "fake-feature
 client.chat(
   message: "Where is the appointment?",
   parameters: {
-    model: "meta-llama/Llama-3.1-8B-Instruct", # This will now be used since "prompt_config_disabled" is true
+    model: "Qwen/Qwen3.8-27B", # This will now be used since "prompt_config_disabled" is true
     prompt_name: "My Prompt With Variables",
     prompt_variables: {
       appointment_id: "1234",

data/lib/nitro_intelligence/client/handlers/audio_transcription_handler.rb

@@ -12,8 +12,9 @@ module NitroIntelligence
           perform_request(audio_file:, message:, parameters:)
         end
 
-        def perform_request(audio_file:, message: "", parameters: {})
+        def perform_request(audio_file:, message: "", parameters: {}, correlation_trace_id: nil)
           add_request_headers(parameters, MODALITY_HEADER => "audio", REQUESTED_MODEL_HEADER => parameters[:model])
+          add_correlation_headers(parameters, trace_id: correlation_trace_id)
           @client.audio.transcriptions.create(
             prompt: message,
             file: audio_file,

data/lib/nitro_intelligence/client/handlers/base_handler.rb

@@ -1,3 +1,5 @@
+require "json"
+
 module NitroIntelligence
   module Client
     module Handlers
@@ -5,6 +7,15 @@ module NitroIntelligence
         MODALITY_HEADER = "nip-modality".freeze
         REQUESTED_MODEL_HEADER = "nip-requested-model".freeze
 
+        # Correlation headers understood by the inference gateway (LiteLLM).
+        # See https://docs.litellm.ai/docs/proxy/request_headers
+        TRACE_ID_HEADER = "x-litellm-trace-id".freeze
+        SPEND_LOGS_METADATA_HEADER = "x-litellm-spend-logs-metadata".freeze
+
+        # Headers over ~8KB are rejected by most proxies. Metadata is caller
+        # supplied, so cap it rather than turning a large hash into a failed request.
+        MAX_SPEND_LOGS_METADATA_BYTES = 4096
+
         def initialize(client:)
           @client = client
         end
@@ -16,6 +27,41 @@ module NitroIntelligence
           (request_options[:extra_headers] ||= {}).merge!(headers.compact)
           parameters
         end
+
+        # Hands the inference gateway what it needs to be matched up with the rest of
+        # the picture: the trace ID the observability platform is recording this
+        # request under, so a Langfuse trace can be found from a LiteLLM request even
+        # when the request fails and never produces a response body, and the caller's
+        # metadata, so gateway spend can be attributed to the work that caused it.
+        #
+        # The two are independent. Metadata is worth sending whether or not anything
+        # is observing, whereas the trace ID is supplied by the observed handlers and
+        # is never read from whatever tracing context happens to be active: a host
+        # application with its own instrumentation has traces of its own, and their
+        # IDs mean nothing to the observability platform.
+        #
+        # add_request_headers drops nil values, so each header appears only when it
+        # has something to say.
+        def add_correlation_headers(parameters, trace_id:)
+          add_request_headers(
+            parameters,
+            TRACE_ID_HEADER => trace_id.presence,
+            SPEND_LOGS_METADATA_HEADER => spend_logs_metadata(parameters[:metadata])
+          )
+        end
+
+        def spend_logs_metadata(metadata)
+          return nil if metadata.blank?
+
+          json = metadata.to_json
+          return json if json.bytesize <= MAX_SPEND_LOGS_METADATA_BYTES
+
+          NitroIntelligence.logger.warn(
+            "#{self.class} metadata is #{json.bytesize} bytes, over the " \
+            "#{MAX_SPEND_LOGS_METADATA_BYTES} byte #{SPEND_LOGS_METADATA_HEADER} limit - omitting it"
+          )
+          nil
+        end
       end
     end
   end

data/lib/nitro_intelligence/client/handlers/chat_handler.rb

@@ -12,8 +12,9 @@ module NitroIntelligence
           perform_request(parameters:)
         end
 
-        def perform_request(parameters: {})
+        def perform_request(parameters: {}, correlation_trace_id: nil)
           add_request_headers(parameters, REQUESTED_MODEL_HEADER => parameters[:model])
+          add_correlation_headers(parameters, trace_id: correlation_trace_id)
           @client.chat.completions.create(**parameters.slice(*ALLOWED_EXTRA_PARAMETERS))
         end
 

data/lib/nitro_intelligence/client/handlers/image_handler.rb

@@ -19,8 +19,9 @@ module NitroIntelligence
           image_generation
         end
 
-        def perform_request(parameters: {})
+        def perform_request(parameters: {}, correlation_trace_id: nil)
           add_request_headers(parameters, MODALITY_HEADER => "image", REQUESTED_MODEL_HEADER => parameters[:model])
+          add_correlation_headers(parameters, trace_id: correlation_trace_id)
           @client.chat.completions.create(**parameters.slice(*ALLOWED_EXTRA_PARAMETERS))
         end
 

data/lib/nitro_intelligence/client/handlers/observed/audio_transcription_handler.rb

@@ -26,7 +26,8 @@ module NitroIntelligence
               type: :generation,
               parameters:,
               trace_name:,
-              prompt:
+              prompt:,
+              input: message
             ) do |generation|
               workflow(generation:, message:, audio_file:, parameters:)
             end
@@ -55,7 +56,9 @@ module NitroIntelligence
           end
 
           def workflow(generation:, message:, audio_file:, parameters:)
-            audio_transcription = @base_handler.perform_request(audio_file:, message:, parameters:)
+            audio_transcription = @base_handler.perform_request(
+              audio_file:, message:, parameters:, correlation_trace_id: generation.trace_id
+            )
 
             audio_file.rewind
             upload_handler = NitroIntelligence::Observability::UploadHandler.new(
@@ -66,9 +69,10 @@ module NitroIntelligence
               upload_queue: Queue.new([NitroIntelligence::Audio.new(audio_file)])
             )
 
+            # The model and input are already recorded on the observation before the
+            # request runs, and the response carries no model of its own
+            # (OpenAI::Models::Audio::Transcription), so neither is set again here.
             trace_attributes = {
-              model: parameters[:model], # Model isn't in response object OpenAI::Models::Audio::Transcription
-              input: message,
               output: audio_transcription.text,
               usage_details: {
                 input_tokens: audio_transcription.usage.input_tokens,

data/lib/nitro_intelligence/client/handlers/observed/chat_handler.rb

@@ -21,9 +21,10 @@ module NitroIntelligence
               type: :generation,
               parameters:,
               trace_name:,
-              prompt:
-            ) do |_generation|
-              workflow(parameters:)
+              prompt:,
+              input: parameters[:messages]
+            ) do |generation|
+              workflow(generation:, parameters:)
             end
           end
 
@@ -46,14 +47,12 @@ module NitroIntelligence
             prompt
           end
 
-          def workflow(parameters:)
-            chat_completion = @base_handler.perform_request(parameters:)
-            input = parameters[:messages]
+          def workflow(generation:, parameters:)
+            chat_completion = @base_handler.perform_request(parameters:, correlation_trace_id: generation.trace_id)
             output = chat_completion.choices.first.message.to_h
 
             trace_attributes = {
               model: chat_completion.model,
-              input:,
               output:,
               usage_details: {
                 prompt_tokens: chat_completion.usage.prompt_tokens,

data/lib/nitro_intelligence/client/handlers/observed/image_handler.rb

@@ -76,7 +76,7 @@ module NitroIntelligence
           end
 
           def workflow(generation:, image_generation:, parameters:)
-            chat_completion = @base_handler.perform_request(parameters:)
+            chat_completion = @base_handler.perform_request(parameters:, correlation_trace_id: generation.trace_id)
 
             image_generation.trace_id = generation.trace_id
             image_generation.parse_file(chat_completion)

data/lib/nitro_intelligence/client/handlers/observed/text_to_speech_handler.rb

@@ -25,7 +25,8 @@ module NitroIntelligence
               type: :generation,
               parameters:,
               trace_name:,
-              prompt:
+              prompt:,
+              input: message
             ) do |generation|
               workflow(message:, parameters:, trace_id: generation.trace_id)
             end
@@ -67,7 +68,7 @@ module NitroIntelligence
           end
 
           def workflow(message:, parameters:, trace_id:)
-            tts = @base_handler.perform_request(message:, parameters:)
+            tts = @base_handler.perform_request(message:, parameters:, correlation_trace_id: trace_id)
             output = ""
 
             Tempfile.create(["tts", ".#{parameters[:response_format]}"]) do |tempfile|
@@ -78,12 +79,10 @@ module NitroIntelligence
               output = handle_text_to_speech_upload(tempfile, trace_id)
             end
 
-            # We only get StringIO object as a response
-            # We dont have usage on tokens and the actual model that was used
-            # We will log the requested model instead
+            # We only get StringIO object as a response, so there are no usage details
+            # and no resolved model to record. The requested model and the input are
+            # already on the observation from before the request ran.
             trace_attributes = {
-              model: parameters[:model],
-              input: message,
               output:,
             }
 

data/lib/nitro_intelligence/client/handlers/text_to_speech_handler.rb

@@ -12,8 +12,9 @@ module NitroIntelligence
           perform_request(message:, parameters:)
         end
 
-        def perform_request(message: "", parameters: {})
+        def perform_request(message: "", parameters: {}, correlation_trace_id: nil)
           add_request_headers(parameters, MODALITY_HEADER => "audio", REQUESTED_MODEL_HEADER => parameters[:model])
+          add_correlation_headers(parameters, trace_id: correlation_trace_id)
           @client.audio.speech.create(
             input: message,
             **parameters.slice(*ALLOWED_EXTRA_PARAMETERS)

data/lib/nitro_intelligence/client/observers/langfuse_observer.rb

@@ -2,16 +2,23 @@ module NitroIntelligence
   module Client
     module Observers
       class LangfuseObserver
+        # The inference gateway returns its own request identifier on every
+        # response, including error responses. Recording it on the observation is
+        # the only way to line a failed generation up with the gateway's logs.
+        # See https://docs.litellm.ai/docs/proxy/response_headers
+        LITELLM_CALL_ID_HEADER = "x-litellm-call-id".freeze
+
+        MAX_STATUS_MESSAGE_LENGTH = 2000
+
         attr_reader :project_client
 
         def initialize(project_client:)
           @project_client = project_client
         end
 
-        def observe(operation_name, type:, parameters:, trace_name:, prompt: nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        def observe(operation_name, type:, parameters:, trace_name:, prompt: nil, input: nil) # rubocop:disable Metrics/AbcSize
           metadata = parameters[:metadata]
           seed = parameters[:trace_seed]
-          user_id = parameters[:user_id] || NitroIntelligence.configuration.observability_user_id
           trace_id = NitroIntelligence::Trace.create_id(seed:) if seed.present?
 
           if prompt
@@ -21,10 +28,7 @@ module NitroIntelligence
 
           metadata = metadata.transform_values(&:to_s)
 
-          Langfuse.propagate_attributes(
-            user_id:,
-            metadata:
-          ) do
+          Langfuse.propagate_attributes(**propagated_attributes(parameters, metadata)) do
             @project_client.observability_client.observe(
               operation_name,
               as_type: type,
@@ -35,8 +39,9 @@ module NitroIntelligence
             ) do |generation|
               generation.update_trace(name: trace_name, release: NitroIntelligence.configuration.current_revision)
               generation.update({ prompt: { name: prompt.name, version: prompt.version } }) if prompt
+              record_input(generation, input)
 
-              result, trace_attributes = yield(generation)
+              result, trace_attributes = observe_failures(generation) { yield(generation) }
 
               if trace_attributes
                 handle_truncation(trace_attributes[:input], trace_attributes[:output], trace_attributes[:model])
@@ -56,6 +61,56 @@ module NitroIntelligence
 
       private
 
+        # Recorded before the request is made so that a request which raises still
+        # shows what was sent. Handlers whose input is not safe to record twice
+        # (image generation sends base64 payloads that are replaced with media
+        # references on success) pass no input and rely on the failure record alone.
+        def record_input(generation, input)
+          return if input.blank?
+
+          generation.input = input
+          generation.update_trace(input:)
+        end
+
+        # `session_id` and `tags` are only forwarded when set so that callers who
+        # pass neither keep the existing propagation payload.
+        def propagated_attributes(parameters, metadata)
+          attributes = {
+            user_id: parameters[:user_id] || NitroIntelligence.configuration.observability_user_id,
+            metadata:,
+          }
+          attributes[:session_id] = parameters[:session_id] if parameters[:session_id].present?
+          attributes[:tags] = parameters[:tags] if parameters[:tags].present?
+          attributes
+        end
+
+        # Without this, a request that raises leaves an observation carrying only
+        # its name and model - no input, no output, no indication anything went
+        # wrong - because langfuse-rb ends the span in an `ensure` and never
+        # records the exception.
+        def observe_failures(generation)
+          yield
+        rescue => e
+          record_failure(generation, e)
+          raise
+        end
+
+        def record_failure(generation, error)
+          generation.update(
+            level: "ERROR",
+            status_message: "#{error.class}: #{error.message}".truncate(MAX_STATUS_MESSAGE_LENGTH)
+          )
+
+          call_id = litellm_call_id(error)
+          generation.metadata = { litellm_call_id: call_id } if call_id
+        end
+
+        def litellm_call_id(error)
+          return nil unless error.respond_to?(:headers)
+
+          error.headers&.[](LITELLM_CALL_ID_HEADER)
+        end
+
         def handle_truncation(_input, output, model_name)
           model = NitroIntelligence.model_catalog.lookup_by_name(model_name)
 

data/lib/nitro_intelligence/version.rb

@@ -1,3 +1,3 @@
 module NitroIntelligence
-  VERSION = "2.2.0".freeze
+  VERSION = "2.3.0".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nitro_intelligence
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.0
 platform: ruby
 authors:
 - Igor Artemenko