ruby_llm-agents 3.7.2 → 3.9.0

data/lib/ruby_llm/agents/pipeline/middleware/instrumentation.rb

@@ -39,45 +39,47 @@ module RubyLLM
           def call(context)
             context.started_at = Time.current
 
-            # Create "running" record immediately (SYNC - must appear on dashboard)
-            execution = create_running_execution(context)
-            context.execution_id = execution&.id
-            emit_start_notification(context)
-            status_update_completed = false
-            raised_exception = nil
-
-            begin
-              @app.call(context)
-              context.completed_at = Time.current
-
-              begin
-                complete_execution(execution, context, status: "success")
-                status_update_completed = true
-              rescue
-                # Let ensure block handle via mark_execution_failed!
-              end
-
-              emit_complete_notification(context, "success")
-            rescue => e
-              context.completed_at = Time.current
-              context.error = e
-              raised_exception = e
+            trace(context) do
+              # Create "running" record immediately (SYNC - must appear on dashboard)
+              execution = create_running_execution(context)
+              context.execution_id = execution&.id
+              emit_start_notification(context)
+              status_update_completed = false
+              raised_exception = nil
 
               begin
-                complete_execution(execution, context, status: determine_error_status(e))
-                status_update_completed = true
-              rescue
-                # Let ensure block handle via mark_execution_failed!
+                @app.call(context)
+                context.completed_at = Time.current
+
+                begin
+                  complete_execution(execution, context, status: "success")
+                  status_update_completed = true
+                rescue
+                  # Let ensure block handle via mark_execution_failed!
+                end
+
+                emit_complete_notification(context, "success")
+              rescue => e
+                context.completed_at = Time.current
+                context.error = e
+                raised_exception = e
+
+                begin
+                  complete_execution(execution, context, status: determine_error_status(e))
+                  status_update_completed = true
+                rescue
+                  # Let ensure block handle via mark_execution_failed!
+                end
+
+                emit_complete_notification(context, determine_error_status(e))
+                raise
+              ensure
+                # Emergency fallback if update failed
+                mark_execution_failed!(execution, error: raised_exception || $!) unless status_update_completed
               end
 
-              emit_complete_notification(context, determine_error_status(e))
-              raise
-            ensure
-              # Emergency fallback if update failed
-              mark_execution_failed!(execution, error: raised_exception || $!) unless status_update_completed
+              context
             end
-
-            context
           end
 
           private
@@ -111,7 +113,7 @@ module RubyLLM
 
             execution
           rescue => e
-            error("Failed to create running execution record: #{e.message}")
+            error("Failed to create running execution record: #{e.message}", context)
             nil
           end
 
@@ -142,7 +144,7 @@ module RubyLLM
             # Save detail data (prompts, responses, tool calls, etc.)
             save_execution_details(execution, context, status)
           rescue => e
-            error("Failed to complete execution record: #{e.message}")
+            error("Failed to complete execution record: #{e.message}", context)
             raise # Re-raise for ensure block to handle via mark_execution_failed!
           end
 
@@ -158,7 +160,7 @@ module RubyLLM
             return unless execution&.id
             return unless execution.status == "running"
 
-            error_message = error ? "#{error.class}: #{error.message}".truncate(1000) : "Unknown error"
+            error_message = build_error_message(error)
 
             update_data = {
               status: "error",
@@ -176,13 +178,35 @@ module RubyLLM
               else
                 RubyLLM::Agents::ExecutionDetail.create!(detail_attrs.merge(execution_id: execution.id))
               end
-            rescue
-              # Non-critical
+            rescue => e
+              debug("Failed to store error detail: #{e.message}")
             end
           rescue => e
             error("CRITICAL: Failed emergency status update for execution #{execution&.id}: #{e.message}")
           end
 
+          # Builds an informative error message including backtrace context
+          #
+          # Preserves the error class, message, and the most relevant
+          # backtrace frames (up to 10) so developers can trace the
+          # failure origin without needing to reproduce it.
+          #
+          # @param error [Exception, nil] The exception
+          # @return [String] Formatted error message with backtrace
+          def build_error_message(error)
+            return "Unknown error" unless error
+
+            parts = ["#{error.class}: #{error.message}"]
+
+            if error.backtrace&.any?
+              relevant_frames = error.backtrace.first(10)
+              parts << "Backtrace (first #{relevant_frames.size} frames):"
+              parts.concat(relevant_frames.map { |frame| "  #{frame}" })
+            end
+
+            parts.join("\n").truncate(5000)
+          end
+
           # Determines the status based on error type
           #
           # @param error [Exception] The exception that occurred
@@ -205,8 +229,8 @@ module RubyLLM
               tenant_id: context.tenant_id,
               execution_id: context.execution_id
             )
-          rescue
-            # Never let notifications break execution
+          rescue => e
+            debug("Start notification failed: #{e.message}", context)
           end
 
           # Emits an AS::Notification for execution completion or error
@@ -242,8 +266,8 @@ module RubyLLM
               error_class: context.error&.class&.name,
               error_message: context.error&.message
             )
-          rescue
-            # Never let notifications break execution
+          rescue => e
+            debug("Complete notification failed: #{e.message}", context)
           end
 
           # Builds data for initial running execution record
@@ -295,6 +319,9 @@ module RubyLLM
               data[:root_execution_id] = context.root_execution_id || context.parent_execution_id
             end
 
+            # Inject tracker request_id and tags
+            inject_tracker_data(context, data)
+
             data
           end
 
@@ -321,7 +348,8 @@ module RubyLLM
 
             context_meta = begin
               context.metadata.dup
-            rescue
+            rescue => e
+              debug("Failed to read context metadata: #{e.message}", context)
               {}
             end
             context_meta.transform_keys!(&:to_s)
@@ -364,7 +392,7 @@ module RubyLLM
             end
 
             if context.error
-              detail_data[:error_message] = truncate_error_message(context.error.message)
+              detail_data[:error_message] = build_error_message(context.error)
             end
 
             if context[:tool_calls].present?
@@ -391,7 +419,7 @@ module RubyLLM
               execution.create_detail!(detail_data)
             end
           rescue => e
-            error("Failed to save execution details: #{e.message}")
+            error("Failed to save execution details: #{e.message}", context)
           end
 
           # Persists execution data to database (legacy fallback)
@@ -411,75 +439,22 @@ module RubyLLM
               create_execution_record(data)
             end
           rescue => e
-            error("Failed to record execution: #{e.message}")
+            error("Failed to record execution: #{e.message}", context)
           end
 
-          # Builds execution data hash
+          # Builds execution data hash for the legacy single-step persistence path.
+          #
+          # Composes from build_running_execution_data and build_completion_data
+          # to avoid duplication.
           #
           # @param context [Context] The execution context
           # @param status [String] "success" or "error"
-          # @return [Hash] Execution data
+          # @return [Hash] Execution data with _detail_data for detail record
           def build_execution_data(context, status)
-            # Merge metadata: agent metadata (base) < middleware metadata (overlay)
-            agent_meta = safe_agent_metadata(context)
-            merged_metadata = agent_meta.transform_keys(&:to_s)
-
-            context_meta = begin
-              context.metadata.dup
-            rescue
-              {}
-            end
-            context_meta.transform_keys!(&:to_s)
-            merged_metadata.merge!(context_meta)
-
-            if context.cached? && context[:cache_key]
-              merged_metadata["response_cache_key"] = context[:cache_key]
-            end
-
-            data = {
-              agent_type: context.agent_class&.name,
-              model_id: context.model,
-              status: determine_status(context, status),
-              duration_ms: context.duration_ms,
-              started_at: context.started_at,
-              completed_at: context.completed_at,
-              cache_hit: context.cached?,
-              input_tokens: context.input_tokens || 0,
-              output_tokens: context.output_tokens || 0,
-              total_cost: context.total_cost || 0,
-              attempts_count: context.attempts_made,
-              metadata: merged_metadata
-            }
-
-            # Extract tracing fields from agent metadata to dedicated columns
-            if agent_meta.any?
-              data[:trace_id] = agent_meta[:trace_id] if agent_meta[:trace_id]
-              data[:request_id] = agent_meta[:request_id] if agent_meta[:request_id]
-              data[:parent_execution_id] = agent_meta[:parent_execution_id] if agent_meta[:parent_execution_id]
-              data[:root_execution_id] = agent_meta[:root_execution_id] if agent_meta[:root_execution_id]
-            end
-
-            # Add tenant_id only if multi-tenancy is enabled and tenant is set
-            if global_config.multi_tenancy_enabled? && context.tenant_id.present?
-              data[:tenant_id] = context.tenant_id
-            end
-
-            # Error class on execution
-            if context.error
-              data[:error_class] = context.error.class.name
-            end
-
-            # Tool calls count on execution
-            if context[:tool_calls].present?
-              data[:tool_calls_count] = context[:tool_calls].size
-            end
-
-            # Attempts count on execution
-            if context[:reliability_attempts].present?
-              data[:attempts_count] = context[:reliability_attempts].size
-            end
+            data = build_running_execution_data(context)
+              .merge(build_completion_data(context, determine_status(context, status)))
 
-            # Store detail data for separate creation
+            # Build detail data for separate creation
             detail_data = {parameters: sanitize_parameters(context)}
             if global_config.persist_prompts
               exec_opts = context.options[:options] || {}
@@ -487,18 +462,16 @@ module RubyLLM
               detail_data[:user_prompt] = context.input.to_s.presence
               detail_data[:assistant_prompt] = exec_opts[:assistant_prefill] if assistant_prompt_column_exists?
             end
-            detail_data[:error_message] = truncate_error_message(context.error.message) if context.error
+            detail_data[:error_message] = build_error_message(context.error) if context.error
             detail_data[:tool_calls] = context[:tool_calls] if context[:tool_calls].present?
             detail_data[:attempts] = context[:reliability_attempts] if context[:reliability_attempts].present?
             if global_config.persist_responses && context.output.respond_to?(:content)
               detail_data[:response] = serialize_response(context)
             end
 
-            # Persist audio data for Speaker executions
             maybe_persist_audio_response(context, detail_data)
 
             data[:_detail_data] = detail_data
-
             data
           end
 
@@ -527,7 +500,8 @@ module RubyLLM
 
             params = begin
               context.agent_instance.send(:options)
-            rescue
+            rescue => e
+              debug("Failed to extract agent options: #{e.message}", context)
               {}
             end
             params = params.dup
@@ -558,10 +532,38 @@ module RubyLLM
             result = context.agent_instance.metadata
             result.is_a?(Hash) ? result : {}
           rescue => e
-            debug("Failed to retrieve agent metadata: #{e.message}")
+            debug("Failed to retrieve agent metadata: #{e.message}", context)
             {}
           end
 
+          # Injects tracker request_id and tags into execution data
+          #
+          # Reads @_track_request_id and @_track_tags from the agent instance,
+          # which are set by BaseAgent#initialize when a Tracker is active.
+          #
+          # @param context [Context] The execution context
+          # @param data [Hash] The execution data hash to modify
+          def inject_tracker_data(context, data)
+            agent = context.agent_instance
+            return unless agent
+
+            # Inject request_id
+            track_request_id = agent.instance_variable_get(:@_track_request_id)
+            if track_request_id && data[:request_id].blank?
+              data[:request_id] = track_request_id
+            end
+
+            # Merge tracker tags into metadata
+            track_tags = agent.instance_variable_get(:@_track_tags)
+            if track_tags.is_a?(Hash) && track_tags.any?
+              data[:metadata] = (data[:metadata] || {}).merge(
+                "tags" => track_tags.transform_keys(&:to_s)
+              )
+            end
+          rescue
+            # Never let tracker data injection break execution
+          end
+
           # Sensitive parameter keys that should be redacted
           SENSITIVE_KEYS = %w[
             password token api_key secret credential auth key
@@ -580,7 +582,7 @@ module RubyLLM
           def truncate_error_message(message)
             return "" if message.nil?
 
-            message.to_s.truncate(1000)
+            message.to_s.truncate(5000)
           rescue
             message.to_s[0, 1000]
           end
@@ -607,7 +609,7 @@ module RubyLLM
 
             response_data
           rescue => e
-            error("Failed to serialize response: #{e.message}")
+            error("Failed to serialize response: #{e.message}", context)
             nil
           end
 
@@ -634,7 +636,7 @@ module RubyLLM
 
             detail_data[:response] = serialize_audio_response(context.output)
           rescue => e
-            error("Failed to persist audio response: #{e.message}")
+            error("Failed to persist audio response: #{e.message}", context)
           end
 
           # Serializes a SpeechResult into a hash for the response column
@@ -689,7 +691,8 @@ module RubyLLM
             else
               cfg.track_executions
             end
-          rescue
+          rescue => e
+            debug("Failed to check tracking config: #{e.message}", context)
             false
           end
 
@@ -698,7 +701,8 @@ module RubyLLM
           # @return [Boolean]
           def track_cache_hits?
             global_config.respond_to?(:track_cache_hits) && global_config.track_cache_hits
-          rescue
+          rescue => e
+            debug("Failed to check track_cache_hits config: #{e.message}")
             false
           end
 
@@ -707,7 +711,8 @@ module RubyLLM
           # @return [Boolean]
           def async_logging?
             global_config.async_logging && defined?(Infrastructure::ExecutionLoggerJob)
-          rescue
+          rescue => e
+            debug("Failed to check async_logging config: #{e.message}")
             false
           end
 
@@ -722,7 +727,8 @@ module RubyLLM
             @_assistant_prompt_column_exists = begin
               defined?(RubyLLM::Agents::ExecutionDetail) &&
                 RubyLLM::Agents::ExecutionDetail.column_names.include?("assistant_prompt")
-            rescue
+            rescue => e
+              debug("Failed to check assistant_prompt column: #{e.message}")
               false
             end
           end

data/lib/ruby_llm/agents/pipeline/middleware/reliability.rb

@@ -41,11 +41,13 @@ module RubyLLM
           def call(context)
             return @app.call(context) unless reliability_enabled?
 
-            config = reliability_config
-            models_to_try = build_models_list(context, config)
-            total_deadline = calculate_deadline(config)
+            trace(context) do
+              config = reliability_config
+              models_to_try = build_models_list(context, config)
+              total_deadline = calculate_deadline(config)
 
-            execute_with_reliability(context, models_to_try, config, total_deadline)
+              execute_with_reliability(context, models_to_try, config, total_deadline)
+            end
           end
 
           private
@@ -103,7 +105,7 @@ module RubyLLM
               # Check circuit breaker for this model
               breaker = get_circuit_breaker(current_model, context)
               if breaker&.open?
-                debug("Circuit breaker open for #{current_model}, skipping")
+                debug("Circuit breaker open for #{current_model}, skipping", context)
                 tracker.record_short_circuit(current_model)
                 next
               end
@@ -237,7 +239,7 @@ module RubyLLM
             return false if attempt_index >= max_retries
             return false if total_deadline && Time.current > total_deadline
             # Don't retry if fallback models are available — move to next model instead
-            return false if has_fallback_models?(config)
+            return false if fallback_models?(config)
 
             retryable_error?(error, config)
           end
@@ -256,7 +258,7 @@ module RubyLLM
           #
           # @param config [Hash] The reliability configuration
           # @return [Boolean]
-          def has_fallback_models?(config)
+          def fallback_models?(config)
             fallbacks = config[:fallback_models]
             fallbacks.is_a?(Array) && fallbacks.any?
           end
@@ -317,8 +319,8 @@ module RubyLLM
               event,
               {agent_type: @agent_class&.name}.merge(extras)
             )
-          rescue
-            # Never let notifications break execution
+          rescue => e
+            debug("Reliability notification failed: #{e.message}")
           end
 
           # Sleeps without blocking other fibers when in async context
@@ -333,8 +335,8 @@ module RubyLLM
             else
               sleep(seconds)
             end
-          rescue
-            # Fall back to regular sleep if async detection fails
+          rescue => e
+            debug("Async sleep failed, falling back to regular sleep: #{e.message}")
             sleep(seconds)
           end
         end

data/lib/ruby_llm/agents/pipeline/middleware/tenant.rb

@@ -41,10 +41,12 @@ module RubyLLM
           # @param context [Context] The execution context
           # @return [Context] The context with tenant fields populated
           def call(context)
-            resolve_tenant!(context)
-            ensure_tenant_record!(context)
-            apply_api_configuration!(context)
-            @app.call(context)
+            trace(context) do
+              resolve_tenant!(context)
+              ensure_tenant_record!(context)
+              apply_api_configuration!(context)
+              @app.call(context)
+            end
           end
 
           private
@@ -112,7 +114,7 @@ module RubyLLM
               ensure_tenant_for_model!(tenant_object)
             else
               # For hash-based or string tenants, ensure a minimal record exists
-              RubyLLM::Agents::Tenant.find_or_create_by!(tenant_id: context.tenant_id)
+              find_or_create_tenant!(context.tenant_id)
             end
           rescue => e
             # Don't fail the execution if tenant record creation fails
@@ -145,6 +147,18 @@ module RubyLLM
               enforcement: options[:enforcement]&.to_s || "soft",
               inherit_global_defaults: options.fetch(:inherit_global, true)
             )
+          rescue ActiveRecord::RecordNotUnique
+            # Race condition: another thread created the record — safe to ignore
+          end
+
+          # Finds or creates a tenant record, handling race conditions
+          #
+          # @param tenant_id [String] The tenant identifier
+          def find_or_create_tenant!(tenant_id)
+            RubyLLM::Agents::Tenant.find_or_create_by!(tenant_id: tenant_id)
+          rescue ActiveRecord::RecordNotUnique
+            # Another thread/process created the record — just find it
+            RubyLLM::Agents::Tenant.find_by!(tenant_id: tenant_id)
           end
 
           # Checks if the tenants table exists (memoized)
@@ -154,7 +168,8 @@ module RubyLLM
             return @tenant_table_exists if defined?(@tenant_table_exists)
 
             @tenant_table_exists = ::ActiveRecord::Base.connection.table_exists?(:ruby_llm_agents_tenants)
-          rescue
+          rescue => e
+            debug("Failed to check tenant table existence: #{e.message}")
             @tenant_table_exists = false
           end
 
@@ -178,7 +193,11 @@ module RubyLLM
             apply_tenant_object_api_keys!(context)
           end
 
-          # Applies API keys from tenant object's llm_api_keys method
+          # Stores tenant API keys on the context for thread-safe per-request use.
+          #
+          # Instead of mutating the global RubyLLM configuration (which is not
+          # thread-safe), keys are stored on the context. The Pipeline::Context#llm
+          # method creates a scoped RubyLLM::Context with these keys when needed.
           #
           # @param context [Context] The execution context
           def apply_tenant_object_api_keys!(context)
@@ -188,34 +207,12 @@ module RubyLLM
             api_keys = tenant_object.llm_api_keys
             return if api_keys.blank?
 
-            apply_api_keys_to_ruby_llm(api_keys)
+            context[:tenant_api_keys] = api_keys
           rescue => e
             # Log but don't fail if API key extraction fails
             warn_api_key_error("tenant object", e)
           end
 
-          # Applies a hash of API keys to RubyLLM configuration
-          #
-          # @param api_keys [Hash] Hash of provider => key mappings
-          def apply_api_keys_to_ruby_llm(api_keys)
-            RubyLLM.configure do |config|
-              api_keys.each do |provider, key|
-                next if key.blank?
-
-                setter = api_key_setter_for(provider)
-                config.public_send(setter, key) if config.respond_to?(setter)
-              end
-            end
-          end
-
-          # Returns the setter method name for a provider's API key
-          #
-          # @param provider [Symbol, String] Provider name (e.g., :openai, :anthropic)
-          # @return [String] Setter method name (e.g., "openai_api_key=")
-          def api_key_setter_for(provider)
-            "#{provider}_api_key="
-          end
-
           # Logs a warning about API key resolution failure
           #
           # @param source [String] Source that failed
@@ -236,7 +233,8 @@ module RubyLLM
             return nil unless tenant.respond_to?(:llm_config)
 
             tenant.llm_config
-          rescue
+          rescue => e
+            debug("Failed to extract tenant config: #{e.message}")
             nil
           end
         end