solarwinds_apm 6.0.0.preV1 → 6.0.0.preV3

data/lib/solarwinds_apm/opentelemetry/solarwinds_sampler.rb

@@ -37,39 +37,36 @@ module SolarWindsAPM
       # parent_context: OpenTelemetry::Context
       def should_sample?(trace_id:, parent_context:, links:, name:, kind:, attributes:)
 
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] should_sample? parameters \n
+        SolarWindsAPM.logger.debug do 
+          "[#{self.class}/#{__method__}] should_sample? start parameters \n
                                         trace_id: #{trace_id.unpack1('H*')}\n
                                         parent_context:  #{parent_context}\n
                                         parent_context.inspect:  #{parent_context.inspect}\n
                                         links: #{links}\n
                                         name: #{name}\n
                                         kind: #{kind}\n
-                                        attributes: #{attributes}"}
+                                        attributes: #{attributes}"
+        end
 
-        # if the upstream has tracestate: sw=.... then it should capture it 
+        parent_span_context = ::OpenTelemetry::Trace.current_span(parent_context).context        
+        xtraceoptions       = ::SolarWindsAPM::XTraceOptions.new(parent_context)
+        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] parent_span_context: #{parent_span_context.inspect}\n xtraceoptions: #{xtraceoptions.inspect}"}
 
-        parent_span_context = ::OpenTelemetry::Trace.current_span(parent_context).context
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] should_sample? parent_span_context: #{parent_span_context.inspect}"}
-        
-        xtraceoptions       = SolarWindsAPM::XTraceOptions.new(parent_context)
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] xtraceoptions: #{xtraceoptions.inspect}"}
-        
         liboboe_decision    = calculate_liboboe_decision(parent_span_context, xtraceoptions, name, kind, attributes)
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] liboboe_decision: #{liboboe_decision.inspect}"}
-
-        otel_decision   = otel_decision_from_liboboe(liboboe_decision)
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] otel_decision: #{otel_decision.inspect}"}
-
-        new_trace_state = calculate_trace_state(liboboe_decision,parent_span_context,xtraceoptions)
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] new_trace_state: #{new_trace_state.inspect}"}
-
-        new_attributes  = calculate_attributes(attributes,liboboe_decision,new_trace_state,parent_span_context,xtraceoptions)
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] new_attributes: #{new_attributes.inspect}"}
+        otel_decision       = otel_decision_from_liboboe(liboboe_decision)
+        new_trace_state     = calculate_trace_state(liboboe_decision, parent_span_context, xtraceoptions)
+        new_attributes      = otel_sampled?(otel_decision)? calculate_attributes(attributes, liboboe_decision, new_trace_state, parent_span_context, xtraceoptions) : nil
+        sampling_result     = ::OpenTelemetry::SDK::Trace::Samplers::Result.new(decision: otel_decision, 
+                                                                                attributes: new_attributes, 
+                                                                                tracestate: new_trace_state)
         
-        sampling_result = ::OpenTelemetry::SDK::Trace::Samplers::Result.new(decision: otel_decision, attributes: new_attributes, tracestate: new_trace_state)
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] sampling_result: #{sampling_result.inspect}"}
-
+        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] should_sample? end with sampling_result: #{sampling_result.inspect} from otel_decision: #{otel_decision.inspect} and new_attributes: #{new_attributes.inspect}"}
         sampling_result
+      rescue StandardError => e
+        SolarWindsAPM.logger.info {"[#{self.class}/#{__method__}] sampler error: #{e.message}"}
+        ::OpenTelemetry::SDK::Trace::Samplers::Result.new(decision: ::OpenTelemetry::SDK::Trace::Samplers::Decision::DROP, 
+                                                          attributes: attributes, 
+                                                          tracestate: ::OpenTelemetry::Trace::Tracestate::DEFAULT)
       end
 
       protected
@@ -78,49 +75,36 @@ module SolarWindsAPM
 
       private
 
-      # return Hash
+      ##
+      # use parent_span_context and xtraceoptions object to feed to liboboe function getDecisions that get liboboe_decision
+      # name, kind and attributes are used for transaction filter caching (to avoid continous calculate_trace_mode calculation)
+      # return decision Hash
       def calculate_liboboe_decision(parent_span_context, xtraceoptions, name, kind, attributes)
+        tracestring = Utils.traceparent_from_context(parent_span_context) if parent_span_context.valid? && parent_span_context.remote?
+        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] tracestring: #{tracestring}"}
 
-        tracestring = nil
-        if parent_span_context.valid? && parent_span_context.remote?
-          tracestring = Transformer.traceparent_from_context(parent_span_context)
-          SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] calculate_liboboe_decision parent_span_context.remote? #{parent_span_context.remote?} with #{tracestring}"}
-        end
-
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] name: #{name}, kind: #{kind}, attributes: #{attributes.inspect}"}
-
-        url = attributes.nil?? '' : attributes['http.host'] || attributes['http.url'] || attributes['net.peer.name']  # otel-ruby contrib use different key to store url info
-        transaction_naming_key = "#{url}-#{name}-#{kind}"
-        
+        # otel-ruby contrib use different key to store url info, currently it's using http.target for path
+        url_path = attributes.nil?? '' : attributes['http.target']
+        transaction_naming_key = "#{url_path}-#{name}-#{kind}"
         tracing_mode           = SolarWindsAPM::TransactionCache.get(transaction_naming_key)
-        
-        if tracing_mode.nil?
-          SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] transaction cache NOT found: #{transaction_naming_key}."}
-          trans_settings = SolarWindsAPM::TransactionSettings.new(url: url, name: name, kind: kind)          
+        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] transaction cache: #{transaction_naming_key}; tracing_mode: #{tracing_mode}."}
+
+        unless tracing_mode
+          trans_settings = SolarWindsAPM::TransactionSettings.new(url_path: url_path, name: name, kind: kind)          
           tracing_mode   = trans_settings.calculate_trace_mode == 1 ? SWO_TRACING_ENABLED : SWO_TRACING_DISABLED
           SolarWindsAPM::TransactionCache.set(transaction_naming_key, tracing_mode)
-        else
-          SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] transaction cache found: #{transaction_naming_key}."}
         end
 
         sw_member_value    = parent_span_context.tracestate[SolarWindsAPM::Constants::INTL_SWO_TRACESTATE_KEY]
-
-        # need to create the config class
         trigger_trace_mode = OboeTracingMode.get_oboe_trigger_trace_mode(@config["trigger_trace"])
         sample_rate        = UNSET
+        options            = xtraceoptions&.options
+        trigger_trace      = xtraceoptions&.intify_trigger_trace || 0
+        signature          = xtraceoptions&.signature
+        timestamp          = xtraceoptions&.timestamp
 
-        options = nil
-        trigger_trace = 0
-        signature = nil
-        timestamp = nil
-        if xtraceoptions
-          options = xtraceoptions.options
-          trigger_trace = xtraceoptions.intify_trigger_trace
-          signature = xtraceoptions.signature
-          timestamp = xtraceoptions.timestamp
-        end
-
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] decision parameters \n
+        SolarWindsAPM.logger.debug do 
+          "[#{self.class}/#{__method__}] get liboboe decision parameters: \n
                                          tracestring: #{tracestring}\n
                                          sw_member_value: #{sw_member_value}\n
                                          tracing_mode:    #{tracing_mode}\n
@@ -129,11 +113,15 @@ module SolarWindsAPM
                                          trigger_trace_mode:    #{trigger_trace_mode}\n
                                          options:      #{options}\n
                                          signature:    #{signature}\n
-                                         timestamp:    #{timestamp}"}
+                                         timestamp:    #{timestamp}"
+        end
+
+        args = [tracestring, sw_member_value, tracing_mode, sample_rate,
+                trigger_trace, trigger_trace_mode, options, signature,timestamp]
 
-        args = [tracestring,sw_member_value,tracing_mode,sample_rate,trigger_trace,trigger_trace_mode,options,signature,timestamp] 
-        do_metrics, do_sample, rate, source, bucket_rate, \
-            bucket_cap, decision_type, auth, status_msg, auth_msg, status = SolarWindsAPM::Context.getDecisions(*args)
+        do_metrics, do_sample, rate, source, bucket_rate,
+          bucket_cap, decision_type, auth, status_msg, auth_msg,
+            status = SolarWindsAPM::Context.getDecisions(*args)
 
         decision = {}
         decision["do_metrics"]    = do_metrics > 0
@@ -158,136 +146,78 @@ module SolarWindsAPM
         elsif liboboe_decision["do_metrics"]
           decision = ::OpenTelemetry::SDK::Trace::Samplers::Decision::RECORD_ONLY
         end
-        SolarWindsAPM.logger.debug {"OTel decision created: #{decision}"}
+        SolarWindsAPM.logger.debug {"otel decision: #{decision} created from liboboe_decision: #{liboboe_decision}"}
         decision
       end
 
-      def create_xtraceoptions_response_value(decision, parent_span_context, xtraceoptions)
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] create_xtraceoptions_response_value decision[auth]: #{decision['auth']}; decision[auth_msg]: #{decision['auth_msg']}; xtraceoptions.trigger_trace: #{xtraceoptions.trigger_trace}"}
-        
-        response = []
-        w3c_sanitized = SolarWindsAPM::Constants::INTL_SWO_EQUALS_W3C_SANITIZED
-        response << [XTRACEOPTIONS_RESP_AUTH, decision['auth_msg']].join(w3c_sanitized) if xtraceoptions.signature && decision['auth_msg']
-        if !decision["auth"] || decision["auth"] < 1
-          trigger_msg = ""
-          tracestring = nil
-          if xtraceoptions.trigger_trace
-            # If a traceparent header was provided then oboe does not generate the message
-            tracestring = Transformer.traceparent_from_context(parent_span_context) if parent_span_context.valid? && parent_span_context.remote?  
-            trigger_msg = tracestring && decision['decision_type'] == 0 ? XTRACEOPTIONS_RESP_TRIGGER_IGNORED : decision['status_msg']
-          else
-            trigger_msg = XTRACEOPTIONS_RESP_TRIGGER_NOT_REQUESTED
-          end
-
-          SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] create_xtraceoptions_response_value parent_span_context: #{parent_span_context}; tracestring: #{tracestring}; trigger_msg: #{trigger_msg}"}
-
-          # e.g. response << trigger-trace####ok
-          response << [XTRACEOPTIONS_RESP_TRIGGER_TRACE, trigger_msg].join(w3c_sanitized)
-
-        end
-
-        # so far the x-trace-options are only used for liboboe calculate decision for x-trace feature
-        # probably not need for remaining services since liboboe decision only calculate once
-        unless xtraceoptions.ignored.empty?
-          ignored_response = [XTRACEOPTIONS_RESP_IGNORED, xtraceoptions.ignored.join(SolarWindsAPM::Constants::INTL_SWO_COMMA_W3C_SANITIZED)]
-          response << ignored_response.join(w3c_sanitized)
-          # e.g. response << ignored####invalidkeys,invalidkeys,invalidkeys
-        end
-
-        response.join(';')  # e.g. trigger-trace####ok;ignored####invalidkeys,invalidkeys,invalidkeys
-      end
-
-      def create_new_trace_state(parent_span_context, decision)
-        decision = sw_from_span_and_decision(parent_span_context, decision)
-        trace_state = ::OpenTelemetry::Trace::Tracestate.from_hash({SolarWindsAPM::Constants::INTL_SWO_TRACESTATE_KEY => decision}) # e.g. sw=3e222c863a04123a-01
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] Created new trace_state: #{trace_state.inspect}"}
-        trace_state
-      end
-
-      # 
-      # calculate_trace_state
-      # This function merely just add sw=value and xtrace_options_response=value into the old/new tracestate 
-      # The return value tracestate will be used in propagating to next services
-      # 
-      def calculate_trace_state(decision, parent_span_context, xtraceoptions)
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] calculate_trace_state parent_span_context: #{parent_span_context.inspect}"}
+      ##
+      # add sw=value and xtrace_options_response=value into the old/new tracestate 
+      # the returned value tracestate will be used in propagating to next services
+      def calculate_trace_state(liboboe_decision, parent_span_context, xtraceoptions)
         if !parent_span_context.valid?
-          
-          trace_state = create_new_trace_state(parent_span_context, decision)
+          trace_state = create_new_trace_state(parent_span_context, liboboe_decision)
+        elsif parent_span_context.tracestate.nil?
+          trace_state = create_new_trace_state(parent_span_context, liboboe_decision)
         else
-
-          parent_trace_state = parent_span_context.tracestate
-          if parent_trace_state.nil?
-            trace_state = create_new_trace_state(parent_span_context, decision)
-          else
-            trace_state = parent_trace_state.set_value(SolarWindsAPM::Constants::INTL_SWO_TRACESTATE_KEY, sw_from_span_and_decision(parent_span_context, decision))
-            SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] Updated trace_state with span_id and sw trace_flags: #{trace_state.inspect}"}
-          end
+          trace_state = parent_span_context.tracestate.set_value(SolarWindsAPM::Constants::INTL_SWO_TRACESTATE_KEY, 
+                                                                 sw_from_span_and_decision(parent_span_context, liboboe_decision))
+          SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] updated trace_state:  #{trace_state.inspect}"}
         end
 
         # for setting up the xtrace_options_response
         if xtraceoptions&.options
-          trace_state = trace_state.set_value(
-            XTraceOptions.sw_xtraceoptions_response_key.to_s,
-            create_xtraceoptions_response_value(decision,parent_span_context,xtraceoptions))
+          trace_state = trace_state.set_value(XTraceOptions.sw_xtraceoptions_response_key.to_s, 
+                                              create_xtraceoptions_response_value(liboboe_decision, parent_span_context, xtraceoptions)) 
         end
+        trace_state
+      end
 
+      ##
+      # 
+      def create_new_trace_state(parent_span_context, liboboe_decision)
+        decision = sw_from_span_and_decision(parent_span_context, liboboe_decision)
+        trace_state = ::OpenTelemetry::Trace::Tracestate.from_hash({SolarWindsAPM::Constants::INTL_SWO_TRACESTATE_KEY => decision}) # e.g. sw=3e222c863a04123a-01
+        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] created new trace_state: #{trace_state.inspect}"}
         trace_state
       end
 
+      ##
       #
-      # SW_TRACESTATE_CAPTURE_KEY = "sw.w3c.tracestate"
-      # sw_xtraceoptions_response_key = "xtrace_options_response"
-      # trace_state is the new trace_state from existing span information
-      # parent_span_context.trace_state is from its parent 
-      #
-      def add_tracestate_capture_to_attributes_dict(attributes_dict, decision, trace_state, parent_span_context)
-        
-        tracestate_capture = attributes_dict[SW_TRACESTATE_CAPTURE_KEY]
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] tracestate_capture #{tracestate_capture.inspect}; attributes_dict #{attributes_dict.inspect}; trace_state #{trace_state.inspect}; parent_span_context: #{parent_span_context.inspect}"        }
+      def create_xtraceoptions_response_value(liboboe_decision, parent_span_context, xtraceoptions)
 
-        # tracestate_capture seems always nil because attributes_dict never have the SW_TRACESTATE_CAPTURE_KEY (sw.w3c.tracestate)
-        # since tracestate_capture is always nil, so the sw always have new value for sw=key
-        if tracestate_capture.nil?
-          trace_state_no_response = trace_state.delete(XTraceOptions.sw_xtraceoptions_response_key)
+        response = []
+        w3c_sanitized = SolarWindsAPM::Constants::INTL_SWO_EQUALS_W3C_SANITIZED
+        w3c_sanitized_comma = SolarWindsAPM::Constants::INTL_SWO_COMMA_W3C_SANITIZED
+        response << [XTRACEOPTIONS_RESP_AUTH, liboboe_decision['auth_msg']].join(w3c_sanitized) if xtraceoptions.signature && liboboe_decision['auth_msg']
         
-        else
-          # Must retain all potential tracestate pairs for attributes
-          attr_trace_state = ::OpenTelemetry::Trace::Tracestate.from_string(tracestate_capture)
+        if !liboboe_decision["auth"] || liboboe_decision["auth"] < 1
+          if xtraceoptions.trigger_trace
+            # If a traceparent header was provided then oboe does not generate the message
+            tracestring = Utils.traceparent_from_context(parent_span_context) if parent_span_context.valid? && parent_span_context.remote?  
+            trigger_msg = tracestring && liboboe_decision['decision_type'] == 0 ? XTRACEOPTIONS_RESP_TRIGGER_IGNORED : liboboe_decision['status_msg']
+            SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] tracestring: #{tracestring}; trigger_msg: #{trigger_msg}"}
+          else
+            trigger_msg = XTRACEOPTIONS_RESP_TRIGGER_NOT_REQUESTED
+          end
 
-          # This step generated the new sw=key for tracestate based on root parent_span_id
-          new_attr_trace_state = attr_trace_state.set_value(SolarWindsAPM::Constants::INTL_SWO_TRACESTATE_KEY, sw_from_span_and_decision(parent_span_context,decision))
-          
-          trace_state_no_response = new_attr_trace_state.delete(XTraceOptions.sw_xtraceoptions_response_key)
+          response << [XTRACEOPTIONS_RESP_TRIGGER_TRACE, trigger_msg].join(w3c_sanitized) # e.g. response << trigger-trace####ok
         end
 
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] trace_state_no_response #{trace_state_no_response.inspect}"}
-
-        # other approach
-        trace_state_no_response = parent_span_context.tracestate.delete(XTraceOptions.sw_xtraceoptions_response_key)
-        no_sw_count = trace_state_no_response.to_h.reject { |k, _v| k == "sw" }.count
-        attributes_dict[SW_TRACESTATE_CAPTURE_KEY] = Transformer.trace_state_header(trace_state_no_response) if no_sw_count > 0 
-
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] attributes_dict #{attributes_dict.inspect}"}
-        attributes_dict
+        # appending ignored value from xtraceoptions to response. e.g. response << ignored####invalidkeys,invalidkeys,invalidkeys
+        response << [XTRACEOPTIONS_RESP_IGNORED, xtraceoptions.ignored.join(w3c_sanitized_comma)].join(w3c_sanitized) unless xtraceoptions.ignored.empty?
+        joined_response = response.join(';')
+        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] final response_value: #{joined_response}"}
+        joined_response
       end
 
       ##
-      # calculate_attributes is used for getting the otel Result class in last step of sampler should_sample? e.g. result = Result.new(decision: otel_decision, attributes: new_attributes, tracestate: new_trace_state)
+      # calculate_attributes is used for getting the otel Result class in last step of sampler should_sample? 
+      # e.g. result = Result.new(decision: otel_decision, attributes: new_attributes, tracestate: new_trace_state)
       # calculate_attributes use new_trace_state that is derived from current span information and old tracestate from parent_span_context.tracestate
       # the sw.w3c.tracestate should perserve the old tracestate value for debugging purpose
-      ##
-      def calculate_attributes(attributes, decision, trace_state, parent_span_context, xtraceoptions)
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] Received attributes: #{attributes.inspect}; decision:#{decision.inspect}; trace_state:#{trace_state.inspect}; parent_span_context:#{parent_span_context.inspect}; xtraceoptions:#{xtraceoptions.inspect}"}
-
-        otel_decision = otel_decision_from_liboboe(decision)
-        return nil if Transformer.sampled?(otel_decision) == false
-        
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] Trace decision is_sampled - setting attributes #{otel_decision.inspect}"}
-
-        new_attributes = {}
-        # Copy existing MappingProxyType KV into new_attributes for modification.
-        attributes&.each {|k,v| new_attributes[k] = v}
+      def calculate_attributes(attributes, liboboe_decision, trace_state, parent_span_context, xtraceoptions)
+        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] new_trace_state: #{trace_state.inspect}"}
+        new_attributes = attributes.dup || {}
 
         # Always (root or is_remote) set _INTERNAL_SW_KEYS if injected
         new_attributes[INTERNAL_SW_KEYS] = xtraceoptions.sw_keys if xtraceoptions.sw_keys
@@ -296,16 +226,14 @@ module SolarWindsAPM
         xtraceoptions.custom_kvs&.each {|k,v| new_attributes[k] = v}
 
         # Always (root or is_remote) set service entry internal KVs       
-        new_attributes[INTERNAL_BUCKET_CAPACITY] = decision["bucket_cap"].to_s
-        new_attributes[INTERNAL_BUCKET_RATE]     = decision["bucket_rate"].to_s
-        new_attributes[INTERNAL_SAMPLE_RATE]     = decision["rate"]
-        new_attributes[INTERNAL_SAMPLE_SOURCE]   = decision["source"]
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] Set attributes with service entry internal KVs: #{new_attributes}"}
+        new_attributes[INTERNAL_BUCKET_CAPACITY] = liboboe_decision["bucket_cap"].to_s
+        new_attributes[INTERNAL_BUCKET_RATE]     = liboboe_decision["bucket_rate"].to_s
+        new_attributes[INTERNAL_SAMPLE_RATE]     = liboboe_decision["rate"]
+        new_attributes[INTERNAL_SAMPLE_SOURCE]   = liboboe_decision["source"]
 
         # set sw.tracestate_parent_id if its tracestate contains "sw"
         sw_value = parent_span_context.tracestate.value(SolarWindsAPM::Constants::INTL_SWO_TRACESTATE_KEY)
-        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] calculate_attributes sw_value: #{sw_value.inspect} parent_span_context.tracestate #{parent_span_context.tracestate.inspect}"}
-        new_attributes[SW_TRACESTATE_ROOT_KEY]= Transformer.span_id_from_sw(sw_value) if sw_value && parent_span_context.remote?
+        new_attributes[SW_TRACESTATE_ROOT_KEY] =  sw_value.split("-")[0] if sw_value && parent_span_context.remote?
 
         # If unsigned or signed TT (root or is_remote), set TriggeredTrace
         new_attributes[SolarWindsAPM::Constants::INTERNAL_TRIGGERED_TRACE] = true if xtraceoptions.trigger_trace
@@ -316,14 +244,43 @@ module SolarWindsAPM
           return new_attributes.freeze || nil
         end
 
-        new_attributes = add_tracestate_capture_to_attributes_dict(new_attributes,decision,trace_state,parent_span_context)
-        # e.g. {"SWKeys"=>"check-id:check-1013,website-id:booking-demo", "BucketCapacity"=>"6.0", "BucketRate"=>"0.1", "SampleRate"=>-1, "SampleSource"=>-1, "http.method"=>"GET", "http.host"=>"0.0.0.0:8002", "http.scheme"=>"http", "http.target"=>"/call_second_rails/", "http.user_agent"=>"curl/7.81.0", "sw.w3c.tracestate"=>"sw=aaaa1111bbbb2222-01"}
+        new_attributes = add_tracestate_capture_to_new_attributes(new_attributes, liboboe_decision, trace_state, parent_span_context)
         new_attributes.freeze
       end
 
-      def sw_from_span_and_decision(parent_span_context, decision)
-        trace_flag = Transformer.trace_flags_from_boolean(decision["do_sample"])
-        Transformer.sw_from_span_and_decision(parent_span_context.hex_span_id, trace_flag)
+      ##
+      # 
+      def add_tracestate_capture_to_new_attributes(new_attributes, liboboe_decision, trace_state, parent_span_context)
+        
+        tracestate_capture = new_attributes[SW_TRACESTATE_CAPTURE_KEY]
+        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] tracestate_capture #{tracestate_capture.inspect}; new_attributes #{new_attributes.inspect}"}
+
+        if tracestate_capture.nil?
+          trace_state_no_response = trace_state.delete(XTraceOptions.sw_xtraceoptions_response_key)
+        else 
+          # retain all potential tracestate pairs for attributes and generate new sw=key for tracestate based on root parent_span_id
+          attr_trace_state        = ::OpenTelemetry::Trace::Tracestate.from_string(tracestate_capture)
+          new_attr_trace_state    = attr_trace_state.set_value(SolarWindsAPM::Constants::INTL_SWO_TRACESTATE_KEY, sw_from_span_and_decision(parent_span_context,liboboe_decision))
+          trace_state_no_response = new_attr_trace_state.delete(XTraceOptions.sw_xtraceoptions_response_key)
+        end
+        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] trace_state_no_response #{trace_state_no_response.inspect}"}
+
+        trace_state_no_response = parent_span_context.tracestate.delete(XTraceOptions.sw_xtraceoptions_response_key)
+        no_sw_count             = trace_state_no_response.to_h.reject { |k, _v| k == "sw" }.count
+        new_attributes[SW_TRACESTATE_CAPTURE_KEY] = Utils.trace_state_header(trace_state_no_response) if no_sw_count > 0 
+        SolarWindsAPM.logger.debug {"[#{self.class}/#{__method__}] new_attributes after add_tracestate_capture_to_new_attributes: #{new_attributes.inspect}"}
+        
+        new_attributes
+      end
+
+      # formats tracestate sw value from span_id and liboboe decision as 16-byte span_id with 8-bit trace_flags e.g. 1a2b3c4d5e6f7g8h-01
+      def sw_from_span_and_decision(parent_span_context, liboboe_decision)
+        trace_flag = (liboboe_decision["do_sample"] == true) ? "01" : "00"
+        [parent_span_context.hex_span_id, trace_flag].join("-")
+      end
+
+      def otel_sampled?(otel_decision)
+        otel_decision == ::OpenTelemetry::SDK::Trace::Samplers::Decision::RECORD_AND_SAMPLE
       end
     end
   end

data/lib/solarwinds_apm/otel_config.rb

@@ -6,14 +6,14 @@ module SolarWindsAPM
   module OTelConfig
     @@config           = {}
     @@config_map       = {}
-    
+
     @@agent_enabled    = true
 
     def self.disable_agent
       return unless @@agent_enabled  # only show the msg once
-      
+
       @@agent_enabled = false
-      SolarWindsAPM.logger.warn {"[#{self.name}/#{__method__}] Agent disabled. No Trace exported."}
+      SolarWindsAPM.logger.warn {"[#{name}/#{__method__}] Agent disabled. No Trace exported."}
     end
 
     def self.validate_service_key
@@ -25,51 +25,69 @@ module SolarWindsAPM
     def self.resolve_sampler
 
       resolve_sampler_config
-      @@config[:sampler] = 
+      @@config[:sampler] =
         ::OpenTelemetry::SDK::Trace::Samplers.parent_based(
           root: SolarWindsAPM::OpenTelemetry::SolarWindsSampler.new(@@config[:sampler_config]),
           remote_parent_sampled: SolarWindsAPM::OpenTelemetry::SolarWindsSampler.new(@@config[:sampler_config]),
           remote_parent_not_sampled: SolarWindsAPM::OpenTelemetry::SolarWindsSampler.new(@@config[:sampler_config]))
     end
 
-    def self.resolve_sampler_config      
+    def self.resolve_sampler_config
       sampler_config = {}
-      sampler_config["trigger_trace"] = "enabled" 
+      sampler_config["trigger_trace"] = "enabled"
       sampler_config["trigger_trace"] = nil if ENV["SW_APM_TRIGGER_TRACING_MODE"] == 'disabled'
       @@config[:sampler_config] = sampler_config
     end
 
     #
     # append/add solarwinds_response_propagator into rack instrumentation
-    # 
-    def self.resolve_for_response_propagator
-      response_propagator = SolarWindsAPM::OpenTelemetry::SolarWindsResponsePropagator::TextMapPropagator.new
-      if @@config_map["OpenTelemetry::Instrumentation::Rack"]
-        if @@config_map["OpenTelemetry::Instrumentation::Rack"][:response_propagators].instance_of?(Array)
-          @@config_map["OpenTelemetry::Instrumentation::Rack"][:response_propagators].append(response_propagator)
+    #
+    def self.resolve_response_propagator
+      response_propagator  = SolarWindsAPM::OpenTelemetry::SolarWindsResponsePropagator::TextMapPropagator.new
+      rack_setting         = @@config_map["OpenTelemetry::Instrumentation::Rack"]
+      
+      if rack_setting
+        if rack_setting[:response_propagators].instance_of?(Array)
+          rack_setting[:response_propagators].append(response_propagator)
+        elsif rack_setting[:response_propagators].nil?
+          rack_setting[:response_propagators] = [response_propagator]
         else
-          @@config_map["OpenTelemetry::Instrumentation::Rack"][:response_propagators] = [response_propagator]
+          SolarWindsAPM.logger.warn {"[#{name}/#{__method__}] Rack response propagator resolve failed. Provided type #{rack_setting[:response_propagators].class}, please provide Array e.g. [#{rack_setting[:response_propagators]}]"}
         end
       else
         @@config_map["OpenTelemetry::Instrumentation::Rack"] = {response_propagators: [response_propagator]}
       end
     end
 
+    def self.obfuscate_helper(instrumentation)
+      if @@config_map[instrumentation] # user provided the option
+        @@config_map[instrumentation][:db_statement] = :obfuscate unless @@config_map[instrumentation][:db_statement] # user provided the db_statement, ignore our default setting 
+      else
+        @@config_map[instrumentation] = {db_statement: :obfuscate}
+      end
+    end
+
+    def self.obfuscate_query
+      obfuscate_helper("OpenTelemetry::Instrumentation::Dalli")
+      obfuscate_helper("OpenTelemetry::Instrumentation::Mysql2")
+      obfuscate_helper("OpenTelemetry::Instrumentation::PG")
+    end
+
     def self.[](key)
       @@config[key.to_sym]
     end
 
     def self.print_config
       @@config.each do |config, value|
-        SolarWindsAPM.logger.warn {"[#{self.name}/#{__method__}] config:     #{config} = #{value}"}
+        SolarWindsAPM.logger.warn {"[#{name}/#{__method__}] config:     #{config} = #{value}"}
       end
       @@config_map.each do |config, value|
-        SolarWindsAPM.logger.warn {"[#{self.name}/#{__method__}] config_map: #{config} = #{value}"}
+        SolarWindsAPM.logger.warn {"[#{name}/#{__method__}] config_map: #{config} = #{value}"}
       end
     end
 
     def self.resolve_solarwinds_processor
-      txn_manager = SolarWindsAPM::OpenTelemetry::TxnNameManager.new
+      txn_manager = SolarWindsAPM::TxnNameManager.new
       exporter    = SolarWindsAPM::OpenTelemetry::SolarWindsExporter.new(txn_manager: txn_manager)
       @@config[:span_processor] = SolarWindsAPM::OpenTelemetry::SolarWindsProcessor.new(exporter, txn_manager)
     end
@@ -84,16 +102,16 @@ module SolarWindsAPM
         return
       end
 
-      SolarWindsAPM.logger.debug {"[#{self.name}/#{__method__}] propagators: #{propagators.map(&:class)}"}
+      SolarWindsAPM.logger.debug {"[#{name}/#{__method__}] propagators: #{propagators.map(&:class)}"}
       unless ([::OpenTelemetry::Trace::Propagation::TraceContext::TextMapPropagator, ::OpenTelemetry::Baggage::Propagation::TextMapPropagator] - propagators.map(&:class)).empty? # rubocop:disable Style/GuardClause
-        SolarWindsAPM.logger.warn {"[#{self.name}/#{__method__}] Missing tracecontext propagator."}
+        SolarWindsAPM.logger.warn {"[#{name}/#{__method__}] Missing tracecontext propagator."}
         disable_agent
       end
     end
 
     def self.initialize
       unless defined?(::OpenTelemetry::SDK::Configurator)
-        SolarWindsAPM.logger.warn {"[#{self.name}/#{__method__}] missing OpenTelemetry::SDK::Configurator; opentelemetry seems not loaded."}
+        SolarWindsAPM.logger.warn {"[#{name}/#{__method__}] missing OpenTelemetry::SDK::Configurator; opentelemetry seems not loaded."}
         disable_agent
         return
       end
@@ -103,39 +121,40 @@ module SolarWindsAPM
       return unless @@agent_enabled
 
       resolve_sampler
-      
+
       resolve_solarwinds_propagator
       resolve_solarwinds_processor
-      resolve_for_response_propagator
+      resolve_response_propagator
+
+      obfuscate_query
 
       print_config if SolarWindsAPM.logger.level.zero?
 
-      ENV['OTEL_TRACES_EXPORTER'] = 'none' if ENV['OTEL_TRACES_EXPORTER'].nil?
-      ::OpenTelemetry::SDK.configure do |c|
-        c.use_all(@@config_map)
-      end
-      
+      ENV['OTEL_TRACES_EXPORTER'] = 'none' if ENV['OTEL_TRACES_EXPORTER'].to_s.empty?
+
+      ::OpenTelemetry::SDK.configure { |c| c.use_all(@@config_map) }
+
       validate_propagator(::OpenTelemetry.propagation.instance_variable_get(:@propagators))
 
       return unless @@agent_enabled
 
       # append our propagators
-      ::OpenTelemetry.propagation.instance_variable_get(:@propagators).append(@@config[:propagators]) 
-      
-      # append our processors (with our exporter)      
+      ::OpenTelemetry.propagation.instance_variable_get(:@propagators).append(@@config[:propagators])
+
+      # append our processors (with our exporter)
       ::OpenTelemetry.tracer_provider.add_span_processor(@@config[:span_processor])
-      
+
       # configure sampler afterwards
       ::OpenTelemetry.tracer_provider.sampler = @@config[:sampler]
       nil
     end
 
-    # 
+    #
     # Allow initialize after set new value to SolarWindsAPM::Config[:key]=value
-    # 
+    #
     # Usage:
-    # 
-    # Default using the use_all to load all instrumentation 
+    #
+    # Default using the use_all to load all instrumentation
     # But with specific instrumentation disabled, use {:enabled: false} in config
     # SolarWindsAPM::OTelConfig.initialize_with_config do |config|
     #   config["OpenTelemetry::Instrumentation::Rack"]  = {"a" => "b"}
@@ -144,14 +163,14 @@ module SolarWindsAPM
     #
     def self.initialize_with_config
       unless block_given?
-        SolarWindsAPM.logger.warn {"[#{self.name}/#{__method__}] Block not given while doing in-code configuration. Agent disabled."}
+        SolarWindsAPM.logger.warn {"[#{name}/#{__method__}] Block not given while doing in-code configuration. Agent disabled."}
         return
       end
 
       yield @@config_map
 
       if @@config_map.empty?
-        SolarWindsAPM.logger.warn {"[#{self.name}/#{__method__}] No configuration given for in-code configuration. Agent disabled."}
+        SolarWindsAPM.logger.warn {"[#{name}/#{__method__}] No configuration given for in-code configuration. Agent disabled."}
         return
       end
 

data/lib/solarwinds_apm/support/lumberjack_formatter.rb

@@ -1,6 +1,21 @@
-# Copyright (c) 2019 SolarWinds, LLC.
+# Copyright (c) 2023 SolarWinds, LLC.
 # All rights reserved.
 
 require_relative 'logger_formatter'
 
-Lumberjack::Formatter.prepend(SolarWindsAPM::Logger::Formatter) if SolarWindsAPM.loaded && defined?(Lumberjack::Formatter)
+module SolarWindsAPM
+  module Lumberjack
+    module LogEntry
+      include SolarWindsAPM::Logger::Formatter # provides #insert_trace_id
+
+      def initialize(time, severity, message, progname, pid, tags)
+        super if SolarWindsAPM::Config[:log_traceId] == :never
+
+        message = insert_trace_id(message)
+        super
+      end
+    end
+  end
+end
+
+Lumberjack::LogEntry.prepend(SolarWindsAPM::Lumberjack::LogEntry) if SolarWindsAPM.loaded && defined?(Lumberjack::LogEntry)

data/lib/solarwinds_apm/support/swomarginalia/comment.rb

@@ -107,7 +107,7 @@ module SolarWindsAPM
         marginalia_job["class"] if marginalia_job.respond_to?(:[])
       end
 
-      DEFAULT_LINES_TO_IGNORE_REGEX = %r{\.rvm|/ruby/gems/|vendor/|marginalia|rbenv|monitor\.rb.*mon_synchronize}.freeze
+      DEFAULT_LINES_TO_IGNORE_REGEX = %r{\.rvm|/ruby/gems/|vendor/|marginalia|rbenv|monitor\.rb.*mon_synchronize}
 
       def self.line
         SWOMarginalia::Comment.lines_to_ignore ||= DEFAULT_LINES_TO_IGNORE_REGEX
@@ -179,7 +179,7 @@ module SolarWindsAPM
           span_id: span_context.hex_span_id,
           trace_flags: trace_flag)
       rescue NameError => e
-        SolarWindsAPM.logger.error {"[#{self.name}/#{__method__}] Couldn't find OpenTelemetry. Error: #{e.message}"}
+        SolarWindsAPM.logger.error {"[#{name}/#{__method__}] Couldn't find OpenTelemetry. Error: #{e.message}"}
       end
 
       if Gem::Version.new(ActiveRecord::VERSION::STRING) < Gem::Version.new('6.1')