datadog 2.7.1 → 2.18.0

data/lib/datadog/tracing/contrib/action_pack/action_dispatch/instrumentation.rb

@@ -9,9 +9,12 @@ module Datadog
         module ActionDispatch
           # Instrumentation for ActionDispatch components
           module Instrumentation
+            SCRIPT_NAME_KEY = 'SCRIPT_NAME'
+            FORMAT_SUFFIX = '(.:format)'
+
             module_function
 
-            def set_http_route_tags(route_spec, script_name)
+            def set_http_route_tags(route_spec, route_path)
               return unless Tracing.enabled?
 
               return unless route_spec
@@ -19,10 +22,10 @@ module Datadog
               request_trace = Tracing.active_trace
               return unless request_trace
 
-              request_trace.set_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE, route_spec.to_s.gsub(/\(.:format\)\z/, ''))
+              request_trace.set_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE, route_spec)
 
-              if script_name && !script_name.empty?
-                request_trace.set_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH, script_name)
+              if route_path && !route_path.empty?
+                request_trace.set_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH, route_path)
               end
             end
 
@@ -40,16 +43,17 @@ module Datadog
               # Instrumentation for ActionDispatch::Journey::Router for Rails versions older than 7.1
               module Router
                 def find_routes(req)
+                  # result is an array of [match, parameters, route] tuples
                   result = super
+                  result.each do |_, _, route|
+                    next unless Instrumentation.dispatcher_route?(route)
 
-                  # result is an array of [match, parameters, route] tuples
-                  routes = result.map(&:last)
+                    http_route = route.path.spec.to_s
+                    http_route.delete_suffix!(FORMAT_SUFFIX)
 
-                  routes.each do |route|
-                    if Instrumentation.dispatcher_route?(route)
-                      Instrumentation.set_http_route_tags(route.path.spec, req.env['SCRIPT_NAME'])
-                      break
-                    end
+                    Instrumentation.set_http_route_tags(http_route, req.env[SCRIPT_NAME_KEY])
+
+                    break
                   end
 
                   result
@@ -62,7 +66,10 @@ module Datadog
                 def find_routes(req)
                   super do |match, parameters, route|
                     if Instrumentation.dispatcher_route?(route)
-                      Instrumentation.set_http_route_tags(route.path.spec, req.env['SCRIPT_NAME'])
+                      http_route = route.path.spec.to_s
+                      http_route.delete_suffix!(FORMAT_SUFFIX)
+
+                      Instrumentation.set_http_route_tags(http_route, req.env[SCRIPT_NAME_KEY])
                     end
 
                     yield [match, parameters, route]

data/lib/datadog/tracing/contrib/action_pack/ext.rb

@@ -16,6 +16,8 @@ module Datadog
           TAG_OPERATION_CONTROLLER = 'controller'
           TAG_ROUTE_ACTION = 'rails.route.action'
           TAG_ROUTE_CONTROLLER = 'rails.route.controller'
+          TAG_DB_RUNTIME = 'rails.db.runtime'
+          TAG_VIEW_RUNTIME = 'rails.view.runtime'
         end
       end
     end

data/lib/datadog/tracing/contrib/action_pack/integration.rb

@@ -4,7 +4,7 @@ require_relative 'configuration/settings'
 require_relative 'patcher'
 require_relative '../integration'
 require_relative '../rails/ext'
-require_relative '../rails/utils'
+require_relative '../../../core/contrib/rails/utils'
 
 module Datadog
   module Tracing
@@ -18,6 +18,9 @@ module Datadog
 
           # @public_api Changing the integration name or integration options can cause breaking changes
           register_as :action_pack, auto_patch: false
+          def self.gem_name
+            'actionpack'
+          end
 
           def self.version
             Gem.loaded_specs['actionpack'] && Gem.loaded_specs['actionpack'].version
@@ -34,7 +37,7 @@ module Datadog
           # enabled by rails integration so should only auto instrument
           # if detected that it is being used without rails
           def auto_instrument?
-            !Contrib::Rails::Utils.railtie_supported?
+            !Core::Contrib::Rails::Utils.railtie_supported?
           end
 
           def new_configuration

data/lib/datadog/tracing/contrib/action_view/integration.rb

@@ -4,7 +4,7 @@ require_relative 'configuration/settings'
 require_relative 'patcher'
 require_relative '../integration'
 require_relative '../rails/ext'
-require_relative '../rails/utils'
+require_relative '../../../core/contrib/rails/utils'
 
 module Datadog
   module Tracing
@@ -18,6 +18,9 @@ module Datadog
 
           # @public_api Changing the integration name or integration options can cause breaking changes
           register_as :action_view, auto_patch: false
+          def self.gem_name
+            'actionview'
+          end
 
           def self.version
             # ActionView is its own gem in Rails 4.1+
@@ -41,7 +44,7 @@ module Datadog
           # enabled by rails integration so should only auto instrument
           # if detected that it is being used without rails
           def auto_instrument?
-            !Contrib::Rails::Utils.railtie_supported?
+            !Core::Contrib::Rails::Utils.railtie_supported?
           end
 
           def new_configuration

data/lib/datadog/tracing/contrib/active_job/integration.rb

@@ -3,7 +3,7 @@
 require_relative 'configuration/settings'
 require_relative 'patcher'
 require_relative '../integration'
-require_relative '../rails/utils'
+require_relative '../../../core/contrib/rails/utils'
 
 module Datadog
   module Tracing
@@ -17,6 +17,9 @@ module Datadog
 
           # @public_api Changing the integration name or integration options can cause breaking changes
           register_as :active_job, auto_patch: false
+          def self.gem_name
+            'activejob'
+          end
 
           def self.version
             Gem.loaded_specs['activejob'] && Gem.loaded_specs['activejob'].version
@@ -33,7 +36,7 @@ module Datadog
           # enabled by rails integration so should only auto instrument
           # if detected that it is being used without rails
           def auto_instrument?
-            !Contrib::Rails::Utils.railtie_supported?
+            !Core::Contrib::Rails::Utils.railtie_supported?
           end
 
           def new_configuration

data/lib/datadog/tracing/contrib/active_record/integration.rb

@@ -7,7 +7,7 @@ require_relative 'patcher'
 require_relative '../component'
 require_relative '../integration'
 require_relative '../rails/ext'
-require_relative '../rails/utils'
+require_relative '../../../core/contrib/rails/utils'
 
 module Datadog
   module Tracing
@@ -22,6 +22,10 @@ module Datadog
           # @public_api Changing the integration name or integration options can cause breaking changes
           register_as :active_record, auto_patch: false
 
+          def self.gem_name
+            'activerecord'
+          end
+
           def self.version
             Gem.loaded_specs['activerecord'] && Gem.loaded_specs['activerecord'].version
           end
@@ -37,7 +41,7 @@ module Datadog
           # enabled by rails integration so should only auto instrument
           # if detected that it is being used without rails
           def auto_instrument?
-            !Contrib::Rails::Utils.railtie_supported?
+            !Core::Contrib::Rails::Utils.railtie_supported?
           end
 
           def new_configuration
@@ -53,7 +57,7 @@ module Datadog
           end
 
           def reset_resolver_cache
-            @resolver&.reset_cache
+            @resolver&.reset_cache if defined?(@resolver)
           end
 
           Contrib::Component.register('activerecord') do |_config|

data/lib/datadog/tracing/contrib/active_support/cache/events/cache.rb

@@ -62,7 +62,10 @@ module Datadog
               end
 
               def on_start(span, event, _id, payload)
-                key = payload[:key]
+                # Since Rails 8, `dd_original_keys` contains the denormalized key provided by the user.
+                # In previous versions, the denormalized key is stored in the official `key` attribute.
+                # We fall back to `key`, even in Rails 8, as a defensive measure.
+                key = payload[:dd_original_keys] || payload[:key]
                 store = payload[:store]
 
                 mapping = MAPPING.fetch(event)
@@ -81,7 +84,9 @@ module Datadog
 
                 span.set_tag('EVENT', event)
 
-                set_cache_key(span, key, mapping[:multi_key])
+                if Datadog.configuration.tracing[:active_support][:cache_key].enabled
+                  set_cache_key(span, key, mapping[:multi_key])
+                end
               rescue StandardError => e
                 Datadog.logger.error(e.message)
                 Datadog::Core::Telemetry::Logger.report(e)

data/lib/datadog/tracing/contrib/active_support/cache/instrumentation.rb

@@ -3,6 +3,7 @@
 require_relative '../../../../core/utils'
 require_relative '../../../metadata/ext'
 require_relative '../ext'
+require_relative 'event'
 
 module Datadog
   module Tracing
@@ -58,7 +59,8 @@ module Datadog
                 end
 
                 span.set_tag(Ext::TAG_CACHE_BACKEND, store) if store
-                set_cache_key(span, key, multi_key)
+
+                set_cache_key(span, key, multi_key) if Datadog.configuration.tracing[:active_support][:cache_key].enabled
 
                 yield
               end
@@ -182,6 +184,39 @@ module Datadog
                 super(operation, key, polyfill_options)
               end
             end
+
+            # Save the original, user-supplied cache key, before it gets normalized.
+            #
+            # Normalized keys can include internal implementation detail,
+            # for example FileStore keys include temp directory names, which
+            # changes on every run, making it impossible to group by the cache key afterward.
+            # Also, the user is never exposed to the normalized key, and only sets/gets using the
+            # original key.
+            module PreserveOriginalKey
+              # Stores the original keys in the options hash, as an array of keys.
+              # It's important to keep all the keys for multi-key operations.
+              # For single-key operations, the key is stored as an array of a single element.
+              def normalize_key(key, options)
+                orig_keys = options[:dd_original_keys] || []
+                orig_keys << key
+                options[:dd_original_keys] = orig_keys
+
+                super
+              end
+
+              # Ensure we don't pollute the default Store instance `options` in {PreserveOriginalKey#normalize_key}.
+              # In most cases, `merged_options` returns a new hash,
+              # but we check for cases where it reuses the instance hash.
+              def merged_options(call_options)
+                ret = super
+
+                if ret.equal?(options)
+                  ret.dup
+                else
+                  ret
+                end
+              end
+            end
           end
         end
       end

data/lib/datadog/tracing/contrib/active_support/cache/patcher.rb

@@ -22,6 +22,10 @@ module Datadog
             def patch
               Events.subscribe!
 
+              if Integration.version >= Gem::Version.new('8.0.0')
+                ::ActiveSupport::Cache::Store.prepend(Cache::Instrumentation::PreserveOriginalKey)
+              end
+
               # Backfill the `:store` key in the ActiveSupport event payload for older Rails.
               if Integration.version < Gem::Version.new('6.1.0')
                 ::ActiveSupport::Cache::Store.prepend(Cache::Instrumentation::Store)

data/lib/datadog/tracing/contrib/active_support/cache/redis.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../../support'
 require_relative 'patcher'
 
 module Datadog
@@ -22,17 +23,26 @@ module Datadog
               # For Rails >= 5.2 w/o redis-activesupport...
               # ActiveSupport includes a Redis cache store internally, and does not require these overrides.
               # https://github.com/rails/rails/blob/master/activesupport/lib/active_support/cache/redis_cache_store.rb
-              def patch_redis?(meth)
+              def patch_redis_store?(meth)
                 !Gem.loaded_specs['redis-activesupport'].nil? \
                   && defined?(::ActiveSupport::Cache::RedisStore) \
                   && ::ActiveSupport::Cache::RedisStore.instance_methods(false).include?(meth)
               end
 
+              # Patches the Rails built-in Redis cache backend `redis_cache_store`, added in Rails 5.2.
+              # We avoid loading the RedisCacheStore class, as it invokes the statement `gem "redis", ">= 4.0.1"` which
+              # fails if the application is using an old version of Redis, or not using Redis at all.
+              # @see https://github.com/rails/rails/blob/d0dcb8fa6073a0c4d42600c15e82e3bb386b27d3/activesupport/lib/active_support/cache/redis_cache_store.rb#L4
+              def patch_redis_cache_store?(meth)
+                Gem.loaded_specs['redis'] &&
+                  Support.fully_loaded?(::ActiveSupport::Cache, :RedisCacheStore) &&
+                  ::ActiveSupport::Cache::RedisCacheStore.instance_methods(false).include?(meth)
+              end
+
               def cache_store_class(meth)
-                if patch_redis?(meth)
+                if patch_redis_store?(meth)
                   [::ActiveSupport::Cache::RedisStore, ::ActiveSupport::Cache::Store]
-                elsif Gem.loaded_specs['redis'] && defined?(::ActiveSupport::Cache::RedisCacheStore) \
-                    && ::ActiveSupport::Cache::RedisCacheStore.instance_methods(false).include?(meth)
+                elsif patch_redis_cache_store?(meth)
                   [::ActiveSupport::Cache::RedisCacheStore, ::ActiveSupport::Cache::Store]
                 else
                   super

data/lib/datadog/tracing/contrib/active_support/configuration/settings.rb

@@ -39,6 +39,16 @@ module Datadog
                 )
               end
             end
+
+            # grouped "cache_key.*" settings
+            settings :cache_key do
+              # enable or disabling the inclusion of the cache_key in the span
+              option :enabled do |o|
+                # cache_key.enabled
+                o.type :bool
+                o.default true
+              end
+            end
           end
         end
       end

data/lib/datadog/tracing/contrib/active_support/integration.rb

@@ -5,7 +5,7 @@ require_relative 'configuration/settings'
 require_relative 'patcher'
 require_relative 'cache/redis'
 require_relative '../rails/ext'
-require_relative '../rails/utils'
+require_relative '../../../core/contrib/rails/utils'
 
 module Datadog
   module Tracing
@@ -19,6 +19,9 @@ module Datadog
 
           # @public_api Changing the integration name or integration options can cause breaking changes
           register_as :active_support, auto_patch: false
+          def self.gem_name
+            'activesupport'
+          end
 
           def self.version
             Gem.loaded_specs['activesupport'] && Gem.loaded_specs['activesupport'].version
@@ -35,7 +38,7 @@ module Datadog
           # enabled by rails integration so should only auto instrument
           # if detected that it is being used without rails
           def auto_instrument?
-            !Contrib::Rails::Utils.railtie_supported?
+            !Core::Contrib::Rails::Utils.railtie_supported?
           end
 
           def new_configuration

data/lib/datadog/tracing/contrib/auto_instrument.rb

@@ -9,10 +9,10 @@ module Datadog
     module Contrib
       # Auto-activate instrumentation
       def self.auto_instrument!
-        require_relative 'rails/utils'
+        require_relative '../../core/contrib/rails/utils'
 
         # Defer to Rails if this is a Rails application
-        if Datadog::Tracing::Contrib::Rails::Utils.railtie_supported?
+        if Datadog::Core::Contrib::Rails::Utils.railtie_supported?
           require_relative 'rails/auto_instrument_railtie'
         else
           AutoInstrument.patch_all!

data/lib/datadog/tracing/contrib/aws/instrumentation.rb

@@ -35,6 +35,16 @@ module Datadog
             span.type = Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
             span.name = Ext::SPAN_COMMAND
             span.resource = context.safely(:resource)
+
+            # Set error on the span if the Response Status Code is in error range
+            if Tracing::Metadata::Ext::HTTP::ERROR_RANGE.cover?(context.safely(:status_code))
+              # At this point we do not have any additional diagnostics
+              # besides the HTTP status code which is recorded in the span tags
+              # later in this method.
+              # Just set the span as errored.
+              span.set_error(nil)
+            end
+
             aws_service = span.resource.split('.')[0]
             span.set_tag(Ext::TAG_AWS_SERVICE, aws_service)
             params = context.safely(:params)

data/lib/datadog/tracing/contrib/aws/integration.rb

@@ -16,6 +16,9 @@ module Datadog
 
           # @public_api Changing the integration name or integration options can cause breaking changes
           register_as :aws, auto_patch: true
+          def self.gem_name
+            'aws-sdk'
+          end
 
           def self.version
             if Gem.loaded_specs['aws-sdk']

data/lib/datadog/tracing/contrib/aws/parsed_context.rb

@@ -26,8 +26,12 @@ module Datadog
             context.params
           end
 
+          def http_response
+            context.http_response
+          end
+
           def status_code
-            context.http_response.status_code
+            http_response.status_code
           end
 
           def http_method

data/lib/datadog/tracing/contrib/concurrent_ruby/integration.rb

@@ -16,6 +16,9 @@ module Datadog
 
           # @public_api Changing the integration name or integration options can cause breaking changes
           register_as :concurrent_ruby
+          def self.gem_name
+            'concurrent-ruby'
+          end
 
           def self.version
             Gem.loaded_specs['concurrent-ruby'] && Gem.loaded_specs['concurrent-ruby'].version

data/lib/datadog/tracing/contrib/configuration/settings.rb

@@ -22,7 +22,7 @@ module Datadog
           end
 
           def configure(options = {})
-            self.class.options.each do |name, _value|
+            self.class.options.each_key do |name|
               self[name] = options[name] if options.key?(name)
             end
 

data/lib/datadog/tracing/contrib/elasticsearch/configuration/settings.rb

@@ -49,6 +49,10 @@ module Datadog
               o.type :string, nilable: true
               o.env Ext::ENV_PEER_SERVICE
             end
+
+            option :on_error do |o|
+              o.type :proc, nilable: true
+            end
           end
         end
       end

data/lib/datadog/tracing/contrib/elasticsearch/patcher.rb

@@ -40,6 +40,7 @@ module Datadog
               # `Client#transport` is the most convenient object both for this integration and the library
               # as users have shared access to it across all `elasticsearch` versions.
               service ||= Datadog.configuration_for(transport, :service_name) || datadog_configuration[:service_name]
+              on_error = Datadog.configuration_for(transport, :on_error) || datadog_configuration[:on_error]
 
               method = args[0]
               path = args[1]
@@ -49,7 +50,11 @@ module Datadog
               url = full_url.path
               response = nil
 
-              Tracing.trace(Datadog::Tracing::Contrib::Elasticsearch::Ext::SPAN_QUERY, service: service) do |span|
+              Tracing.trace(
+                Datadog::Tracing::Contrib::Elasticsearch::Ext::SPAN_QUERY,
+                service: service,
+                on_error: on_error
+              ) do |span|
                 begin
                   connection = transport.connections.first
                   host = connection.host[:host] if connection

data/lib/datadog/tracing/contrib/ethon/easy_patch.rb

@@ -110,11 +110,10 @@ module Datadog
 
               datadog_tag_request
 
-              if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(datadog_trace)
-                datadog_trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
-              end
-
-              if datadog_configuration[:distributed_tracing]
+              if Tracing::Distributed::PropagationPolicy.enabled?(
+                global_config: datadog_configuration,
+                trace: datadog_trace
+              )
                 @datadog_original_headers ||= {}
                 Contrib::HTTP.inject(datadog_trace, @datadog_original_headers)
                 self.headers = @datadog_original_headers

data/lib/datadog/tracing/contrib/excon/middleware.rb

@@ -30,10 +30,12 @@ module Datadog
                 trace = Tracing.active_trace
                 datum[:datadog_span] = span
                 annotate!(span, datum)
-                if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                  trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
+                if Tracing::Distributed::PropagationPolicy.enabled?(
+                  global_config: @options,
+                  trace: trace
+                )
+                  propagate!(trace, span, datum)
                 end
-                propagate!(trace, span, datum) if distributed_tracing?
 
                 span
               end

data/lib/datadog/tracing/contrib/ext.rb

@@ -48,6 +48,7 @@ module Datadog
 
         module Messaging
           TAG_SYSTEM = 'messaging.system'
+          TAG_DESTINATION = 'messaging.destination'
           PEER_SERVICE_SOURCES = Array[Tracing::Metadata::Ext::NET::TAG_DESTINATION_NAME,
             Tracing::Metadata::Ext::TAG_PEER_HOSTNAME,
             Tracing::Metadata::Ext::NET::TAG_TARGET_HOST,].freeze

data/lib/datadog/tracing/contrib/extensions.rb

@@ -110,6 +110,13 @@ module Datadog
           module Settings
             InvalidIntegrationError = Class.new(StandardError)
 
+            # Used to avoid concurrency issues between registering integrations (e.g. mutation) and reporting the
+            # current integrations for logging/debugging/telemetry purposes (e.g. iteration) in the
+            # `@instrumented_integrations` hash.
+            #
+            # See https://github.com/DataDog/dd-trace-rb/issues/2851 for details on the original issue.
+            INSTRUMENTED_INTEGRATIONS_LOCK = Mutex.new
+
             def self.included(base)
               base.class_eval do
                 settings :contrib do
@@ -123,6 +130,20 @@ module Datadog
                     o.default({})
                   end
 
+                  # Enables population of default in the `peer.service` span tag.
+                  # Explicitly setting the `peer.service` for an integration will
+                  # still be honored with this option disabled.
+                  #
+                  # Also when disabled, other peer service related configurations have no effect.
+                  #
+                  # @default `DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED` environment variable, otherwise `false`
+                  # @return [Boolean]
+                  option :peer_service_defaults do |o|
+                    o.env Tracing::Configuration::Ext::SpanAttributeSchema::ENV_PEER_SERVICE_DEFAULTS_ENABLED
+                    o.type :bool
+                    o.default false
+                  end
+
                   # Global service name behavior
                   settings :global_default_service_name do
                     # Overrides default service name to global service name
@@ -161,7 +182,10 @@ module Datadog
                 configuration_name = options[:describes] || :default
                 filtered_options = options.reject { |k, _v| k == :describes }
                 integration.configure(configuration_name, filtered_options, &block)
-                instrumented_integrations[integration_name] = integration
+                INSTRUMENTED_INTEGRATIONS_LOCK.synchronize do
+                  @instrumented_integrations ||= {}
+                  @instrumented_integrations[integration_name] = integration
+                end
 
                 # Add to activation list
                 integrations_pending_activation << integration
@@ -192,14 +216,16 @@ module Datadog
               @integrations_pending_activation ||= Set.new
             end
 
+            # This method is only for logging/debugging/telemetry purposes (e.g. iteration) in the
+            # `@instrumented_integrations` hash.
             # @!visibility private
             def instrumented_integrations
-              @instrumented_integrations ||= {}
+              INSTRUMENTED_INTEGRATIONS_LOCK.synchronize { (@instrumented_integrations&.dup || {}).freeze }
             end
 
             # @!visibility private
             def reset!
-              instrumented_integrations.clear
+              INSTRUMENTED_INTEGRATIONS_LOCK.synchronize { @instrumented_integrations&.clear }
               super
             end
 

data/lib/datadog/tracing/contrib/faraday/middleware.rb

@@ -29,10 +29,12 @@ module Datadog
 
             Tracing.trace(Ext::SPAN_REQUEST, on_error: request_options[:on_error]) do |span, trace|
               annotate!(span, env, request_options)
-              if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
+              if Tracing::Distributed::PropagationPolicy.enabled?(
+                global_config: request_options,
+                trace: trace
+              )
+                propagate!(trace, span, env)
               end
-              propagate!(trace, span, env) if request_options[:distributed_tracing] && Tracing.enabled?
               app.call(env).on_complete { |resp| handle_response(span, resp, request_options) }
             end
           end

data/lib/datadog/tracing/contrib/graphql/configuration/error_extension_env_parser.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Contrib
+      module GraphQL
+        module Configuration
+          # Parses the environment variable `DD_TRACE_GRAPHQL_ERROR_EXTENSIONS` for error extension names declaration.
+          class ErrorExtensionEnvParser
+            # Parses the environment variable `DD_TRACE_GRAPHQL_ERROR_EXTENSIONS` into an array of error extension names.
+            def self.call(values)
+              # Split by comma, remove leading and trailing whitespaces,
+              # remove empty values, and remove repeated values.
+              values.split(',').each(&:strip!).reject(&:empty?).uniq
+            end
+          end
+        end
+      end
+    end
+  end
+end