ddtrace 1.12.1 → 1.23.2

data/lib/datadog/kit/appsec/events.rb

@@ -9,109 +9,159 @@ module Datadog
       module Events
         LOGIN_SUCCESS_EVENT = 'users.login.success'
         LOGIN_FAILURE_EVENT = 'users.login.failure'
-
-        # Attach login success event information to the trace
-        #
-        # This method is experimental and may change in the future.
-        #
-        # @param trace [TraceOperation] Trace to attach data to. Defaults to
-        #   active trace.
-        # @param span [SpanOperation] Span to attach data to. Defaults to
-        #   active span on trace. Note that this should be a service entry span.
-        #   When AppSec is enabled, the expected span and trace are automatically
-        #   used as defaults.
-        # @param user [Hash<Symbol, String>] User information to pass to
-        #   Datadog::Kit::Identity.set_user. Must contain at least :id as key.
-        # @param others [Hash<String || Symbol, String>] Additional free-form
-        #   event information to attach to the trace.
-        def self.track_login_success(trace = nil, span = nil, user:, **others)
-          if (appsec_scope = Datadog::AppSec.active_scope)
-            trace = appsec_scope.trace
-            span = appsec_scope.service_entry_span
+        SIGNUP_EVENT = 'users.signup'
+
+        class << self
+          # Attach login success event information to the trace
+          #
+          # This method is experimental and may change in the future.
+          #
+          # @param trace [TraceOperation] Trace to attach data to. Defaults to
+          #   active trace.
+          # @param span [SpanOperation] Span to attach data to. Defaults to
+          #   active span on trace. Note that this should be a service entry span.
+          #   When AppSec is enabled, the expected span and trace are automatically
+          #   used as defaults.
+          # @param user [Hash<Symbol, String>] User information to pass to
+          #   Datadog::Kit::Identity.set_user. Must contain at least :id as key.
+          # @param others [Hash<String || Symbol, String>] Additional free-form
+          #   event information to attach to the trace.
+          def track_login_success(trace = nil, span = nil, user:, **others)
+            set_trace_and_span_context('track_login_success', trace, span) do |active_trace, active_span|
+              user_options = user.dup
+              user_id = user_options.delete(:id)
+
+              raise ArgumentError, 'missing required key: :user => { :id }' if user_id.nil?
+
+              track(LOGIN_SUCCESS_EVENT, active_trace, active_span, **others)
+
+              Kit::Identity.set_user(active_trace, active_span, id: user_id, **user_options)
+            end
           end
 
-          trace ||= Datadog::Tracing.active_trace
-          span ||= trace.active_span || Datadog::Tracing.active_span
-
-          raise ArgumentError, "span #{span.span_id} does not belong to trace #{trace.id}" if trace.id != span.trace_id
-
-          track(LOGIN_SUCCESS_EVENT, trace, span, **others)
-
-          user_options = user.dup
-          user_id = user_options.delete(:id)
-
-          raise ArgumentError, 'missing required key: :user => { :id }' if user_id.nil?
+          # Attach login failure event information to the trace
+          #
+          # This method is experimental and may change in the future.
+          #
+          # @param trace [TraceOperation] Trace to attach data to. Defaults to
+          #   active trace.
+          # @param span [SpanOperation] Span to attach data to. Defaults to
+          #   active span on trace. Note that this should be a service entry span.
+          #   When AppSec is enabled, the expected span and trace are automatically
+          #   used as defaults.
+          # @param user_id [String] User id that attempted login
+          # @param user_exists [bool] Whether the user id that did a login attempt exists.
+          # @param others [Hash<String || Symbol, String>] Additional free-form
+          #   event information to attach to the trace.
+          def track_login_failure(trace = nil, span = nil, user_id:, user_exists:, **others)
+            set_trace_and_span_context('track_login_failure', trace, span) do |active_trace, active_span|
+              raise ArgumentError, 'user_id cannot be nil' if user_id.nil?
+
+              track(LOGIN_FAILURE_EVENT, active_trace, active_span, **others)
+
+              active_span.set_tag('appsec.events.users.login.failure.usr.id', user_id)
+              active_span.set_tag('appsec.events.users.login.failure.usr.exists', user_exists)
+            end
+          end
 
-          Kit::Identity.set_user(trace, span, id: user_id, **user_options)
-        end
+          # Attach signup event information to the trace
+          #
+          # This method is experimental and may change in the future.
+          #
+          # @param trace [TraceOperation] Trace to attach data to. Defaults to
+          #   active trace.
+          # @param span [SpanOperation] Span to attach data to. Defaults to
+          #   active span on trace. Note that this should be a service entry span.
+          #   When AppSec is enabled, the expected span and trace are automatically
+          #   used as defaults.
+          # @param user [Hash<Symbol, String>] User information to pass to
+          #   Datadog::Kit::Identity.set_user. Must contain at least :id as key.
+          # @param others [Hash<String || Symbol, String>] Additional free-form
+          #   event information to attach to the trace.
+          def track_signup(trace = nil, span = nil, user:, **others)
+            set_trace_and_span_context('track_signup', trace, span) do |active_trace, active_span|
+              user_options = user.dup
+              user_id = user_options.delete(:id)
+
+              raise ArgumentError, 'missing required key: :user => { :id }' if user_id.nil?
+
+              track(SIGNUP_EVENT, active_trace, active_span, **others)
+
+              Kit::Identity.set_user(trace, id: user_id, **user_options)
+            end
+          end
 
-        # Attach login failure event information to the trace
-        #
-        # This method is experimental and may change in the future.
-        #
-        # @param trace [TraceOperation] Trace to attach data to. Defaults to
-        #   active trace.
-        # @param span [SpanOperation] Span to attach data to. Defaults to
-        #   active span on trace. Note that this should be a service entry span.
-        #   When AppSec is enabled, the expected span and trace are automatically
-        #   used as defaults.
-        # @param user_id [String] User id that attempted login
-        # @param user_exists [bool] Whether the user id that did a login attempt exists.
-        # @param others [Hash<String || Symbol, String>] Additional free-form
-        #   event information to attach to the trace.
-        def self.track_login_failure(trace = nil, span = nil, user_id:, user_exists:, **others)
-          if (appsec_scope = Datadog::AppSec.active_scope)
-            trace = appsec_scope.trace
-            span = appsec_scope.service_entry_span
+          # Attach custom event information to the trace
+          #
+          # This method is experimental and may change in the future.
+          #
+          # @param event [String] Mandatory. Event code.
+          # @param trace [TraceOperation] Trace to attach data to. Defaults to
+          #   active trace.
+          # @param span [SpanOperation] Span to attach data to. Defaults to
+          #   active span on trace. Note that this should be a service entry span.
+          #   When AppSec is enabled, the expected span and trace are automatically
+          #   used as defaults.
+          # @param others [Hash<Symbol, String>] Additional free-form
+          #   event information to attach to the trace. Key must not
+          #   be :track.
+          def track(event, trace = nil, span = nil, **others)
+            if trace && span
+              check_trace_span_integrity(trace, span)
+
+              span.set_tag("appsec.events.#{event}.track", 'true')
+              span.set_tag("_dd.appsec.events.#{event}.sdk", 'true')
+
+              others.each do |k, v|
+                raise ArgumentError, 'key cannot be :track' if k.to_sym == :track
+
+                span.set_tag("appsec.events.#{event}.#{k}", v) unless v.nil?
+              end
+
+              trace.keep!
+            else
+              set_trace_and_span_context('track', trace, span) do |active_trace, active_span|
+                active_span.set_tag("appsec.events.#{event}.track", 'true')
+                active_span.set_tag("_dd.appsec.events.#{event}.sdk", 'true')
+
+                others.each do |k, v|
+                  raise ArgumentError, 'key cannot be :track' if k.to_sym == :track
+
+                  active_span.set_tag("appsec.events.#{event}.#{k}", v) unless v.nil?
+                end
+
+                active_trace.keep!
+              end
+            end
           end
 
-          trace ||= Datadog::Tracing.active_trace
-          span ||= trace.active_span || Datadog::Tracing.active_span
+          private
 
-          raise ArgumentError, "span #{span.span_id} does not belong to trace #{trace.id}" if trace.id != span.trace_id
+          def set_trace_and_span_context(method, trace = nil, span = nil)
+            if (appsec_scope = Datadog::AppSec.active_scope)
+              trace = appsec_scope.trace
+              span = appsec_scope.service_entry_span
+            end
 
-          track(LOGIN_FAILURE_EVENT, trace, span, **others)
+            trace ||= Datadog::Tracing.active_trace
+            span ||=  trace && trace.active_span || Datadog::Tracing.active_span
 
-          raise ArgumentError, 'user_id cannot be nil' if user_id.nil?
+            unless trace && span
+              Datadog.logger.debug(
+                "Tracing not enabled. Method ##{method} is a no-op. Please enable tracing if you want ##{method}"\
+                ' to track this events'
+              )
+              return
+            end
 
-          span.set_tag('appsec.events.users.login.failure.usr.id', user_id)
-          span.set_tag('appsec.events.users.login.failure.usr.exists', user_exists)
-        end
+            check_trace_span_integrity(trace, span)
 
-        # Attach custom event information to the trace
-        #
-        # This method is experimental and may change in the future.
-        #
-        # @param event [String] Mandatory. Event code.
-        # @param trace [TraceOperation] Trace to attach data to. Defaults to
-        #   active trace.
-        # @param span [SpanOperation] Span to attach data to. Defaults to
-        #   active span on trace. Note that this should be a service entry span.
-        #   When AppSec is enabled, the expected span and trace are automatically
-        #   used as defaults.
-        # @param others [Hash<Symbol, String>] Additional free-form
-        #   event information to attach to the trace. Key must not
-        #   be :track.
-        def self.track(event, trace = nil, span = nil, **others)
-          if (appsec_scope = Datadog::AppSec.active_scope)
-            trace = appsec_scope.trace
-            span = appsec_scope.service_entry_span
+            yield(trace, span)
           end
 
-          trace ||= Datadog::Tracing.active_trace
-          span ||= trace.active_span || Datadog::Tracing.active_span
-
-          raise ArgumentError, "span #{span.span_id} does not belong to trace #{trace.id}" if trace.id != span.trace_id
-
-          span.set_tag("appsec.events.#{event}.track", 'true')
-
-          others.each do |k, v|
-            raise ArgumentError, 'key cannot be :track' if k.to_sym == :track
-
-            span.set_tag("appsec.events.#{event}.#{k}", v) unless v.nil?
+          def check_trace_span_integrity(trace, span)
+            raise ArgumentError, "span #{span.span_id} does not belong to trace #{trace.id}" if trace.id != span.trace_id
           end
-
-          trace.keep!
         end
       end
     end

data/lib/datadog/kit/enable_core_dumps.rb

@@ -16,11 +16,13 @@ module Datadog
             '(Could not open /proc/sys/kernel/core_pattern)'
           end
 
+        enabled_status = "Maximum size: #{maximum_size} Output pattern: '#{core_pattern}'"
+
         if maximum_size <= 0
           Kernel.warn("[ddtrace] Could not enable core dumps on crash, maximum size is #{maximum_size} (disabled).")
           return
         elsif maximum_size == current_size
-          Kernel.warn('[ddtrace] Core dumps already enabled, nothing to do!')
+          Kernel.warn("[ddtrace] Core dumps already enabled, nothing to do. #{enabled_status}")
           return
         end
 
@@ -35,12 +37,9 @@ module Datadog
         end
 
         if current_size == 0
-          Kernel.warn("[ddtrace] Enabled core dumps. Maximum size: #{maximum_size} Output pattern: '#{core_pattern}'")
+          Kernel.warn("[ddtrace] Enabled core dumps. #{enabled_status}")
         else
-          Kernel.warn(
-            "[ddtrace] Raised core dump limit. Old size: #{current_size} " \
-            "Maximum size: #{maximum_size} Output pattern: '#{core_pattern}'"
-          )
+          Kernel.warn("[ddtrace] Raised core dump limit. Old size: #{current_size} #{enabled_status}")
         end
       end
     end

data/lib/datadog/kit/identity.rb

@@ -6,84 +6,99 @@ module Datadog
   module Kit
     # Tracking identity via traces
     module Identity
-      # Attach user information to the trace
-      #
-      # @param trace [TraceOperation] Trace to attach data to. Defaults to
-      #   active trace.
-      # @param span [SpanOperation] Span to attach data to. Defaults to
-      #   active span on trace. Note that this should be a service entry span.
-      #   When AppSec is enabled, the expected span and trace are automatically
-      #   used as defaults.
-      # @param id [String] Mandatory. Username or client id extracted
-      #   from the access token or Authorization header in the inbound request
-      #   from outside the system.
-      # @param email [String] Email of the authenticated user associated
-      #   to the trace.
-      # @param name [String] User-friendly name. To be displayed in the
-      #   UI if set.
-      # @param session_id [String] Session ID of the authenticated user.
-      # @param role [String] Actual/assumed role the client is making
-      #   the request under extracted from token or application security
-      #   context.
-      # @param scope [String] Scopes or granted authorities the client
-      #   currently possesses extracted from token or application security
-      #   context. The value would come from the scope associated with an OAuth
-      #   2.0 Access Token or an attribute value in a SAML 2.0 Assertion.
-      # @param others [Hash<Symbol, String>] Additional free-form
-      #   user information to attach to the trace.
-      #
-      # rubocop:disable Metrics/CyclomaticComplexity
-      # rubocop:disable Metrics/PerceivedComplexity
-      # rubocop:disable Metrics/AbcSize
-      def self.set_user(
-        trace = nil, span = nil, id:, email: nil, name: nil, session_id: nil, role: nil, scope: nil, **others
-      )
-        raise ArgumentError, 'missing required key: :id' if id.nil?
+      class << self
+        # Attach user information to the trace
+        #
+        # @param trace [TraceOperation] Trace to attach data to. Defaults to
+        #   active trace.
+        # @param span [SpanOperation] Span to attach data to. Defaults to
+        #   active span on trace. Note that this should be a service entry span.
+        #   When AppSec is enabled, the expected span and trace are automatically
+        #   used as defaults.
+        # @param id [String] Mandatory. Username or client id extracted
+        #   from the access token or Authorization header in the inbound request
+        #   from outside the system.
+        # @param email [String] Email of the authenticated user associated
+        #   to the trace.
+        # @param name [String] User-friendly name. To be displayed in the
+        #   UI if set.
+        # @param session_id [String] Session ID of the authenticated user.
+        # @param role [String] Actual/assumed role the client is making
+        #   the request under extracted from token or application security
+        #   context.
+        # @param scope [String] Scopes or granted authorities the client
+        #   currently possesses extracted from token or application security
+        #   context. The value would come from the scope associated with an OAuth
+        #   2.0 Access Token or an attribute value in a SAML 2.0 Assertion.
+        # @param others [Hash<Symbol, String>] Additional free-form
+        #   user information to attach to the trace.
+        #
+        # rubocop:disable Metrics/CyclomaticComplexity
+        # rubocop:disable Metrics/PerceivedComplexity
+        def set_user(
+          trace = nil, span = nil, id:, email: nil, name: nil, session_id: nil, role: nil, scope: nil, **others
+        )
+          raise ArgumentError, 'missing required key: :id' if id.nil?
 
-        # enforce types
+          # enforce types
 
-        raise TypeError, ':id must be a String'         unless id.is_a?(String)
-        raise TypeError, ':email must be a String'      unless email.nil? || email.is_a?(String)
-        raise TypeError, ':name must be a String'       unless name.nil? || name.is_a?(String)
-        raise TypeError, ':session_id must be a String' unless session_id.nil? || session_id.is_a?(String)
-        raise TypeError, ':role must be a String'       unless role.nil? || role.is_a?(String)
-        raise TypeError, ':scope must be a String'      unless scope.nil? || scope.is_a?(String)
+          raise TypeError, ':id must be a String'         unless id.is_a?(String)
+          raise TypeError, ':email must be a String'      unless email.nil? || email.is_a?(String)
+          raise TypeError, ':name must be a String'       unless name.nil? || name.is_a?(String)
+          raise TypeError, ':session_id must be a String' unless session_id.nil? || session_id.is_a?(String)
+          raise TypeError, ':role must be a String'       unless role.nil? || role.is_a?(String)
+          raise TypeError, ':scope must be a String'      unless scope.nil? || scope.is_a?(String)
 
-        others.each do |k, v|
-          raise TypeError, "#{k.inspect} must be a String" unless v.nil? || v.is_a?(String)
-        end
+          others.each do |k, v|
+            raise TypeError, "#{k.inspect} must be a String" unless v.nil? || v.is_a?(String)
+          end
+
+          set_trace_and_span_context('set_user', trace, span) do |_active_trace, active_span|
+            # set tags once data is known consistent
+            active_span.set_tag('usr.id', id)
+            active_span.set_tag('usr.email', email)           unless email.nil?
+            active_span.set_tag('usr.name', name)             unless name.nil?
+            active_span.set_tag('usr.session_id', session_id) unless session_id.nil?
+            active_span.set_tag('usr.role', role)             unless role.nil?
+            active_span.set_tag('usr.scope', scope)           unless scope.nil?
 
-        if (appsec_scope = Datadog::AppSec.active_scope)
-          trace = appsec_scope.trace
-          span = appsec_scope.service_entry_span
+            others.each do |k, v|
+              active_span.set_tag("usr.#{k}", v) unless v.nil?
+            end
+
+            if Datadog::AppSec.active_scope
+              user = ::Datadog::AppSec::Instrumentation::Gateway::User.new(id)
+              ::Datadog::AppSec::Instrumentation.gateway.push('identity.set_user', user)
+            end
+          end
         end
+        # rubocop:enable Metrics/PerceivedComplexity
+        # rubocop:enable Metrics/CyclomaticComplexity
 
-        trace ||= Datadog::Tracing.active_trace
-        span ||= trace.active_span || Datadog::Tracing.active_span
+        private
 
-        raise ArgumentError, "span #{span.span_id} does not belong to trace #{trace.id}" if trace.id != span.trace_id
+        def set_trace_and_span_context(method, trace = nil, span = nil)
+          if (appsec_scope = Datadog::AppSec.active_scope)
+            trace = appsec_scope.trace
+            span = appsec_scope.service_entry_span
+          end
 
-        # set tags once data is known consistent
+          trace ||= Datadog::Tracing.active_trace
+          span ||= trace && trace.active_span || Datadog::Tracing.active_span
 
-        span.set_tag('usr.id', id)
-        span.set_tag('usr.email', email)           unless email.nil?
-        span.set_tag('usr.name', name)             unless name.nil?
-        span.set_tag('usr.session_id', session_id) unless session_id.nil?
-        span.set_tag('usr.role', role)             unless role.nil?
-        span.set_tag('usr.scope', scope)           unless scope.nil?
+          unless trace && span
+            Datadog.logger.debug(
+              "Tracing not enabled. Method ##{method} is a no-op. Please enable tracing if you want ##{method}"\
+              ' to track this events'
+            )
+            return
+          end
 
-        others.each do |k, v|
-          span.set_tag("usr.#{k}", v) unless v.nil?
-        end
+          raise ArgumentError, "span #{span.span_id} does not belong to trace #{trace.id}" if trace.id != span.trace_id
 
-        if appsec_scope
-          user = ::Datadog::AppSec::Instrumentation::Gateway::User.new(id)
-          ::Datadog::AppSec::Instrumentation.gateway.push('identity.set_user', user)
+          yield(trace, span)
         end
       end
-      # rubocop:enable Metrics/AbcSize
-      # rubocop:enable Metrics/PerceivedComplexity
-      # rubocop:enable Metrics/CyclomaticComplexity
     end
   end
 end

data/lib/datadog/opentelemetry/api/context.rb

@@ -2,6 +2,7 @@
 
 require_relative 'trace/span'
 require_relative '../../tracing/trace_operation'
+require_relative '../trace'
 
 module Datadog
   module OpenTelemetry
@@ -66,9 +67,15 @@ module Datadog
         # @param [Object] value Object to be stored under key
         # @return [Context]
         def set_values(values)
-          trace = nil
-          if (current_span = values[CURRENT_SPAN_KEY])
-            trace = current_span.datadog_trace
+          if (current_span = values[CURRENT_SPAN_KEY]) && current_span.datadog_trace
+            # Because `#set_value` returns new context, we have to create
+            # a new copy of the active trace to ensure there's no conflict with
+            # other users of the active trace.
+            # It effectively becomes an internal trace propagation.
+            trace = Datadog::OpenTelemetry::Trace.start_trace_copy(
+              current_span.datadog_trace,
+              parent_span: current_span.datadog_span
+            )
           end
 
           existing_values = @trace && @trace.otel_values || {}

data/lib/datadog/opentelemetry/sdk/propagator.rb

@@ -29,9 +29,10 @@ module Datadog
           carrier, context: ::OpenTelemetry::Context.current,
           getter: ::OpenTelemetry::Context::Propagation.text_map_getter
         )
-          unless getter == ::OpenTelemetry::Context::Propagation.text_map_getter
+          if getter != ::OpenTelemetry::Context::Propagation.text_map_getter &&
+              getter != ::OpenTelemetry::Common::Propagation.rack_env_getter
             Datadog.logger.error(
-              'Custom getter is not supported. Please inform the `ddtrace` team at ' \
+              "Custom getter #{getter} is not supported. Please inform the `ddtrace` team at " \
             ' https://github.com/DataDog/dd-trace-rb of your use case so we can best support you. Using the default ' \
             'OpenTelemetry::Context::Propagation.text_map_getter as a fallback getter.'
             )
@@ -81,8 +82,9 @@ module Datadog
         private
 
         # Converts the {Numeric} Datadog id object to OpenTelemetry's byte array format.
+        # This method currently converts an unsigned 64-bit Integer to a binary String.
         def to_otel_id(dd_id)
-          Array(dd_id).pack('S')
+          Array(dd_id).pack('Q')
         end
       end
     end

data/lib/datadog/opentelemetry/sdk/span_processor.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'trace/span'
+
 module Datadog
   module OpenTelemetry
     module SDK
@@ -27,7 +29,7 @@ module Datadog
         #
         # @param [Span] span the {Span} that just ended.
         def on_finish(span)
-          span.datadog_span.finish
+          span.datadog_span.finish(ns_to_time(span.end_timestamp))
         end
 
         # Export all ended spans to the configured `Exporter` that have not yet
@@ -76,15 +78,56 @@ module Datadog
         end
 
         def start_datadog_span(span)
-          tags = span.resource.attribute_enumerator.to_h
+          attributes = span.attributes.dup # Dup to allow modification of frozen Hash
+
+          name, kwargs = span_arguments(span, attributes)
 
-          kind = span.kind || 'internal'
-          tags[Tracing::Metadata::Ext::TAG_KIND] = kind
+          datadog_span = Tracing.trace(name, **kwargs)
 
-          datadog_span = Tracing.trace(span.name, tags: tags)
           datadog_span.set_error([nil, span.status.description]) unless span.status.ok?
+          datadog_span.set_tags(span.attributes)
+
           datadog_span
         end
+
+        # Some special attributes can override Datadog Span fields
+        def span_arguments(span, attributes)
+          if attributes.key?('analytics.event') && (analytics_event = attributes['analytics.event']).respond_to?(:casecmp)
+            attributes[Tracing::Metadata::Ext::Analytics::TAG_SAMPLE_RATE] = analytics_event.casecmp('true') == 0 ? 1 : 0
+          end
+          attributes[Tracing::Metadata::Ext::TAG_KIND] = span.kind || 'internal'
+
+          kwargs = { start_time: ns_to_time(span.start_timestamp) }
+
+          name = if attributes.key?('operation.name')
+                   attributes['operation.name']
+                 elsif (rich_name = Datadog::OpenTelemetry::Trace::Span.enrich_name(span.kind, attributes))
+                   rich_name.downcase
+                 else
+                   span.kind
+                 end
+
+          kwargs[:resource] = attributes.key?('resource.name') ? attributes['resource.name'] : span.name
+          kwargs[:service] = attributes['service.name'] if attributes.key?('service.name')
+          kwargs[:type] = attributes['span.type'] if attributes.key?('span.type')
+
+          attributes.reject! { |key, _| OpenTelemetry::Trace::Span::DATADOG_SPAN_ATTRIBUTE_OVERRIDES.include?(key) }
+
+          # DEV: There's no `flat_map!`; we have to split it into two operations
+          attributes = attributes.map do |key, value|
+            Datadog::OpenTelemetry::Trace::Span.serialize_attribute(key, value)
+          end
+          attributes.flatten!(1)
+
+          kwargs[:tags] = attributes
+
+          [name, kwargs]
+        end
+
+        # From nanoseconds, used by OpenTelemetry, to a {Time} object, used by the Datadog Tracer.
+        def ns_to_time(timestamp_ns)
+          Time.at(timestamp_ns / 1000000000.0)
+        end
       end
     end
   end