chef-handler-datadog-demo 0.1.0

data/gemfiles/chef_10.14.4.gemfile

@@ -0,0 +1,17 @@
+# This file was generated by Appraisal
+
+source "http://rubygems.org"
+
+gem "chef", "10.14.4"
+gem "ohai", "< 8.0"
+
+group :localdev do
+  gem "guard"
+  gem "guard-rspec"
+  gem "guard-rubocop"
+  gem "pry"
+  gem "terminal-notifier-guard"
+  gem "travis-lint"
+end
+
+gemspec :path => "../"

data/gemfiles/chef_10.gemfile

@@ -0,0 +1,16 @@
+# This file was generated by Appraisal
+
+source "http://rubygems.org"
+
+gem "chef", "~> 10.0"
+
+group :localdev do
+  gem "guard"
+  gem "guard-rspec"
+  gem "guard-rubocop"
+  gem "pry"
+  gem "terminal-notifier-guard"
+  gem "travis-lint"
+end
+
+gemspec :path => "../"

data/gemfiles/chef_11.gemfile

@@ -0,0 +1,16 @@
+# This file was generated by Appraisal
+
+source "http://rubygems.org"
+
+gem "chef", "~> 11.0"
+
+group :localdev do
+  gem "guard"
+  gem "guard-rspec"
+  gem "guard-rubocop"
+  gem "pry"
+  gem "terminal-notifier-guard"
+  gem "travis-lint"
+end
+
+gemspec :path => "../"

data/gemfiles/chef_12.gemfile

@@ -0,0 +1,16 @@
+# This file was generated by Appraisal
+
+source "http://rubygems.org"
+
+gem "chef", "~> 12.0"
+
+group :localdev do
+  gem "guard"
+  gem "guard-rspec"
+  gem "guard-rubocop"
+  gem "pry"
+  gem "terminal-notifier-guard"
+  gem "travis-lint"
+end
+
+gemspec :path => "../"

data/lib/chef/handler/datadog_demo.rb

@@ -0,0 +1,123 @@
+# encoding: utf-8
+require 'rubygems'
+require 'chef/handler'
+require 'chef/mash'
+require 'dogapi'
+require_relative 'datadog_demo_chef_metrics'
+require_relative 'datadog_demo_chef_tags'
+require_relative 'datadog_demo_chef_events'
+
+class Chef
+  class Handler
+    # Datadog handler to send Chef run details to Datadog
+    class DatadogDemo < Chef::Handler
+      attr_reader :config
+
+      # For the tags to work, the client must have created an Application Key on the
+      # "Account Settings" page here: https://app.datadoghq.com/account/settings
+      # It should be passed along from the node/role/environemnt attributes, as the default is nil.
+      def initialize(config = {})
+        @config = Mash.new(config)
+        # If *any* api_key is not provided, this will fail immediately.
+        @dog = Dogapi::Client.new(@config[:api_key], @config[:application_key])
+      end
+
+      def report
+        # use datadog agent proxy settings, if available
+        use_agent_proxy unless ENV['DATADOG_PROXY'].nil?
+
+        # prepare the metrics, event, and tags information to be reported
+        prepare_report_for_datadog
+        # post the report information to the datadog service
+        send_report_to_datadog
+      ensure
+        # restore the env proxy settings before leaving to avoid downstream side-effects
+        restore_env_proxies unless ENV['DATADOG_PROXY'].nil?
+      end
+
+      private
+
+      # prepare metrics, event, and tags data for posting to datadog
+      def prepare_report_for_datadog
+        # uses class method accessors for run_status and config
+        hostname = resolve_correct_hostname
+        # prepare chef run metrics
+        @metrics =
+            DatadogChefMetrics.new
+            .with_dogapi_client(@dog)
+            .with_hostname(hostname)
+            .with_run_status(run_status)
+
+        # Collect and prepare tags
+        @tags =
+            DatadogChefTags.new
+            .with_dogapi_client(@dog)
+            .with_hostname(hostname)
+            .with_run_status(run_status)
+            .with_application_key(config[:application_key])
+            .with_tag_prefix(config[:tag_prefix])
+            .with_retries(config[:tags_submission_retries])
+
+        # Build the chef event information
+        @event =
+            DatadogChefEvents.new
+            .with_dogapi_client(@dog)
+            .with_hostname(hostname)
+            .with_run_status(run_status)
+            .with_failure_notifications(@config['notify_on_failure'])
+            .with_tags(@tags.combined_host_tags)
+      end
+
+      # Submit metrics, event, and tags information to datadog
+      def send_report_to_datadog
+        @metrics.emit_to_datadog
+        @event.emit_to_datadog
+        @tags.send_update_to_datadog
+      rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
+        Chef::Log.error("Could not connect to Datadog. Connection error:\n" + e)
+        Chef::Log.error('Data to be submitted was:')
+        Chef::Log.error(@event.event_title)
+        Chef::Log.error(@event.event_body)
+        Chef::Log.error('Tags to be set for this run:')
+        Chef::Log.error(@tags.combined_host_tags)
+      end
+
+      # Select which hostname to report back to Datadog.
+      # Makes decision based on inputs from `config` and when absent, use the
+      # node's `ec2` attribute existence to make the decision.
+      #
+      # @return [String] the hostname decided upon
+      def resolve_correct_hostname
+        node = run_status.node
+        use_ec2_instance_id = !config.key?(:use_ec2_instance_id) ||
+                              (config.key?(:use_ec2_instance_id) && config[:use_ec2_instance_id])
+
+        if config[:hostname]
+          puts "found hostname #{config[:hostname]} in config object"
+          config[:hostname]
+        elsif use_ec2_instance_id && node.attribute?('ec2') && node.ec2.attribute?('instance_id')
+          node.ec2.instance_id
+        else
+          node.name
+        end
+      end
+
+      # Using the agent proxy settings requires setting http(s)_proxy
+      # env vars.  However, original env var settings need to be
+      # preserved for restoration at the end of the handler.
+      def use_agent_proxy
+        Chef::Log.info('Using agent proxy settings')
+        @env_http_proxy = ENV['http_proxy']
+        @env_https_proxy = ENV['https_proxy']
+        ENV['http_proxy'] = ENV['DATADOG_PROXY']
+        ENV['https_proxy'] = ENV['DATADOG_PROXY']
+      end
+
+      # Restore environment proxy settings to pre-report values
+      def restore_env_proxies
+        ENV['http_proxy'] = @env_http_proxy
+        ENV['https_proxy'] = @env_https_proxy
+      end
+    end # end class Datadog
+  end # end class Handler
+end # end class Chef

data/lib/chef/handler/datadog_demo_chef_events.rb

@@ -0,0 +1,174 @@
+# encoding: utf-8
+require 'rubygems'
+require 'chef/handler'
+require 'chef/mash'
+require 'dogapi'
+
+# helper class for sending events about chef runs
+class DatadogChefEvents
+  def initialize
+    @dog = nil
+    @hostname = nil
+    @run_status = nil
+    @failure_notfications = nil
+
+    @alert_type = ''
+    @event_priority = ''
+    @event_title = ''
+    # TODO: refactor how event_body is constructed in the class methods
+    #       handling of the event_body is a bit clunky and depends on the order of
+    #       method calls
+    @event_body = ''
+  end
+
+  # set the dogapi client handle
+  #
+  # @param dogapi_client [Dogapi::Client] datadog api client handle
+  # @return [DatadogChefTags] instance reference to self enabling method chaining
+  def with_dogapi_client(dogapi_client)
+    @dog = dogapi_client
+    self
+  end
+
+  # set the target hostname (chef node name)
+  #
+  # @param hostname [String] hostname to use for the handler report
+  # @return [DatadogChefTags] instance reference to self enabling method chaining
+  def with_hostname(hostname)
+    @hostname = hostname
+    self
+  end
+
+  # set the chef run status used for the report
+  #
+  # @param run_status [Chef::RunStatus] current chef run status
+  # @return [DatadogChefTags] instance reference to self enabling method chaining
+  def with_run_status(run_status)
+    @run_status = run_status
+    self
+  end
+
+  # set the failure notification list
+  #
+  # @param failure_notifications [Array] set of datadog notification handles
+  # @return [DatadogChefTags] instance reference to self enabling method chaining
+  def with_failure_notifications(failure_notifications)
+    @failure_notifications = failure_notifications
+    self
+  end
+
+  # set the datadog host tags associated with the event
+  #
+  # @param [Array] the set of host tags
+  # @return [DatadogChefTags] instance reference to self enabling method chaining
+  def with_tags(tags)
+    @tags = tags
+    self
+  end
+
+  # Emit Chef event to Datadog
+  def emit_to_datadog
+    @event_body = ''
+    build_event_data
+    evt = @dog.emit_event(Dogapi::Event.new(@event_body,
+                                            msg_title: @event_title,
+                                            event_type: 'config_management.run',
+                                            event_object: @hostname,
+                                            alert_type: @alert_type,
+                                            priority: @event_priority,
+                                            source_type_name: 'chef',
+                                            tags: @tags
+                                           ), host: @hostname)
+
+    begin
+      # FIXME: nice-to-have: abstract format of return value away a bit
+      # in dogapi directly. See https://github.com/DataDog/dogapi-rb/issues/18
+      if evt.length < 2
+        Chef::Log.warn("Unexpected response from Datadog Event API: #{evt}")
+      else
+        # [http_response_code, {"event" => {"url" => "...", ...}}]
+        # 2xx means ok
+        if evt[0].to_i / 100 != 2
+          Chef::Log.warn("Could not submit event to Datadog (HTTP call failed): #{evt[0]}")
+        else
+          Chef::Log.debug("Successfully submitted Chef event to Datadog for #{@hostname} at #{evt[1]['event']['url']}")
+        end
+      end
+    rescue
+      Chef::Log.warn("Could not determine whether chef run was successfully submitted to Datadog: #{evt}")
+    end
+  end
+
+  private
+
+  def pluralize(number, noun)
+    case number
+    when 0..1
+      "less than 1 #{noun}"
+    else
+      "#{number.round} #{noun}s"
+    end
+  rescue
+    Chef::Log.warn("Cannot make #{number} more legible")
+    "#{number} #{noun}s"
+  end
+
+  # Compose a list of resources updated during a run.
+  def update_resource_list
+    # No resources updated?
+    return unless @run_status.updated_resources.length.to_i > 0
+
+    if @run_status.failed?
+      # Shorten the list when there is a failure for stacktrace debugging
+      report_resources = @run_status.updated_resources.last(5)
+    else
+      report_resources = @run_status.updated_resources
+    end
+
+    @event_body = "\n$$$\n"
+    report_resources.each do |r|
+      @event_body << "- #{r} (#{r.defined_at})\n"
+    end
+    @event_body << "\n$$$\n"
+  end
+
+  # Marshal the Event data for submission
+  def build_event_data
+    # bail early in case of a compiletime failure
+    # OPTIMIZE: Use better inspectors to handle failure scenarios, refactor needed.
+    if @run_status.elapsed_time.nil?
+      @alert_type = 'error'
+      @event_title = "Chef failed during compile phase on #{@hostname} "
+      @event_priority = 'normal'
+      @event_body = 'Chef was unable to complete a run, an error during compilation may have occurred.'
+    else
+      run_time = pluralize(@run_status.elapsed_time, 'second')
+
+      # This is the first line of the Event body, the rest is appended here.
+      @event_body = "Chef updated #{@run_status.updated_resources.length} resources out of #{@run_status.all_resources.length} resources total."
+
+      # Update resource list, truncated when failed to 5
+      # update will add to the event_body
+      update_resource_list
+
+      if @run_status.success?
+        @alert_type = 'success'
+        @event_priority = 'low'
+        @event_title = "Chef completed in #{run_time} on #{@hostname} "
+      else
+        @alert_type = 'error'
+        @event_priority = 'normal'
+        @event_title = "Chef failed in #{run_time} on #{@hostname} "
+
+        if @failure_notifications
+          handles = @failure_notifications
+          # convert the notification handle array to a string
+          @event_body << "\nAlerting: #{handles.join(' ')}\n"
+        end
+
+        @event_body << "\n$$$\n#{@run_status.formatted_exception}\n$$$\n"
+        @event_body << "\n$$$\n#{@run_status.backtrace.join("\n")}\n$$$\n"
+      end
+    end
+  end
+end # end module DatadogChefEvent

data/lib/chef/handler/datadog_demo_chef_metrics.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+require 'dogapi'
+
+# helper class for sending datadog metrics from a chef run
+class DatadogChefMetrics
+  def initialize
+    @dog = nil
+    @hostname = ''
+    @run_status = nil
+  end
+
+  # set the dogapi client handle
+  #
+  # @param dogapi_client [Dogapi::Client] datadog api client
+  # @return [DatadogChefMetrics] instance reference to self enabling method chaining
+  def with_dogapi_client(dogapi_client)
+    @dog = dogapi_client
+    self
+  end
+
+  # set the target hostname (chef node name)
+  #
+  # @param hostname [String] hostname used for reporting metrics
+  # @return [DatadogChefMetrics] instance reference to self enabling method chaining
+  def with_hostname(hostname)
+    @hostname = hostname
+    self
+  end
+
+  # set the chef run status used for the report
+  #
+  # @param run_status [Chef::RunStatus] current run status
+  # @return [DatadogChefMetrics] instance reference to self enabling method chaining
+  def with_run_status(run_status)
+    @run_status = run_status
+    self
+  end
+
+  # Emit Chef metrics to Datadog
+  def emit_to_datadog
+    # Send base success/failure metric
+    @dog.emit_point(@run_status.success? ? 'chef.run.success' : 'chef.run.failure', 1, host: @hostname, type: 'counter')
+
+    # If there is a failure during compile phase, a large portion of
+    # run_status may be unavailable. Bail out here
+    warn_msg = 'Error during compile phase, no Datadog metrics available.'
+    return Chef::Log.warn(warn_msg) if compile_error?
+
+    @dog.emit_point('chef.resources.total', @run_status.all_resources.length, host: @hostname)
+    @dog.emit_point('chef.resources.updated', @run_status.updated_resources.length, host: @hostname)
+    @dog.emit_point('chef.resources.elapsed_time', @run_status.elapsed_time, host: @hostname)
+    Chef::Log.debug('Submitted Chef metrics back to Datadog')
+  rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
+    Chef::Log.error("Could not send metrics to Datadog. Connection error:\n" + e)
+  end
+
+  private
+
+  def compile_error?
+    @run_status.all_resources.nil? || @run_status.elapsed_time.nil? || @run_status.updated_resources.nil?
+  end
+end # end class DatadogChefMetrics

data/lib/chef/handler/datadog_demo_chef_tags.rb

@@ -0,0 +1,140 @@
+# encoding: utf-8
+require 'rubygems'
+require 'chef/handler'
+require 'chef/mash'
+require 'dogapi'
+
+# helper class for sending datadog tags from chef runs
+class DatadogChefTags
+  def initialize
+    @node = nil
+    @run_status = nil
+    @application_key = nil
+    @tag_prefix = 'tag:'
+    @retries = 0
+    @combined_host_tags = nil
+  end
+
+  # set the dogapi client handle
+  #
+  # @param dogapi_client [Dogapi::Client] datadog api client handle
+  # @return [DatadogChefTags] instance reference to self enabling method chaining
+  def with_dogapi_client(dogapi_client)
+    @dog = dogapi_client
+    self
+  end
+
+  # set the chef run status used for the report
+  #
+  # @param run_status [Chef::RunStatus] current chef run status
+  # @return [DatadogChefTags] instance reference to self enabling method chaining
+  def with_run_status(run_status)
+    @run_status = run_status
+    # Build up an array of Chef tags that will be sent back
+    # Selects all [env, roles, tags] from the Node's object and reformats
+    # them to `key:value` e.g. `role:database-master`.
+    @node = run_status.node
+    self
+  end
+
+  # set the target hostname (chef node name)
+  #
+  # @param hostname [String] hostname to use for the handler report
+  # @return [DatadogChefTags] instance reference to self enabling method chaining
+  def with_hostname(hostname)
+    @hostname = hostname
+    self
+  end
+
+  # set the datadog application key
+  #
+  # TODO: the application key is only needed for error checking, e.g. an app key exists
+  #   would be cleaner to push this check up to the data prep method in the
+  #   calling handler class
+  #
+  # @param application_key [String] datadog application key used for chef reports
+  # @return [DatadogChefTags] instance reference to self enabling method chaining
+  def with_application_key(application_key)
+    @application_key = application_key
+    if @application_key.nil?
+      Chef::Log.warn('You need an application key to let Chef tag your nodes ' \
+              'in Datadog. Visit https://app.datadoghq.com/account/settings#api to ' \
+                'create one and update your datadog attributes in the datadog cookbook.'
+                    )
+      fail ArgumentError, 'Missing Datadog Application Key'
+    end
+    self
+  end
+
+  # set the prefix to be added to all Chef tags
+  #
+  # @param tag_prefix [String] prefix to be added to all Chef tags
+  # @return [DatadogChefTags] instance reference to self enabling method chaining
+  def with_tag_prefix(tag_prefix)
+    @tag_prefix = tag_prefix unless tag_prefix.nil?
+    self
+  end
+
+  # set the number of retries when sending tags, when the host is not yet present
+  # on Datadog
+  #
+  # @param retries [Integer] number of retries
+  # @return [DatadogChefTags] instance reference to self enabling method chaining
+  def with_retries(retries)
+    @retries = retries unless retries.nil?
+    self
+  end
+
+  # send updated chef run generated tags to Datadog
+  def send_update_to_datadog
+    tags = combined_host_tags
+    retries = @retries
+    begin
+      loop do
+        should_retry = false
+        rc = @dog.update_tags(@hostname, tags, 'chef')
+        # See FIXME in DatadogChefEvents::emit_to_datadog about why I feel dirty repeating this code here
+        if rc.length < 2
+          Chef::Log.warn("Unexpected response from Datadog Tags API: #{rc}")
+        else
+          if retries > 0 && rc[0].to_i == 404
+            Chef::Log.debug("Host #{@hostname} not yet present on Datadog, re-submitting tags in 2 seconds")
+            sleep 2
+            retries -= 1
+            should_retry = true
+          elsif rc[0].to_i / 100 != 2
+            Chef::Log.warn("Could not submit #{tags} tags for #{@hostname} to Datadog: #{rc}")
+          else
+            Chef::Log.debug("Successfully updated #{@hostname}'s tags to #{tags.join(', ')}")
+          end
+        end
+        break unless should_retry
+      end
+    rescue
+      Chef::Log.warn("Could not determine whether #{@hostname}'s tags were successfully submitted to Datadog: #{rc}")
+    end
+  end
+
+  # return a combined array of tags that should be sent to Datadog
+  #
+  # @return [Array] the set of host tags based off the chef run
+  def combined_host_tags
+    # Combine (union) all arrays. Removes duplicates if found.
+    node_env.split | node_roles | node_tags
+  end
+
+  private
+
+  def node_roles
+    @node.run_list.roles.map! { |role| 'role:' + role }
+  end
+
+  def node_env
+    'env:' + @node.chef_environment if @node.respond_to?('chef_environment')
+  end
+
+  def node_tags
+    return [] unless @node.tags
+    @node.tags.map { |tag| "#{@tag_prefix}#{tag}" }
+  end
+end # end class DatadogChefTags