chef_sous_vide 0.1.0

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler/gem_tasks"
+require "cucumber/rake/task"
+
+Cucumber::Rake::Task.new
+
+task :default => [ "cucumber" ]
+
+begin
+  require "kitchen/rake_tasks"
+  Kitchen::RakeTasks.new
+rescue LoadError
+  puts ">>>>> Kitchen gem not loaded, omitting tasks" unless ENV["CI"]
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "sous_vide"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/kitchen.yml

@@ -0,0 +1,98 @@
+---
+<% if (ARGV[0] == "converge") %>
+transport:
+  name: rsync
+<% end %>
+
+driver:
+  name: docker
+  use_sudo: false
+  provision_command:
+    # prevent APT from deleting the APT folder
+    - rm /etc/apt/apt.conf.d/docker-clean
+    - apt-get install -y -q apt-transport-https rsync
+    # allow test-kitchen to use root user (connects via ssh)
+    - sed -i 's/prohibit-password/yes/' /etc/ssh/sshd_config
+    # disable systemd since it doesn't work in docker
+    - ln -sf /bin/true /bin/systemctl
+    # install chef-client as part of the image, save time in the runtime
+    - curl -LO https://omnitruck.chef.io/install.sh && bash ./install.sh -v 12.17.44 && rm install.sh
+    - ln /opt/chef/bin/chef-client /bin/chef-client
+  username: root
+  password: root
+
+verifier:
+  sudo: false
+
+provisioner:
+  install_strategy: 'skip'
+
+platforms:
+  - name: ubuntu-16.04
+  - name: ubuntu-18.04
+
+suites:
+  - name: default
+    includes:
+      - ubuntu-16.04
+    driver:
+      hostname: elasticsearch
+      instance_name: elasticsearch
+      forward:
+        - "5601:5601" # kibana
+        - "9200:9200" # es
+    run_list:
+      - sous_vide::install
+      - sous_vide::default
+    attributes:
+      kitchen:
+        roles:
+          - elasticsearch
+      java:
+        jdk_version: 8
+      elasticsearch:
+        install:
+          version: "6.6.2"
+  - name: tomcat
+    includes:
+      - ubuntu-18.04
+      - ubuntu-16.04
+    driver:
+      links:
+        - elasticsearch
+    run_list:
+      - sous_vide::install
+      - sous_vide::tomcat
+    attributes:
+      kitchen:
+        roles:
+          - tomcat-example
+      java:
+        jdk_version: 8
+  - name: nginx
+    includes:
+      - ubuntu-16.04
+    run_list:
+      - sous_vide::install
+      - sous_vide::nginx
+    attributes:
+      kitchen:
+        roles:
+          - nginx-example
+      nginx:
+        install_method: "source"
+        modules:
+          - nginx::headers_more_module
+          - nginx::http_auth_request_module
+          - nginx::http_echo_module
+          - nginx::http_geoip_module
+          - nginx::http_gzip_static_module
+          - nginx::http_realip_module
+          - nginx::http_v2_module
+          - nginx::http_ssl_module
+          - nginx::http_stub_status_module
+          - nginx::naxsi_module
+          - nginx::ngx_devel_module
+          - nginx::ngx_lua_module
+          - nginx::openssl_source
+          - nginx::upload_progress_module

data/lib/sous_vide.rb

@@ -0,0 +1,10 @@
+require "chef/handler"
+require "chef/http"
+require "sous_vide/version"
+
+require "sous_vide/tracked_resource"
+require "sous_vide/handler"
+
+module SousVide
+  class Error < StandardError; end
+end

data/lib/sous_vide/event_methods.rb

@@ -0,0 +1,176 @@
+module SousVide
+  # This module implements Chef event methods. It's based on Chef::EventDispatch::Base.
+  #
+  # Nested resources are explicitly ignored and the code flow will be as follows:
+  #
+  # 1. resource_action_start
+  # 2. resource_* events (possibly multiple)
+  # 3. resource_action_complete
+  #
+  # The code is intentionally procedural and explicit. If :resource_action_start assigned
+  # @processing_now only then other events will work. :resource_action_completed unassigns
+  # @processing_now so all events will be ignored until :resource_action_start is called
+  # again.
+  module EventMethods
+    # This hook will always fire whenever chef is about to converge a resource, including why_run
+    # mode, notifications or skipped resources.
+    def resource_action_start(new_resource, action, notification_type, notifying_resource)
+      if nested?(new_resource) # ignore nested resources
+        new_r_name = "#{new_resource.resource_name}[#{new_resource.name}]##{action}"
+        debug("Received :resource_action_start on #{new_r_name}.",
+              "It's a nested resource and will be ignored.")
+        return false
+      end
+
+      # This is a delayed notification. From now on we are in 'delayed' run phase.
+      if notification_type == :delayed && @run_phase != "delayed"
+        debug("Changed run phase to 'delayed'.")
+        @run_phase = "delayed"
+      end
+
+      @processing_now = create(chef_resource: new_resource, action: action)
+      debug("Received :resource_action_start on #{@processing_now}.")
+
+      @execution_order += 1
+      @processing_now.execution_order = @execution_order
+      @processing_now.execution_phase = @run_phase
+      @processing_now.started_at = Time.now.strftime("%F %T")
+
+      @processing_now.chef_resource_handle = new_resource
+
+      @processing_now.before_notifications = new_resource.before_notifications.size
+      @processing_now.immediate_notifications = new_resource.immediate_notifications.size
+      @processing_now.delayed_notifications = new_resource.delayed_notifications.size
+
+      # When notifying resource is present notification_type will also be present.  It is nil for
+      # delayed notifications.
+      if notifying_resource
+        _name = "#{notifying_resource.resource_name}[#{notifying_resource.name}]"
+        debug("Notified from #{_name} (:#{notification_type})")
+        @processing_now.notifying_resource = _name
+      end
+      @processing_now.notification_type = notification_type
+      true
+    end
+
+    def resource_updated(new_resource, _action)
+      return false if @processing_now.nil? || nested?(new_resource) # ignore nested resources
+
+      debug("Received :resource_updated on #{@processing_now}")
+      @processing_now.status = "updated"
+      true
+    end
+
+    # Resource is skipped when a guard instruction stops the converge process or when
+    # `action :nothing` is used (it's a guard too).
+    def resource_skipped(new_resource, _action, conditional)
+      return false if @processing_now.nil? || nested?(new_resource) # ignore nested resources
+
+      debug("Received :resource_skipped on #{@processing_now}", "(#{conditional.to_text})")
+      @processing_now.guard_description = conditional.to_text
+      @processing_now.status = "skipped"
+      true
+    end
+
+    def resource_up_to_date(new_resource, _action)
+      return false if @processing_now.nil? || nested?(new_resource) # ignore nested resources
+
+      debug("Received :resource_up_to_date on #{@processing_now}")
+      @processing_now.status = "up-to-date"
+      true
+    end
+
+    def resource_completed(new_resource)
+      return false if @processing_now.nil? || nested?(new_resource) # ignore nested resources
+
+      debug("Received :resource_completed on #{@processing_now}")
+      @processing_now.duration_ms = (new_resource.elapsed_time.to_f * 1000).to_i
+
+      # If a resource has notifications Chef will converge it in a forced_why_run mode to
+      # determine if any update will happen and if the notifications should be called.
+      #
+      # When this event was fired in why_run mode we override it's status to `why-run`.
+      #
+      # This is also how Chef::Runner#focred_why_run is implemented so it should be reliable.
+      #
+      # TODO: what happens when :why_run for notification fails?
+      if ::Chef::Config[:why_run]
+        debug("Resource #{@processing_now.name}##{@processing_now.action} marked why-run",
+              "because Chef::Config[:why_run] is true.")
+        @processing_now.status = "why-run"
+      end
+
+      # This resource was not notified by another and is a subject of normal ordered converge
+      # process. @resource_collection_cursor is pointing to next resource according to the
+      # expanded resource collection.
+      #
+      # When chef-client fails we will take remaining entries and add to the report as
+      # 'unprocessed'. It works because resources are ordered and we can keep track where we are.
+      #
+      # why-run mode and notifications are not in natural order and must not move the cursor.
+      #
+      # Having it pointing ahead is relevant because current resource has just been converged
+      # (technically 'failed') and it is not 'unprocessed'.
+      if !@processing_now.notifying_resource && # not notified
+         !::Chef::Config[:why_run] &&           # not why-run
+         @run_phase == "converge"               # only converge phase
+
+        @resource_collection_cursor += 1
+      end
+
+      @processing_now.completed_at = Time.now.strftime("%F %T")
+      @processed << @processing_now
+      @processing_now = nil
+      true
+    end
+
+    def resource_failed(new_resource, _action, exception)
+      return false if @processing_now.nil? || nested?(new_resource) # ignore nested resources
+
+      debug("Received :resource_failed on #{@processing_now}")
+      @processing_now.status = "failed"
+      @processing_now.error_source = new_resource.to_text
+      @processing_now.error_output = exception.message
+      true
+    end
+
+    # Resources with retries can succeed on subsequent attempts or ignore_failure option may be
+    # set and it's the only place we can capture intermittent errors.
+    #
+    # This event can fire multiple times, but we capture only the most recent error.
+    def resource_failed_retriable(new_resource, _action, _remaining_retries, exception)
+      return false if @processing_now.nil? || nested?(new_resource) # ignore nested resources
+
+      debug("Received :resource_failed_retriable on #{@processing_now}")
+      @processing_now.retries += 1
+      @processing_now.error_source = new_resource.to_text
+      @processing_now.error_output = exception.message
+      true
+    end
+
+    def converge_start
+      debug("Received :converge_start")
+      debug("Changed run phase to 'converge'.")
+      @run_phase = "converge"
+      @run_name ||= [@run_started_at, @chef_node_role, @chef_node_ipv4, @run_id].join(" ")
+    end
+
+    def converge_complete
+      debug("Received :converge_completed")
+      @run_success = true
+      @run_completed_at = Time.now.strftime("%F %T")
+      send_to_output!
+    end
+
+    def converge_failed(_exception)
+      debug("Received :converge_failed")
+      @run_success = false
+      @run_completed_at = Time.now.strftime("%F %T")
+      @run_phase = "post-converge"
+      debug("Changed run phase to 'post-converge'.")
+
+      consume_unprocessed_resources!
+      send_to_output!
+    end
+  end
+end

data/lib/sous_vide/handler.rb

@@ -0,0 +1,186 @@
+require "sous_vide/event_methods"
+require "sous_vide/outputs/json_file"
+require "sous_vide/outputs/json_http"
+require "sous_vide/outputs/logger"
+require "sous_vide/outputs/multi"
+
+require "securerandom"
+require "singleton"
+
+module SousVide
+  # == SousVide::Handler
+  #
+  # The Handler receives event data from chef-client and keeps track of the converge process. It's
+  # essentially a stream parser hooked into Chef::EventDispatch.
+  #
+  # Event methods are all in SousVide::EventMethods module. This file contains logic that does not
+  # deal with events directly.
+  class Handler
+    include Singleton
+    include EventMethods
+
+    attr_accessor :chef_run_context,
+                  :logger,
+                  :sous_output,
+                  :processing_now,
+                  :processed,
+                  :run_phase,
+                  :run_id,
+                  :run_name
+
+    # Enables the handler. Call it anywhere in the recipe, ideally as early as possible.
+    #
+    #   SousVide::Handler.register(node.run_context)
+    #
+    # All converge-time resources will be included in the report regardless at what point
+    # registration happens.
+    #
+    # Compile-time resources defined before registration will not be included.
+    # TODO: see client.rb start_handlers as an option.
+    #
+    # `chef_handler` resource does not support subscribing to :events so we have give up DSL and
+    # use Chef API.
+    #
+    # The `Chef.event_handler` DSL could be used but dealing with returns and exceptions in procs
+    # is a pain.
+    def self.register(run_context)
+      ::Chef::Log.info "Registering SousVide"
+
+      instance.chef_run_context = run_context
+      instance.post_initialize
+
+      run_context.events.register(instance)
+    end
+
+    def initialize
+      @execution_order = 0
+      @resource_collection_cursor = 0
+
+      @processed = []
+      @processing_now = nil
+
+      @run_started_at = Time.now.strftime("%F %T")
+
+      # Not related to RunStatus#run_id, it's our internal run id.
+      @run_id = SecureRandom.uuid.split("-").first # => 596e9d00
+      @run_phase = "compile"
+
+      # Default to Chef logger, but can be changed to anything that responds to #call
+      @logger = ::Chef::Log
+      @sous_output = Outputs::Logger.new(logger: @logger)
+    end
+
+    # This is called in #register, as soon as @chef_run_context is available.
+    def post_initialize
+      @chef_node_ipv4 = @chef_run_context.node["ipaddress"] || "<no ip>"
+      @chef_node_role = @chef_run_context.node["roles"].first || "<no role>"
+      @chef_node_instance_id = @chef_run_context.node.name
+    end
+
+
+    def run_data
+      {
+        chef_run_id: @run_id,
+        chef_run_name: @run_name,
+        chef_run_started_at: @run_started_at,
+        chef_run_completed_at: @run_completed_at,
+        chef_run_success: @run_success
+      }
+    end
+
+    def node_data
+      {
+        chef_node_ipv4: @chef_node_ipv4,
+        chef_node_instance_id: @chef_node_instance_id,
+        chef_node_role: @chef_node_role
+      }
+    end
+
+    def create(chef_resource:, action:)
+      tracked = TrackedResource.new(action: action,
+                                    name: chef_resource.name,
+                                    type: chef_resource.resource_name)
+
+      tracked.cookbook_name = chef_resource.cookbook_name || "<Dynamically Defined Resource>"
+      tracked.cookbook_recipe = chef_resource.recipe_name || "<Dynamically Defined Resource>"
+      tracked.source_line = chef_resource.source_line || "<Dynamically Defined Resource>"
+      tracked.chef_resource_handle = chef_resource
+      tracked
+    end
+
+    # We will ignore nested resources. Once a top level resource triggered :resource_action
+    # started any events not related to it will be ignored.
+    def nested?(chef_resource)
+      @processing_now &&
+        @processing_now.chef_resource_handle != chef_resource
+    end
+
+    # When chef-client fails we haven't seen all resources and need to backfill the handler.
+    def consume_unprocessed_resources!
+      all_known_resources = expand_chef_resources!
+
+      # No unprocessed resources left. Failure likely occured on last resource or in a delayed
+      # notification.
+      # TODO: check delayed notification failure
+      return if @resource_collection_cursor >= all_known_resources.size
+
+      unprocessed = all_known_resources[@resource_collection_cursor..-1]
+
+      # We will pass unprocessed resources via  :resource_action_start and :resource_completed so
+      # they will end up in @processed array, but with status set to 'unprocessed' and execution
+      # phase 'post-converge'.
+      #
+      # TODO: consider placing these resources before delayed notification, currently they are
+      # always at the very end. It _maybe_ makes sense.
+      unprocessed.each do |tracked|
+        resource_action_start(
+          tracked.chef_resource_handle, # new_resource
+          tracked.action,               # action
+          nil,                          # notification_type
+          nil                           # notifying_resource
+        )
+
+        resource_completed(
+          tracked.chef_resource_handle  # new_resource
+        )
+      end
+    end
+
+    # Resources with multiple actions must be expanded, ie. given resource:
+    #
+    #   service 'nginx' do
+    #     action [:enable, :start]
+    #   end
+    #
+    # After expansion we should have 2 resources:
+    #
+    #   * service[nginx] with action :enable
+    #   * service[nginx] with action :start
+    #
+    # We keep track of the progress and on failure we will pick up unprocessed resources from here
+    # to feed the handler in post-converge stage.
+    #
+    # On a successful chef-client run @processed will contain all resources and this method won't
+    # be called.
+    def expand_chef_resources!
+      chef_run_context.resource_collection.flat_map do |chef_resource|
+        Array(chef_resource.action).map do |action|
+          create(chef_resource: chef_resource, action: action)
+        end
+      end
+    end
+
+    def send_to_output!
+      @sous_output.call(run_data: run_data, node_data: node_data, resources_data: @processed)
+    end
+
+    def debug(*args)
+      message = args.compact.join(" ")
+      logger.debug(message)
+    end
+
+    def logger
+      @logger ||= ::Chef::Log
+    end
+  end
+end