motherbrain 0.0.0.placeholder → 0.13.1

data/lib/mb/berkshelf.rb

@@ -0,0 +1,50 @@
+module MotherBrain
+  # This module contains integration code for Berkshelf into motherbrain
+  module Berkshelf
+    class << self
+      # An array of Pathnames representing cookbooks in the Berkshelf
+      #
+      # @option options [Boolean] :with_plugin
+      #   only return cookbooks that contain a plugin
+      #
+      # @return [Array<Pathname>]
+      def cookbooks(options = {})
+        cookbooks = cookbooks_path.children
+
+        if options[:with_plugin]
+          cookbooks.select! { |cb_path| Dir.has_mb_plugin?(cb_path) }
+        end
+
+        cookbooks
+      end
+
+      # Location of the cookbooks directory in the Berkshelf
+      #
+      # @return [Pathname]
+      def cookbooks_path
+        path.join('cookbooks')
+      end
+
+      # The default location of the Berkshelf. This is in your user directory
+      # unless explicitly specified by the environment variable 'BERKSHELF_PATH'
+      #
+      # @return [String]
+      def default_path
+        ENV["BERKSHELF_PATH"] || File.expand_path("~/.berkshelf")
+      end
+
+      # Create the directory structure for the Berkshelf
+      def init
+        FileUtils.mkdir_p(path)
+        FileUtils.mkdir_p(cookbooks_path)
+      end
+
+      # The location of the Berkshelf
+      #
+      # @return [Pathname]
+      def path
+        Pathname.new(Application.config.berkshelf.path || default_path)
+      end
+    end
+  end
+end

data/lib/mb/bootstrap.rb

@@ -0,0 +1,9 @@
+module MotherBrain
+  module Bootstrap
+    autoload :Manager, 'mb/bootstrap/manager'
+    autoload :Manifest, 'mb/bootstrap/manifest'
+    autoload :Routine, 'mb/bootstrap/routine'
+    autoload :Worker, 'mb/bootstrap/worker'
+    autoload :Template, 'mb/bootstrap/template'
+  end
+end

data/lib/mb/bootstrap/manager.rb

@@ -0,0 +1,250 @@
+module MotherBrain
+  module Bootstrap
+    class Manager
+      class << self
+        # @raise [Celluloid::DeadActorError] if Bootstrap Manager has not been started
+        #
+        # @return [Celluloid::Actor(Bootstrap::Manager)]
+        def instance
+          MB::Application[:bootstrap_manager] or raise Celluloid::DeadActorError, "bootstrap manager not running"
+        end
+      end
+
+      include Celluloid
+      include MB::Logging
+      include MB::Mixin::Locks
+      include MB::Mixin::AttributeSetting
+      include MB::Mixin::Services
+
+      finalizer :finalize_callback
+
+      def initialize
+        log.debug { "Bootstrap Manager starting..." }
+        @worker_pool = Bootstrap::Worker.pool(size: 50)
+      end
+
+      # Asynchronously bootstrap a collection of nodes described in the given manifest in the proper order
+      #
+      # @param [String] environment
+      #   name of the environment to bootstrap nodes to
+      # @param [Bootstrap::Manifest] manifest
+      #   manifest of nodes and what they should become
+      # @param [Plugin] plugin
+      #   a motherbrain plugin with a bootstrap routine to follow
+      #
+      # @option options [Hash] :component_versions (Hash.new)
+      #   Hash of components and the versions to set them to
+      # @option options [Hash] :cookbook_versions (Hash.new)
+      #   Hash of cookbooks and the versions to set them to
+      # @option options [Hash] :environment_attributes (Hash.new)
+      #   Hash of additional attributes to set on the environment
+      # @option options [Boolean] :force
+      #   ignore and bypass any existing locks on an environment
+      # @option options [Hash] :attributes (Hash.new)
+      #   a hash of node level attributes to set on the bootstrapped nodes
+      # @option options [Array] :run_list (Array.new)
+      #   an initial run list to bootstrap with
+      # @option options [String] :chef_version
+      #   version of Chef to install on the node
+      # @option options [Hash] :hints (Hash.new)
+      #   a hash of Ohai hints to place on the bootstrapped node
+      # @option options [Boolean] :sudo (true)
+      #   bootstrap with sudo
+      # @option options [String] :bootstrap_proxy (nil)
+      #   URL to a proxy server to bootstrap through
+      #
+      # @return [MB::JobRecord]
+      #   a reference to the executing job
+      def async_bootstrap(environment, manifest, plugin, options = {})
+        job = Job.new(:bootstrap)
+        async(:bootstrap, job, environment, manifest, plugin, options)
+
+        job.ticket
+      end
+
+      # Bootstrap a collection of nodes described in the given manifest in the proper order
+      #
+      # @param [MB::Job] job
+      #   a job to send progress updates to
+      # @param [String] environment
+      #   name of the environment to bootstrap nodes to
+      # @param [MB::Bootstrap::Manifest] manifest
+      #   manifest of nodes and what they should become
+      # @param [MB::Plugin] plugin
+      #   a motherbrain plugin with a bootstrap routine to follow
+      #
+      # @see {#async_bootstrap} for options
+      def bootstrap(job, environment, manifest, plugin, options = {})
+        options = options.reverse_merge(
+          cookbook_versions: Hash.new,
+          component_versions: Hash.new,
+          environment_attributes: Hash.new,
+          hints: Hash.new,
+          bootstrap_proxy: Application.config[:chef][:bootstrap_proxy],
+          sudo: Application.config[:ssh][:sudo],
+          force: false
+        )
+        options[:environment] = environment
+
+        job.report_running
+
+        validate_bootstrap_configuration!(manifest, plugin)
+
+        job.set_status("Searching for environment")
+        unless chef_connection.environment.find(environment)
+          return job.report_failure("Environment '#{environment}' not found")
+        end
+
+        job.set_status("Starting bootstrap of nodes on: #{environment}")
+        task_queue = plugin.bootstrap_routine.task_queue.dup
+
+        chef_synchronize(chef_environment: environment, force: options[:force], job: job) do
+          if options[:component_versions].any?
+            job.set_status("Setting component versions")
+            set_component_versions(environment, plugin, options[:component_versions])
+          end
+
+          if options[:cookbook_versions].any?
+            job.set_status("Setting cookbook versions")
+            set_cookbook_versions(environment, options[:cookbook_versions])
+          end
+
+          if options[:environment_attributes].any?
+            job.set_status("Setting environment attributes")
+            set_environment_attributes(environment, options[:environment_attributes])
+          end
+
+          unless options[:environment_attributes_file].nil?
+            job.set_status("Setting environment attributes from file")
+            set_environment_attributes_from_file(environment, options[:environment_attributes_file])
+          end
+
+          while tasks = task_queue.shift
+            host_errors = Hash.new
+            group_names = Array(tasks).collect(&:group_name).join(', ')
+
+            instructions = Bootstrap::Routine.map_instructions(tasks, manifest)
+            if instructions.empty?
+              log.info "Skipping bootstrap of group(s): #{group_names}. No hosts defined in manifest to bootstrap for " +
+              "these groups."
+              next
+            end
+
+            job.set_status("Bootstrapping group(s): #{group_names}")
+
+            concurrent_bootstrap(job, manifest, instructions, options).each do |host, host_info|
+              if host_info[:result][:status] == :error
+                host_errors[host] = host_info
+              end
+            end
+
+            unless host_errors.empty?
+              abort GroupBootstrapError.new(host_errors)
+            end
+
+            job.set_status("Finished bootstrapping group(s): #{group_names}")
+          end
+        end
+
+        job.report_success
+      rescue => ex
+        job.report_failure(ex)
+      ensure
+        job.terminate if job && job.alive?
+      end
+
+      # Concurrently bootstrap a grouped collection of nodes from a manifest and return
+      # their results. This function will block until all nodes have finished
+      # bootstrapping.
+      #
+      # @param [MB::Job] job
+      #   a job to send progress updates to
+      # @param [Bootstrap::Manifest] manifest
+      #   a hash where the keys are node group names and the values are arrays of hostnames
+      # @param [Hash] instructions
+      #   a hash containing an entry for every host to bootstrap and the groups it belongs to, the
+      #   run list it should be bootstrapped with, and the chef attributes to be applied to the node
+      #   for it's first run.
+      #
+      # @option options [Hash] :attributes (Hash.new)
+      #   a hash of attributes to use in the first Chef run
+      # @option options [Array] :run_list (Array.new)
+      #   an initial run list to bootstrap with
+      # @option options [String] :chef_version
+      #   version of Chef to install on the node
+      # @option options [Hash] :hints (Hash.new)
+      #   a hash of Ohai hints to place on the bootstrapped node
+      # @option options [Boolean] :sudo (true)
+      #   bootstrap with sudo
+      # @option options [String] :bootstrap_proxy (nil)
+      #   URL to a proxy server to bootstrap through
+      #
+      # @return [Hash]
+      #   a hash where keys are group names and their values are the results of {Bootstrap::Worker#run}
+      #
+      # @example
+      #   {
+      #     "cloud-1.riotgames.com" => {
+      #       groups: ["proxy_server::default", "app_server::default"],
+      #       result: {
+      #         status: :ok,
+      #         message: "",
+      #         bootstrap_type: :full
+      #       }
+      #     },
+      #     "cloud-2.riotgames.com" => {
+      #       groups: ["database_master::default"],
+      #       result: {
+      #         status: :error,
+      #         message: "client verification error"
+      #         bootstrap_type: :partial
+      #       }
+      #     },
+      #     "cloud-3.riotgames.com" => {
+      #       groups: ["database_slave::default"],
+      #       result: {
+      #         status: :ok
+      #         message: ""
+      #         bootstrap_type: :partial
+      #       }
+      #     }
+      #   }
+      def concurrent_bootstrap(job, manifest, instructions, options = {})
+        response     = Hash.new
+
+        instructions.each do |host, host_info|
+          boot_options = options.merge(host_info[:options])
+          boot_options.merge!(manifest[:options]) if manifest[:options]
+
+          job.set_status("Bootstrapping #{host} with group(s): #{host_info[:groups]}")
+
+          response[host] = {
+            groups: host_info[:groups],
+            result: worker_pool.future(:run, host, boot_options)
+          }
+        end
+
+        response.each { |host, host_info| host_info[:result] = host_info[:result].value }
+      end
+
+      private
+
+        attr_reader :worker_pool
+
+        def finalize_callback
+          log.debug { "Bootstrap Manager stopping..." }
+          worker_pool.terminate if worker_pool && worker_pool.alive?
+        end
+
+        def validate_bootstrap_configuration!(manifest, plugin)
+          manifest.validate!(plugin)
+
+          validator_path = File.expand_path(config_manager.config.chef[:validator_path])
+
+          unless File.exists?(validator_path)
+            raise ValidatorPemNotFound.new(validator_path)
+          end
+        end
+    end
+  end
+end

data/lib/mb/bootstrap/manifest.rb

@@ -0,0 +1,131 @@
+module MotherBrain
+  module Bootstrap
+    # Manifest for bootstrapping a collection of nodes as a specified node group
+    #
+    # @example valid manifest structure
+    #   {
+    #     "nodes": [
+    #       {
+    #         "groups": ["activemq::master"],
+    #         "hosts": [
+    #           "euca-10-20-37-171.eucalyptus.cloud.riotgames.com",
+    #           "euca-10-20-37-172.eucalyptus.cloud.riotgames.com"
+    #         ]
+    #       },
+    #       {
+    #         "groups": ["activemq::slave"],
+    #         "hosts": [
+    #           "euca-10-20-37-168.eucalyptus.cloud.riotgames.com"
+    #          ]
+    #       }
+    #     ]
+    #   }
+    #
+    class Manifest < MotherBrain::Manifest
+      class << self
+        # Create a new instance of {Bootstrap::Manifest} from the returning
+        # value from calling {MotherBrain::Provisioner#up} on a provisioner and
+        # the provision manifest sent to {MotherBrain::Provisioner#up}.
+        #
+        # @param [Hash] nodes
+        #   the result from calling {MotherBrain::Provisioner#up}
+        # @param [Provisioner::Manifest] provisioner_manifest
+        #   the manifest sent to the provisioner performing
+        #   {MotherBrain::Provisioner#up}
+        #
+        # @return [Bootstrap::Manifest]
+        def from_provisioner(nodes, provisioner_manifest, path = nil)
+          nodes      = nodes.dup
+          attributes = provisioner_manifest.dup
+
+          new.tap do |boot_manifest|
+            boot_manifest.path    = path
+            boot_manifest[:nodes] = Array.new
+            boot_manifest[:options] = provisioner_manifest[:options]
+
+            attributes.node_groups.each do |node_group|
+              instance_type = node_group[:type]
+              count         = node_group[:count] || 1
+              groups        = node_group[:groups]
+              hosts         = Set.new
+
+              used_instances = nodes.select { |node|
+                node[:instance_type] == instance_type
+              }.take(count)
+
+              used_instances.each do |used_instance|
+                hosts.add(used_instance[:public_hostname])
+                nodes.delete(used_instance)
+              end
+
+              boot_manifest[:nodes] << {
+                groups: groups,
+                hosts: hosts.to_a
+              }
+            end
+          end
+        end
+
+        # Validates that the instance of manifest describes a layout for the given routine
+        #
+        # @param [Bootstrap::Manifest] manifest
+        # @param [Plugin] plugin
+        #
+        # @raise [InvalidBootstrapManifest]
+        # @raise [NoBootstrapRoutine]
+        def validate!(manifest, plugin)
+          unless plugin.bootstrap_routine
+            raise NoBootstrapRoutine,
+              "Plugin '#{plugin.name}' (#{plugin.version}) does not contain a bootstrap routine"
+          end
+
+          unless manifest[:nodes].is_a?(Array)
+            msg = "Manifest should contain an array of nodes"
+            raise InvalidBootstrapManifest, msg
+          end
+
+          manifest.node_groups.each do |node_group|
+            groups = node_group[:groups]
+
+            groups.each do |group|
+              match = group.match(Plugin::NODE_GROUP_ID_REGX)
+
+              unless match
+                msg = "Manifest contained the entry: '#{group}' which is not"
+                msg << " in the proper node group format: 'component::group'"
+                raise InvalidBootstrapManifest, msg
+              end
+
+              unless plugin.bootstrap_routine.has_task?(group)
+                msg = "Manifest describes the node group '#{group}' which is not found"
+                msg << " in the given routine for '#{plugin}'"
+                raise InvalidBootstrapManifest, msg
+              end
+            end
+          end
+        end
+      end
+
+      # Finds all hosts to be bootstrapped with a set of groups
+      #
+      # @return [Array] of hosts
+      def hosts_for_group(group)
+        hosts = node_groups.select do |node_group|
+          node_group[:groups].include?(group)
+        end.collect { |node_group| node_group[:hosts] }.flatten
+
+        MB::NodeFilter.expand_ipranges(hosts)
+      end
+
+      # Validates that the instance of manifest describes a layout for the given routine
+      #
+      # @param [Plugin] plugin
+      #
+      # @raise [InvalidBootstrapManifest]
+      # @raise [NoBootstrapRoutine]
+      def validate!(plugin)
+        self.class.validate!(self, plugin)
+      end
+    end
+  end
+end

data/lib/mb/bootstrap/routine.rb

@@ -0,0 +1,199 @@
+module MotherBrain
+  module Bootstrap
+    class Routine
+      # Container for a bootstrap task defined in a bootstrap routine
+      #
+      # @api private
+      class Task
+        class << self
+          # Create a new bootstrap routine task from a group path
+          #
+          # @param [MB::Plugin] plugin
+          #   the plugin to find the group in
+          # @param [#to_s] group_path
+          #   a string representing the path to a group in a component ("nginx::master")
+          #
+          # @raise [MB::PluginSyntaxError] if the group or component is not found on the plugin
+          #
+          # @return [Routine::Task]
+          def from_group_path(plugin, group_path)
+            component_id, group_id = group_path.to_s.split('::')
+            group                  = plugin.component!(component_id).group!(group_id)
+            from_group(group_path, group)
+          rescue ComponentNotFound, GroupNotFound => ex
+            raise PluginSyntaxError, ex
+          end
+
+          # @param [#to_s] group_path
+          #   a string representing the path to a group in a component ("nginx::master")
+          # @param [MB::Group] group
+          #
+          # @return [Routine::Task]
+          def from_group(group_path, group)
+            new(group_path, run_list: group.run_list, chef_attributes: group.chef_attributes)
+          end
+        end
+
+        # @return [String]
+        attr_reader :group_name
+        # @return [Array]
+        attr_reader :run_list
+        # @return [Hashie::Mash]
+        attr_reader :chef_attributes
+
+        # @param [String] group_name
+        def initialize(group_name, options = {})
+          @group_name      = group_name
+          @run_list        = options[:run_list] || Array.new
+          @chef_attributes = options[:chef_attributes] || Hashie::Mash.new
+        end
+      end
+
+      class << self
+        # Takes a task, or array of tasks, and a Bootstrap::Manifest and returns a Hash containing instructions
+        # for how to bootstrap each node found in the manifest based on the set of tasks, or task, given.
+        #
+        # @param [Bootstrap::Routine::Task, Array<Bootstrap::Routine::Task>] tasks
+        # @param [Bootstrap::Manifest] manifest
+        #
+        # @return [Hash]
+        #   A hash containing an entry for every host to bootstrap and the groups it belongs to, the
+        #   run list it should be bootstrapped with, and the chef attributes to be applied to the node
+        #   for it's first run.
+        #
+        #   {
+        #     "euca-10-20-37-171.eucalyptus.cloud.riotgames.com" => {
+        #       groups: [ "app_server::default" ],
+        #       options: {
+        #         run_list: Array.new,
+        #         chef_attributes: Hashie::Hash.new
+        #       }
+        #     },
+        #     "euca-10-20-37-172.eucalyptus.cloud.riotgames.com" => {
+        #       groups: [ "app_server::default", "database_slave::default" ],
+        #       options: {
+        #         run_list: Array.new,
+        #         chef_attributes: Hashie::Hash.new
+        #       }
+        #     },
+        #     "euca-10-20-37-168.eucalyptus.cloud.riotgames.com" => {
+        #       groups: [ "database_master::default" ],
+        #       options: {
+        #         run_list: Array.new,
+        #         chef_attributes: Hashie::Hash.new
+        #       }
+        #     }
+        #   }
+        def map_instructions(tasks, manifest)
+          {}.tap do |nodes|
+            Array(tasks).each do |task|
+              manifest.hosts_for_group(task.group_name).each do |host|
+                nodes[host] ||= {
+                  groups: Array.new,
+                  options: {
+                    run_list: Array.new,
+                    chef_attributes: Hashie::Mash.new
+                  }
+                }
+
+                nodes[host][:groups] << task.group_name unless nodes[host][:groups].include?(task.group_name)
+                nodes[host][:options][:run_list]        = nodes[host][:options][:run_list] | task.run_list
+                nodes[host][:options][:chef_attributes] =
+                  nodes[host][:options][:chef_attributes].deep_merge(task.chef_attributes)
+              end
+            end
+          end
+        end
+      end
+
+      # @return [MB::Plugin]
+      attr_reader :plugin
+
+      # @param [MB::Plugin] plugin
+      def initialize(plugin, &block)
+        @plugin     = plugin
+        @task_procs = Array.new
+
+        if block_given?
+          dsl_eval(&block)
+        end
+      end
+
+      # Returns an array of groups or an array of an array groups representing the order in
+      # which the cluster should be bootstrapped in. Groups which can be bootstrapped together
+      # are contained within an array. Groups should be bootstrapped starting from index 0 of
+      # the returned array.
+      #
+      # @return [Array<Bootstrap::Routine::Task>, Array<Array<Bootstrap::Routine::Task>>]
+      def task_queue
+        @task_queue ||= MB.expand_procs(task_procs)
+      end
+
+      # Checks if the routine contains a boot task for the given node group
+      #
+      # @param [String] node_group
+      #   name for a bootstrap task to check for
+      #
+      # @return [Boolean]
+      def has_task?(node_group, task_queue = self.task_queue)
+        task_queue.find do |task|
+          if task.is_a?(Array)
+            has_task?(node_group, task)
+          else
+            task.group_name == node_group
+          end
+        end
+      end
+
+      private
+
+        # @return [Array<Proc>]
+        attr_reader :task_procs
+
+        def dsl_eval(&block)
+          room = CleanRoom.new(self)
+          room.instance_eval(&block)
+          @task_procs = room.send(:task_procs)
+        end
+
+      # @api private
+      class CleanRoom < CleanRoomBase
+        # Add a Bootstrap::Routine::Task for bootstrapping nodes in the given node group to the {Routine}
+        #
+        # @example
+        #   Routine.new(...) do
+        #     bootstrap("mysql::master")
+        #   end
+        #
+        # @param [String] group_path
+        #   a group path
+        def bootstrap(group_path)
+          self.task_procs.push -> { Task.from_group_path(real_model.plugin, group_path) }
+        end
+
+        # Add an array of Bootstrap::Routine::Task(s) to be executed asyncronously to the {Routine}
+        #
+        # @example
+        #   Routine.new(...) do
+        #     async do
+        #       bootstrap("mysql::master")
+        #       bootstrap("myapp::webserver")
+        #     end
+        #   end
+        def async(&block)
+          room = self.class.new(real_model)
+          room.instance_eval(&block)
+
+          self.task_procs.push room.task_procs
+        end
+
+        protected
+
+          # @return [Array<Proc>]
+          def task_procs
+            @task_procs ||= Array.new
+          end
+      end
+    end
+  end
+end