motherbrain 0.0.0.placeholder → 0.13.1

data/lib/mb/provisioner.rb

@@ -0,0 +1,186 @@
+module MotherBrain
+  module Provisioner
+    autoload :Manager, 'mb/provisioner/manager'
+    autoload :Manifest, 'mb/provisioner/manifest'
+    autoload :ProvisionData, 'mb/provisioner/provision_data'
+
+    class << self
+      attr_reader :default_id
+
+      # @param [Class] klass
+      # @option options [Boolean] :default
+      #
+      # @raise [ProvisionerRegistrationError] if a provisioner is registered as the default provisioner when
+      #   a default provisioner already exists
+      #
+      # @return [Set]
+      def register(klass, options = {})
+        validate_provisioner_class(klass)
+
+        unless get(klass.provisioner_id).nil?
+          raise ProvisionerRegistrationError,
+            "A provisioner with the id '#{klass.provisioner_id}' has already been registered"
+        end
+
+        if options[:default]
+          unless @default_id.nil?
+            raise ProvisionerRegistrationError, "A default provisioner has already been defined (#{default_id})"
+          end
+
+          @default_id = klass.provisioner_id
+        end
+
+        all.add(klass)
+      end
+
+      # List of all the registered provisioners
+      #
+      # @return [Set<MB::Provisioner::Base>]
+      def all
+        @all ||= Set.new
+      end
+
+      # Get registered provisioner class from the given ID. Return nil if no provisioner with
+      # the corresponding ID is found
+      #
+      # @param [#to_sym] id
+      #
+      # @return [Class, nil]
+      def get(id)
+        all.find { |klass| klass.provisioner_id == id.to_sym }
+      end
+
+      # Get registered provisioner class fromt he given ID. Raise an error if no provisioner with
+      # the corresponding ID is found
+      #
+      # @raise [ProvisionerNotRegistered] if no provisioner with the corresponding ID is found
+      #
+      # @return [Class]
+      def get!(id)
+        provisioner = get(id)
+
+        if provisioner.nil?
+          raise ProvisionerNotRegistered, "No provisioner registered with the ID: '#{id}'"
+        end
+
+        provisioner
+      end
+
+      # Return the default provisioner if one has been registered as the default
+      #
+      # @return [Class, nil]
+      def default
+        _default_id = ENV['MB_DEFAULT_PROVISIONER'] || self.default_id
+        get!(_default_id) if _default_id
+      end
+
+      # Clears all of the registered Provisioners.
+      #
+      # @return [Set]
+      #   an empty Set
+      def clear!
+        @default_id = nil
+        @all        = Set.new
+      end
+
+      # @param [Symbol] klass
+      #
+      # @raise [InvalidProvisionerClass] if the class does not respond to provisioner_id
+      #
+      # @return [Boolean]
+      def validate_provisioner_class(klass)
+        unless klass.respond_to?(:provisioner_id)
+          raise InvalidProvisionerClass, "Cannot register provisioner: all provisioners must respond to ':provisioner_id'"
+        end
+
+        if klass.provisioner_id.nil?
+          raise InvalidProvisionerClass, "Cannot register provisioner: invalid provisioner_id '#{klass.provisioner_id}'"
+        end
+
+        true
+      end
+    end
+
+    class Base
+      class << self
+        # The identifier for the Provisioner
+        #
+        # @return [Symbol]
+        attr_reader :provisioner_id
+
+        # @param [#to_sym] provisioner_id
+        def register_provisioner(provisioner_id, options = {})
+          @provisioner_id = provisioner_id.to_sym
+          Provisioner.register(self, options)
+        end
+
+        # Validate that the return created nodes contains the expected number of nodes and the proper
+        # instance types
+        #
+        # @param [Array<Hash>] created
+        # @param [Provisioner::Manifest] manifest
+        #
+        # @raise [UnexpectedProvisionCount] if an unexpected amount of nodes was returned by the
+        #   request to the provisioner
+        def validate_create(created, manifest)
+          unless created.length == manifest.node_count
+            raise UnexpectedProvisionCount.new(manifest.node_count, created.length)
+          end
+        end
+      end
+
+      include Celluloid
+      include MB::Logging
+      include MB::Mixin::Services
+
+      attr_reader :options
+
+      def initialize(options = {})
+        @options = options
+      end
+
+      # Request a provisioner to generate a set of nodes described by the given manifest
+      #
+      # @param [MB::Job] job
+      # @param [String] env_name
+      #   name of the set of nodes to be created
+      # @param [MB::Provisioner::Manifest] manifest
+      #   manifest describing how many and what kind of nodes to create
+      # @param [MB::Plugin] plugin
+      # @param [Hash] options
+      #
+      # @example
+      #   [
+      #     {
+      #       instance_type: "m1.large",
+      #       public_hostname: "cloud-1.riotgames.com"
+      #     },
+      #     {
+      #       instance_type: "m1.small",
+      #       public_hostname: "cloud-2.riotgames.com"
+      #     }
+      #   ]
+      #
+      # @return [Array]
+      #   an array of hashes representing nodes generated of given sizes
+      def up(job, env_name, manifest, plugin, options = {})
+        raise AbstractFunction
+      end
+
+      # Destroy a set of provisioned nodes
+      #
+      # @param [MB::Job] job
+      # @param [String] environment_name
+      #   name of the set of nodes to destroy
+      # @param [Hash] options
+      #
+      # @raise [MB::ProvisionError]
+      #   if a caught error occurs during provisioning
+      #
+      # @return [Boolean]
+      def down(job, environment_name, options = {})
+        raise AbstractFunction
+      end
+    end
+  end
+end

data/lib/mb/provisioner/manager.rb

@@ -0,0 +1,213 @@
+module MotherBrain
+  class ProvisionerSupervisor < Celluloid::SupervisionGroup; end
+
+  module Provisioner
+    # Handles provisioning of nodes and joining them to a Chef Server. Requests are
+    # delegated to a provisioner of the desired type or 'Environment Factory' by
+    # default.
+    class Manager
+      class << self
+        # @raise [Celluloid::DeadActorError] if Provisioner Manager has not been started
+        #
+        # @return [Celluloid::Actor(Provisioner::Manager)]
+        def instance
+          MB::Application[:provisioner_manager] or raise Celluloid::DeadActorError, "provisioner manager not running"
+        end
+      end
+
+      WORKER_OPTS = [
+        :component_versions,
+        :cookbook_versions,
+        :environment_attributes,
+        :environment_attributes_file,
+        :skip_bootstrap,
+        :force
+      ]
+
+      include Celluloid
+      include MB::Logging
+      include MB::Mixin::Locks
+      include MB::Mixin::Services
+
+      finalizer :finalize_callback
+
+      attr_reader :provisioner_registry
+
+      def initialize
+        log.debug { "Provision Manager starting..." }
+        @provisioner_registry   = Celluloid::Registry.new
+        @provisioner_supervisor = ProvisionerSupervisor.new_link(@provisioner_registry)
+        Provisioner.all.each do |provisioner|
+          @provisioner_supervisor.supervise_as(provisioner.provisioner_id, provisioner)
+        end
+      end
+
+      # Asynchronously destroy an environment that was created with motherbrain
+      #
+      # @param [String] environment
+      #   name of the environment to destroy
+      #
+      # @option options [#to_sym] :with
+      #   id of provisioner to use
+      def async_destroy(environment, options = {})
+        job = Job.new(:destroy)
+        async(:destroy, job, environment, options)
+
+        job.ticket
+      end
+
+      # Asynchronously create a new environment
+      #
+      # @param [#to_s] environment
+      #   name of the environment to create or append to
+      # @param [MB::Provisioner::Manifest] manifest
+      #   manifest of nodes to create
+      # @param [MB::Plugin] plugin
+      #   the plugin we are creating these nodes for
+      #
+      # @see {#provision} for options
+      #
+      # @return [MB::JobRecord]
+      #   a reference to the executing job
+      def async_provision(environment, manifest, plugin, options = {})
+        options = options.reverse_merge(skip_bootstrap: false)
+
+        job_type = options[:skip_bootstrap] ? :provision : :provision_and_bootstrap
+        job      = Job.new(job_type)
+
+        async(:provision, job, environment, manifest, plugin, options)
+
+        job.ticket
+      end
+
+      # Retrieve the running provisioner for the given ID. The default provisioner will be returned
+      # if nil is provided.
+      #
+      # @param [#to_sym, nil] id ({Provisioner.default_id})
+      #
+      # @raise [ProvisionerNotStarted]
+      #   if no provisioner is registered with the given ID
+      #
+      # @return [MB::Provisioner::Base]
+      def choose_provisioner(id)
+        id ||= Provisioner.default_id
+
+        unless provisioner = @provisioner_registry[id.to_sym]
+          abort ProvisionerNotStarted.new(id)
+        end
+
+        provisioner
+      end
+
+      # Destroy an environment that was created with motherbrain
+      #
+      # @param [MB::Job] job
+      #   a job to update with progress
+      # @param [String] environment
+      #   name of the environment to destroy
+      #
+      # @option options [#to_sym] :with
+      #   id of provisioner to use
+      def destroy(job, environment, options = {})
+        job.report_running
+        options = options.dup
+
+        chef_synchronize(chef_environment: environment, force: options[:force]) do
+          provision_data = ProvisionData.new(environment)
+
+          provision_data.provisioners.each do |provisioner_name|
+            provisioner = choose_provisioner(provisioner_name)
+            job.set_status "Destroying nodes provisioned with #{provisioner_name.upcase}"
+            provisioner.down job, environment, options
+          end
+
+          job.set_status "Removing nodes and clients from Chef"
+          environment_manager.purge_nodes(environment)
+
+          job.set_status "Destroying Chef environment #{environment}"
+          environment_manager.destroy(environment)
+
+          job.set_status "Destroying provision data"
+          provision_data.destroy
+        end
+
+        job.report_success("Environment #{environment} destroyed")
+      rescue => ex
+        job.report_failure(ex)
+      ensure
+        job.terminate if job && job.alive?
+      end
+
+      # Create a new environment
+      #
+      # @param [MB::Job] job
+      #   a job to update with progress
+      # @param [String] environment
+      #   name of the environment to create or append to
+      # @param [MB::Provisioner::Manifest] manifest
+      #   manifest of nodes to create
+      # @param [MB::Plugin] plugin
+      #   the plugin we are creating these nodes for
+      #
+      # @option options [String] :chef_version
+      #   version of Chef to install on the node
+      # @option options [Hash] :component_versions (Hash.new)
+      #   Hash of components and the versions to set them to
+      # @option options [Hash] :cookbook_versions (Hash.new)
+      #   Hash of cookbooks and the versions to set them to
+      # @option options [Hash] :environment_attributes (Hash.new)
+      #   Hash of additional attributes to set on the environment
+      # @option options [Boolean] :skip_bootstrap (false)
+      #   skip automatic bootstrapping of the created environment
+      # @option options [Boolean] :force (false)
+      #   force provisioning nodes to the environment even if the environment is locked
+      def provision(job, environment, manifest, plugin, options = {})
+        job.report_running("preparing to provision")
+
+        options = options.reverse_merge(
+          component_versions: Hash.new,
+          cookbook_versions: Hash.new,
+          environment_attributes: Hash.new,
+          skip_bootstrap: false,
+          force: false
+        )
+
+        manifest.validate!(plugin)
+        response = choose_provisioner(manifest.provisioner).up(job, environment, manifest, plugin, options)
+
+        unless options[:skip_bootstrap]
+          bootstrap_manifest = Bootstrap::Manifest.from_provisioner(response, manifest)
+          write_bootstrap_manifest(job, environment, bootstrap_manifest, plugin)
+          bootstrapper.bootstrap(job, environment, bootstrap_manifest, plugin, options)
+        end
+
+        job.report_success unless job.completed?
+      rescue => ex
+        job.report_failure(ex)
+      ensure
+        job.terminate if job && job.alive?
+      end
+
+      private
+
+        # @param [MB::Job] job
+        # @param [String] environment
+        # @param [MB::Manifest] manifest
+        # @param [MB::Plugin] plugin
+        def write_bootstrap_manifest(job, environment, manifest, plugin)
+          filename = "#{plugin.name}_#{environment}_#{Time.now.to_i}.json"
+          path = MB::FileSystem.manifests.join(filename)
+          contents = JSON.pretty_generate(manifest.to_hash)
+
+          job.set_status("Writing bootstrap manifest to #{path}")
+
+          File.open(path, 'w') { |file| file.write contents }
+        end
+
+        def finalize_callback
+          log.debug { "Provision Manager stopping..." }
+          @provisioner_supervisor.terminate
+        end
+    end
+  end
+end

data/lib/mb/provisioner/manifest.rb

@@ -0,0 +1,125 @@
+module MotherBrain
+  module Provisioner
+    # Manifest for creating a set of nodes of a given instance type for a set of node groups
+    #
+    # @example valid manifest structure
+    #   {
+    #     "provisioner": "aws",
+    #     "options": {
+    #       "provisioner_specific_option": "yellow",
+    #     },
+    #     "nodes": [
+    #       {
+    #         "groups": ["activemq::master"],
+    #         "type": "m1.large",
+    #         "count": 4
+    #       },
+    #       {
+    #         "groups": ["activemq::slave"],
+    #         "type": "m1.large",
+    #         "count": 2
+    #       }
+    #     ]
+    #   }
+    class Manifest < MotherBrain::Manifest
+      class << self
+        # Validate the given parameter contains a Hash or Manifest with a valid structure
+        #
+        # @param [Provisioner::Manifest] manifest
+        # @param [MotherBrain::Plugin] plugin
+        #
+        # @raise [InvalidProvisionManifest] if the given manifest is not well formed
+        def validate!(manifest, plugin)
+          unless manifest.is_a?(Hash)
+            raise InvalidProvisionManifest,
+              "The provisioner manifest needs to be a hash, but you provided a(n) #{manifest.class}"
+          end
+
+          nodes = manifest[:nodes]
+
+          unless nodes
+            raise InvalidProvisionManifest,
+              "The provisioner manifest needs to have a key 'nodes' containing an array"
+          end
+
+          unless nodes.is_a?(Array)
+            raise InvalidProvisionManifest,
+              "The provisioner manifest needs to have a key 'nodes' containing an array, but it was a(n) #{nodes.class}"
+          end
+
+          nodes.each do |node|
+            unless node.is_a?(Hash)
+              raise InvalidProvisionManifest,
+                "The provisioner manifest needs to have an array of hashes at 'nodes', but there was a #{node.class}: #{node.inspect}"
+            end
+
+            unless node.has_key?(:type) && !node[:type].nil?
+              raise InvalidProvisionManifest,
+                "A node entry in a provision manifest needs to contain key 'type' with a value."
+            end
+
+            unless node.has_key?(:groups) && !node[:groups].nil?
+              raise InvalidProvisionManifest,
+                "A node entry in a provision manifest needs to contain a key 'groups' with a value."
+            end
+
+            type   = node[:type]
+            count  = node[:count]
+            groups = node[:groups]
+
+            unless type.to_s.match(/\w+\.\w+/)
+              raise InvalidProvisionManifest,
+                "Provision manifest contained an entry not in the proper format: '#{type}'. Expected: 'a1.size'"
+            end
+
+            if count
+              unless count.is_a?(Fixnum) && count >= 0
+                raise InvalidProvisionManifest,
+                  "Provision manifest contained an invalid value for count: '#{count}'. Expected an integer greater than 0."
+              end
+            end
+
+            unless groups.is_a?(Array)
+              raise InvalidProvisionManifest,
+                "The provisioner manifest contains a node group without an array of groups: #{node.inspect}"
+            end
+
+            groups.each do |group|
+              match = group.match(Plugin::NODE_GROUP_ID_REGX)
+
+              unless match
+                raise InvalidProvisionManifest,
+                  "Provision manifest contained an entry not in the proper format: '#{name}'. Expected: 'component::group'"
+              end
+
+              component = match[1]
+              group = match[2]
+
+              unless plugin.has_component?(component)
+                raise InvalidProvisionManifest,
+                  "Provision manifest describes the component: '#{component}' but '#{plugin.name}' does not have this component"
+              end
+
+              unless plugin.component(component).has_group?(group)
+                raise InvalidProvisionManifest,
+                  "Provision manifest describes the group: '#{group}' in the component '#{component}' but that component does not have this group"
+              end
+            end
+          end
+        end
+      end
+
+      # @param [Plugin] plugin
+      #
+      # @raise [InvalidProvisionManifest] if the given manifest is not well formed
+      def validate!(plugin)
+        self.class.validate!(self, plugin)
+      end
+
+      # @return [String] the provisioner token from the manifest
+      def provisioner
+        self[:provisioner]
+      end
+    end
+  end
+end