motherbrain 0.14.3 → 0.14.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 70223232856120fcbd6223d4adc1852e6b32a723
-  data.tar.gz: 955e58f67c7a1701dde56c2554d81d0bc75381ec
+  metadata.gz: d2a478e99e516d43785867b8e067d0dc47ea676c
+  data.tar.gz: 3b13b3e672777cc022ab3ffa2a7d789fb066b36e
 SHA512:
-  metadata.gz: 12e3706fd4d81fbaf71a7876bdeac05615dc1f630b9df69789e93e7022c315416d19f11d6a582fbba762a63d56ba54f768c0789b291f3e7e87a7b02324071571
-  data.tar.gz: bd384cc5e449e4243f880f2b68f18a7ced77049c18a08455481a840756a4b4269fb2d3c018025bfdfe49a81763b2e7be1318f68c7d615932f2349c1b1aaa4896
+  metadata.gz: 8357c35313e3402ca32181ff777f9498c387306545d5d1619267127b0481b479b228243d48051648d4ffeece01c46c5c70c24d8bc4e8c6642483c1b9ecce217b
+  data.tar.gz: 672b31a6fb744bdca06e4a235f7a8c6594d1bb0db29dfa515981721f40a19c44d8b7ed4bdd9d73ea0f8cfbcbc543d6e41284b613cd140357b4d1bb97ba351351

data/.travis.yml

@@ -1,9 +1,8 @@
+--- 
+env: 
+  - TEST_SUITE=unit
+  - TEST_SUITE=acceptance
 language: ruby
-rvm:
-  - 1.9.3
-  - 2.0.0
-  - jruby-19mode
-env:
-  - TEST_SUITE=unit CI=true
-  - TEST_SUITE=acceptance CI=true
-script: 'bundle exec thor specc:$TEST_SUITE'
+rvm: 
+  - "2.0.0"
+script: "bundle exec thor spec:$TEST_SUITE"

data/Gemfile

@@ -22,6 +22,7 @@ group :development do
   gem 'ronn', platforms: :ruby
   gem 'chef-zero', '~> 1.5.0'
   gem 'pry-nav'
+  gem 'ruby-prof'
 end
 
 group :test do

data/README.md

@@ -1,5 +1,7 @@
 # motherbrain
 
+[![Build Status](https://travis-ci.org/RiotGames/motherbrain.png?branch=master)](https://travis-ci.org/RiotGames/motherbrain)
+
 motherbrain is an orchestration framework for Chef. In the same way that you
 would use Chef's Knife command to create a single node, you can use
 motherbrain to create and control an entire application environment.
@@ -13,7 +15,7 @@ motherbrain to create and control an entire application environment.
 
 ## Requirements
 
-* Ruby 1.9.3+
+* Ruby 2.0.0+
 * Chef Server 10 or 11, or Hosted Chef
 
 ## Installation
@@ -256,9 +258,9 @@ component "app" do
   versioned
 
   service "app" do
-    service\_group "app"
-    service\_recipe "myface::service"
-    service\_attribute "myface.app.state"
+    service_group "app"
+    service_recipe "myface::service"
+    service_attribute "myface.app.state"
   end
 
   group "app" do

data/bin/mb_prof

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'ruby-prof'
+
+out_path = File.join(File.expand_path("../../profile", __FILE__), Time.now.strftime("#{([$PROGRAM_NAME.split("/").last].concat ARGV).join("_")}-%Y-%m-%d_%I:%M:%S"))
+
+RubyProf.start
+
+begin 
+  $:.push File.expand_path("../../lib", __FILE__)
+  require 'motherbrain'
+  
+  MB::Cli::Runner.new(ARGV).execute!
+ensure
+  printer = RubyProf::MultiPrinter.new(RubyProf.stop)
+
+  FileUtils.mkdir_p(out_path)
+  printer.print(path: out_path)
+end

data/lib/mb/api/v1/jobs_endpoint.rb

@@ -10,7 +10,7 @@ module MotherBrain::API
       namespace 'jobs' do
         desc "list all jobs (completed and active)"
         get do
-          job_manager.list
+          job_manager.list.to_a
         end
 
         desc "list all active jobs"

data/lib/mb/cli/sub_command/plugin.rb

@@ -185,6 +185,13 @@ module MotherBrain
                   default: false,
                   desc: "Perform upgrade even if the environment is locked",
                   aliases: "-f"
+                method_option :concurrency,
+                  type: :numeric,
+                  desc: "The max number of nodes to upgrade concurrently.",
+                  aliases: "-C"
+                method_option :stack_order,
+                  type: :boolean,
+                  desc: "The upgrade will be constrained to the order defined in the plugin's stack_order."
                 desc("upgrade", "Upgrade an environment to the specified versions")
                 define_method(:upgrade) do
                   upgrade_options = Hash.new.merge(options).deep_symbolize_keys
@@ -206,7 +213,7 @@ module MotherBrain
                 end
 
                 desc("attributes", "View available attributes for plugin.")
-                define_method(:attributes) do 
+                define_method(:attributes) do
                   ui.say "\n"
                   ui.say "** listing attributes for #{plugin}:"
                   ui.say "\n"

data/lib/mb/upgrade/worker.rb

@@ -117,32 +117,121 @@ module MotherBrain
           options[:environment_attributes_file]
         end
 
+        def max_concurrency
+          options[:concurrency]
+        end
+
+        def upgrade_in_stack_order?
+          options[:stack_order] == true
+        end
+
         # @return [Array<String>]
         def nodes
-          return @nodes if @nodes
+          @nodes ||= begin
+            job.set_status("Looking for nodes")
+
+            nodes = plugin.nodes(environment_name)
+            nodes.each do |component_name, group|
+              group.each do |group_name, nodes|
+                group[group_name] = nodes.map(&:public_hostname)
+              end
+            end
+
+            log.info("Found nodes #{nodes.inspect}")
+            nodes = bucket(nodes)
+            nodes = slice_for_concurrency(nodes)
 
-          job.set_status("Looking for nodes")
+            log.info("Sliced nodes into concurrency buckets #{nodes.inspect}")
 
-          @nodes = plugin.nodes(environment_name).collect { |component, groups|
-            groups.collect { |group, nodes|
-              nodes.collect(&:public_hostname)
-            }
-          }.flatten.compact.uniq
+            unless nodes.any?
+              log.info "No nodes in environment '#{environment_name}'"
+            end
 
-          unless @nodes.any?
-            log.info "No nodes in environment '#{environment_name}'"
+            nodes
           end
+        end
+
+        # Places hosts into buckets. The buckets depend on whether the
+        # stack_order option is true. If it is true then the plugin
+        # is consulted to obtain the stack_order tasks and a bucket is
+        # created for each group in the bootstrap order. If the stack_order
+        # option is not true then one bucket will be created with all the
+        # nodes. If more than one group specifies the same host the duplicates
+        # will be removed. If the stack_order is true and more than one group
+        # specifies the same host it will appear in the bucket for the
+        # first group it is found in and will be removed from all others.
+        #
+        # @example
+        #   stack_order do
+        #     bootstrap('some_component::db')
+        #     bootstrap('some_component::app')
+        #   end
+        #
+        #   And given:
+        #   {
+        #     some_component => {
+        #       db => ['db1', 'db2', 'db3'],
+        #       app => ['app1', 'app2', 'app3']
+        #     }
+        #   }
+        #
+        #  Then with stack_order == true
+        #
+        #    [['db1', 'db2', 'db3'],['app1','app2','app3']]
+        #
+        #  With stack_order == false
+        #
+        #    [['db1', 'db2', 'db3', 'app1','app2','app3']]
+        #
+        def bucket(nodes)
+          if upgrade_in_stack_order?
+            task_queue = plugin.bootstrap_routine.task_queue
+
+            seen = []
+            task_queue.collect do |task|
+              component_name, group_name = task.group_name.split('::')
+              group_nodes = nodes[component_name][group_name]
+              group_nodes = group_nodes - seen
+              seen += group_nodes
+              group_nodes
+            end
+          else
+            [] << nodes.collect do |component, groups|
+              groups.collect do |group, nodes|
+                nodes
+              end
+            end.flatten.compact.uniq
+          end
+        end
 
-          @nodes
+        # Takes an array of buckets and slices the buckets based on the
+        # value of max_concurrency.
+        #
+        # @example
+        #
+        #   With a max_concurrency of two and an input of
+        #     [['db1', 'db2', 'db3'],['app1','app2','app3']]
+        #
+        #   Returns
+        #     [['db1', 'db2'], ['db3'], ['app1','app2'], ['app3']]
+        def slice_for_concurrency(nodes)
+          if max_concurrency
+            nodes.inject([]) { |buckets, g| buckets += g.each_slice(max_concurrency).to_a }
+          else
+            nodes
+          end
         end
 
         def run_chef
           log.info "Running Chef on #{nodes}"
           job.set_status("Running Chef on nodes")
 
-          nodes.concurrent_map { |node|
-            node_querier.chef_run(node)
-          }
+          nodes.map do |group|
+            log.info("Running chef concurrently on nodes #{group.inspect}")
+            group.concurrent_map do |node|
+              node_querier.chef_run(node)
+            end
+          end
         end
     end
   end

data/lib/mb/version.rb

@@ -1,3 +1,3 @@
 module MotherBrain
-  VERSION = '0.14.3'
+  VERSION = '0.14.4'
 end

data/motherbrain.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |s|
   s.name          = "motherbrain"
   s.require_paths = ["lib"]
   s.version       = MotherBrain::VERSION
-  s.required_ruby_version = ">= 1.9.3"
+  s.required_ruby_version = ">= 2.0.0"
 
   s.add_dependency 'celluloid', '~> 0.15'
   # s.add_dependency 'dcell', '~> 0.14.0'
@@ -43,7 +43,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'net-ssh'
   s.add_dependency 'net-sftp'
   s.add_dependency 'solve', '>= 0.4.4'
-  s.add_dependency 'ridley-connectors', '~> 1.4.0'
+  s.add_dependency 'ridley-connectors', '~> 1.5.0'
   s.add_dependency 'thor', '~> 0.18.0'
   s.add_dependency 'faraday', '< 0.9.0'
   s.add_dependency 'faraday_middleware'

data/spec/unit/mb/api/v1/plugins_endpoint_spec.rb

@@ -97,9 +97,7 @@ describe MB::API::V1::PluginsEndpoint do
     it "returns the latest plugin version" do
       response = JSON.parse(last_response.body)
       response["name"].should eql("apple")
-      response["version"]["major"].should eql(2)
-      response["version"]["minor"].should eql(0)
-      response["version"]["patch"].should eql(0)
+      response["version"].should eql("2.0.0")
     end
   end
 

data/spec/unit/mb/upgrade/worker_spec.rb

@@ -194,13 +194,80 @@ describe MB::Upgrade::Worker do
   end
 
   describe "#nodes" do
-    pending "This should not be the responsibility of MB::Upgrade"
+    # pending "This should not be the responsibility of MB::Upgrade"
+
+    let(:nodes) {
+      {
+        'some_component' => {
+          'app' => [
+            Hashie::Mash.new({public_hostname: 'app1'}),
+            Hashie::Mash.new({public_hostname: 'app2'}),
+            Hashie::Mash.new({public_hostname: 'app3'}),
+            Hashie::Mash.new({public_hostname: 'db1'})],
+          'db' => [
+            Hashie::Mash.new({public_hostname: 'db1'}),
+            Hashie::Mash.new({public_hostname: 'db2'}),
+            Hashie::Mash.new({public_hostname: 'db3'})]
+        }
+      }
+    }
+
+    let(:db_task) { double MB::Bootstrap::Routine::Task, group_name: 'some_component::db'}
+    let(:app_task) { double MB::Bootstrap::Routine::Task, group_name: 'some_component::app'}
+    let(:tasks) { [db_task, app_task] }
+    let(:bootstrap_routine) { double MB::Bootstrap::Routine, task_queue: tasks }
+
+    before do
+      worker.unstub :nodes
+      plugin.stub(:bootstrap_routine).and_return(bootstrap_routine)
+      plugin.stub(:nodes).with(environment_name).and_return { nodes }
+    end
+
+    subject { worker.send(:nodes) }
+
+    context "when neither the stack_order or concurrency options are given" do
+      it "should create one concurrency bucket with all of the nodes" do
+        subject.should == [['app1', 'app2', 'app3', 'db1', 'db2', 'db3']]
+      end
+    end
+
+    context "when the stack_order option is given" do
+      before { options[:stack_order] = true }
+      it "should create a bucket for each group in the stack_order" do
+        subject.should == [['db1', 'db2', 'db3'],['app1', 'app2', 'app3']]
+      end
+    end
+
+    context "when the max_concurrency option is given" do
+      before { options[:concurrency] = 2 }
+      it "should create buckets of max_concurrency size" do
+        subject.should == [["app1", "app2"], ["app3", "db1"], ["db2", "db3"]]
+      end
+
+      context "when the max_concurrency option is 1" do
+        before { options[:concurrency] = 1 }
+        it "should create buckets of max_concurrency size" do
+          subject.should == [["app1"], ["app2"], ["app3"], ["db1"], ["db2"], ["db3"]]
+        end
+      end
+    end
+
+    context "when the stack_order and max_concurrency option is given" do
+      before do
+        options[:stack_order] = true
+        options[:concurrency] = 2
+      end
+
+      it "should create buckets of max_concurrency size and the stack_order is maintained" do
+        subject.should == [['db1', 'db2'], ['db3'], ['app1', 'app2'], ['app3']]
+      end
+    end
   end
 
   describe "#run_chef" do
     subject(:run_chef) { worker.send :run_chef }
 
-    let(:nodes) { %w[node1 node2 node3] }
+    let(:nodes) { [['node1', 'node2', 'node3']] }
 
     before do
       worker.unstub :run_chef
@@ -209,8 +276,10 @@ describe MB::Upgrade::Worker do
     end
 
     it "runs chef on the nodes" do
-      nodes.each do |node|
-        MB::Application.node_querier.should_receive(:chef_run).with(node)
+      nodes.each do |group|
+        group.each do |node|
+          MB::Application.node_querier.should_receive(:chef_run).with(node)
+        end
       end
 
       run_chef

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: motherbrain
 version: !ruby/object:Gem::Version
-  version: 0.14.3
+  version: 0.14.4
 platform: ruby
 authors:
 - Jamie Winsor
@@ -15,7 +15,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-21 00:00:00.000000000 Z
+date: 2014-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: celluloid
@@ -121,14 +121,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 1.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 1.5.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -296,6 +296,7 @@ email:
 executables:
 - boot
 - mb
+- mb_prof
 - mbsrv
 extensions: []
 extra_rdoc_files: []
@@ -318,6 +319,7 @@ files:
 - VAGRANT.md
 - bin/boot
 - bin/mb
+- bin/mb_prof
 - bin/mbsrv
 - config.json
 - features/cli/bootstrap_command/configurable_scripts.feature
@@ -451,6 +453,7 @@ files:
 - man/mb.1.html
 - man/mb.1.ronn.erb
 - motherbrain.gemspec
+- profile/.gitkeep
 - scripts/node_name.rb
 - spec/fixtures/cb_metadata.json
 - spec/fixtures/cb_metadata.rb
@@ -547,7 +550,6 @@ files:
 - spec/unit/mb/provisioner/provision_data_spec.rb
 - spec/unit/mb/provisioner_spec.rb
 - spec/unit/mb/provisioners/aws_spec.rb
-- spec/unit/mb/provisioners/environment_factory_spec.rb
 - spec/unit/mb/rest_gateway_spec.rb
 - spec/unit/mb/ridley_ext/cookbook_object_spec.rb
 - spec/unit/mb/srv_ctl_spec.rb
@@ -568,7 +570,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
@@ -696,7 +698,6 @@ test_files:
 - spec/unit/mb/provisioner/provision_data_spec.rb
 - spec/unit/mb/provisioner_spec.rb
 - spec/unit/mb/provisioners/aws_spec.rb
-- spec/unit/mb/provisioners/environment_factory_spec.rb
 - spec/unit/mb/rest_gateway_spec.rb
 - spec/unit/mb/ridley_ext/cookbook_object_spec.rb
 - spec/unit/mb/srv_ctl_spec.rb

data/spec/unit/mb/provisioners/environment_factory_spec.rb

@@ -1,108 +0,0 @@
-require 'spec_helper'
-
-describe MB::Provisioner::EnvironmentFactory do
-  let(:manifest) {
-    MB::Provisioner::Manifest.new.from_json(manifest_hash.to_json)
-  }
-  let(:manifest_hash) {
-    {
-      nodes: [
-        {
-          type: "m1.large",
-          count: 4,
-          components: ["activemq::master"]
-        },
-        {
-          type: "m1.large",
-          count: 2,
-          components: ["activemq::slave"]
-        },
-        {
-          type: "m1.small",
-          count: 2,
-          components: ["nginx::server"]
-        }
-      ]
-    }
-  }
-
-  describe "ClassMethods" do
-    subject { described_class }
-
-    describe "::convert_manifest" do
-      it "returns an array of hashes" do
-        subject.convert_manifest(manifest).should be_a(Array)
-        subject.convert_manifest(manifest).should each be_a(Hash)
-      end
-
-      it "contains an element for the amount of each node group and instance type" do
-        subject.convert_manifest(manifest).should have(8).items
-      end
-
-      describe "with different ordering" do
-        let(:manifest_hash) {
-          {
-            nodes: [
-              { groups: "default", type: "none" },
-              { type: "none", groups: "default", count: 2 }
-            ]
-          }
-        }
-
-        it "it still works" do
-          subject.convert_manifest(manifest).should be_a(Array)
-        end
-      end
-    end
-  end
-
-  let(:options) do
-    {
-      api_url: "https://ef.riotgames.com",
-      api_key: "58dNU5xBxDKjR15W71Lp",
-      ssl: {
-        verify: false
-      }
-    }
-  end
-
-  subject { described_class.new(options) }
-
-  describe "#up" do
-    let(:job) { MB::Job.new(:provision) }
-    let(:env_name) { "mbtest" }
-    let(:plugin) { double('plugin') }
-    let(:options) { Hash.new }
-
-    it "skips the bootstrap process" do
-      connection         = double('connection')
-      environment        = double('environment')
-      converted_manifest = Hash.new
-
-      subject.should_receive(:new_connection).and_return(connection)
-      described_class.should_receive(:convert_manifest).with(manifest).and_return(converted_manifest)
-      described_class.should_receive(:handle_created).with(environment).and_return(Array.new)
-      described_class.should_receive(:validate_create).and_return(true)
-      connection.stub_chain(:environment, :create).with(env_name, converted_manifest).and_return(Hash.new)
-      connection.stub_chain(:environment, :created?).with(env_name).and_return(true)
-      connection.stub_chain(:environment, :find).with(env_name, force: true).and_return(environment)
-
-      subject.up(job, env_name, manifest, plugin, options)
-    end
-  end
-
-  describe "#down" do
-    let(:job) { MB::Job.new(:destroy) }
-    let(:env_name) { "mbtest" }
-
-    it "sends a destroy command to environment factory with the given environment" do
-      connection = double('connection')
-
-      subject.should_receive(:new_connection).and_return(connection)
-      subject.should_receive(:destroyed?).with(connection, env_name).and_return(true)
-      connection.stub_chain(:environment, :destroy).with(env_name).and_return(Hash.new)
-
-      subject.down(job, env_name, options)
-    end
-  end
-end