motherbrain 0.14.3 → 0.14.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 70223232856120fcbd6223d4adc1852e6b32a723
4
- data.tar.gz: 955e58f67c7a1701dde56c2554d81d0bc75381ec
3
+ metadata.gz: d2a478e99e516d43785867b8e067d0dc47ea676c
4
+ data.tar.gz: 3b13b3e672777cc022ab3ffa2a7d789fb066b36e
5
5
  SHA512:
6
- metadata.gz: 12e3706fd4d81fbaf71a7876bdeac05615dc1f630b9df69789e93e7022c315416d19f11d6a582fbba762a63d56ba54f768c0789b291f3e7e87a7b02324071571
7
- data.tar.gz: bd384cc5e449e4243f880f2b68f18a7ced77049c18a08455481a840756a4b4269fb2d3c018025bfdfe49a81763b2e7be1318f68c7d615932f2349c1b1aaa4896
6
+ metadata.gz: 8357c35313e3402ca32181ff777f9498c387306545d5d1619267127b0481b479b228243d48051648d4ffeece01c46c5c70c24d8bc4e8c6642483c1b9ecce217b
7
+ data.tar.gz: 672b31a6fb744bdca06e4a235f7a8c6594d1bb0db29dfa515981721f40a19c44d8b7ed4bdd9d73ea0f8cfbcbc543d6e41284b613cd140357b4d1bb97ba351351
data/.travis.yml CHANGED
@@ -1,9 +1,8 @@
1
+ ---
2
+ env:
3
+ - TEST_SUITE=unit
4
+ - TEST_SUITE=acceptance
1
5
  language: ruby
2
- rvm:
3
- - 1.9.3
4
- - 2.0.0
5
- - jruby-19mode
6
- env:
7
- - TEST_SUITE=unit CI=true
8
- - TEST_SUITE=acceptance CI=true
9
- script: 'bundle exec thor specc:$TEST_SUITE'
6
+ rvm:
7
+ - "2.0.0"
8
+ script: "bundle exec thor spec:$TEST_SUITE"
data/Gemfile CHANGED
@@ -22,6 +22,7 @@ group :development do
22
22
  gem 'ronn', platforms: :ruby
23
23
  gem 'chef-zero', '~> 1.5.0'
24
24
  gem 'pry-nav'
25
+ gem 'ruby-prof'
25
26
  end
26
27
 
27
28
  group :test do
data/README.md CHANGED
@@ -1,5 +1,7 @@
1
1
  # motherbrain
2
2
 
3
+ [![Build Status](https://travis-ci.org/RiotGames/motherbrain.png?branch=master)](https://travis-ci.org/RiotGames/motherbrain)
4
+
3
5
  motherbrain is an orchestration framework for Chef. In the same way that you
4
6
  would use Chef's Knife command to create a single node, you can use
5
7
  motherbrain to create and control an entire application environment.
@@ -13,7 +15,7 @@ motherbrain to create and control an entire application environment.
13
15
 
14
16
  ## Requirements
15
17
 
16
- * Ruby 1.9.3+
18
+ * Ruby 2.0.0+
17
19
  * Chef Server 10 or 11, or Hosted Chef
18
20
 
19
21
  ## Installation
@@ -256,9 +258,9 @@ component "app" do
256
258
  versioned
257
259
 
258
260
  service "app" do
259
- service\_group "app"
260
- service\_recipe "myface::service"
261
- service\_attribute "myface.app.state"
261
+ service_group "app"
262
+ service_recipe "myface::service"
263
+ service_attribute "myface.app.state"
262
264
  end
263
265
 
264
266
  group "app" do
data/bin/mb_prof ADDED
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'ruby-prof'
4
+
5
+ out_path = File.join(File.expand_path("../../profile", __FILE__), Time.now.strftime("#{([$PROGRAM_NAME.split("/").last].concat ARGV).join("_")}-%Y-%m-%d_%I:%M:%S"))
6
+
7
+ RubyProf.start
8
+
9
+ begin
10
+ $:.push File.expand_path("../../lib", __FILE__)
11
+ require 'motherbrain'
12
+
13
+ MB::Cli::Runner.new(ARGV).execute!
14
+ ensure
15
+ printer = RubyProf::MultiPrinter.new(RubyProf.stop)
16
+
17
+ FileUtils.mkdir_p(out_path)
18
+ printer.print(path: out_path)
19
+ end
@@ -10,7 +10,7 @@ module MotherBrain::API
10
10
  namespace 'jobs' do
11
11
  desc "list all jobs (completed and active)"
12
12
  get do
13
- job_manager.list
13
+ job_manager.list.to_a
14
14
  end
15
15
 
16
16
  desc "list all active jobs"
@@ -185,6 +185,13 @@ module MotherBrain
185
185
  default: false,
186
186
  desc: "Perform upgrade even if the environment is locked",
187
187
  aliases: "-f"
188
+ method_option :concurrency,
189
+ type: :numeric,
190
+ desc: "The max number of nodes to upgrade concurrently.",
191
+ aliases: "-C"
192
+ method_option :stack_order,
193
+ type: :boolean,
194
+ desc: "The upgrade will be constrained to the order defined in the plugin's stack_order."
188
195
  desc("upgrade", "Upgrade an environment to the specified versions")
189
196
  define_method(:upgrade) do
190
197
  upgrade_options = Hash.new.merge(options).deep_symbolize_keys
@@ -206,7 +213,7 @@ module MotherBrain
206
213
  end
207
214
 
208
215
  desc("attributes", "View available attributes for plugin.")
209
- define_method(:attributes) do
216
+ define_method(:attributes) do
210
217
  ui.say "\n"
211
218
  ui.say "** listing attributes for #{plugin}:"
212
219
  ui.say "\n"
@@ -117,32 +117,121 @@ module MotherBrain
117
117
  options[:environment_attributes_file]
118
118
  end
119
119
 
120
+ def max_concurrency
121
+ options[:concurrency]
122
+ end
123
+
124
+ def upgrade_in_stack_order?
125
+ options[:stack_order] == true
126
+ end
127
+
120
128
  # @return [Array<String>]
121
129
  def nodes
122
- return @nodes if @nodes
130
+ @nodes ||= begin
131
+ job.set_status("Looking for nodes")
132
+
133
+ nodes = plugin.nodes(environment_name)
134
+ nodes.each do |component_name, group|
135
+ group.each do |group_name, nodes|
136
+ group[group_name] = nodes.map(&:public_hostname)
137
+ end
138
+ end
139
+
140
+ log.info("Found nodes #{nodes.inspect}")
141
+ nodes = bucket(nodes)
142
+ nodes = slice_for_concurrency(nodes)
123
143
 
124
- job.set_status("Looking for nodes")
144
+ log.info("Sliced nodes into concurrency buckets #{nodes.inspect}")
125
145
 
126
- @nodes = plugin.nodes(environment_name).collect { |component, groups|
127
- groups.collect { |group, nodes|
128
- nodes.collect(&:public_hostname)
129
- }
130
- }.flatten.compact.uniq
146
+ unless nodes.any?
147
+ log.info "No nodes in environment '#{environment_name}'"
148
+ end
131
149
 
132
- unless @nodes.any?
133
- log.info "No nodes in environment '#{environment_name}'"
150
+ nodes
134
151
  end
152
+ end
153
+
154
+ # Places hosts into buckets. The buckets depend on whether the
155
+ # stack_order option is true. If it is true then the plugin
156
+ # is consulted to obtain the stack_order tasks and a bucket is
157
+ # created for each group in the bootstrap order. If the stack_order
158
+ # option is not true then one bucket will be created with all the
159
+ # nodes. If more than one group specifies the same host the duplicates
160
+ # will be removed. If the stack_order is true and more than one group
161
+ # specifies the same host it will appear in the bucket for the
162
+ # first group it is found in and will be removed from all others.
163
+ #
164
+ # @example
165
+ # stack_order do
166
+ # bootstrap('some_component::db')
167
+ # bootstrap('some_component::app')
168
+ # end
169
+ #
170
+ # And given:
171
+ # {
172
+ # some_component => {
173
+ # db => ['db1', 'db2', 'db3'],
174
+ # app => ['app1', 'app2', 'app3']
175
+ # }
176
+ # }
177
+ #
178
+ # Then with stack_order == true
179
+ #
180
+ # [['db1', 'db2', 'db3'],['app1','app2','app3']]
181
+ #
182
+ # With stack_order == false
183
+ #
184
+ # [['db1', 'db2', 'db3', 'app1','app2','app3']]
185
+ #
186
+ def bucket(nodes)
187
+ if upgrade_in_stack_order?
188
+ task_queue = plugin.bootstrap_routine.task_queue
189
+
190
+ seen = []
191
+ task_queue.collect do |task|
192
+ component_name, group_name = task.group_name.split('::')
193
+ group_nodes = nodes[component_name][group_name]
194
+ group_nodes = group_nodes - seen
195
+ seen += group_nodes
196
+ group_nodes
197
+ end
198
+ else
199
+ [] << nodes.collect do |component, groups|
200
+ groups.collect do |group, nodes|
201
+ nodes
202
+ end
203
+ end.flatten.compact.uniq
204
+ end
205
+ end
135
206
 
136
- @nodes
207
+ # Takes an array of buckets and slices the buckets based on the
208
+ # value of max_concurrency.
209
+ #
210
+ # @example
211
+ #
212
+ # With a max_concurrency of two and an input of
213
+ # [['db1', 'db2', 'db3'],['app1','app2','app3']]
214
+ #
215
+ # Returns
216
+ # [['db1', 'db2'], ['db3'], ['app1','app2'], ['app3']]
217
+ def slice_for_concurrency(nodes)
218
+ if max_concurrency
219
+ nodes.inject([]) { |buckets, g| buckets += g.each_slice(max_concurrency).to_a }
220
+ else
221
+ nodes
222
+ end
137
223
  end
138
224
 
139
225
  def run_chef
140
226
  log.info "Running Chef on #{nodes}"
141
227
  job.set_status("Running Chef on nodes")
142
228
 
143
- nodes.concurrent_map { |node|
144
- node_querier.chef_run(node)
145
- }
229
+ nodes.map do |group|
230
+ log.info("Running chef concurrently on nodes #{group.inspect}")
231
+ group.concurrent_map do |node|
232
+ node_querier.chef_run(node)
233
+ end
234
+ end
146
235
  end
147
236
  end
148
237
  end
data/lib/mb/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module MotherBrain
2
- VERSION = '0.14.3'
2
+ VERSION = '0.14.4'
3
3
  end
data/motherbrain.gemspec CHANGED
@@ -33,7 +33,7 @@ Gem::Specification.new do |s|
33
33
  s.name = "motherbrain"
34
34
  s.require_paths = ["lib"]
35
35
  s.version = MotherBrain::VERSION
36
- s.required_ruby_version = ">= 1.9.3"
36
+ s.required_ruby_version = ">= 2.0.0"
37
37
 
38
38
  s.add_dependency 'celluloid', '~> 0.15'
39
39
  # s.add_dependency 'dcell', '~> 0.14.0'
@@ -43,7 +43,7 @@ Gem::Specification.new do |s|
43
43
  s.add_dependency 'net-ssh'
44
44
  s.add_dependency 'net-sftp'
45
45
  s.add_dependency 'solve', '>= 0.4.4'
46
- s.add_dependency 'ridley-connectors', '~> 1.4.0'
46
+ s.add_dependency 'ridley-connectors', '~> 1.5.0'
47
47
  s.add_dependency 'thor', '~> 0.18.0'
48
48
  s.add_dependency 'faraday', '< 0.9.0'
49
49
  s.add_dependency 'faraday_middleware'
data/profile/.gitkeep ADDED
File without changes
@@ -97,9 +97,7 @@ describe MB::API::V1::PluginsEndpoint do
97
97
  it "returns the latest plugin version" do
98
98
  response = JSON.parse(last_response.body)
99
99
  response["name"].should eql("apple")
100
- response["version"]["major"].should eql(2)
101
- response["version"]["minor"].should eql(0)
102
- response["version"]["patch"].should eql(0)
100
+ response["version"].should eql("2.0.0")
103
101
  end
104
102
  end
105
103
 
@@ -194,13 +194,80 @@ describe MB::Upgrade::Worker do
194
194
  end
195
195
 
196
196
  describe "#nodes" do
197
- pending "This should not be the responsibility of MB::Upgrade"
197
+ # pending "This should not be the responsibility of MB::Upgrade"
198
+
199
+ let(:nodes) {
200
+ {
201
+ 'some_component' => {
202
+ 'app' => [
203
+ Hashie::Mash.new({public_hostname: 'app1'}),
204
+ Hashie::Mash.new({public_hostname: 'app2'}),
205
+ Hashie::Mash.new({public_hostname: 'app3'}),
206
+ Hashie::Mash.new({public_hostname: 'db1'})],
207
+ 'db' => [
208
+ Hashie::Mash.new({public_hostname: 'db1'}),
209
+ Hashie::Mash.new({public_hostname: 'db2'}),
210
+ Hashie::Mash.new({public_hostname: 'db3'})]
211
+ }
212
+ }
213
+ }
214
+
215
+ let(:db_task) { double MB::Bootstrap::Routine::Task, group_name: 'some_component::db'}
216
+ let(:app_task) { double MB::Bootstrap::Routine::Task, group_name: 'some_component::app'}
217
+ let(:tasks) { [db_task, app_task] }
218
+ let(:bootstrap_routine) { double MB::Bootstrap::Routine, task_queue: tasks }
219
+
220
+ before do
221
+ worker.unstub :nodes
222
+ plugin.stub(:bootstrap_routine).and_return(bootstrap_routine)
223
+ plugin.stub(:nodes).with(environment_name).and_return { nodes }
224
+ end
225
+
226
+ subject { worker.send(:nodes) }
227
+
228
+ context "when neither the stack_order or concurrency options are given" do
229
+ it "should create one concurrency bucket with all of the nodes" do
230
+ subject.should == [['app1', 'app2', 'app3', 'db1', 'db2', 'db3']]
231
+ end
232
+ end
233
+
234
+ context "when the stack_order option is given" do
235
+ before { options[:stack_order] = true }
236
+ it "should create a bucket for each group in the stack_order" do
237
+ subject.should == [['db1', 'db2', 'db3'],['app1', 'app2', 'app3']]
238
+ end
239
+ end
240
+
241
+ context "when the max_concurrency option is given" do
242
+ before { options[:concurrency] = 2 }
243
+ it "should create buckets of max_concurrency size" do
244
+ subject.should == [["app1", "app2"], ["app3", "db1"], ["db2", "db3"]]
245
+ end
246
+
247
+ context "when the max_concurrency option is 1" do
248
+ before { options[:concurrency] = 1 }
249
+ it "should create buckets of max_concurrency size" do
250
+ subject.should == [["app1"], ["app2"], ["app3"], ["db1"], ["db2"], ["db3"]]
251
+ end
252
+ end
253
+ end
254
+
255
+ context "when the stack_order and max_concurrency option is given" do
256
+ before do
257
+ options[:stack_order] = true
258
+ options[:concurrency] = 2
259
+ end
260
+
261
+ it "should create buckets of max_concurrency size and the stack_order is maintained" do
262
+ subject.should == [['db1', 'db2'], ['db3'], ['app1', 'app2'], ['app3']]
263
+ end
264
+ end
198
265
  end
199
266
 
200
267
  describe "#run_chef" do
201
268
  subject(:run_chef) { worker.send :run_chef }
202
269
 
203
- let(:nodes) { %w[node1 node2 node3] }
270
+ let(:nodes) { [['node1', 'node2', 'node3']] }
204
271
 
205
272
  before do
206
273
  worker.unstub :run_chef
@@ -209,8 +276,10 @@ describe MB::Upgrade::Worker do
209
276
  end
210
277
 
211
278
  it "runs chef on the nodes" do
212
- nodes.each do |node|
213
- MB::Application.node_querier.should_receive(:chef_run).with(node)
279
+ nodes.each do |group|
280
+ group.each do |node|
281
+ MB::Application.node_querier.should_receive(:chef_run).with(node)
282
+ end
214
283
  end
215
284
 
216
285
  run_chef
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: motherbrain
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.3
4
+ version: 0.14.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jamie Winsor
@@ -15,7 +15,7 @@ authors:
15
15
  autorequire:
16
16
  bindir: bin
17
17
  cert_chain: []
18
- date: 2014-02-21 00:00:00.000000000 Z
18
+ date: 2014-03-20 00:00:00.000000000 Z
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency
21
21
  name: celluloid
@@ -121,14 +121,14 @@ dependencies:
121
121
  requirements:
122
122
  - - ~>
123
123
  - !ruby/object:Gem::Version
124
- version: 1.4.0
124
+ version: 1.5.0
125
125
  type: :runtime
126
126
  prerelease: false
127
127
  version_requirements: !ruby/object:Gem::Requirement
128
128
  requirements:
129
129
  - - ~>
130
130
  - !ruby/object:Gem::Version
131
- version: 1.4.0
131
+ version: 1.5.0
132
132
  - !ruby/object:Gem::Dependency
133
133
  name: thor
134
134
  requirement: !ruby/object:Gem::Requirement
@@ -296,6 +296,7 @@ email:
296
296
  executables:
297
297
  - boot
298
298
  - mb
299
+ - mb_prof
299
300
  - mbsrv
300
301
  extensions: []
301
302
  extra_rdoc_files: []
@@ -318,6 +319,7 @@ files:
318
319
  - VAGRANT.md
319
320
  - bin/boot
320
321
  - bin/mb
322
+ - bin/mb_prof
321
323
  - bin/mbsrv
322
324
  - config.json
323
325
  - features/cli/bootstrap_command/configurable_scripts.feature
@@ -451,6 +453,7 @@ files:
451
453
  - man/mb.1.html
452
454
  - man/mb.1.ronn.erb
453
455
  - motherbrain.gemspec
456
+ - profile/.gitkeep
454
457
  - scripts/node_name.rb
455
458
  - spec/fixtures/cb_metadata.json
456
459
  - spec/fixtures/cb_metadata.rb
@@ -547,7 +550,6 @@ files:
547
550
  - spec/unit/mb/provisioner/provision_data_spec.rb
548
551
  - spec/unit/mb/provisioner_spec.rb
549
552
  - spec/unit/mb/provisioners/aws_spec.rb
550
- - spec/unit/mb/provisioners/environment_factory_spec.rb
551
553
  - spec/unit/mb/rest_gateway_spec.rb
552
554
  - spec/unit/mb/ridley_ext/cookbook_object_spec.rb
553
555
  - spec/unit/mb/srv_ctl_spec.rb
@@ -568,7 +570,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
568
570
  requirements:
569
571
  - - '>='
570
572
  - !ruby/object:Gem::Version
571
- version: 1.9.3
573
+ version: 2.0.0
572
574
  required_rubygems_version: !ruby/object:Gem::Requirement
573
575
  requirements:
574
576
  - - '>='
@@ -696,7 +698,6 @@ test_files:
696
698
  - spec/unit/mb/provisioner/provision_data_spec.rb
697
699
  - spec/unit/mb/provisioner_spec.rb
698
700
  - spec/unit/mb/provisioners/aws_spec.rb
699
- - spec/unit/mb/provisioners/environment_factory_spec.rb
700
701
  - spec/unit/mb/rest_gateway_spec.rb
701
702
  - spec/unit/mb/ridley_ext/cookbook_object_spec.rb
702
703
  - spec/unit/mb/srv_ctl_spec.rb
@@ -1,108 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe MB::Provisioner::EnvironmentFactory do
4
- let(:manifest) {
5
- MB::Provisioner::Manifest.new.from_json(manifest_hash.to_json)
6
- }
7
- let(:manifest_hash) {
8
- {
9
- nodes: [
10
- {
11
- type: "m1.large",
12
- count: 4,
13
- components: ["activemq::master"]
14
- },
15
- {
16
- type: "m1.large",
17
- count: 2,
18
- components: ["activemq::slave"]
19
- },
20
- {
21
- type: "m1.small",
22
- count: 2,
23
- components: ["nginx::server"]
24
- }
25
- ]
26
- }
27
- }
28
-
29
- describe "ClassMethods" do
30
- subject { described_class }
31
-
32
- describe "::convert_manifest" do
33
- it "returns an array of hashes" do
34
- subject.convert_manifest(manifest).should be_a(Array)
35
- subject.convert_manifest(manifest).should each be_a(Hash)
36
- end
37
-
38
- it "contains an element for the amount of each node group and instance type" do
39
- subject.convert_manifest(manifest).should have(8).items
40
- end
41
-
42
- describe "with different ordering" do
43
- let(:manifest_hash) {
44
- {
45
- nodes: [
46
- { groups: "default", type: "none" },
47
- { type: "none", groups: "default", count: 2 }
48
- ]
49
- }
50
- }
51
-
52
- it "it still works" do
53
- subject.convert_manifest(manifest).should be_a(Array)
54
- end
55
- end
56
- end
57
- end
58
-
59
- let(:options) do
60
- {
61
- api_url: "https://ef.riotgames.com",
62
- api_key: "58dNU5xBxDKjR15W71Lp",
63
- ssl: {
64
- verify: false
65
- }
66
- }
67
- end
68
-
69
- subject { described_class.new(options) }
70
-
71
- describe "#up" do
72
- let(:job) { MB::Job.new(:provision) }
73
- let(:env_name) { "mbtest" }
74
- let(:plugin) { double('plugin') }
75
- let(:options) { Hash.new }
76
-
77
- it "skips the bootstrap process" do
78
- connection = double('connection')
79
- environment = double('environment')
80
- converted_manifest = Hash.new
81
-
82
- subject.should_receive(:new_connection).and_return(connection)
83
- described_class.should_receive(:convert_manifest).with(manifest).and_return(converted_manifest)
84
- described_class.should_receive(:handle_created).with(environment).and_return(Array.new)
85
- described_class.should_receive(:validate_create).and_return(true)
86
- connection.stub_chain(:environment, :create).with(env_name, converted_manifest).and_return(Hash.new)
87
- connection.stub_chain(:environment, :created?).with(env_name).and_return(true)
88
- connection.stub_chain(:environment, :find).with(env_name, force: true).and_return(environment)
89
-
90
- subject.up(job, env_name, manifest, plugin, options)
91
- end
92
- end
93
-
94
- describe "#down" do
95
- let(:job) { MB::Job.new(:destroy) }
96
- let(:env_name) { "mbtest" }
97
-
98
- it "sends a destroy command to environment factory with the given environment" do
99
- connection = double('connection')
100
-
101
- subject.should_receive(:new_connection).and_return(connection)
102
- subject.should_receive(:destroyed?).with(connection, env_name).and_return(true)
103
- connection.stub_chain(:environment, :destroy).with(env_name).and_return(Hash.new)
104
-
105
- subject.down(job, env_name, options)
106
- end
107
- end
108
- end