cloud-mu 3.4.0 → 3.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (104) hide show
  1. checksums.yaml +4 -4
  2. data/ansible/roles/mu-nat/tasks/main.yml +3 -0
  3. data/bin/mu-aws-setup +41 -7
  4. data/bin/mu-azure-setup +34 -0
  5. data/bin/mu-configure +214 -119
  6. data/bin/mu-gcp-setup +37 -2
  7. data/bin/mu-node-manage +3 -0
  8. data/bin/mu-refresh-ssl +67 -0
  9. data/bin/mu-run-tests +14 -4
  10. data/bin/mu-self-update +30 -10
  11. data/bin/mu-upload-chef-artifacts +30 -26
  12. data/cloud-mu.gemspec +8 -6
  13. data/cookbooks/mu-master/attributes/default.rb +5 -1
  14. data/cookbooks/mu-master/metadata.rb +2 -2
  15. data/cookbooks/mu-master/recipes/default.rb +81 -26
  16. data/cookbooks/mu-master/recipes/init.rb +197 -62
  17. data/cookbooks/mu-master/recipes/update_nagios_only.rb +1 -1
  18. data/cookbooks/mu-master/recipes/vault.rb +78 -77
  19. data/cookbooks/mu-master/templates/default/mods/rewrite.conf.erb +1 -0
  20. data/cookbooks/mu-master/templates/default/nagios.conf.erb +103 -0
  21. data/cookbooks/mu-master/templates/default/web_app.conf.erb +14 -30
  22. data/cookbooks/mu-tools/attributes/default.rb +5 -0
  23. data/cookbooks/mu-tools/files/centos-6/CentOS-Base.repo +47 -0
  24. data/cookbooks/mu-tools/libraries/helper.rb +12 -2
  25. data/cookbooks/mu-tools/libraries/monkey.rb +1 -1
  26. data/cookbooks/mu-tools/recipes/apply_security.rb +6 -0
  27. data/cookbooks/mu-tools/recipes/aws_api.rb +6 -4
  28. data/cookbooks/mu-tools/recipes/base_repositories.rb +1 -1
  29. data/cookbooks/mu-tools/recipes/gcloud.rb +2 -9
  30. data/cookbooks/mu-tools/recipes/google_api.rb +5 -2
  31. data/cookbooks/mu-tools/resources/disk.rb +108 -58
  32. data/extras/Gemfile.lock.bootstrap +394 -0
  33. data/extras/bucketstubs/error.html +0 -0
  34. data/extras/bucketstubs/index.html +0 -0
  35. data/extras/clean-stock-amis +9 -9
  36. data/extras/git_rpm/build.sh +20 -0
  37. data/extras/git_rpm/mugit.spec +53 -0
  38. data/extras/image-generators/VMWare/centos8.yaml +15 -0
  39. data/extras/openssl_rpm/build.sh +19 -0
  40. data/extras/openssl_rpm/mussl.spec +46 -0
  41. data/extras/python_rpm/muthon.spec +14 -4
  42. data/extras/ruby_rpm/muby.spec +9 -5
  43. data/extras/sqlite_rpm/build.sh +19 -0
  44. data/extras/sqlite_rpm/muqlite.spec +47 -0
  45. data/install/installer +7 -5
  46. data/modules/mu.rb +12 -5
  47. data/modules/mu/cloud/machine_images.rb +1 -1
  48. data/modules/mu/cloud/providers.rb +6 -1
  49. data/modules/mu/cloud/resource_base.rb +1 -1
  50. data/modules/mu/cloud/ssh_sessions.rb +4 -0
  51. data/modules/mu/config.rb +28 -12
  52. data/modules/mu/config/database.rb +2 -2
  53. data/modules/mu/config/firewall_rule.rb +1 -1
  54. data/modules/mu/config/ref.rb +2 -2
  55. data/modules/mu/config/schema_helpers.rb +12 -3
  56. data/modules/mu/config/server.rb +10 -4
  57. data/modules/mu/config/server_pool.rb +2 -2
  58. data/modules/mu/config/vpc.rb +10 -10
  59. data/modules/mu/defaults/AWS.yaml +32 -32
  60. data/modules/mu/deploy.rb +23 -10
  61. data/modules/mu/groomers/chef.rb +2 -2
  62. data/modules/mu/master.rb +49 -3
  63. data/modules/mu/mommacat.rb +8 -5
  64. data/modules/mu/mommacat/naming.rb +2 -2
  65. data/modules/mu/mommacat/storage.rb +22 -27
  66. data/modules/mu/providers/aws.rb +142 -48
  67. data/modules/mu/providers/aws/alarm.rb +3 -3
  68. data/modules/mu/providers/aws/bucket.rb +19 -19
  69. data/modules/mu/providers/aws/cache_cluster.rb +22 -22
  70. data/modules/mu/providers/aws/cdn.rb +2 -2
  71. data/modules/mu/providers/aws/collection.rb +14 -14
  72. data/modules/mu/providers/aws/container_cluster.rb +27 -27
  73. data/modules/mu/providers/aws/database.rb +40 -39
  74. data/modules/mu/providers/aws/dnszone.rb +5 -5
  75. data/modules/mu/providers/aws/endpoint.rb +35 -35
  76. data/modules/mu/providers/aws/firewall_rule.rb +26 -23
  77. data/modules/mu/providers/aws/function.rb +28 -28
  78. data/modules/mu/providers/aws/group.rb +7 -7
  79. data/modules/mu/providers/aws/habitat.rb +2 -2
  80. data/modules/mu/providers/aws/job.rb +6 -6
  81. data/modules/mu/providers/aws/loadbalancer.rb +34 -34
  82. data/modules/mu/providers/aws/log.rb +14 -14
  83. data/modules/mu/providers/aws/msg_queue.rb +10 -10
  84. data/modules/mu/providers/aws/nosqldb.rb +8 -8
  85. data/modules/mu/providers/aws/notifier.rb +7 -7
  86. data/modules/mu/providers/aws/role.rb +17 -15
  87. data/modules/mu/providers/aws/search_domain.rb +10 -10
  88. data/modules/mu/providers/aws/server.rb +176 -95
  89. data/modules/mu/providers/aws/server_pool.rb +65 -105
  90. data/modules/mu/providers/aws/storage_pool.rb +17 -9
  91. data/modules/mu/providers/aws/user.rb +1 -1
  92. data/modules/mu/providers/aws/vpc.rb +103 -51
  93. data/modules/mu/providers/aws/vpc_subnet.rb +43 -39
  94. data/modules/mu/providers/azure.rb +78 -12
  95. data/modules/mu/providers/azure/server.rb +18 -3
  96. data/modules/mu/providers/cloudformation/server.rb +1 -1
  97. data/modules/mu/providers/google.rb +19 -4
  98. data/modules/mu/providers/google/folder.rb +6 -2
  99. data/modules/mu/providers/google/function.rb +65 -30
  100. data/modules/mu/providers/google/role.rb +1 -1
  101. data/modules/mu/providers/google/vpc.rb +27 -2
  102. data/modules/tests/aws-servers-with-handrolled-iam.yaml +37 -0
  103. data/modules/tests/k8s.yaml +1 -1
  104. metadata +24 -8
@@ -7,7 +7,7 @@ long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
7
7
  source_url 'https://github.com/cloudamatic/mu'
8
8
  issues_url 'https://github.com/cloudamatic/mu/issues'
9
9
  chef_version '>= 12.1' if respond_to?(:chef_version)
10
- version '0.9.6'
10
+ version '0.9.7'
11
11
 
12
12
  %w( centos amazon redhat ).each do |os|
13
13
  supports os
@@ -28,4 +28,4 @@ depends 'consul-cluster', '~> 2.0.0'
28
28
  depends 'chef-sugar' # undeclared dependency of consul 2.1, which can't be upgraded without creating a conflict with consul-cluster and vault-cluster -zr2d2
29
29
  depends 'hostsfile', '~> 3.0.1'
30
30
  depends 'chef-vault', '~> 3.1.1'
31
- depends 'apache2', '< 6.0.0'
31
+ depends 'apache2', '< 8.0.0'
@@ -217,24 +217,25 @@ if !node['update_nagios_only']
217
217
  if !$MU_CFG['public_address'].match(/^\d+\.\d+\.\d+\.\d+$/)
218
218
  svrname = $MU_CFG['public_address']
219
219
  end
220
-
221
- # nagios keeps disabling the default vhost, so let's make another one
222
- include_recipe "apache2::mod_proxy"
223
- include_recipe "apache2::mod_proxy_http"
224
- include_recipe "apache2::mod_rewrite"
225
-
226
- if node['platform_family'] == "rhel" and node['platform_version'].split('.')[0].to_i == 6
227
- package "httpd24-mod_ldap"
228
- apache_module 'ldap' do
229
- conf true
230
- end
231
- else
232
- include_recipe "apache2::mod_authnz_ldap"
220
+ apache2_install "" do
221
+ docroot_dir "/var/www/html"
222
+ modules %w{status alias auth_basic authn_core authn_file authz_core authz_groupfile authz_host authz_user autoindex deflate dir env mime negotiation setenvif log_config logio unixd systemd headers proxy proxy_http rewrite ssl ldap authnz_ldap slotmem_shm}
233
223
  end
224
+ package "mod_ldap"
225
+
226
+ # add stock .conf files to the mix where applicable
227
+ apache2_mod_proxy ""
228
+ apache2_mod_ldap ""
229
+ apache2_mod_cgid ""
230
+ apache2_mod_ssl ""
234
231
 
235
- apache_site "default" do
236
- enable false
232
+ apache2_mod "php"
233
+ apache2_default_site "" do
234
+ action :enable
235
+ notifies :start, "service[apache2]", :delayed
237
236
  end
237
+
238
+ # nagios keeps disabling the default vhost, so let's make another one
238
239
  execute "Allow net connect to local for apache" do
239
240
  command "/usr/sbin/setsebool -P httpd_can_network_connect on"
240
241
  not_if "/usr/sbin/getsebool httpd_can_network_connect | grep -cim1 ^.*on$"
@@ -242,22 +243,76 @@ if !node['update_nagios_only']
242
243
  notifies :reload, "service[apache2]", :delayed
243
244
  end
244
245
 
246
+ aliases = [node['fqdn'], node['hostname'], node['local_hostname'], node['local_ipv4'], node['public_hostname'], node['public_ipv4']]
247
+ if node['ec2']
248
+ aliases << node['ec2']['local_ipv4']
249
+ aliases << node['ec2']['local_hostname']
250
+ aliases << node['ec2']['public_ipv4']
251
+ aliases << node['ec2']['public_hostname']
252
+ end
253
+ aliases.uniq!
254
+ aliases.reject! { |a| a.nil? or a.empty? }
255
+
256
+ service 'apache2' do
257
+ extend Apache2::Cookbook::Helpers
258
+ service_name lazy { apache_platform_service_name }
259
+ supports restart: true, status: true, reload: true
260
+ action :enable
261
+ end
245
262
 
246
- web_app "mu_docs" do
247
- server_name svrname
248
- server_aliases [node['fqdn'], node['hostname'], node['local_hostname'], node['local_ipv4'], node['public_hostname'], node['public_ipv4']]
249
- docroot "/var/www/html"
250
- cookbook "mu-master"
263
+ template '/etc/httpd/sites-available/mu_docs.conf' do
264
+ variables(
265
+ server_name: svrname,
266
+ server_port: "80",
267
+ server_aliases: aliases,
268
+ docroot: "/var/www/html"
269
+ )
270
+ cookbook 'mu-master'
271
+ source 'web_app.conf.erb'
251
272
  notifies :reload, "service[apache2]", :delayed
252
273
  end
253
- web_app "https_proxy" do
254
- server_name svrname
255
- server_port "443"
256
- server_aliases [node['fqdn'], node['hostname'], node['local_hostname'], node['local_ipv4'], node['public_hostname'], node['public_ipv4']]
257
- docroot "/var/www/html"
258
- cookbook "mu-master"
274
+ apache2_site "mu_docs"
275
+ template '/etc/httpd/sites-available/https_proxy.conf' do
276
+ variables(
277
+ server_name: svrname,
278
+ server_port: "443",
279
+ server_aliases: aliases,
280
+ docroot: "/var/www/html"
281
+ )
282
+ cookbook 'mu-master'
283
+ source 'web_app.conf.erb'
284
+ notifies :reload, "service[apache2]", :delayed
285
+ end
286
+ apache2_site "https_proxy"
287
+
288
+ # configure the appropriate authentication method for the web server
289
+ case node['nagios']['server_auth_method']
290
+ when 'openid'
291
+ apache2_mod 'auth_openid'
292
+ when 'cas'
293
+ apache2_mod 'auth_cas'
294
+ end
295
+
296
+ # apache2_conf "nagios" do
297
+ # server_name svrname
298
+ # server_aliases aliases
299
+ # template 'nagios.conf.erb'
300
+ # cookbook "mu-master"
301
+ # notifies :reload, "service[apache2]", :delayed
302
+ # action :enable
303
+ # end
304
+ template '/etc/httpd/sites-available/nagios.conf' do
305
+ variables(
306
+ server_name: svrname,
307
+ server_port: "443",
308
+ server_aliases: aliases,
309
+ docroot: "/var/www/html"
310
+ )
311
+ cookbook 'mu-master'
312
+ source 'nagios.conf.erb'
259
313
  notifies :reload, "service[apache2]", :delayed
260
314
  end
315
+ apache2_site "nagios"
261
316
 
262
317
  link "/etc/nagios3" do
263
318
  to "/etc/nagios"
@@ -35,20 +35,20 @@ ENV['PATH'] = ENV['PATH']+":/bin:/opt/opscode/embedded/bin"
35
35
 
36
36
  # XXX We want to be able to override these things when invoked from chef-apply,
37
37
  # but, like, how?
38
- CHEF_SERVER_VERSION="12.17.15-1"
39
- CHEF_CLIENT_VERSION="14.13.11"
38
+ CHEF_SERVER_VERSION="14.0.65-1"
39
+ CHEF_CLIENT_VERSION="16.9.29"
40
40
  KNIFE_WINDOWS="1.9.0"
41
41
  MU_BASE="/opt/mu"
42
- MU_BRANCH="master" # GIT HOOK EDITABLE DO NOT TOUCH
43
- realbranch=`cd #{MU_BASE}/lib && git rev-parse --abbrev-ref HEAD` # ~FC048
44
42
 
45
- if ENV.key?('MU_BRANCH')
46
- MU_BRANCH = ENV['MU_BRANCH']
43
+ MU_BRANCH = if ENV.key?('MU_BRANCH')
44
+ ENV['MU_BRANCH']
47
45
  elsif $?.exitstatus == 0
48
- MU_BRANCH=realbranch.chomp
46
+ realbranch=`cd #{MU_BASE}/lib && git rev-parse --abbrev-ref HEAD` # ~FC048
47
+ realbranch.chomp
49
48
  else
50
- MU_BRANCH="master"
49
+ "master"
51
50
  end
51
+
52
52
  begin
53
53
  resources('service[sshd]')
54
54
  rescue Chef::Exceptions::ResourceNotFound
@@ -77,6 +77,12 @@ service "iptables" do
77
77
  only_if "( /bin/systemctl -l --no-pager | grep iptables.service ) || ( /sbin/chkconfig --list | grep ^iptables )"
78
78
  end
79
79
 
80
+ service "firewalld" do
81
+ ignore_failure true
82
+ action :nothing
83
+ only_if "/bin/systemctl -l --no-pager | grep firewalld.service"
84
+ end
85
+
80
86
  # These guys are a workaround for Opscode bugs that seems to affect some Chef
81
87
  # Server upgrades.
82
88
  directory "/var/run/postgresql" do
@@ -126,28 +132,39 @@ file "use a clean /etc/hosts during install" do
126
132
  not_if { ::Dir.exist?("#{MU_BASE}/lib/.git") }
127
133
  end
128
134
 
135
+ execute "modprobe br_netfilter" do
136
+ action :nothing
137
+ end
138
+
129
139
  execute "reconfigure Chef server" do
130
- command "/opt/opscode/bin/chef-server-ctl reconfigure"
140
+ command "CHEF_LICENSE=\"accept\" /opt/opscode/bin/chef-server-ctl reconfigure"
131
141
  action :nothing
132
142
  notifies :stop, "service[iptables]", :before
143
+ notifies :stop, "service[firewalld]", :before
133
144
  # notifies :create, "link[/tmp/.s.PGSQL.5432]", :before
134
145
  notifies :create, "link[/var/run/postgresql/.s.PGSQL.5432]", :before
135
146
  notifies :restart, "service[chef-server]", :immediately
136
147
  if !RUNNING_STANDALONE
137
148
  notifies :start, "service[iptables]", :immediately
149
+ notifies :start, "service[firewalld]", :immediately
150
+ else
151
+ notifies :run, "execute[Chef Server rabbitmq workaround]", :before
138
152
  end
139
153
  only_if { RUNNING_STANDALONE }
140
154
  end
141
155
  execute "upgrade Chef server" do
142
- command "/opt/opscode/bin/chef-server-ctl upgrade"
156
+ command "CHEF_LICENSE=\"accept\" /opt/opscode/bin/chef-server-ctl upgrade"
143
157
  action :nothing
144
158
  timeout 1200 # this can take a while
145
159
  notifies :stop, "service[iptables]", :before
160
+ notifies :stop, "service[firewalld]", :before
161
+ notifies :run, "execute[modprobe br_netfilter]", :before
146
162
  notifies :run, "execute[Chef Server rabbitmq workaround]", :before
147
163
  # notifies :create, "link[/tmp/.s.PGSQL.5432]", :before
148
164
  notifies :create, "link[/var/run/postgresql/.s.PGSQL.5432]", :before
149
165
  if !RUNNING_STANDALONE
150
166
  notifies :start, "service[iptables]", :immediately
167
+ notifies :start, "service[firewalld]", :immediately
151
168
  end
152
169
  only_if { RUNNING_STANDALONE }
153
170
  end
@@ -160,8 +177,10 @@ service "chef-server" do
160
177
  # notifies :create, "link[/tmp/.s.PGSQL.5432]", :before
161
178
  # notifies :create, "link[/var/run/postgresql/.s.PGSQL.5432]", :before
162
179
  notifies :stop, "service[iptables]", :before
180
+ notifies :stop, "service[firewalld]", :before
163
181
  if !RUNNING_STANDALONE
164
182
  notifies :start, "service[iptables]", :immediately
183
+ notifies :start, "service[firewalld]", :immediately
165
184
  end
166
185
  only_if { RUNNING_STANDALONE }
167
186
  end
@@ -173,7 +192,7 @@ dpkgs = {}
173
192
 
174
193
  elversion = node['platform_version'].split('.')[0]
175
194
 
176
- rhelbase = ["git", "curl", "diffutils", "patch", "gcc", "gcc-c++", "make", "postgresql-devel", "libyaml", "libffi-devel", "tcl", "tk"]
195
+ rhelbase = ["git", "curl", "diffutils", "patch", "gcc", "gcc-c++", "make", "postgresql-devel", "libyaml", "libffi-devel", "tcl", "tk", "xfsprogs"]
177
196
 
178
197
  case node['platform_family']
179
198
  when 'rhel'
@@ -182,11 +201,11 @@ when 'rhel'
182
201
 
183
202
  case node['platform_version'].split('.')[0].to_i
184
203
  when 6
185
- basepackages.concat(["cryptsetup-luks", "mysql-devel", "centos-release-scl"])
204
+ basepackages.concat(["cryptsetup-luks", "mysql-devel", "centos-release-scl", "perl-WWW-Curl"])
186
205
  removepackages = ["nagios"]
187
206
 
188
207
  when 7
189
- basepackages.concat(['libX11', 'mariadb-devel', 'cryptsetup'])
208
+ basepackages.concat(['policycoreutils-python', 'libX11', 'mariadb-devel', 'cryptsetup', 'tcl-devel', 'gdbm-devel', 'sqlite-devel', 'tk-devel', 'perl-CGI', 'perl-DBI', 'perl-Data-Dumper', 'perl-Digest-MD5', 'perl-Git-SVN', 'perl-YAML', 'nvme-cli'])
190
209
  removepackages = ['nagios', 'firewalld']
191
210
 
192
211
  when 8
@@ -222,8 +241,18 @@ rpms = {
222
241
  "chef-server-core" => "https://packages.chef.io/files/stable/chef-server/#{CHEF_SERVER_VERSION.sub(/\-\d+$/, "")}/el/#{elversion}/chef-server-core-#{CHEF_SERVER_VERSION}.el#{elversion}.x86_64.rpm"
223
242
  }
224
243
 
225
- rpms["ruby25"] = "https://s3.amazonaws.com/cloudamatic/muby-2.5.3-1.el#{elversion}.x86_64.rpm"
226
- rpms["python27"] = "https://s3.amazonaws.com/cloudamatic/muthon-2.7.16-1.el#{elversion}.x86_64.rpm"
244
+ rpms["ruby27"] = "https://s3.amazonaws.com/cloudamatic/muby-2.7.2-1.el#{elversion}.x86_64.rpm"
245
+ if elversion.to_i == 6
246
+ rpms["openssl"] = "https://s3.amazonaws.com/cloudamatic/mussl-1.1.1h-1.el6.x86_64.rpm"
247
+ rpms["sqlite"] = "https://s3.amazonaws.com/cloudamatic/muqlite-3.33-1.el6.x86_64.rpm"
248
+ end
249
+ if elversion.to_i == 7
250
+ rpms["mugit"] = "https://s3.amazonaws.com/cloudamatic/mugit-2.30.0-1.el7.x86_64.rpm"
251
+ end
252
+ # this takes up a huge amount of space, save it until we're fully operational
253
+ if !RUNNING_STANDALONE
254
+ rpms["python38"] = "https://s3.amazonaws.com/cloudamatic/muthon-3.8.3-1.el#{elversion}.x86_64.rpm"
255
+ end
227
256
 
228
257
  package basepackages
229
258
 
@@ -298,25 +327,25 @@ rpm_package "Chef Server upgrade package" do
298
327
  only_if { RUNNING_STANDALONE }
299
328
  end
300
329
 
301
- # REMOVE OLD RUBYs
302
- execute "clean up old Ruby 2.1.6" do
303
- command "rm -rf /opt/rubies/ruby-2.1.6"
304
- ignore_failure true
305
- only_if { ::Dir.exist?("/opt/rubies/ruby-2.1.6") }
306
- end
307
-
308
- execute "Kill ruby-2.3.1" do
330
+ execute "clean up old ruby-2.3.1 package" do
309
331
  command "yum erase ruby23-2.3.1-1.el7.centos.x86_64 -y; rpm -e ruby23"
310
332
  ignore_failure true
311
333
  only_if { ::Dir.exist?("/opt/rubies/ruby-2.3.1") }
312
334
  end
313
-
314
- execute "clean up old ruby-2.3.1" do
315
- command "rm -rf /opt/rubies/ruby-2.3.1"
335
+ execute "clean up old muby-2.5.3 package" do
336
+ command "yum erase muby-2.5.3-1.el7.x86_64 -y"
316
337
  ignore_failure true
317
- only_if { ::Dir.exist?("/opt/rubies/ruby-2.3.1") }
338
+ only_if "rpm -q muby-2.5.3"
318
339
  end
319
340
 
341
+ %w{2.1.6 2.3.1 2.5.3}.each { |v|
342
+ execute "clean up old ruby-#{v} directory" do
343
+ command "rm -rf /opt/rubies/ruby-#{v}"
344
+ ignore_failure true
345
+ only_if { ::Dir.exist?("/opt/rubies/ruby-#{v}") }
346
+ end
347
+ }
348
+
320
349
  execute "yum makecache" do
321
350
  action :nothing
322
351
  end
@@ -325,11 +354,14 @@ end
325
354
  rpms.each_pair { |pkg, src|
326
355
  rpm_package pkg do
327
356
  source src
328
- if pkg == "ruby25"
357
+ if pkg == "ruby27"
329
358
  options '--prefix=/opt/rubies/'
330
359
  end
331
360
  if pkg == "epel-release"
332
361
  notifies :run, "execute[yum makecache]", :immediately
362
+ if elversion.to_i == 6
363
+ not_if "rpm -q epel-release"
364
+ end
333
365
  end
334
366
  if pkg == "chef-server-core"
335
367
  notifies :stop, "service[iptables]", :before
@@ -352,6 +384,10 @@ package removepackages do
352
384
  end
353
385
 
354
386
 
387
+ if rpms["mugit"]
388
+ ENV['PATH'] = "/usr/local/git-current/bin:"+ENV['PATH']
389
+ end
390
+
355
391
 
356
392
  file "initial chef-server.rb" do
357
393
  path "/etc/opscode/chef-server.rb"
@@ -364,6 +400,7 @@ nginx['ssl_port'] = 7443
364
400
  nginx['ssl_ciphers'] = 'HIGH:MEDIUM:!LOW:!kEDH:!aNULL:!ADH:!eNULL:!EXP:!SSLv2:!SEED:!CAMELLIA:!PSK'
365
401
  nginx['ssl_protocols'] = 'TLSv1.2'
366
402
  bookshelf['external_url'] = 'https://127.0.0.1:7443'
403
+ bookshelf['vip'] = server_name
367
404
  bookshelf['vip_port'] = 7443\n"
368
405
  not_if { ::File.size?("/etc/opscode/chef-server.rb") }
369
406
  notifies :run, "execute[reconfigure Chef server]", :immediately
@@ -406,13 +443,71 @@ remote_file "#{MU_BASE}/bin/mu-self-update" do
406
443
  mode 0755
407
444
  end
408
445
 
409
- bash "install modules for our built-in Python" do
410
- code <<-EOH
411
- /usr/local/python-current/bin/pip install -r #{MU_BASE}/lib/requirements.txt
412
- EOH
446
+ # Skip this during initial installs, it's space-hungry
447
+ if !RUNNING_STANDALONE
448
+ bash "install modules for our built-in Python" do
449
+ code <<-EOH
450
+ /usr/local/python-current/bin/pip install -r #{MU_BASE}/lib/requirements.txt
451
+ EOH
452
+ end
453
+ end
454
+
455
+ # bundle a less heavy version of our Gemfile during initial installation, so we
456
+ # can actually fit on normal root disks until we have enough code and
457
+ # credentials to roll a dedicated /opt.
458
+ TMPDIR = Dir.mktmpdir
459
+ gemfile_dir = if RUNNING_STANDALONE and !File.readlines("/etc/mtab").grep(/\s\/opt\s/).any?
460
+ ruby_block "set up alternate install-time Gemfile" do # ~FC014
461
+ block do
462
+ exclude_gems = %w{aws-sdk azure_sdk google-api-client}
463
+
464
+ ["/sys/hypervisor/uuid",
465
+ "/sys/devices/virtual/dmi/id/product_uuid",
466
+ "/sys/devices/virtual/dmi/id/board_asset_tag"].each { |src|
467
+ if File.exists?(src)
468
+ uuid = File.read(src).chomp
469
+ if uuid and uuid =~ /^ec2/i
470
+ exclude_gems.delete("aws-sdk")
471
+ end
472
+ break
473
+ end
474
+ }
475
+ dmiout = shell_out!(%Q{PATH=/sbin:/usr/sbin:/bin:/usr/bin dmidecode})
476
+ if dmiout.match(/Google/)
477
+ exclude_gems.delete("google-api-client")
478
+ end
479
+
480
+ if File.exists?("/var/log/waagent.log") and File.read("/var/log/waagent.log") =~ /added Azure fabric/
481
+ exclude_gems.delete("azure_sdk")
482
+ end
483
+
484
+ f = File.open("#{TMPDIR}/cloud-mu.gemspec", "w")
485
+ File.read("#{MU_BASE}/lib/cloud-mu.gemspec").each_line { |l|
486
+ skipme = false
487
+ if l=~ /s\.add_runtime_dependency/
488
+ exclude_gems.each { |gem|
489
+ if l =~ /\b#{gem}\b/
490
+ skipme = true
491
+ end
492
+ }
493
+ next if skipme
494
+ end
495
+ f.puts l.chomp
496
+ }
497
+ f.close
498
+
499
+ Dir.mkdir("#{TMPDIR}/modules")
500
+ FileUtils.cp("#{MU_BASE}/lib/modules/Gemfile", "#{TMPDIR}/modules")
501
+ end
502
+ end
503
+ "#{TMPDIR}/modules"
504
+ else
505
+ "#{MU_BASE}/lib/modules"
413
506
  end
414
507
 
415
- ["/usr/local/ruby-current", "/opt/chef/embedded"].each { |rubydir|
508
+ rubies = ["/usr/local/ruby-current", "/opt/chef/embedded"]
509
+
510
+ rubies.each { |rubydir|
416
511
  gembin = rubydir+"/bin/gem"
417
512
  gemdir = Dir.glob("#{rubydir}/lib/ruby/gems/?.?.?/gems").last
418
513
  bundler_path = gembin.sub(/gem$/, "bundle")
@@ -424,19 +519,24 @@ end
424
519
  EOH
425
520
  action :nothing
426
521
  end
427
- gem_package bundler_path do
522
+
523
+ gem_package "bundler for #{rubydir}" do
428
524
  gem_binary gembin
429
525
  package_name "bundler"
430
526
  if rubydir == "/usr/local/ruby-current" or File.exists?(bundler_path)
431
527
  action :upgrade
432
528
  ignore_failure true
433
529
  end
530
+ version "~> 2.1.4"
434
531
  notifies :run, "bash[fix #{rubydir} gem permissions]", :delayed
435
532
  end
436
- execute "#{bundler_path} install" do
437
- cwd "#{MU_BASE}/lib/modules"
438
- umask 0022
439
- not_if "#{bundler_path} check"
533
+ execute "#{bundler_path} install from #{gemfile_dir} for #{rubydir}" do
534
+ command "PATH=/usr/local/git-current/bin:/usr/local/git-current/libexec/git-core:${PATH} #{bundler_path} install"
535
+ cwd gemfile_dir
536
+ umask "0022"
537
+ if !RUNNING_STANDALONE
538
+ not_if { system("cd #{gemfile_dir} && #{bundler_path} check 2>&1"); $?.exitstatus }
539
+ end
440
540
  notifies :run, "bash[fix #{rubydir} gem permissions]", :delayed
441
541
  notifies :restart, "service[chef-server]", :delayed if rubydir == "/opt/opscode/embedded"
442
542
  # XXX notify mommacat if we're *not* in chef-apply... RUNNING_STANDALONE
@@ -463,7 +563,7 @@ end
463
563
  # This is mostly to make sure Berkshelf has a clean and current environment to
464
564
  # live with.
465
565
  execute "/usr/local/ruby-current/bin/bundle clean --force" do
466
- cwd "#{MU_BASE}/lib/modules"
566
+ cwd gemfile_dir
467
567
  only_if { RUNNING_STANDALONE }
468
568
  end
469
569
 
@@ -490,29 +590,31 @@ require "simple-password-gen"
490
590
  # XXX this would make an awesome library
491
591
  execute "create mu Chef user" do
492
592
  command "/opt/opscode/bin/chef-server-ctl user-create mu Mu Master root@example.com #{Password.pronounceable} -f #{MU_BASE}/var/users/mu/mu.user.key"
493
- umask 0277
593
+ umask "0277"
494
594
  not_if "/opt/opscode/bin/chef-server-ctl user-list | grep '^mu$'"
595
+ notifies :start, "service[chef-server]", :before
495
596
  end
496
597
  execute "create mu Chef org" do
497
598
  command "/opt/opscode/bin/chef-server-ctl org-create mu mu -a mu -f #{MU_BASE}/var/orgs/mu/mu.org.key"
498
- umask 0277
599
+ umask "0277"
499
600
  not_if "/opt/opscode/bin/chef-server-ctl org-list | grep '^mu$'"
601
+ notifies :start, "service[chef-server]", :before
500
602
  end
501
603
  # TODO copy in ~/.chef/mu.*.key to /opt/mu/var/users/mu if the stuff already exists
502
604
  file "initial root knife.rb" do
503
605
  path "/root/.chef/knife.rb"
504
606
  content "
505
- node_name 'mu'
506
- client_key '#{MU_BASE}/var/users/mu/mu.user.key'
507
- validation_client_name 'mu-validator'
508
- validation_key '#{MU_BASE}/var/orgs/mu/mu.org.key'
509
- chef_server_url 'https://127.0.0.1:7443/organizations/mu'
510
- chef_server_root 'https://127.0.0.1:7443/organizations/mu'
511
- syntax_check_cache_path '/root/.chef/syntax_check_cache'
512
- cookbook_path [ '/root/.chef/cookbooks', '/root/.chef/site_cookbooks' ]
513
- ssl_verify_mode :verify_none
514
- knife[:vault_mode] = 'client'
515
- knife[:vault_admins] = ['mu']\n"
607
+ node_name 'mu'
608
+ client_key '#{MU_BASE}/var/users/mu/mu.user.key'
609
+ validation_client_name 'mu-validator'
610
+ validation_key '#{MU_BASE}/var/orgs/mu/mu.org.key'
611
+ chef_server_url 'https://127.0.0.1:7443/organizations/mu'
612
+ chef_server_root 'https://127.0.0.1:7443/organizations/mu'
613
+ syntax_check_cache_path '/root/.chef/syntax_check_cache'
614
+ cookbook_path [ '/root/.chef/cookbooks', '/root/.chef/site_cookbooks' ]
615
+ ssl_verify_mode :verify_none
616
+ knife[:vault_mode] = 'client'
617
+ knife[:vault_admins] = ['mu']\n"
516
618
  only_if { !::File.size?("/root/.chef/knife.rb") }
517
619
  notifies :run, "execute[initial Chef artifact upload]", :immediately
518
620
  end
@@ -530,36 +632,64 @@ if SSH_DIR != ROOT_SSH_DIR
530
632
  mode 0700
531
633
  end
532
634
  end
533
- bash "add localhost ssh to authorized_keys and config" do
635
+ bash "add localhost ssh to config" do
534
636
  code <<-EOH
535
- cat #{ROOT_SSH_DIR}/id_rsa.pub >> #{SSH_DIR}/authorized_keys
536
637
  echo "Host localhost" >> #{ROOT_SSH_DIR}/config
537
638
  echo " IdentityFile #{ROOT_SSH_DIR}/id_rsa" >> #{ROOT_SSH_DIR}/config
538
639
  EOH
539
640
  action :nothing
540
641
  end
541
642
  execute "ssh-keygen -N '' -f #{ROOT_SSH_DIR}/id_rsa" do
542
- umask 0177
643
+ umask "0177"
543
644
  not_if { ::File.exist?("#{ROOT_SSH_DIR}/id_rsa") }
544
- notifies :run, "bash[add localhost ssh to authorized_keys and config]", :immediately
645
+ notifies :run, "bash[add localhost ssh to config]", :immediately
646
+ notifies :run, "execute[add localhost key to authorized_keys]", :immediately
647
+ end
648
+ execute "add localhost key to authorized_keys" do
649
+ command "cat #{ROOT_SSH_DIR}/id_rsa.pub >> #{SSH_DIR}/authorized_keys"
650
+ only_if {
651
+ found = false
652
+ pubkey = if File.exists?("#{SSH_DIR}/authorized_keys")
653
+ File.read("#{ROOT_SSH_DIR}/id_rsa.pub").chomp
654
+ end
655
+ if pubkey and File.exists?("#{SSH_DIR}/authorized_keys")
656
+ authfile = File.read("#{ROOT_SSH_DIR}/authorized_keys")
657
+ authfile.each_line { |l|
658
+ if l =~ /#{Regexp.quote(pubkey)}/
659
+ found = true
660
+ end
661
+ }
662
+ end
663
+ !found
664
+ }
545
665
  end
546
- file "/etc/chef/client.pem" do
666
+ # XXX foodcritic says this is a repeat declaration, but it's... not
667
+ file "/etc/chef/client.pem" do # ~FC005
547
668
  action :nothing
548
669
  end
549
670
  file "/etc/chef/validation.pem" do
550
671
  action :nothing
551
672
  end
673
+ file "/etc/chef/client.rb" do
674
+ action :nothing
675
+ end
676
+
677
+ knife_cfg = "-c /root/.chef/knife.rb"
552
678
 
553
679
  execute "create MU-MASTER Chef client" do
680
+ # XXX I dislike --ssh-verify-host-key=never intensely, but the CLI-documented 'accept_new' doesn't actually work
554
681
  if SSH_USER == "root"
555
- command "/opt/chef/bin/knife bootstrap -N MU-MASTER --no-node-verify-api-cert --node-ssl-verify-mode=none 127.0.0.1"
682
+ command "/opt/chef/bin/knife bootstrap #{knife_cfg} -N MU-MASTER --no-node-verify-api-cert --node-ssl-verify-mode=none -U #{SSH_USER} --ssh-identity-file=/root/.ssh/id_rsa --ssh-verify-host-key=never 127.0.0.1"
556
683
  else
557
- command "/opt/chef/bin/knife bootstrap -N MU-MASTER --no-node-verify-api-cert --node-ssl-verify-mode=none -x #{SSH_USER} --sudo 127.0.0.1"
684
+ command "/opt/chef/bin/knife bootstrap #{knife_cfg} -N MU-MASTER --no-node-verify-api-cert --node-ssl-verify-mode=none -U #{SSH_USER} --ssh-identity-file=/root/.ssh/id_rsa --ssh-verify-host-key=never --sudo 127.0.0.1"
558
685
  end
559
- not_if "/opt/chef/bin/knife node list | grep '^MU-MASTER$'"
560
- only_if "/opt/chef/bin/knife ssl check" # make sure we don't wipe ourselves due to unrelated SSL issues
686
+ only_if "/opt/chef/bin/knife node #{knife_cfg} list" # don't do crazy stuff just because knife isn't working
687
+ not_if "/opt/chef/bin/knife node #{knife_cfg} list | grep '^MU-MASTER$'"
688
+ notifies :run, "execute[add localhost key to authorized_keys]", :before
689
+ notifies :delete, "file[/etc/chef/client.rb]", :before
561
690
  notifies :delete, "file[/etc/chef/client.pem]", :before
562
691
  notifies :delete, "file[/etc/chef/validation.pem]", :before
692
+ notifies :start, "service[chef-server]", :before
563
693
  only_if { RUNNING_STANDALONE }
564
694
  end
565
695
 
@@ -575,7 +705,7 @@ end
575
705
 
576
706
  # Community cookbooks keep touching gems, and none of them are smart about our
577
707
  # default umask. We have to clean up after them every time.
578
- ["/usr/local/ruby-current", "/opt/chef/embedded"].each { |rubydir|
708
+ rubies.each { |rubydir|
579
709
  execute "trigger permission fix in #{rubydir}" do
580
710
  command "ls /etc/motd > /dev/null"
581
711
  notifies :run, "bash[fix #{rubydir} gem permissions]", :delayed
@@ -588,3 +718,8 @@ bash "fix misc permissions" do
588
718
  chmod go+rx #{MU_BASE}/lib/bin/* #{MU_BASE}/lib/extras/*-stock-* #{MU_BASE}/lib/extras/vault_tools/*.sh
589
719
  EOH
590
720
  end
721
+
722
+ directory TMPDIR do
723
+ action :delete
724
+ recursive true
725
+ end