cloud-mu 3.4.0 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. checksums.yaml +4 -4
  2. data/ansible/roles/mu-nat/tasks/main.yml +3 -0
  3. data/bin/mu-aws-setup +41 -7
  4. data/bin/mu-azure-setup +34 -0
  5. data/bin/mu-configure +214 -119
  6. data/bin/mu-gcp-setup +37 -2
  7. data/bin/mu-node-manage +3 -0
  8. data/bin/mu-refresh-ssl +67 -0
  9. data/bin/mu-run-tests +14 -4
  10. data/bin/mu-self-update +30 -10
  11. data/bin/mu-upload-chef-artifacts +30 -26
  12. data/cloud-mu.gemspec +8 -6
  13. data/cookbooks/mu-master/attributes/default.rb +5 -1
  14. data/cookbooks/mu-master/metadata.rb +2 -2
  15. data/cookbooks/mu-master/recipes/default.rb +81 -26
  16. data/cookbooks/mu-master/recipes/init.rb +197 -62
  17. data/cookbooks/mu-master/recipes/update_nagios_only.rb +1 -1
  18. data/cookbooks/mu-master/recipes/vault.rb +78 -77
  19. data/cookbooks/mu-master/templates/default/mods/rewrite.conf.erb +1 -0
  20. data/cookbooks/mu-master/templates/default/nagios.conf.erb +103 -0
  21. data/cookbooks/mu-master/templates/default/web_app.conf.erb +14 -30
  22. data/cookbooks/mu-tools/attributes/default.rb +5 -0
  23. data/cookbooks/mu-tools/files/centos-6/CentOS-Base.repo +47 -0
  24. data/cookbooks/mu-tools/libraries/helper.rb +12 -2
  25. data/cookbooks/mu-tools/libraries/monkey.rb +1 -1
  26. data/cookbooks/mu-tools/recipes/apply_security.rb +6 -0
  27. data/cookbooks/mu-tools/recipes/aws_api.rb +6 -4
  28. data/cookbooks/mu-tools/recipes/base_repositories.rb +1 -1
  29. data/cookbooks/mu-tools/recipes/gcloud.rb +2 -9
  30. data/cookbooks/mu-tools/recipes/google_api.rb +5 -2
  31. data/cookbooks/mu-tools/resources/disk.rb +108 -58
  32. data/extras/Gemfile.lock.bootstrap +394 -0
  33. data/extras/bucketstubs/error.html +0 -0
  34. data/extras/bucketstubs/index.html +0 -0
  35. data/extras/clean-stock-amis +9 -9
  36. data/extras/git_rpm/build.sh +20 -0
  37. data/extras/git_rpm/mugit.spec +53 -0
  38. data/extras/image-generators/VMWare/centos8.yaml +15 -0
  39. data/extras/openssl_rpm/build.sh +19 -0
  40. data/extras/openssl_rpm/mussl.spec +46 -0
  41. data/extras/python_rpm/muthon.spec +14 -4
  42. data/extras/ruby_rpm/muby.spec +9 -5
  43. data/extras/sqlite_rpm/build.sh +19 -0
  44. data/extras/sqlite_rpm/muqlite.spec +47 -0
  45. data/install/installer +7 -5
  46. data/modules/mu.rb +12 -5
  47. data/modules/mu/cloud/machine_images.rb +1 -1
  48. data/modules/mu/cloud/providers.rb +6 -1
  49. data/modules/mu/cloud/resource_base.rb +1 -1
  50. data/modules/mu/cloud/ssh_sessions.rb +4 -0
  51. data/modules/mu/config.rb +28 -12
  52. data/modules/mu/config/database.rb +2 -2
  53. data/modules/mu/config/firewall_rule.rb +1 -1
  54. data/modules/mu/config/ref.rb +2 -2
  55. data/modules/mu/config/schema_helpers.rb +12 -3
  56. data/modules/mu/config/server.rb +10 -4
  57. data/modules/mu/config/server_pool.rb +2 -2
  58. data/modules/mu/config/vpc.rb +10 -10
  59. data/modules/mu/defaults/AWS.yaml +32 -32
  60. data/modules/mu/deploy.rb +23 -10
  61. data/modules/mu/groomers/chef.rb +2 -2
  62. data/modules/mu/master.rb +49 -3
  63. data/modules/mu/mommacat.rb +8 -5
  64. data/modules/mu/mommacat/naming.rb +2 -2
  65. data/modules/mu/mommacat/storage.rb +22 -27
  66. data/modules/mu/providers/aws.rb +142 -48
  67. data/modules/mu/providers/aws/alarm.rb +3 -3
  68. data/modules/mu/providers/aws/bucket.rb +19 -19
  69. data/modules/mu/providers/aws/cache_cluster.rb +22 -22
  70. data/modules/mu/providers/aws/cdn.rb +2 -2
  71. data/modules/mu/providers/aws/collection.rb +14 -14
  72. data/modules/mu/providers/aws/container_cluster.rb +27 -27
  73. data/modules/mu/providers/aws/database.rb +40 -39
  74. data/modules/mu/providers/aws/dnszone.rb +5 -5
  75. data/modules/mu/providers/aws/endpoint.rb +35 -35
  76. data/modules/mu/providers/aws/firewall_rule.rb +26 -23
  77. data/modules/mu/providers/aws/function.rb +28 -28
  78. data/modules/mu/providers/aws/group.rb +7 -7
  79. data/modules/mu/providers/aws/habitat.rb +2 -2
  80. data/modules/mu/providers/aws/job.rb +6 -6
  81. data/modules/mu/providers/aws/loadbalancer.rb +34 -34
  82. data/modules/mu/providers/aws/log.rb +14 -14
  83. data/modules/mu/providers/aws/msg_queue.rb +10 -10
  84. data/modules/mu/providers/aws/nosqldb.rb +8 -8
  85. data/modules/mu/providers/aws/notifier.rb +7 -7
  86. data/modules/mu/providers/aws/role.rb +17 -15
  87. data/modules/mu/providers/aws/search_domain.rb +10 -10
  88. data/modules/mu/providers/aws/server.rb +176 -95
  89. data/modules/mu/providers/aws/server_pool.rb +65 -105
  90. data/modules/mu/providers/aws/storage_pool.rb +17 -9
  91. data/modules/mu/providers/aws/user.rb +1 -1
  92. data/modules/mu/providers/aws/vpc.rb +103 -51
  93. data/modules/mu/providers/aws/vpc_subnet.rb +43 -39
  94. data/modules/mu/providers/azure.rb +78 -12
  95. data/modules/mu/providers/azure/server.rb +18 -3
  96. data/modules/mu/providers/cloudformation/server.rb +1 -1
  97. data/modules/mu/providers/google.rb +19 -4
  98. data/modules/mu/providers/google/folder.rb +6 -2
  99. data/modules/mu/providers/google/function.rb +65 -30
  100. data/modules/mu/providers/google/role.rb +1 -1
  101. data/modules/mu/providers/google/vpc.rb +27 -2
  102. data/modules/tests/aws-servers-with-handrolled-iam.yaml +37 -0
  103. data/modules/tests/k8s.yaml +1 -1
  104. metadata +24 -8
@@ -7,7 +7,7 @@ long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
7
7
  source_url 'https://github.com/cloudamatic/mu'
8
8
  issues_url 'https://github.com/cloudamatic/mu/issues'
9
9
  chef_version '>= 12.1' if respond_to?(:chef_version)
10
- version '0.9.6'
10
+ version '0.9.7'
11
11
 
12
12
  %w( centos amazon redhat ).each do |os|
13
13
  supports os
@@ -28,4 +28,4 @@ depends 'consul-cluster', '~> 2.0.0'
28
28
  depends 'chef-sugar' # undeclared dependency of consul 2.1, which can't be upgraded without creating a conflict with consul-cluster and vault-cluster -zr2d2
29
29
  depends 'hostsfile', '~> 3.0.1'
30
30
  depends 'chef-vault', '~> 3.1.1'
31
- depends 'apache2', '< 6.0.0'
31
+ depends 'apache2', '< 8.0.0'
@@ -217,24 +217,25 @@ if !node['update_nagios_only']
217
217
  if !$MU_CFG['public_address'].match(/^\d+\.\d+\.\d+\.\d+$/)
218
218
  svrname = $MU_CFG['public_address']
219
219
  end
220
-
221
- # nagios keeps disabling the default vhost, so let's make another one
222
- include_recipe "apache2::mod_proxy"
223
- include_recipe "apache2::mod_proxy_http"
224
- include_recipe "apache2::mod_rewrite"
225
-
226
- if node['platform_family'] == "rhel" and node['platform_version'].split('.')[0].to_i == 6
227
- package "httpd24-mod_ldap"
228
- apache_module 'ldap' do
229
- conf true
230
- end
231
- else
232
- include_recipe "apache2::mod_authnz_ldap"
220
+ apache2_install "" do
221
+ docroot_dir "/var/www/html"
222
+ modules %w{status alias auth_basic authn_core authn_file authz_core authz_groupfile authz_host authz_user autoindex deflate dir env mime negotiation setenvif log_config logio unixd systemd headers proxy proxy_http rewrite ssl ldap authnz_ldap slotmem_shm}
233
223
  end
224
+ package "mod_ldap"
225
+
226
+ # add stock .conf files to the mix where applicable
227
+ apache2_mod_proxy ""
228
+ apache2_mod_ldap ""
229
+ apache2_mod_cgid ""
230
+ apache2_mod_ssl ""
234
231
 
235
- apache_site "default" do
236
- enable false
232
+ apache2_mod "php"
233
+ apache2_default_site "" do
234
+ action :enable
235
+ notifies :start, "service[apache2]", :delayed
237
236
  end
237
+
238
+ # nagios keeps disabling the default vhost, so let's make another one
238
239
  execute "Allow net connect to local for apache" do
239
240
  command "/usr/sbin/setsebool -P httpd_can_network_connect on"
240
241
  not_if "/usr/sbin/getsebool httpd_can_network_connect | grep -cim1 ^.*on$"
@@ -242,22 +243,76 @@ if !node['update_nagios_only']
242
243
  notifies :reload, "service[apache2]", :delayed
243
244
  end
244
245
 
246
+ aliases = [node['fqdn'], node['hostname'], node['local_hostname'], node['local_ipv4'], node['public_hostname'], node['public_ipv4']]
247
+ if node['ec2']
248
+ aliases << node['ec2']['local_ipv4']
249
+ aliases << node['ec2']['local_hostname']
250
+ aliases << node['ec2']['public_ipv4']
251
+ aliases << node['ec2']['public_hostname']
252
+ end
253
+ aliases.uniq!
254
+ aliases.reject! { |a| a.nil? or a.empty? }
255
+
256
+ service 'apache2' do
257
+ extend Apache2::Cookbook::Helpers
258
+ service_name lazy { apache_platform_service_name }
259
+ supports restart: true, status: true, reload: true
260
+ action :enable
261
+ end
245
262
 
246
- web_app "mu_docs" do
247
- server_name svrname
248
- server_aliases [node['fqdn'], node['hostname'], node['local_hostname'], node['local_ipv4'], node['public_hostname'], node['public_ipv4']]
249
- docroot "/var/www/html"
250
- cookbook "mu-master"
263
+ template '/etc/httpd/sites-available/mu_docs.conf' do
264
+ variables(
265
+ server_name: svrname,
266
+ server_port: "80",
267
+ server_aliases: aliases,
268
+ docroot: "/var/www/html"
269
+ )
270
+ cookbook 'mu-master'
271
+ source 'web_app.conf.erb'
251
272
  notifies :reload, "service[apache2]", :delayed
252
273
  end
253
- web_app "https_proxy" do
254
- server_name svrname
255
- server_port "443"
256
- server_aliases [node['fqdn'], node['hostname'], node['local_hostname'], node['local_ipv4'], node['public_hostname'], node['public_ipv4']]
257
- docroot "/var/www/html"
258
- cookbook "mu-master"
274
+ apache2_site "mu_docs"
275
+ template '/etc/httpd/sites-available/https_proxy.conf' do
276
+ variables(
277
+ server_name: svrname,
278
+ server_port: "443",
279
+ server_aliases: aliases,
280
+ docroot: "/var/www/html"
281
+ )
282
+ cookbook 'mu-master'
283
+ source 'web_app.conf.erb'
284
+ notifies :reload, "service[apache2]", :delayed
285
+ end
286
+ apache2_site "https_proxy"
287
+
288
+ # configure the appropriate authentication method for the web server
289
+ case node['nagios']['server_auth_method']
290
+ when 'openid'
291
+ apache2_mod 'auth_openid'
292
+ when 'cas'
293
+ apache2_mod 'auth_cas'
294
+ end
295
+
296
+ # apache2_conf "nagios" do
297
+ # server_name svrname
298
+ # server_aliases aliases
299
+ # template 'nagios.conf.erb'
300
+ # cookbook "mu-master"
301
+ # notifies :reload, "service[apache2]", :delayed
302
+ # action :enable
303
+ # end
304
+ template '/etc/httpd/sites-available/nagios.conf' do
305
+ variables(
306
+ server_name: svrname,
307
+ server_port: "443",
308
+ server_aliases: aliases,
309
+ docroot: "/var/www/html"
310
+ )
311
+ cookbook 'mu-master'
312
+ source 'nagios.conf.erb'
259
313
  notifies :reload, "service[apache2]", :delayed
260
314
  end
315
+ apache2_site "nagios"
261
316
 
262
317
  link "/etc/nagios3" do
263
318
  to "/etc/nagios"
@@ -35,20 +35,20 @@ ENV['PATH'] = ENV['PATH']+":/bin:/opt/opscode/embedded/bin"
35
35
 
36
36
  # XXX We want to be able to override these things when invoked from chef-apply,
37
37
  # but, like, how?
38
- CHEF_SERVER_VERSION="12.17.15-1"
39
- CHEF_CLIENT_VERSION="14.13.11"
38
+ CHEF_SERVER_VERSION="14.0.65-1"
39
+ CHEF_CLIENT_VERSION="16.9.29"
40
40
  KNIFE_WINDOWS="1.9.0"
41
41
  MU_BASE="/opt/mu"
42
- MU_BRANCH="master" # GIT HOOK EDITABLE DO NOT TOUCH
43
- realbranch=`cd #{MU_BASE}/lib && git rev-parse --abbrev-ref HEAD` # ~FC048
44
42
 
45
- if ENV.key?('MU_BRANCH')
46
- MU_BRANCH = ENV['MU_BRANCH']
43
+ MU_BRANCH = if ENV.key?('MU_BRANCH')
44
+ ENV['MU_BRANCH']
47
45
  elsif $?.exitstatus == 0
48
- MU_BRANCH=realbranch.chomp
46
+ realbranch=`cd #{MU_BASE}/lib && git rev-parse --abbrev-ref HEAD` # ~FC048
47
+ realbranch.chomp
49
48
  else
50
- MU_BRANCH="master"
49
+ "master"
51
50
  end
51
+
52
52
  begin
53
53
  resources('service[sshd]')
54
54
  rescue Chef::Exceptions::ResourceNotFound
@@ -77,6 +77,12 @@ service "iptables" do
77
77
  only_if "( /bin/systemctl -l --no-pager | grep iptables.service ) || ( /sbin/chkconfig --list | grep ^iptables )"
78
78
  end
79
79
 
80
+ service "firewalld" do
81
+ ignore_failure true
82
+ action :nothing
83
+ only_if "/bin/systemctl -l --no-pager | grep firewalld.service"
84
+ end
85
+
80
86
  # These guys are a workaround for Opscode bugs that seems to affect some Chef
81
87
  # Server upgrades.
82
88
  directory "/var/run/postgresql" do
@@ -126,28 +132,39 @@ file "use a clean /etc/hosts during install" do
126
132
  not_if { ::Dir.exist?("#{MU_BASE}/lib/.git") }
127
133
  end
128
134
 
135
+ execute "modprobe br_netfilter" do
136
+ action :nothing
137
+ end
138
+
129
139
  execute "reconfigure Chef server" do
130
- command "/opt/opscode/bin/chef-server-ctl reconfigure"
140
+ command "CHEF_LICENSE=\"accept\" /opt/opscode/bin/chef-server-ctl reconfigure"
131
141
  action :nothing
132
142
  notifies :stop, "service[iptables]", :before
143
+ notifies :stop, "service[firewalld]", :before
133
144
  # notifies :create, "link[/tmp/.s.PGSQL.5432]", :before
134
145
  notifies :create, "link[/var/run/postgresql/.s.PGSQL.5432]", :before
135
146
  notifies :restart, "service[chef-server]", :immediately
136
147
  if !RUNNING_STANDALONE
137
148
  notifies :start, "service[iptables]", :immediately
149
+ notifies :start, "service[firewalld]", :immediately
150
+ else
151
+ notifies :run, "execute[Chef Server rabbitmq workaround]", :before
138
152
  end
139
153
  only_if { RUNNING_STANDALONE }
140
154
  end
141
155
  execute "upgrade Chef server" do
142
- command "/opt/opscode/bin/chef-server-ctl upgrade"
156
+ command "CHEF_LICENSE=\"accept\" /opt/opscode/bin/chef-server-ctl upgrade"
143
157
  action :nothing
144
158
  timeout 1200 # this can take a while
145
159
  notifies :stop, "service[iptables]", :before
160
+ notifies :stop, "service[firewalld]", :before
161
+ notifies :run, "execute[modprobe br_netfilter]", :before
146
162
  notifies :run, "execute[Chef Server rabbitmq workaround]", :before
147
163
  # notifies :create, "link[/tmp/.s.PGSQL.5432]", :before
148
164
  notifies :create, "link[/var/run/postgresql/.s.PGSQL.5432]", :before
149
165
  if !RUNNING_STANDALONE
150
166
  notifies :start, "service[iptables]", :immediately
167
+ notifies :start, "service[firewalld]", :immediately
151
168
  end
152
169
  only_if { RUNNING_STANDALONE }
153
170
  end
@@ -160,8 +177,10 @@ service "chef-server" do
160
177
  # notifies :create, "link[/tmp/.s.PGSQL.5432]", :before
161
178
  # notifies :create, "link[/var/run/postgresql/.s.PGSQL.5432]", :before
162
179
  notifies :stop, "service[iptables]", :before
180
+ notifies :stop, "service[firewalld]", :before
163
181
  if !RUNNING_STANDALONE
164
182
  notifies :start, "service[iptables]", :immediately
183
+ notifies :start, "service[firewalld]", :immediately
165
184
  end
166
185
  only_if { RUNNING_STANDALONE }
167
186
  end
@@ -173,7 +192,7 @@ dpkgs = {}
173
192
 
174
193
  elversion = node['platform_version'].split('.')[0]
175
194
 
176
- rhelbase = ["git", "curl", "diffutils", "patch", "gcc", "gcc-c++", "make", "postgresql-devel", "libyaml", "libffi-devel", "tcl", "tk"]
195
+ rhelbase = ["git", "curl", "diffutils", "patch", "gcc", "gcc-c++", "make", "postgresql-devel", "libyaml", "libffi-devel", "tcl", "tk", "xfsprogs"]
177
196
 
178
197
  case node['platform_family']
179
198
  when 'rhel'
@@ -182,11 +201,11 @@ when 'rhel'
182
201
 
183
202
  case node['platform_version'].split('.')[0].to_i
184
203
  when 6
185
- basepackages.concat(["cryptsetup-luks", "mysql-devel", "centos-release-scl"])
204
+ basepackages.concat(["cryptsetup-luks", "mysql-devel", "centos-release-scl", "perl-WWW-Curl"])
186
205
  removepackages = ["nagios"]
187
206
 
188
207
  when 7
189
- basepackages.concat(['libX11', 'mariadb-devel', 'cryptsetup'])
208
+ basepackages.concat(['policycoreutils-python', 'libX11', 'mariadb-devel', 'cryptsetup', 'tcl-devel', 'gdbm-devel', 'sqlite-devel', 'tk-devel', 'perl-CGI', 'perl-DBI', 'perl-Data-Dumper', 'perl-Digest-MD5', 'perl-Git-SVN', 'perl-YAML', 'nvme-cli'])
190
209
  removepackages = ['nagios', 'firewalld']
191
210
 
192
211
  when 8
@@ -222,8 +241,18 @@ rpms = {
222
241
  "chef-server-core" => "https://packages.chef.io/files/stable/chef-server/#{CHEF_SERVER_VERSION.sub(/\-\d+$/, "")}/el/#{elversion}/chef-server-core-#{CHEF_SERVER_VERSION}.el#{elversion}.x86_64.rpm"
223
242
  }
224
243
 
225
- rpms["ruby25"] = "https://s3.amazonaws.com/cloudamatic/muby-2.5.3-1.el#{elversion}.x86_64.rpm"
226
- rpms["python27"] = "https://s3.amazonaws.com/cloudamatic/muthon-2.7.16-1.el#{elversion}.x86_64.rpm"
244
+ rpms["ruby27"] = "https://s3.amazonaws.com/cloudamatic/muby-2.7.2-1.el#{elversion}.x86_64.rpm"
245
+ if elversion.to_i == 6
246
+ rpms["openssl"] = "https://s3.amazonaws.com/cloudamatic/mussl-1.1.1h-1.el6.x86_64.rpm"
247
+ rpms["sqlite"] = "https://s3.amazonaws.com/cloudamatic/muqlite-3.33-1.el6.x86_64.rpm"
248
+ end
249
+ if elversion.to_i == 7
250
+ rpms["mugit"] = "https://s3.amazonaws.com/cloudamatic/mugit-2.30.0-1.el7.x86_64.rpm"
251
+ end
252
+ # this takes up a huge amount of space, save it until we're fully operational
253
+ if !RUNNING_STANDALONE
254
+ rpms["python38"] = "https://s3.amazonaws.com/cloudamatic/muthon-3.8.3-1.el#{elversion}.x86_64.rpm"
255
+ end
227
256
 
228
257
  package basepackages
229
258
 
@@ -298,25 +327,25 @@ rpm_package "Chef Server upgrade package" do
298
327
  only_if { RUNNING_STANDALONE }
299
328
  end
300
329
 
301
- # REMOVE OLD RUBYs
302
- execute "clean up old Ruby 2.1.6" do
303
- command "rm -rf /opt/rubies/ruby-2.1.6"
304
- ignore_failure true
305
- only_if { ::Dir.exist?("/opt/rubies/ruby-2.1.6") }
306
- end
307
-
308
- execute "Kill ruby-2.3.1" do
330
+ execute "clean up old ruby-2.3.1 package" do
309
331
  command "yum erase ruby23-2.3.1-1.el7.centos.x86_64 -y; rpm -e ruby23"
310
332
  ignore_failure true
311
333
  only_if { ::Dir.exist?("/opt/rubies/ruby-2.3.1") }
312
334
  end
313
-
314
- execute "clean up old ruby-2.3.1" do
315
- command "rm -rf /opt/rubies/ruby-2.3.1"
335
+ execute "clean up old muby-2.5.3 package" do
336
+ command "yum erase muby-2.5.3-1.el7.x86_64 -y"
316
337
  ignore_failure true
317
- only_if { ::Dir.exist?("/opt/rubies/ruby-2.3.1") }
338
+ only_if "rpm -q muby-2.5.3"
318
339
  end
319
340
 
341
+ %w{2.1.6 2.3.1 2.5.3}.each { |v|
342
+ execute "clean up old ruby-#{v} directory" do
343
+ command "rm -rf /opt/rubies/ruby-#{v}"
344
+ ignore_failure true
345
+ only_if { ::Dir.exist?("/opt/rubies/ruby-#{v}") }
346
+ end
347
+ }
348
+
320
349
  execute "yum makecache" do
321
350
  action :nothing
322
351
  end
@@ -325,11 +354,14 @@ end
325
354
  rpms.each_pair { |pkg, src|
326
355
  rpm_package pkg do
327
356
  source src
328
- if pkg == "ruby25"
357
+ if pkg == "ruby27"
329
358
  options '--prefix=/opt/rubies/'
330
359
  end
331
360
  if pkg == "epel-release"
332
361
  notifies :run, "execute[yum makecache]", :immediately
362
+ if elversion.to_i == 6
363
+ not_if "rpm -q epel-release"
364
+ end
333
365
  end
334
366
  if pkg == "chef-server-core"
335
367
  notifies :stop, "service[iptables]", :before
@@ -352,6 +384,10 @@ package removepackages do
352
384
  end
353
385
 
354
386
 
387
+ if rpms["mugit"]
388
+ ENV['PATH'] = "/usr/local/git-current/bin:"+ENV['PATH']
389
+ end
390
+
355
391
 
356
392
  file "initial chef-server.rb" do
357
393
  path "/etc/opscode/chef-server.rb"
@@ -364,6 +400,7 @@ nginx['ssl_port'] = 7443
364
400
  nginx['ssl_ciphers'] = 'HIGH:MEDIUM:!LOW:!kEDH:!aNULL:!ADH:!eNULL:!EXP:!SSLv2:!SEED:!CAMELLIA:!PSK'
365
401
  nginx['ssl_protocols'] = 'TLSv1.2'
366
402
  bookshelf['external_url'] = 'https://127.0.0.1:7443'
403
+ bookshelf['vip'] = server_name
367
404
  bookshelf['vip_port'] = 7443\n"
368
405
  not_if { ::File.size?("/etc/opscode/chef-server.rb") }
369
406
  notifies :run, "execute[reconfigure Chef server]", :immediately
@@ -406,13 +443,71 @@ remote_file "#{MU_BASE}/bin/mu-self-update" do
406
443
  mode 0755
407
444
  end
408
445
 
409
- bash "install modules for our built-in Python" do
410
- code <<-EOH
411
- /usr/local/python-current/bin/pip install -r #{MU_BASE}/lib/requirements.txt
412
- EOH
446
+ # Skip this during initial installs, it's space-hungry
447
+ if !RUNNING_STANDALONE
448
+ bash "install modules for our built-in Python" do
449
+ code <<-EOH
450
+ /usr/local/python-current/bin/pip install -r #{MU_BASE}/lib/requirements.txt
451
+ EOH
452
+ end
453
+ end
454
+
455
+ # bundle a less heavy version of our Gemfile during initial installation, so we
456
+ # can actually fit on normal root disks until we have enough code and
457
+ # credentials to roll a dedicated /opt.
458
+ TMPDIR = Dir.mktmpdir
459
+ gemfile_dir = if RUNNING_STANDALONE and !File.readlines("/etc/mtab").grep(/\s\/opt\s/).any?
460
+ ruby_block "set up alternate install-time Gemfile" do # ~FC014
461
+ block do
462
+ exclude_gems = %w{aws-sdk azure_sdk google-api-client}
463
+
464
+ ["/sys/hypervisor/uuid",
465
+ "/sys/devices/virtual/dmi/id/product_uuid",
466
+ "/sys/devices/virtual/dmi/id/board_asset_tag"].each { |src|
467
+ if File.exists?(src)
468
+ uuid = File.read(src).chomp
469
+ if uuid and uuid =~ /^ec2/i
470
+ exclude_gems.delete("aws-sdk")
471
+ end
472
+ break
473
+ end
474
+ }
475
+ dmiout = shell_out!(%Q{PATH=/sbin:/usr/sbin:/bin:/usr/bin dmidecode})
476
+ if dmiout.match(/Google/)
477
+ exclude_gems.delete("google-api-client")
478
+ end
479
+
480
+ if File.exists?("/var/log/waagent.log") and File.read("/var/log/waagent.log") =~ /added Azure fabric/
481
+ exclude_gems.delete("azure_sdk")
482
+ end
483
+
484
+ f = File.open("#{TMPDIR}/cloud-mu.gemspec", "w")
485
+ File.read("#{MU_BASE}/lib/cloud-mu.gemspec").each_line { |l|
486
+ skipme = false
487
+ if l=~ /s\.add_runtime_dependency/
488
+ exclude_gems.each { |gem|
489
+ if l =~ /\b#{gem}\b/
490
+ skipme = true
491
+ end
492
+ }
493
+ next if skipme
494
+ end
495
+ f.puts l.chomp
496
+ }
497
+ f.close
498
+
499
+ Dir.mkdir("#{TMPDIR}/modules")
500
+ FileUtils.cp("#{MU_BASE}/lib/modules/Gemfile", "#{TMPDIR}/modules")
501
+ end
502
+ end
503
+ "#{TMPDIR}/modules"
504
+ else
505
+ "#{MU_BASE}/lib/modules"
413
506
  end
414
507
 
415
- ["/usr/local/ruby-current", "/opt/chef/embedded"].each { |rubydir|
508
+ rubies = ["/usr/local/ruby-current", "/opt/chef/embedded"]
509
+
510
+ rubies.each { |rubydir|
416
511
  gembin = rubydir+"/bin/gem"
417
512
  gemdir = Dir.glob("#{rubydir}/lib/ruby/gems/?.?.?/gems").last
418
513
  bundler_path = gembin.sub(/gem$/, "bundle")
@@ -424,19 +519,24 @@ end
424
519
  EOH
425
520
  action :nothing
426
521
  end
427
- gem_package bundler_path do
522
+
523
+ gem_package "bundler for #{rubydir}" do
428
524
  gem_binary gembin
429
525
  package_name "bundler"
430
526
  if rubydir == "/usr/local/ruby-current" or File.exists?(bundler_path)
431
527
  action :upgrade
432
528
  ignore_failure true
433
529
  end
530
+ version "~> 2.1.4"
434
531
  notifies :run, "bash[fix #{rubydir} gem permissions]", :delayed
435
532
  end
436
- execute "#{bundler_path} install" do
437
- cwd "#{MU_BASE}/lib/modules"
438
- umask 0022
439
- not_if "#{bundler_path} check"
533
+ execute "#{bundler_path} install from #{gemfile_dir} for #{rubydir}" do
534
+ command "PATH=/usr/local/git-current/bin:/usr/local/git-current/libexec/git-core:${PATH} #{bundler_path} install"
535
+ cwd gemfile_dir
536
+ umask "0022"
537
+ if !RUNNING_STANDALONE
538
+ not_if { system("cd #{gemfile_dir} && #{bundler_path} check 2>&1"); $?.exitstatus }
539
+ end
440
540
  notifies :run, "bash[fix #{rubydir} gem permissions]", :delayed
441
541
  notifies :restart, "service[chef-server]", :delayed if rubydir == "/opt/opscode/embedded"
442
542
  # XXX notify mommacat if we're *not* in chef-apply... RUNNING_STANDALONE
@@ -463,7 +563,7 @@ end
463
563
  # This is mostly to make sure Berkshelf has a clean and current environment to
464
564
  # live with.
465
565
  execute "/usr/local/ruby-current/bin/bundle clean --force" do
466
- cwd "#{MU_BASE}/lib/modules"
566
+ cwd gemfile_dir
467
567
  only_if { RUNNING_STANDALONE }
468
568
  end
469
569
 
@@ -490,29 +590,31 @@ require "simple-password-gen"
490
590
  # XXX this would make an awesome library
491
591
  execute "create mu Chef user" do
492
592
  command "/opt/opscode/bin/chef-server-ctl user-create mu Mu Master root@example.com #{Password.pronounceable} -f #{MU_BASE}/var/users/mu/mu.user.key"
493
- umask 0277
593
+ umask "0277"
494
594
  not_if "/opt/opscode/bin/chef-server-ctl user-list | grep '^mu$'"
595
+ notifies :start, "service[chef-server]", :before
495
596
  end
496
597
  execute "create mu Chef org" do
497
598
  command "/opt/opscode/bin/chef-server-ctl org-create mu mu -a mu -f #{MU_BASE}/var/orgs/mu/mu.org.key"
498
- umask 0277
599
+ umask "0277"
499
600
  not_if "/opt/opscode/bin/chef-server-ctl org-list | grep '^mu$'"
601
+ notifies :start, "service[chef-server]", :before
500
602
  end
501
603
  # TODO copy in ~/.chef/mu.*.key to /opt/mu/var/users/mu if the stuff already exists
502
604
  file "initial root knife.rb" do
503
605
  path "/root/.chef/knife.rb"
504
606
  content "
505
- node_name 'mu'
506
- client_key '#{MU_BASE}/var/users/mu/mu.user.key'
507
- validation_client_name 'mu-validator'
508
- validation_key '#{MU_BASE}/var/orgs/mu/mu.org.key'
509
- chef_server_url 'https://127.0.0.1:7443/organizations/mu'
510
- chef_server_root 'https://127.0.0.1:7443/organizations/mu'
511
- syntax_check_cache_path '/root/.chef/syntax_check_cache'
512
- cookbook_path [ '/root/.chef/cookbooks', '/root/.chef/site_cookbooks' ]
513
- ssl_verify_mode :verify_none
514
- knife[:vault_mode] = 'client'
515
- knife[:vault_admins] = ['mu']\n"
607
+ node_name 'mu'
608
+ client_key '#{MU_BASE}/var/users/mu/mu.user.key'
609
+ validation_client_name 'mu-validator'
610
+ validation_key '#{MU_BASE}/var/orgs/mu/mu.org.key'
611
+ chef_server_url 'https://127.0.0.1:7443/organizations/mu'
612
+ chef_server_root 'https://127.0.0.1:7443/organizations/mu'
613
+ syntax_check_cache_path '/root/.chef/syntax_check_cache'
614
+ cookbook_path [ '/root/.chef/cookbooks', '/root/.chef/site_cookbooks' ]
615
+ ssl_verify_mode :verify_none
616
+ knife[:vault_mode] = 'client'
617
+ knife[:vault_admins] = ['mu']\n"
516
618
  only_if { !::File.size?("/root/.chef/knife.rb") }
517
619
  notifies :run, "execute[initial Chef artifact upload]", :immediately
518
620
  end
@@ -530,36 +632,64 @@ if SSH_DIR != ROOT_SSH_DIR
530
632
  mode 0700
531
633
  end
532
634
  end
533
- bash "add localhost ssh to authorized_keys and config" do
635
+ bash "add localhost ssh to config" do
534
636
  code <<-EOH
535
- cat #{ROOT_SSH_DIR}/id_rsa.pub >> #{SSH_DIR}/authorized_keys
536
637
  echo "Host localhost" >> #{ROOT_SSH_DIR}/config
537
638
  echo " IdentityFile #{ROOT_SSH_DIR}/id_rsa" >> #{ROOT_SSH_DIR}/config
538
639
  EOH
539
640
  action :nothing
540
641
  end
541
642
  execute "ssh-keygen -N '' -f #{ROOT_SSH_DIR}/id_rsa" do
542
- umask 0177
643
+ umask "0177"
543
644
  not_if { ::File.exist?("#{ROOT_SSH_DIR}/id_rsa") }
544
- notifies :run, "bash[add localhost ssh to authorized_keys and config]", :immediately
645
+ notifies :run, "bash[add localhost ssh to config]", :immediately
646
+ notifies :run, "execute[add localhost key to authorized_keys]", :immediately
647
+ end
648
+ execute "add localhost key to authorized_keys" do
649
+ command "cat #{ROOT_SSH_DIR}/id_rsa.pub >> #{SSH_DIR}/authorized_keys"
650
+ only_if {
651
+ found = false
652
+ pubkey = if File.exists?("#{SSH_DIR}/authorized_keys")
653
+ File.read("#{ROOT_SSH_DIR}/id_rsa.pub").chomp
654
+ end
655
+ if pubkey and File.exists?("#{SSH_DIR}/authorized_keys")
656
+ authfile = File.read("#{ROOT_SSH_DIR}/authorized_keys")
657
+ authfile.each_line { |l|
658
+ if l =~ /#{Regexp.quote(pubkey)}/
659
+ found = true
660
+ end
661
+ }
662
+ end
663
+ !found
664
+ }
545
665
  end
546
- file "/etc/chef/client.pem" do
666
+ # XXX foodcritic says this is a repeat declaration, but it's... not
667
+ file "/etc/chef/client.pem" do # ~FC005
547
668
  action :nothing
548
669
  end
549
670
  file "/etc/chef/validation.pem" do
550
671
  action :nothing
551
672
  end
673
+ file "/etc/chef/client.rb" do
674
+ action :nothing
675
+ end
676
+
677
+ knife_cfg = "-c /root/.chef/knife.rb"
552
678
 
553
679
  execute "create MU-MASTER Chef client" do
680
+ # XXX I dislike --ssh-verify-host-key=never intensely, but the CLI-documented 'accept_new' doesn't actually work
554
681
  if SSH_USER == "root"
555
- command "/opt/chef/bin/knife bootstrap -N MU-MASTER --no-node-verify-api-cert --node-ssl-verify-mode=none 127.0.0.1"
682
+ command "/opt/chef/bin/knife bootstrap #{knife_cfg} -N MU-MASTER --no-node-verify-api-cert --node-ssl-verify-mode=none -U #{SSH_USER} --ssh-identity-file=/root/.ssh/id_rsa --ssh-verify-host-key=never 127.0.0.1"
556
683
  else
557
- command "/opt/chef/bin/knife bootstrap -N MU-MASTER --no-node-verify-api-cert --node-ssl-verify-mode=none -x #{SSH_USER} --sudo 127.0.0.1"
684
+ command "/opt/chef/bin/knife bootstrap #{knife_cfg} -N MU-MASTER --no-node-verify-api-cert --node-ssl-verify-mode=none -U #{SSH_USER} --ssh-identity-file=/root/.ssh/id_rsa --ssh-verify-host-key=never --sudo 127.0.0.1"
558
685
  end
559
- not_if "/opt/chef/bin/knife node list | grep '^MU-MASTER$'"
560
- only_if "/opt/chef/bin/knife ssl check" # make sure we don't wipe ourselves due to unrelated SSL issues
686
+ only_if "/opt/chef/bin/knife node #{knife_cfg} list" # don't do crazy stuff just because knife isn't working
687
+ not_if "/opt/chef/bin/knife node #{knife_cfg} list | grep '^MU-MASTER$'"
688
+ notifies :run, "execute[add localhost key to authorized_keys]", :before
689
+ notifies :delete, "file[/etc/chef/client.rb]", :before
561
690
  notifies :delete, "file[/etc/chef/client.pem]", :before
562
691
  notifies :delete, "file[/etc/chef/validation.pem]", :before
692
+ notifies :start, "service[chef-server]", :before
563
693
  only_if { RUNNING_STANDALONE }
564
694
  end
565
695
 
@@ -575,7 +705,7 @@ end
575
705
 
576
706
  # Community cookbooks keep touching gems, and none of them are smart about our
577
707
  # default umask. We have to clean up after them every time.
578
- ["/usr/local/ruby-current", "/opt/chef/embedded"].each { |rubydir|
708
+ rubies.each { |rubydir|
579
709
  execute "trigger permission fix in #{rubydir}" do
580
710
  command "ls /etc/motd > /dev/null"
581
711
  notifies :run, "bash[fix #{rubydir} gem permissions]", :delayed
@@ -588,3 +718,8 @@ bash "fix misc permissions" do
588
718
  chmod go+rx #{MU_BASE}/lib/bin/* #{MU_BASE}/lib/extras/*-stock-* #{MU_BASE}/lib/extras/vault_tools/*.sh
589
719
  EOH
590
720
  end
721
+
722
+ directory TMPDIR do
723
+ action :delete
724
+ recursive true
725
+ end