cloud-mu 2.0.4 → 2.1.0beta

data/cookbooks/mu-master/recipes/init.rb

@@ -173,7 +173,7 @@ dpkgs = {}
 
 elversion = node['platform_version'].to_i > 2000 ? 6 : node['platform_version'].to_i
 if platform_family?("rhel")
-  basepackages = ["git", "curl", "diffutils", "patch", "gcc", "gcc-c++", "make", "postgresql-devel", "libyaml", "libffi-devel"]
+  basepackages = ["git", "curl", "diffutils", "patch", "gcc", "gcc-c++", "make", "postgresql-devel", "libyaml", "libffi-devel", "tcl", "tk"]
 #        package epel-release-6-8.9.amzn1.noarch (which is newer than epel-release-6-8.noarch) is already installed
 
   rpms = {
@@ -189,13 +189,15 @@ if platform_family?("rhel")
   elsif elversion < 7
     basepackages.concat(["mysql-devel"])
     rpms["ruby25"] = "https://s3.amazonaws.com/cloudamatic/muby-2.5.3-1.el6.x86_64.rpm"
+    rpms["python27"] = "https://s3.amazonaws.com/cloudamatic/muthon-2.7.16-1.el6.x86_64.rpm"
     
     removepackages = ["nagios"]
 
   # RHEL7, CentOS7
   elsif elversion < 8
-    basepackages.concat(["libX11", "tcl", "tk", "mariadb-devel", "cryptsetup"])
+    basepackages.concat(["libX11", "mariadb-devel", "cryptsetup"])
     rpms["ruby25"] = "https://s3.amazonaws.com/cloudamatic/muby-2.5.3-1.el7.x86_64.rpm"
+    rpms["python27"] = "https://s3.amazonaws.com/cloudamatic/muthon-2.7.16-1.el7.x86_64.rpm"
     removepackages = ["nagios", "firewalld"]
   end
   # Amazon Linux
@@ -284,20 +286,19 @@ end
 # REMOVE OLD RUBYs
 execute "clean up old Ruby 2.1.6" do
   command "rm -rf /opt/rubies/ruby-2.1.6"
+  ignore_failure true
   only_if { ::Dir.exist?("/opt/rubies/ruby-2.1.6") }
 end
 
-yum_package 'ruby23-2.3.1-1.el7.centos.x86_64' do
-  action :purge
-end
-
 execute "Kill ruby-2.3.1" do
   command "yum erase ruby23-2.3.1-1.el7.centos.x86_64 -y; rpm -e ruby23"
+  ignore_failure true
   only_if { ::Dir.exist?("/opt/rubies/ruby-2.3.1") }
 end
 
 execute "clean up old ruby-2.3.1" do
   command "rm -rf /opt/rubies/ruby-2.3.1"
+  ignore_failure true
   only_if { ::Dir.exist?("/opt/rubies/ruby-2.3.1") }
 end
 
@@ -320,6 +321,7 @@ rpms.each_pair { |pkg, src|
     end
   end
 }
+
 package ["jq"] do
   ignore_failure true # sometimes we can't see EPEL immediately
 end
@@ -382,6 +384,12 @@ remote_file "#{MU_BASE}/bin/mu-self-update" do
   mode 0755
 end
 
+bash "install modules for our built-in Python" do
+  code <<-EOH
+    /usr/local/python-current/bin/pip install -r #{MU_BASE}/lib/requirements.txt
+  EOH
+end
+
 ["/usr/local/ruby-current", "/opt/chef/embedded"].each { |rubydir|
   gembin = rubydir+"/bin/gem"
   gemdir = Dir.glob("#{rubydir}/lib/ruby/gems/?.?.?/gems").last
@@ -399,7 +407,6 @@ end
     package_name "bundler"
     action :upgrade if rubydir == "/usr/local/ruby-current"
     notifies :run, "bash[fix #{rubydir} gem permissions]", :delayed
-    options('-q --no-documentation')
   end
   execute "#{bundler_path} install" do
     cwd "#{MU_BASE}/lib/modules"
@@ -421,7 +428,6 @@ end
         action :remove
         only_if { ::Dir.exist?(dir) }
         only_if { ::Dir.exist?(gemdir) }
-        options('-q --no-documentation')
       end
       execute "rm -rf #{gemdir}/knife-windows-#{Regexp.last_match[1]}"
     }
@@ -546,7 +552,7 @@ end
 file "#{MU_BASE}/etc/mu.rc" do
   content %Q{export MU_INSTALLDIR="#{MU_BASE}"
 export MU_DATADIR="#{MU_BASE}/var"
-export PATH="#{MU_BASE}/bin:/usr/local/ruby-current/bin:${PATH}:/opt/opscode/embedded/bin"
+export PATH="#{MU_BASE}/bin:/usr/local/ruby-current/bin:/usr/local/python-current/bin:${PATH}:/opt/opscode/embedded/bin"
 }
   mode 0644
   action :create_if_missing

data/cookbooks/mu-master/templates/default/mu.rc.erb

@@ -1,7 +1,7 @@
 # bash/sh environment support for Mu tools. Intended for the system (root)
 # user. Regular users get a .murc installed by mu-user-manage, from the template
 # in <%= @installdir %>/lib/install/user-dot-murc.erb
-export PATH="<%= @installdir %>/bin:/usr/local/ruby-current/bin:${PATH}:/opt/opscode/embedded/bin"
+export PATH="<%= @installdir %>/bin:/usr/local/ruby-current/bin:/usr/local/python-current/bin:${PATH}:/opt/opscode/embedded/bin"
 export MU_INSTALLDIR="<%= @installdir %>"
 export MU_DATADIR="<%= @installdir %>/var"
 <% if @repos %>

data/cookbooks/mu-master/templates/default/web_app.conf.erb

@@ -30,10 +30,6 @@
   ProxyPass /scratchpad https://localhost:2260/scratchpad
   ProxyPassReverse /scratchpad https://localhost:2260/scratchpad
 
-  # Jenkins CI web interface
-  ProxyPass /jenkins http://localhost:8080/jenkins
-  ProxyPassReverse /jenkins http://localhost:8080/jenkins
-
   # Nagios web UI
   ProxyPass /nagios/ https://localhost:8443/nagios/
   ProxyPassReverse /nagios/ https://localhost:8443/nagios/

data/cookbooks/mu-php54/metadata.rb

@@ -8,7 +8,7 @@ long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
 source_url 'https://github.com/cloudamatic/mu'
 issues_url 'https://github.com/cloudamatic/mu/issues'
 chef_version '>= 14.0' if respond_to?(:chef_version)
-version '0.3.1'
+version '0.3.0'
 
 %w( centos ubuntu ).each do |os|
 	supports os
@@ -18,4 +18,4 @@ depends 'mu-utility'
 depends 'simple_iptables', '~> 0.8.0'
 depends 'apache2', '< 4.0'
 depends 'mysql', '~> 8.5.1'
-depends 'yum-epel', '~> 3.2.0'
+depends 'yum-epel', '~> 3.2.0'

data/cookbooks/mu-php54/recipes/default.rb

@@ -38,9 +38,7 @@ case node['platform']
 
     # What we really mean is "chef_gem" but that insists on running
     # at compile time, before any of its dependencies are ready.
-    gem_package "mysql" do
-      options('-q --no-documentation')
-    end
+    gem_package "mysql"
 
     # Sundry libraries for PHP
     ["libmcrypt", "libmcrypt-devel", "php-devel", "php-pdo", "php-mysql", "php-pgsql", "php-gd", "php-pspell", "php-snmp", "php-xmlrpc", "php-xml", "php-mbstring", "php-mcrypt", "php-pear"].each { |pkg|

data/cookbooks/mu-tools/recipes/eks.rb

@@ -93,7 +93,7 @@ EOH
     source "https://s3-us-west-2.amazonaws.com/amazon-eks/1.10.3/2018-06-05/eks-2017-11-01.normal.json"
   end
 
-  execute "aws configure add-model --service-model file://root/.aws/eks/eks-2017-11-01.normal.json --service-name eks"
+  execute "aws configure add-model --service-model file:///root/.aws/eks/eks-2017-11-01.normal.json --service-name eks"
 
   execute "systemctl daemon-reload" do
     action :nothing
@@ -112,10 +112,33 @@ EOH
     notifies :restart, "service[kubelet]", :delayed
   end
 
+  file "/etc/systemd/system/kubelet.service.d/10-kubelet-args.conf" do
+    content "[Service]
+Environment='KUBELET_ARGS=--node-ip=#{get_aws_metadata("meta-data/local-ipv4")} --pod-infra-container-image=602401143452.dkr.ecr.#{region}.amazonaws.com/eks/pause-amd64:3.1'"
+    notifies :run, "execute[systemctl daemon-reload]", :immediately
+    notifies :restart, "service[kubelet]", :delayed
+  end
+
+  template "/etc/kubernetes/kubelet/kubelet-config.json" do
+    source "kubelet-config.json.erb"
+    variables(
+      :dns => get_first_nameserver(),
+    )
+    notifies :restart, "service[kubelet]", :delayed
+  end
+
+  file "/etc/systemd/system/kubelet.service.d/30-kubelet-extra-args.conf" do
+    content "[Service]
+Environment='KUBELET_EXTRA_ARGS=$KUBELET_EXTRA_ARGS'
+"
+    notifies :restart, "service[kubelet]", :delayed
+    notifies :run, "execute[systemctl daemon-reload]", :immediately
+  end
+
   directory "/root/.kube"
 
   remote_file "/usr/bin/aws-iam-authenticator" do
-    source "https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator"
+    source "https://amazon-eks.s3-us-west-2.amazonaws.com/1.12.7/2019-03-27/bin/linux/amd64/aws-iam-authenticator"
     mode 0755
     not_if "test -f /usr/bin/aws-iam-authenticator"
   end

data/cookbooks/mu-tools/recipes/nrpe.rb

@@ -82,7 +82,12 @@ if !node['application_attributes']['skip_recipes'].include?('nrpe')
     service "nrpe" do
       action [:enable, :start]
     end
-  
+
+    # Workaround for Amazon Linux/Chef 14 problem in nrpe cookbook
+    # https://github.com/sous-chefs/nrpe/issues/96
+    node.normal['nrpe']['plugin_dir'] = "/usr/lib64/nagios/plugins"
+    node.save
+
     nrpe_check "check_disk" do
       command "#{node['nrpe']['plugin_dir']}/check_disk"
       warning_condition '15%'

data/cookbooks/mu-tools/recipes/set_mu_hostname.rb

@@ -60,6 +60,14 @@ if !node['application_attributes']['skip_recipes'].include?('set_mu_hostname')
         file "/etc/hostname" do
           content $hostname
         end
+      elsif node['platform'] == "amazon"
+        file "/etc/hostname" do
+          content $hostname
+        end
+        execute "set hostname" do
+          command "hostname #{$hostname}"
+          not_if "test \"`hostname`\" = \"#{$hostname}\" "
+        end
       else
         execute "set hostname" do
           command "hostname #{$hostname}"

data/cookbooks/mu-tools/templates/default/etc_hosts.erb

@@ -6,7 +6,7 @@
 # doing only private IPs although that can be problematic 
 # if the same deploy has cross VPC or cross region resources
   if n.name != @hostname %>
-<%= n['ipaddress'] %> <%= n.name %>
+<%= n.ipaddress %> <%= n.name %>
 <%
   end
   }

data/cookbooks/mu-tools/templates/default/kubeconfig.erb

@@ -3,10 +3,10 @@ clusters:
 - cluster:
     server: <%= @endpoint %>
     certificate-authority-data: <%= @cacert %>
-  name: kubernetes
+  name: <%= @cluster %>
 contexts:
 - context:
-    cluster: kubernetes
+    cluster: <%= @cluster %>
     user: aws
   name: aws
 current-context: aws

data/cookbooks/mu-tools/templates/default/kubelet-config.json.erb

@@ -0,0 +1,35 @@
+{
+  "kind": "KubeletConfiguration",
+  "apiVersion": "kubelet.config.k8s.io/v1beta1",
+  "address": "0.0.0.0",
+  "clusterDNS": "<%= @dns %>",
+  "authentication": {
+    "anonymous": {
+      "enabled": false
+    },
+    "webhook": {
+      "cacheTTL": "2m0s",
+      "enabled": true
+    },
+    "x509": {
+      "clientCAFile": "/etc/kubernetes/pki/ca.crt"
+    }
+  },
+  "authorization": {
+    "mode": "Webhook",
+    "webhook": {
+      "cacheAuthorizedTTL": "5m0s",
+      "cacheUnauthorizedTTL": "30s"
+    }
+  },
+  "clusterDomain": "cluster.local",
+  "hairpinMode": "hairpin-veth",
+  "cgroupDriver": "cgroupfs",
+  "cgroupRoot": "/",
+  "featureGates": {
+    "RotateKubeletServerCertificate": true
+  },
+  "serializeImagePulls": false,
+  "serverTLSBootstrap": true,
+  "configMapAndSecretChangeDetectionStrategy": "Cache"
+}

data/extras/clean-stock-amis

@@ -18,16 +18,22 @@ require 'json'
 require File.realpath(File.expand_path(File.dirname(__FILE__)+"/../bin/mu-load-config.rb"))
 require 'mu'
 
+credentials = if ARGV[0] and !ARGV[0].empty?
+  ARGV[0]
+else
+  nil
+end
+
 filters = [
   {
     name: "owner-id",
-    values: [MU.account_number]
+    values: [MU::Cloud::AWS.credToAcct(credentials)]
   }
 ]
 
 
 MU::Cloud::AWS.listRegions.each { | r|
-  images = MU::Cloud::AWS.ec2(r).describe_images(
+  images = MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_images(
     filters: filters + [{ "name" => "state", "values" => ["available"]}]
   ).images
   images.each { |ami|
@@ -39,9 +45,9 @@ MU::Cloud::AWS.listRegions.each { | r|
 				end
 			}
 			MU.log "Deregistering #{ami.name} (#{ami.creation_date})", MU::WARN, details: snaps
-			MU::Cloud::AWS.ec2(r).deregister_image(image_id: ami.image_id)
+			MU::Cloud::AWS.ec2(region: r, credentials: credentials).deregister_image(image_id: ami.image_id)
 			snaps.each { |snap_id|
-				MU::Cloud::AWS.ec2(r).delete_snapshot(snapshot_id: snap_id)
+				MU::Cloud::AWS.ec2(region: r, credentials: credentials).delete_snapshot(snapshot_id: snap_id)
 			}
 		end
   }

data/extras/list-stock-amis

@@ -0,0 +1,64 @@
+#!/usr/local/ruby-current/bin/ruby
+# Copyright:: Copyright (c) 2019 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'optimist'
+require 'json'
+require 'yaml'
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/../bin/mu-load-config.rb"))
+require 'mu'
+
+credentials = if ARGV[0] and !ARGV[0].empty?
+  ARGV[0]
+else
+  nil
+end
+
+filters = [
+  {
+    name: "owner-id",
+    values: [MU::Cloud::AWS.credToAcct(credentials)]
+  }
+]
+
+platforms = {}
+
+MU::Cloud::AWS.listRegions.each { | r|
+  images = MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_images(
+    filters: filters + [{ "name" => "state", "values" => ["available"]}]
+  ).images
+  images.each { |ami|
+		if (DateTime.now.to_time - DateTime.parse(ami.creation_date).to_time) < 15552000 and ami.name.match(/^MU-PROD-\d{10}-[A-Z]{2}-(.*)/)
+      platform = Regexp.last_match[1].downcase
+      next if !platform
+      platforms[platform] ||= {}
+      if !platforms[platform][r] or
+         DateTime.parse(ami.creation_date).to_time > platforms[platform][r]['date']
+        platforms[platform][r] = {
+          "date" => DateTime.parse(ami.creation_date).to_time,
+          "ami" => ami.image_id,
+          "name" => ami.name
+        }
+      end
+		end
+  }
+}
+
+platforms.each_pair { |p, r_data|
+  r_data.each_pair { |r, data|
+    r_data[r] = data["ami"]
+  }
+}
+
+puts platforms.to_yaml

data/extras/python_rpm/build.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+rpm -q rpm-build || yum -y install rpm-build
+
+base="/opt/mu/lib/extras/python_rpm"
+
+for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS;do
+  mkdir -p ~/rpmbuild/$d
+done
+cd ~/rpmbuild
+
+echo "Temporarily deleting /usr/local/python-current so rpmbuild can create it"
+link="`readlink /usr/local/python-current`"
+rm -f /usr/local/python-current
+chmod 000 /usr/bin/python # otherwise this brain-dead build system tries to compile parts of itself with the wrong executable
+env -i PATH="/bin:/usr/bin" /usr/bin/rpmbuild -ba $base/muthon.spec
+chmod 755 /usr/bin/python
+find ~/rpmbuild/ -type f -name 'muthon*' -exec ls -la {} \;
+if [ "$link" != "" ];then
+  ln -s "$link" /usr/local/python-current
+fi

data/extras/python_rpm/muthon.spec

@@ -0,0 +1,68 @@
+Summary: Python for Mu
+BuildArch: x86_64
+Name: muthon
+Version: 2.7.16
+Release: 1%{dist}
+Group: Development/Languages
+License: Ruby License/GPL - see COPYING
+URL: https://www.python.org/
+Prefix: /opt/pythons
+Source: https://www.python.org/ftp/python/%{version}/Python-%{version}.tgz
+
+# auto-require inserts nonsensical things, like a dependency on our own
+# executable, so I guess we'll declare dependencies by package ourselves
+AutoReq: no
+# XXX these don't work for some reason
+#%global __requires_exclude ^/usr/local/bin/python$
+#%global __requires_exclude ^/opt/pythons/Python-%{version}/bin/python.*$
+
+BuildRequires: zlib-devel
+BuildRequires: tcl-devel
+BuildRequires: gdbm-devel
+BuildRequires: openssl-devel
+BuildRequires: sqlite-devel
+BuildRequires: tk-devel
+Requires: zlib
+Requires: gdbm
+Requires: tcl
+Requires: openssl
+Requires: glibc
+Requires: ncurses-libs
+Requires: sqlite
+Requires: tk
+
+%description
+I was sober when I wrote this spec file
+    
+%prep
+rm -rf $RPM_BUILD_DIR/Python-%{version}
+rm -rf %{prefix}
+test -f $RPM_SOURCE_DIR/Python-%{version}.tgz || ( cd $RPM_SOURCE_DIR && curl -O https://www.python.org/ftp/python/%{version}/Python-%{version}.tgz )
+curl https://bootstrap.pypa.io/get-pip.py -o $RPM_SOURCE_DIR/get-pip.py
+tar -xzvf $RPM_SOURCE_DIR/Python-%{version}.tgz
+mkdir -p $RPM_BUILD_ROOT%{prefix}
+rm -rf $RPM_BUILD_ROOT%{prefix}/Python-%{version}
+ln -s %{prefix}/Python-%{version} $RPM_BUILD_ROOT%{prefix}/Python-%{version}
+    
+%build
+cd $RPM_BUILD_DIR/Python-%{version}
+mkdir -p %{prefix}/Python-%{version}
+env -i PATH="/bin:/usr/bin" ./configure --prefix=%{prefix}/Python-%{version} --exec-prefix=%{prefix}/Python-%{version} --enable-shared LDFLAGS=-Wl,-rpath=%{prefix}/Python-%{version}/lib
+env -i PATH="/bin:/usr/bin" make
+
+%install
+cd $RPM_BUILD_DIR/Python-%{version}
+env -i PATH="/bin:/usr/bin" make install
+%{prefix}/Python-%{version}/bin/python $RPM_SOURCE_DIR/get-pip.py --prefix %{prefix}/Python-%{version}/
+mkdir -p $RPM_BUILD_ROOT%{prefix}
+mv %{prefix}/Python-%{version} $RPM_BUILD_ROOT%{prefix}/
+mkdir -p $RPM_BUILD_ROOT/usr/local/
+ln -s %{prefix}/Python-%{version} $RPM_BUILD_ROOT/usr/local/python-current
+
+%clean
+cd $RPM_BUILD_DIR/Python-%{version}
+make clean
+    
+%files
+%{prefix}/Python-%{version}/*
+/usr/local/python-current