cloud-mu 3.4.0 → 3.5.0

data/extras/clean-stock-amis

@@ -54,15 +54,15 @@ MU::Cloud::AWS.listRegions.each { | r|
         MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).deregister_image(image_id: ami.image_id)
       rescue Aws::EC2::Errors::InvalidAMIIDUnavailable
       end
-        snaps.each { |snap_id|
-          begin
-            MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).delete_snapshot(snapshot_id: snap_id)
-          rescue Aws::EC2::Errors::InvalidSnapshotInUse
-            sleep 5
-            retry
-          rescue Aws::EC2::Errors::InvalidSnapshotNotFound
-          end
-        }
+      snaps.each { |snap_id|
+        begin
+          MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).delete_snapshot(snapshot_id: snap_id)
+        rescue Aws::EC2::Errors::InvalidSnapshotInUse
+          sleep 5
+          retry
+        rescue Aws::EC2::Errors::InvalidSnapshotNotFound
+        end
+      }
     end
   }
 }

data/extras/git_rpm/build.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+rpm -q rpm-build || yum -y install rpm-build
+rpm -q libcurl-devel || yum -y install libcurl-devel
+
+base="/opt/mu/lib/extras/git_rpm"
+
+for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS;do
+  mkdir -p ~/rpmbuild/$d
+done
+cd ~/rpmbuild
+
+echo "Temporarily deleting /usr/local/git-current so rpmbuild can create it"
+link="`readlink /usr/local/git-current`"
+rm -f /usr/local/git-current
+env -i PATH="/bin:/usr/bin" /usr/bin/rpmbuild -ba $base/mugit.spec
+find ~/rpmbuild/ -type f -name 'mugit*' -exec ls -la {} \;
+if [ "$link" != "" ];then
+  ln -s "$link" /usr/local/git-current
+fi

data/extras/git_rpm/mugit.spec

@@ -0,0 +1,53 @@
+Summary: Git for Mu on RHEL7-compatible systems
+BuildArch: x86_64
+Name: mugit
+Version: 2.30.0
+Release: 1%{dist}
+Group: Development/Languages
+License: https://git-scm.com/about/free-and-open-source
+URL: https://git-scm.com/
+Prefix: /opt/git
+Source: https://github.com/git/git.git
+
+AutoReq: yes
+
+%description
+I was surprisingly chill when I wrote this spec file
+    
+%prep
+rm -rf $RPM_BUILD_DIR/git-%{version}
+rm -rf $RPM_SOURCE_DIR/git*
+rm -rf %{prefix}
+test -f $RPM_SOURCE_DIR/git-%{version}/GIT-VERSION-GEN || ( cd $RPM_SOURCE_DIR && git clone https://github.com/git/git.git )
+cd $RPM_SOURCE_DIR && mv git $RPM_BUILD_DIR/git-%{version}
+mkdir -p $RPM_BUILD_ROOT%{prefix}
+rm -rf $RPM_BUILD_ROOT%{prefix}/git-%{version}
+ln -s %{prefix}/git-%{version} $RPM_BUILD_ROOT%{prefix}/git-%{version}
+    
+%build
+cd $RPM_BUILD_DIR/git-%{version}
+mkdir -p %{prefix}/git-%{version}
+env -i PATH="/bin:/usr/bin" git checkout v%{version}
+env -i PATH="/bin:/usr/bin" rm -rf .git
+env -i PATH="/bin:/usr/bin" make configure
+env -i PATH="/bin:/usr/bin" ./configure --prefix=%{prefix}/git-%{version}
+env -i PATH="/bin:/usr/bin" make all
+
+%install
+cd $RPM_BUILD_DIR/git-%{version}
+env -i PATH="/bin:/usr/bin" make install
+mkdir -p $RPM_BUILD_ROOT%{prefix}
+mv %{prefix}/git-%{version} $RPM_BUILD_ROOT%{prefix}/
+mkdir -p $RPM_BUILD_ROOT/usr/local/
+ln -s %{prefix}/git-%{version} $RPM_BUILD_ROOT/usr/local/git-current
+# some idiot utility expects this to be present because reasons
+touch /rpmbuild/SOURCES/git.git
+
+%clean
+cd $RPM_BUILD_DIR/git-%{version}
+make clean
+rm -f /rpmbuild/SOURCES/git.git
+    
+%files
+%{prefix}/git-%{version}/*
+/usr/local/git-current

data/extras/image-generators/VMWare/centos8.yaml

@@ -0,0 +1,15 @@
+---
+  appname: mu
+  servers: 
+    - 
+      name: centos7
+      platform: centos7
+      size: m3.medium
+      scrub_groomer: true
+      run_list:
+      - recipe[mu-tools::apply_security]
+      - recipe[mu-tools::updates]
+      - recipe[mu-tools::split_var_partitions]
+      create_image:
+        image_then_destroy: true
+        public: true

data/extras/openssl_rpm/build.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+rpm -q rpm-build || yum -y install rpm-build
+
+base="/opt/mu/lib/extras/openssl_rpm"
+
+for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS;do
+  mkdir -p ~/rpmbuild/$d
+done
+cd ~/rpmbuild
+
+echo "Temporarily deleting /usr/local/openssl-current so rpmbuild can create it"
+link="`readlink /usr/local/openssl-current`"
+rm -f /usr/local/openssl-current
+env -i PATH="/bin:/usr/bin" /usr/bin/rpmbuild -ba $base/mussl.spec
+find ~/rpmbuild/ -type f -name 'mussl*' -exec ls -la {} \;
+if [ "$link" != "" ];then
+  ln -s "$link" /usr/local/openssl-current
+fi

data/extras/openssl_rpm/mussl.spec

@@ -0,0 +1,46 @@
+Summary: OpenSSL for Mu on RHEL6-compatible systems
+BuildArch: x86_64
+Name: mussl
+Version: 1.1.1h
+Release: 1%{dist}
+Group: Development/Languages
+License: https://www.openssl.org/source/license-openssl-ssleay.txt
+URL: https://www.openssl.org/
+Prefix: /opt/openssl
+Source: https://www.openssl.org/source/openssl-%{version}.tar.gz
+
+AutoReq: yes
+
+%description
+I was agitated when I wrote this spec file
+    
+%prep
+rm -rf $RPM_BUILD_DIR/openssl-%{version}
+rm -rf %{prefix}
+test -f $RPM_SOURCE_DIR/openssl-%{version}.tar.gz || ( cd $RPM_SOURCE_DIR && curl -O https://www.openssl.org/source/openssl-%{version}.tar.gz )
+tar -xzvf $RPM_SOURCE_DIR/openssl-%{version}.tar.gz
+mkdir -p $RPM_BUILD_ROOT%{prefix}
+rm -rf $RPM_BUILD_ROOT%{prefix}/openssl-%{version}
+ln -s %{prefix}/openssl-%{version} $RPM_BUILD_ROOT%{prefix}/openssl-%{version}
+    
+%build
+cd $RPM_BUILD_DIR/openssl-%{version}
+mkdir -p %{prefix}/openssl-%{version}
+env -i PATH="/bin:/usr/bin" ./config --prefix=%{prefix}/openssl-%{version}
+env -i PATH="/bin:/usr/bin" make
+
+%install
+cd $RPM_BUILD_DIR/openssl-%{version}
+env -i PATH="/bin:/usr/bin" make install
+mkdir -p $RPM_BUILD_ROOT%{prefix}
+mv %{prefix}/openssl-%{version} $RPM_BUILD_ROOT%{prefix}/
+mkdir -p $RPM_BUILD_ROOT/usr/local/
+ln -s %{prefix}/openssl-%{version} $RPM_BUILD_ROOT/usr/local/openssl-current
+
+%clean
+cd $RPM_BUILD_DIR/openssl-%{version}
+make clean
+    
+%files
+%{prefix}/openssl-%{version}/*
+/usr/local/openssl-current

data/extras/python_rpm/muthon.spec

@@ -1,10 +1,10 @@
 Summary: Python for Mu
 BuildArch: x86_64
 Name: muthon
-Version: 2.7.16
+Version: 3.8.3
 Release: 1%{dist}
 Group: Development/Languages
-License: Ruby License/GPL - see COPYING
+License: https://docs.python.org/3/license.html
 URL: https://www.python.org/
 Prefix: /opt/pythons
 Source: https://www.python.org/ftp/python/%{version}/Python-%{version}.tgz
@@ -16,12 +16,16 @@ AutoReq: no
 #%global __requires_exclude ^/usr/local/bin/python$
 #%global __requires_exclude ^/opt/pythons/Python-%{version}/bin/python.*$
 
+%{?el6:BuildRequires: mussl}
+%{?el6:BuildRequires: muqlite}
 BuildRequires: zlib-devel
 BuildRequires: tcl-devel
 BuildRequires: gdbm-devel
 BuildRequires: openssl-devel
 BuildRequires: sqlite-devel
 BuildRequires: tk-devel
+%{?el6:Requires: mussl}
+%{?el6:Requires: muqlite}
 Requires: zlib
 Requires: gdbm
 Requires: tcl
@@ -47,13 +51,19 @@ ln -s %{prefix}/Python-%{version} $RPM_BUILD_ROOT%{prefix}/Python-%{version}
 %build
 cd $RPM_BUILD_DIR/Python-%{version}
 mkdir -p %{prefix}/Python-%{version}
-env -i PATH="/bin:/usr/bin" ./configure --prefix=%{prefix}/Python-%{version} --exec-prefix=%{prefix}/Python-%{version} --enable-shared LDFLAGS=-Wl,-rpath=%{prefix}/Python-%{version}/lib
+%if 0%{?el6}
+# The SQLite library location logic is dain-bramaged
+sed -i "s/sqlite_inc_paths = \[ '\/usr\/include'/sqlite_inc_paths = \[ '\/usr\/local\/sqlite-current\/include'/" setup.py
+env -i PATH="/bin:/usr/bin" LDFLAGS="-L/usr/local/openssl-current/lib" ./configure --prefix=%{prefix}/Python-%{version} --exec-prefix=%{prefix}/Python-%{version} --enable-shared LDFLAGS=-Wl,-rpath=%{prefix}/Python-%{version}/lib,-rpath=/usr/local/openssl-current/lib,-rpath=/usr/local/sqlite-current/lib --with-openssl=/usr/local/openssl-current --enable-loadable-sqlite-extensions
+%else
+env -i PATH="/bin:/usr/bin" ./configure --prefix=%{prefix}/Python-%{version} --exec-prefix=%{prefix}/Python-%{version} --enable-shared LDFLAGS=-Wl,-rpath=%{prefix}/Python-%{version}/lib --enable-loadable-sqlite-extensions
+%endif
 env -i PATH="/bin:/usr/bin" make
 
 %install
 cd $RPM_BUILD_DIR/Python-%{version}
 env -i PATH="/bin:/usr/bin" make install
-%{prefix}/Python-%{version}/bin/python $RPM_SOURCE_DIR/get-pip.py --prefix %{prefix}/Python-%{version}/
+%{prefix}/Python-%{version}/bin/python3 $RPM_SOURCE_DIR/get-pip.py --prefix %{prefix}/Python-%{version}/ || ( ldd %{prefix}/Python-%{version}/bin/python3 ; exit 1 )
 mkdir -p $RPM_BUILD_ROOT%{prefix}
 mv %{prefix}/Python-%{version} $RPM_BUILD_ROOT%{prefix}/
 mkdir -p $RPM_BUILD_ROOT/usr/local/

data/extras/ruby_rpm/muby.spec

@@ -1,13 +1,13 @@
 Summary: Ruby for Mu(by)
 BuildArch: x86_64
 Name: muby
-Version: 2.5.5
+Version: 2.7.2
 Release: 1%{dist}
 Group: Development/Languages
 License: Ruby License/GPL - see COPYING
 URL: http://www.ruby-lang.org/
 Prefix: /opt/rubies
-Source: https://cache.ruby-lang.org/pub/ruby/2.5/ruby-%{version}.tar.gz
+Source: https://cache.ruby-lang.org/pub/ruby/2.7/ruby-%{version}.tar.gz
 
 BuildRequires: zlib
 BuildRequires: zlib-devel
@@ -19,7 +19,7 @@ I was drunk when I wrote this spec file
 %prep
 rm -rf $RPM_BUILD_DIR/ruby-%{version}
 rm -rf %{prefix}
-test -f $RPM_SOURCE_DIR/ruby-%{version}.tar.gz || ( cd $RPM_SOURCE_DIR && curl -O https://cache.ruby-lang.org/pub/ruby/2.5/ruby-%{version}.tar.gz )
+test -f $RPM_SOURCE_DIR/ruby-%{version}.tar.gz || ( cd $RPM_SOURCE_DIR && curl -O https://cache.ruby-lang.org/pub/ruby/2.7/ruby-%{version}.tar.gz )
 tar -xzvf $RPM_SOURCE_DIR/ruby-%{version}.tar.gz
 mkdir -p $RPM_BUILD_ROOT%{prefix}
 ln -s %{prefix}/ruby-%{version} $RPM_BUILD_ROOT%{prefix}/ruby-%{version}
@@ -33,11 +33,14 @@ make
 cd $RPM_BUILD_DIR/ruby-%{version}
 make install
 mkdir -p %{prefix}
-%{prefix}/ruby-%{version}/bin/gem install bundler
+yes | %{prefix}/ruby-%{version}/bin/gem install bundler --version '~> 2.1.4' --force
 mkdir -p $RPM_BUILD_ROOT%{prefix}
 mv %{prefix}/ruby-%{version} $RPM_BUILD_ROOT%{prefix}/
-mkdir -p $RPM_BUILD_ROOT/usr/local/
+mkdir -p $RPM_BUILD_ROOT/usr/local/bin
 ln -s %{prefix}/ruby-%{version} $RPM_BUILD_ROOT/usr/local/ruby-current
+mkdir -p /usr/local/bin
+rm -f $RPM_BUILD_ROOT/usr/local/bin/ruby
+ln -s /usr/local/ruby-current/bin/ruby $RPM_BUILD_ROOT/usr/local/bin/ruby
 
 %clean
 cd $RPM_BUILD_DIR/ruby-%{version}
@@ -46,3 +49,4 @@ make clean
 %files
 %{prefix}/ruby-%{version}/*
 /usr/local/ruby-current
+/usr/local/bin/ruby

data/extras/sqlite_rpm/build.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+rpm -q rpm-build || yum -y install rpm-build
+
+base="/opt/mu/lib/extras/sqlite_rpm"
+
+for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS;do
+  mkdir -p ~/rpmbuild/$d
+done
+cd ~/rpmbuild
+
+echo "Temporarily deleting /usr/local/sqlite-current so rpmbuild can create it"
+link="`readlink /usr/local/sqlite-current`"
+rm -f /usr/local/sqlite-current
+env -i PATH="/bin:/usr/bin" /usr/bin/rpmbuild -ba $base/muqlite.spec
+find ~/rpmbuild/ -type f -name 'muqlite*' -exec ls -la {} \;
+if [ "$link" != "" ];then
+  ln -s "$link" /usr/local/sqlite-current
+fi

data/extras/sqlite_rpm/muqlite.spec

@@ -0,0 +1,47 @@
+Summary: SQLite for Mu on RHEL6-compatible systems
+BuildArch: x86_64
+Name: muqlite
+Version: 3.33
+Release: 1%{dist}
+Group: Development/Languages
+License: https://www.sqlite.org/copyright.html
+URL: https://sqlite.org/
+Prefix: /opt/sqlite
+Source: https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=branch-%{version}
+
+AutoReq: yes
+
+%description
+I was surprisingly chill when I wrote this spec file
+    
+%prep
+rm -rf $RPM_BUILD_DIR/sqlite-%{version}
+rm -rf %{prefix}
+test -f $RPM_SOURCE_DIR/sqlite.tar.gz?r=branch-%{version} || ( cd $RPM_SOURCE_DIR && curl -O https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=branch-%{version} )
+tar -xzvf $RPM_SOURCE_DIR/sqlite.tar.gz?r=branch-%{version}
+mv sqlite sqlite-%{version}
+mkdir -p $RPM_BUILD_ROOT%{prefix}
+rm -rf $RPM_BUILD_ROOT%{prefix}/sqlite-%{version}
+ln -s %{prefix}/sqlite-%{version} $RPM_BUILD_ROOT%{prefix}/sqlite-%{version}
+    
+%build
+cd $RPM_BUILD_DIR/sqlite-%{version}
+mkdir -p %{prefix}/sqlite-%{version}
+env -i PATH="/bin:/usr/bin" ./configure --prefix=%{prefix}/sqlite-%{version}
+env -i PATH="/bin:/usr/bin" make
+
+%install
+cd $RPM_BUILD_DIR/sqlite-%{version}
+env -i PATH="/bin:/usr/bin" make install
+mkdir -p $RPM_BUILD_ROOT%{prefix}
+mv %{prefix}/sqlite-%{version} $RPM_BUILD_ROOT%{prefix}/
+mkdir -p $RPM_BUILD_ROOT/usr/local/
+ln -s %{prefix}/sqlite-%{version} $RPM_BUILD_ROOT/usr/local/sqlite-current
+
+%clean
+cd $RPM_BUILD_DIR/sqlite-%{version}
+make clean
+    
+%files
+%{prefix}/sqlite-%{version}/*
+/usr/local/sqlite-current

data/install/installer

@@ -2,7 +2,7 @@
 
 BOLD=`tput bold`
 NORM=`tput sgr0`
-CHEF_CLIENT_VERSION="14.13.11"
+CHEF_CLIENT_VERSION="16.9.29"
 if [ "$MU_BRANCH" == "" ];then
   MU_BRANCH="master"
   mydir="`dirname $0`"
@@ -23,12 +23,8 @@ elif [ "$DIST_VERSION" == "server" ];then # funny package name in RHEL6
 fi
 
 if [ "$DIST_VERSION" == 7 ];then
-  OPSCODE_CHEF_PKG="chef-server-core-$CHEF_SERVER_VERSION.el7.x86_64"
-  OPSCODE_CHEF_DL="https://packages.chef.io/stable/el/7/${OPSCODE_CHEF_PKG}.rpm"
   CHEF_CLIENT_PKG="chef-$CHEF_CLIENT_VERSION.el7.x86_64"
 else
-  OPSCODE_CHEF_PKG="chef-server-core-$CHEF_SERVER_VERSION.el6.x86_64"
-  OPSCODE_CHEF_DL="https://packages.chef.io/stable/el/6/${OPSCODE_CHEF_PKG}.rpm"
   CHEF_CLIENT_PKG="chef-$CHEF_CLIENT_VERSION.el6.x86_64"
 fi
 
@@ -43,7 +39,9 @@ if [ ! -f /opt/chef/bin/chef-apply ];then
 fi
 set -x
 set -e
+
 if ! /bin/rpm -q $CHEF_CLIENT_PKG > /dev/null ;then
+
   # Drop any old Chef packages laying around, first
   /usr/bin/yum -y erase chef || /bin/rpm -e chef
   /bin/rm -rf /opt/chef ~/.berkshelf ~/.chef /etc/chef
@@ -51,6 +49,10 @@ if ! /bin/rpm -q $CHEF_CLIENT_PKG > /dev/null ;then
   /bin/sh /root/chef-install.sh -v $CHEF_CLIENT_VERSION
 fi
 
+mkdir -p /etc/chef
+grep ^chef_license /etc/chef/client.rb || echo "chef_license 'accept'" >> /etc/chef/client.rb
+export CHEF_LICENSE="accept"
+
 
 if [ -d /opt/mu/lib/cookbooks/mu-master/recipes ];then
   /opt/chef/bin/chef-apply /opt/mu/lib/cookbooks/mu-master/recipes/init.rb

data/modules/mu.rb

@@ -16,8 +16,6 @@ require 'bundler/setup'
 require 'yaml'
 require 'socket'
 require 'net/http'
-gem 'aws-sdk-core'
-autoload :Aws, "aws-sdk-core"
 gem 'nokogiri'
 autoload :Nokogiri, "nokogiri"
 gem 'simple-password-gen'
@@ -320,14 +318,14 @@ module MU
   # @param always [Proc]: Optional block of code to invoke before returning or failing, a bit like +ensure+
   # @param loop_if [Proc]: Optional block of code to invoke which will cause our block to be rerun until true
   # @param loop_msg [String]: Message to display every third attempt
-  def self.retrier(catchme = nil, wait: 30, max: 0, ignoreme: [], on_retry: nil, always: nil, loop_if: nil, loop_msg: nil)
+  def self.retrier(catchme = nil, wait: 30, max: 0, ignoreme: [], on_retry: nil, always: nil, loop_if: nil, loop_msg: nil, logmsg_interval: 3)
 
     loop_if ||= Proc.new { false }
 
     retries = 0
     begin
       retries += 1
-      loglevel = ((retries % 3) == 0) ? MU::NOTICE : MU::DEBUG
+      loglevel = (logmsg_interval > 0 and (retries % logmsg_interval) == 0) ? MU::NOTICE : MU::DEBUG
       log_attempts = retries.to_s
       log_attempts += (max > 0 ? "/"+max.to_s : "")
       yield(retries, wait) if block_given?
@@ -702,7 +700,8 @@ module MU
   if !$MU_CFG.nil? and !$MU_CFG['public_address'].nil? and
      !$MU_CFG['public_address'].empty? and @@my_public_ip != $MU_CFG['public_address']
     @@mu_public_addr = $MU_CFG['public_address']
-    if !@@mu_public_addr.match(/^\d+\.\d+\.\d+\.\d+$/)
+    if !@@mu_public_addr.match(/^\d+\.\d+\.\d+\.\d+$/) and
+       File.exists?("/etc/hostname") and File.exists?("/etc/hosts")
       hostname = IO.readlines("/etc/hostname")[0].gsub(/\n/, '')
 
       hostlines = File.open('/etc/hosts').grep(/.*#{hostname}.*/)
@@ -887,22 +886,30 @@ module MU
   # @param stringify_keys [Boolean]
   # @return [Hash]
   def self.structToHash(struct, stringify_keys: false)
+    return nil if struct.nil?
+
     google_struct = false
     begin
       google_struct = struct.class.ancestors.include?(::Google::Apis::Core::Hashable)
     rescue NameError
+    rescue TypeError
+      return struct
     end
 
     aws_struct = false
     begin
       aws_struct = struct.class.ancestors.include?(::Seahorse::Client::Response)
     rescue NameError
+    rescue TypeError
+      return struct
     end
 
     azure_struct = false
     begin
       azure_struct = struct.class.ancestors.include?(::MsRestAzure) or struct.class.name.match(/Azure::.*?::Mgmt::.*?::Models::/)
     rescue NameError
+    rescue TypeError
+      return struct
     end
 
     if struct.is_a?(Struct) or struct.class.ancestors.include?(Struct) or