cloud-mu 2.1.0beta → 3.0.0beta

data/cookbooks/mu-tools/resources/disk.rb

@@ -1,6 +1,7 @@
 
 property :mountpoint, String, name_property: true
 property :device, String, required: true
+property :delete_on_termination, :kind_of => [TrueClass, FalseClass], default: true
 property :preserve_data, :kind_of => [TrueClass, FalseClass], :required => false, :default => false
 property :reboot_after_create, :kind_of => [TrueClass, FalseClass], :required => false, :default => false
 property :size, Integer, default: 8
@@ -22,7 +23,8 @@ action :create do
     request "add_volume"
     passparams(
       :dev => devicename,
-      :size => new_resource.size
+      :size => new_resource.size,
+      :delete_on_termination => new_resource.delete_on_termination
     )
     not_if { ::File.exist?(device) }
   end

data/cookbooks/mu-tools/templates/amazon/sshd_config.erb

@@ -165,4 +165,4 @@ UseDNS no
 
 # CAP Mod, restrict ciphers
 Ciphers aes128-ctr,aes192-ctr,aes256-ctr
-AllowUsers root
+AllowUsers ec2-user root

data/cookbooks/mu-tools/templates/default/etc_hosts.erb

@@ -6,7 +6,7 @@
 # doing only private IPs although that can be problematic 
 # if the same deploy has cross VPC or cross region resources
   if n.name != @hostname %>
-<%= n.ipaddress %> <%= n.name %>
+<%= n['ipaddress'] %> <%= n.name %>
 <%
   end
   }

data/cookbooks/mu-tools/templates/default/kubeconfig-gke.erb

@@ -0,0 +1,27 @@
+apiVersion: v1
+clusters:
+- cluster:
+    server: <%= @endpoint %>
+    certificate-authority-data: <%= @cacert %>
+  name: <%= @cluster %>
+kind: Config
+preferences: {}
+contexts:
+- context:
+    cluster: <%= @cluster %>
+    user: client
+  name: client
+<% if @username and @password %>- context:
+    cluster: <%= @cluster %>
+    user: <%= @username %>
+  name: <%= @username %>
+current-context: <%= @username %><% else %>current-context: client<% end %>
+users:
+- name: client
+  user:
+    client-certificate-data: <%= @clientcert %>
+    client-key-data: <%= @clientkey %>
+<% if @username and @password %>- name: <%= @username %>
+  user:
+    username: <%= @username %>
+    password: <%= @password %><% end %>

data/cookbooks/mu-tools/templates/windows-10/sshd_config.erb

@@ -0,0 +1,137 @@
+# $OpenBSD: sshd_config,v 1.99 2016/07/11 03:19:44 tedu Exp $
+
+# This is the sshd server system-wide configuration file.  See
+# sshd_config(5) for more information.
+
+# This sshd was compiled with PATH=/bin:/usr/sbin:/sbin:/usr/bin
+
+# The strategy used for options in the default sshd_config shipped with
+# OpenSSH is to specify options with their default value where
+# possible, but leave them commented.  Uncommented options override the
+# default value.
+
+#Port 22
+#AddressFamily any
+#ListenAddress 0.0.0.0
+#ListenAddress ::
+
+# The default requires explicit activation of protocol 1
+#Protocol 2
+
+# HostKey for protocol version 1
+#HostKey C:\ProgramData\ssh\ssh_host_key
+# HostKeys for protocol version 2
+HostKey C:\ProgramData\ssh\ssh_host_rsa_key
+HostKey C:\ProgramData\ssh\ssh_host_dsa_key
+HostKey C:\ProgramData\ssh\ssh_host_ecdsa_key
+HostKey C:\ProgramData\ssh\ssh_host_ed25519_key
+
+# Lifetime and size of ephemeral version 1 server key
+#KeyRegenerationInterval 1h
+#ServerKeyBits 1024
+
+# Ciphers and keying
+#RekeyLimit default none
+
+Ciphers aes256-ctr
+KexAlgorithms diffie-hellman-group-exchange-sha256
+MACs hmac-sha2-256
+
+# Logging
+#SyslogFacility AUTH
+#LogLevel ERROR 
+
+# Authentication:
+
+#LoginGraceTime 2m
+#PermitRootLogin prohibit-password
+StrictModes no
+#MaxAuthTries 6
+#MaxSessions 10
+
+#RSAAuthentication yes
+#PubkeyAuthentication yes
+
+# The default is to check both .ssh/authorized_keys and .ssh/authorized_keys2
+# but this is overridden so installations will only check .ssh/authorized_keys
+AuthorizedKeysFile .ssh/authorized_keys
+
+#AuthorizedPrincipalsFile none
+
+#AuthorizedKeysCommand none
+#AuthorizedKeysCommandUser nobody
+
+# For this to work you will also need host keys in /etc/ssh_known_hosts
+#RhostsRSAAuthentication no
+# similar for protocol version 2
+#HostbasedAuthentication no
+# Change to yes if you don't trust ~/.ssh/known_hosts for
+# RhostsRSAAuthentication and HostbasedAuthentication
+#IgnoreUserKnownHosts no
+# Don't read the user's ~/.rhosts and ~/.shosts files
+#IgnoreRhosts yes
+
+# To disable tunneled clear text passwords, change to no here!
+PasswordAuthentication no
+#PermitEmptyPasswords no
+
+# Change to no to disable s/key passwords
+#ChallengeResponseAuthentication yes
+
+# Kerberos options
+#KerberosAuthentication no
+#KerberosOrLocalPasswd yes
+#KerberosTicketCleanup yes
+#KerberosGetAFSToken no
+
+# GSSAPI options
+#GSSAPIAuthentication no
+#GSSAPICleanupCredentials yes
+
+# Set this to 'yes' to enable PAM authentication, account processing,
+# and session processing. If this is enabled, PAM authentication will
+# be allowed through the ChallengeResponseAuthentication and
+# PasswordAuthentication.  Depending on your PAM configuration,
+# PAM authentication via ChallengeResponseAuthentication may bypass
+# the setting of "PermitRootLogin without-password".
+# If you just want the PAM account and session checks to run without
+# PAM authentication, then enable this but set PasswordAuthentication
+# and ChallengeResponseAuthentication to 'no'.
+#UsePAM no
+
+#AllowAgentForwarding yes
+#AllowTcpForwarding yes
+#GatewayPorts no
+#X11Forwarding no
+#X11DisplayOffset 10
+#X11UseLocalhost yes
+#PermitTTY yes
+#PrintMotd yes
+#PrintLastLog yes
+#TCPKeepAlive yes
+#UseLogin no
+#PermitUserEnvironment no
+#Compression delayed
+#ClientAliveInterval 0
+#ClientAliveCountMax 3
+#UseDNS no
+#PidFile /var/run/sshd.pid
+#MaxStartups 10:30:100
+#PermitTunnel no
+#ChrootDirectory none
+#VersionAddendum none
+
+# no default banner path
+#Banner none
+
+# override default of no subsystems
+Subsystem sftp  /usr/sbin/sftp-server
+
+# Example of overriding settings on a per-user basis
+#Match User anoncvs
+# X11Forwarding no
+# AllowTcpForwarding no
+# PermitTTY no
+# ForceCommand cvs server
+
+AllowGroups Administrators sshusers

data/cookbooks/mu-utility/recipes/nat.rb

@@ -56,6 +56,10 @@ else
       raw "-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT"
       position 97
     end
+    firewall_rule "inbound from NAT network" do
+      raw "-A INPUT -s #{$ip_block} -j ACCEPT"
+      position 98
+    end
     firewall_rule "NAT forwarding" do
       raw "-A FORWARD -s #{$ip_block} -j ACCEPT"
       position 98

data/extras/clean-stock-amis

@@ -46,8 +46,8 @@ MU::Cloud::AWS.listRegions.each { | r|
 			}
 			MU.log "Deregistering #{ami.name} (#{ami.creation_date})", MU::WARN, details: snaps
 			MU::Cloud::AWS.ec2(region: r, credentials: credentials).deregister_image(image_id: ami.image_id)
-			snaps.each { |snap_id|
-				MU::Cloud::AWS.ec2(region: r, credentials: credentials).delete_snapshot(snapshot_id: snap_id)
+ 		  snaps.each { |snap_id|
+			  MU::Cloud::AWS.ec2(region: r, credentials: credentials).delete_snapshot(snapshot_id: snap_id)
 			}
 		end
   }

data/extras/generate-stock-images

@@ -0,0 +1,131 @@
+#!/usr/local/ruby-current/bin/ruby
+# Copyright:: Copyright (c) 2019 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/../bin/mu-load-config.rb"))
+# now we have our global config available as the read-only hash $MU_CFG
+
+require 'rubygems'
+require 'bundler/setup'
+require 'optimist'
+require 'mu'
+
+bok_dir = MU.myRoot+"/extras/image-generators"
+
+available_clouds = {}
+Dir.foreach(bok_dir) { |d|
+  next if d == "." or d == ".."
+  next if !Dir.exist?(MU.myRoot+"/extras/image-generators/"+d)
+  available_clouds[d] = []
+  Dir.foreach(bok_dir+"/"+d) { |yamlfile|
+    next if !yamlfile.match(/(.+?)\.yaml$/)
+    platform = Regexp.last_match[1]
+    available_clouds[d] << platform
+  }
+}
+
+$opts = Optimist::options do
+  banner <<-EOS
+Usage:
+#{$0} [-c <cloud>] [-p <platform>]
+  EOS
+  opt :clouds, "Clouds for which to generate images", :require => false, :type => :strings, :default => available_clouds.keys
+  opt :platforms, "Platforms for which to generate images", :require => false, :type => :strings, :default => available_clouds.values.flatten.sort.uniq
+  opt :environment, "Environment with which to tag our generated images.", :require => false, :type => :string, :default => "prod"
+  opt :age, "Minimum age, in days, at which we will replace existing images. Set to 0 to force a new build regardless of age.", :require => false, :type => :integer, :default => 30
+  if available_clouds.keys.include?("AWS")
+    opt :upload_to, "AWS S3 bucket and path to which we should upload our updated image list.", :require => false, :type => :string, :default => "s3://"+MU::Cloud::BASE_IMAGE_BUCKET+MU::Cloud::BASE_IMAGE_PATH
+  end
+  available_clouds.keys.each { |cloud|
+    opt (cloud.downcase+"_creds").to_sym, "Credentials to use when creating images in #{cloud}.", :require => false, :type => :string
+  }
+  opt :dryrun, "Don't actually run our deploy.", :require => false, :type => :boolean, :default => false
+end
+
+pwd = Dir.pwd
+
+if !available_clouds.keys.include?("AWS") # XXX or if we don't have permissions to write $opt[:upload_to]
+  MU.log "No AWS credentials available- I have nowhere to upload new imaged lists. Will print to STDOUT instead.", MU::WARN
+end
+
+now = DateTime.now
+
+exitcode = 0
+$opts[:clouds].each { |cloud|
+  current_images = MU::Cloud.getStockImage(cloud, fail_hard: true)
+  $opts[:platforms].each { |platform|
+    if File.exists?(bok_dir+"/"+cloud+"/"+platform+".yaml")
+      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get("Server")
+      if current_images[platform].is_a?(String)
+        age = cloudclass.imageTimeStamp(current_images[platform])
+        if (now - age) >= $opts[:age]
+          MU.log "#{cloud} image for #{platform} was last built #{age.to_s}, refreshing", MU::NOTICE
+        else
+          next
+        end
+      else
+        needed = false
+        if !current_images[platform]
+          needed = true
+        else
+          current_images[platform].each_pair { |r, img_id|
+            age = cloudclass.imageTimeStamp(img_id, region: r)
+            if (now - age) >= $opts[:age]
+              MU.log "#{cloud} image for #{platform} was last built #{age.to_s}, refreshing", MU::NOTICE
+              needed = true
+              break
+            end
+          }
+        end
+        next if !needed
+      end
+      conf_engine = MU::Config.new(
+        bok_dir+"/"+cloud+"/"+platform+".yaml",
+        default_credentials: $opts[(cloud.downcase+"_creds").to_sym]
+      )
+      stack_conf = conf_engine.config
+
+      if $opts[:dryrun]
+        puts stack_conf.to_yaml
+      else
+        begin
+          deployer = MU::Deploy.new(
+            $opts[:environment],
+            stack_conf: stack_conf
+          )
+          deployer.run
+          MU.log "New images for #{cloud}:#{platform}", MU::NOTICE, details: deployer.mommacat.deployment['images']
+          current_images[platform] ||= {}
+          current_images.deep_merge!(deployer.mommacat.deployment['images'])
+
+          # Scrub any loose metadata left over from our image deployment. It's
+          # ok, this won't touch the images we just made.
+          MU::Cleanup.run(deployer.mommacat.deploy_id, skipsnapshots: true, verbosity: MU::Logger::QUIET)
+        rescue Exception => e
+          MU.log e.message, MU::ERR
+          exitcode = 1
+        end
+      end
+    end
+  }
+
+  if !available_clouds.keys.include?("AWS") # XXX or if we don't have permissions
+    puts current_images.to_yaml
+  elsif !$opts[:dryrun]
+    MU::Cloud::AWS::Bucket.upload($opts[:upload_to]+"/"+cloud+".yaml", data: current_images.to_yaml, credentials: $opts[:aws_creds], acl: "public-read")
+  end
+}
+
+exit exitcode

data/extras/image-generators/AWS/centos6.yaml

@@ -0,0 +1,17 @@
+---
+appname: mu
+servers: 
+- name: centos6
+  platform: centos6
+  size: m3.medium
+  scrub_groomer: true
+  run_list:
+  - recipe[mu-tools::cloudinit]
+  - recipe[mu-tools::apply_security]
+  - recipe[mu-tools::updates]
+  - recipe[mu-tools::split_var_partitions]
+  create_image:
+    image_then_destroy: true
+    public: true
+    copy_to_regions:
+    - "#ALL"

data/extras/image-generators/{gcp → Google}/centos6.yaml

@@ -3,6 +3,7 @@
   servers: 
   - name: centos6
     cloud: Google
+    image_id: "centos-cloud/centos-6"
     platform: centos6
     ssh_user: centos
     size: g1-small

data/extras/image-generators/Google/centos7.yaml

@@ -0,0 +1,18 @@
+---
+  appname: mu
+  servers: 
+  - name: centos7
+    cloud: Google
+    image_id: "centos-cloud/centos-7"
+    platform: centos6
+    ssh_user: centos
+    size: g1-small
+    associate_public_ip: true
+    run_list:
+    - recipe[mu-tools::cloudinit]
+    - recipe[mu-tools::apply_security]
+    - recipe[mu-tools::updates]
+    - recipe[mu-tools::split_var_partitions]
+    create_image:
+      image_then_destroy: true
+      public: true

data/extras/ruby_rpm/muby.spec

@@ -1,7 +1,7 @@
 Summary: Ruby for Mu(by)
 BuildArch: x86_64
 Name: muby
-Version: 2.5.3
+Version: 2.5.5
 Release: 1%{dist}
 Group: Development/Languages
 License: Ruby License/GPL - see COPYING

data/install/README.md

@@ -1,8 +1,46 @@
 # Cloudamatic Mu Master Installation
-There are two paths to creating a Mu Master.  
+There are two paths to creating a Mu Master. _Typical Installation_ and _CloudFormation Installation_
 
-- **Typical Installation**: The simplest and recommended path is to use our CloudFormation script to configure an appropriate Virtual Private Cloud and master with all features enabled, including both a command line and Jenkins GUI user interface.
-- **Custom Installation:** If you prefer, you can also create your own VPC and manually provision a Mu Master.  This gives you more control over the shape of the master VPC and individual settings
+## Typical Instalation
+In the standard instsatation create your original VPC and manually provision a Mu Master instance.
 
-For detailed instructions on both installation techniques see [our Wiki Installation page](https://github.com/cloudamatic/mu/wiki/Install-Home)
-For mu master usage instructions see [our Wiki usage page](https://github.com/cloudamatic/mu/wiki/Usage)
+### Prerequisites
+1. Fully configured networking for the Mu Master
+	* Must have access to the internet
+	* Must manually configure any security on the networking
+1. Properly configured instance
+	* Supported OS `CentOS 6-7`, `RHEL 6-7`, or `Amazon Linux 2`
+	* API credentials to grant proper Mu-Master permissions. (Cloud provider roles recomended when hosted in the same cloud you intend to work in.)
+
+### Installation 
+
+**To Install From Master**
+```
+curl https://raw.githubusercontent.com/cloudamatic/mu/master/install/installer > installer
+chmod +x installer
+./installer
+```
+
+**To Install From Development or Other Branch**
+```
+curl https://raw.githubusercontent.com/cloudamatic/mu/development/install/installer > installer
+chmod +x installer
+MU_BRANCH=development ./installer
+```
+
+**Silent Install**
+```
+TODO: @zr2d2
+```
+>For detailed instructions on installation techniques see [our Wiki Installation page](https://github.com/cloudamatic/mu/wiki/Install-Home)
+
+## CloudFormation Installation
+> This method is depricated and may be removed from future releases
+
+The simplest path is to use our CloudFormation script to configure an appropriate Virtual Private Cloud and master with all features enabled. 
+
+### Get Started by Clicking the Launch Button!!
+
+[![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/new?stackName=CloudamaticInstaller&templateURL=https://s3.amazonaws.com/mu-cfn-installer/cfn_create_mu_master.json)
+
+>All  AWS resources Created in `us-east-1` region.