cloud-mu 2.0.1 → 2.0.2

data/cookbooks/mu-tools/recipes/rsyslog.rb

@@ -18,7 +18,7 @@
 
 if !node['application_attributes']['skip_recipes'].include?('rsyslog')
   case node['platform_family']
-  when "rhel", "debian"
+  when "rhel", "debian", "amazon"
     package "rsyslog"
     package "rsyslog-gnutls"
     execute "chcon -R -h -t var_log_t /Mu_Logs" do
@@ -29,7 +29,7 @@ if !node['application_attributes']['skip_recipes'].include?('rsyslog')
       action [:enable, :start]
       notifies :run, "execute[chcon -R -h -t var_log_t /Mu_Logs]", :immediately
     end
-    if platform_family?("rhel")
+    if platform_family?("rhel") or platform_family?("amazon")
       $rsyslog_ssl_ca_path = "/etc/pki/Mu_CA.pem"
       if !platform?("amazon")
         package node['platform_version'].to_i < 6 ? "policycoreutils" : "policycoreutils-python"

data/cookbooks/mu-tools/recipes/set_local_fw.rb

@@ -16,42 +16,51 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+if !node['application_attributes']['skip_recipes'].include?('set_local_fw')
+  master_ips = get_mu_master_ips
+  case node['platform_family']
+  when 'rhel', 'amazon'
+    include_recipe 'mu-firewall'
 
-master_ips = get_mu_master_ips
-case node['platform']
-when platform_family?('rhel')
-  include_recipe 'mu-firewall'
-
-  if elversion >= 7 # Can use firewalld, but not if iptables is already rigged
-    package "firewall-config" do
-      not_if "/bin/systemctl list-units | grep iptables.service"
+    if elversion >= 7 and node['platform_family'] != "amazon" # Can use firewalld, but not if iptables is already rigged
+      package "firewall-config" do
+        not_if "/bin/systemctl list-units | grep iptables.service"
+      end
+      execute "restart FirewallD" do # ...but only if iptables isn't live
+        command "/bin/firewall-cmd --reload"
+        action :nothing
+        not_if "/bin/systemctl list-units | grep iptables.service"
+        only_if { ::File.exist?("/bin/firewall-cmd") }
+      end
     end
-    execute "restart FirewallD" do # ...but only if iptables isn't live
-      command "/bin/firewall-cmd --reload"
-      action :nothing
-      not_if "/bin/systemctl list-units | grep iptables.service"
-      only_if { ::File.exist?("/bin/firewall-cmd") }
+
+    if elversion <= 6
+      firewall_rule "Allow loopback in" do
+        raw "-A INPUT -i lo -j ACCEPT"
+      end
+
+      firewall_rule "Allow loopback out" do
+        raw "-A OUTPUT -o lo -j ACCEPT"
+      end
     end
-  end
 
-  if elversion <= 6
-    firewall_rule "Allow loopback in" do
-      raw "-A INPUT -i lo -j ACCEPT"
+    firewall_rule "Allow eth0 out" do
+      raw "-A OUTPUT -o eth0 -j ACCEPT"
     end
 
-    firewall_rule "Allow loopback out" do
-      raw "-A OUTPUT -o lo -j ACCEPT"
+    firewall_rule "Allow established connections" do
+      raw "-A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT"
     end
-  end
 
-  opento = master_ips.map { |x| "#{x}/32"}
+    opento = master_ips.map { |x| "#{x}/32"}
 
-  opento.uniq.each { |src|
-    [:tcp, :udp, :icmp].each { |proto|
-      firewall_rule "allow all #{src} #{proto} traffic" do
-        source src
-        protocol proto
-      end
+    opento.uniq.each { |src|
+      [:tcp, :udp, :icmp].each { |proto|
+        firewall_rule "allow all #{src} #{proto} traffic" do
+          source src
+          protocol proto
+        end
+      }
     }
-  }
+  end
 end

data/environments/dev.json

@@ -1,5 +1,5 @@
 {
-	"name": "DEV",
+	"name": "dev",
 	"default_attributes": {
 	  },
 	"json_class": "Chef::Environment",

data/environments/prod.json

@@ -1,5 +1,5 @@
 {
-	"name": "PROD",
+	"name": "prod",
 	"default_attributes": {
 	  },
 	"json_class": "Chef::Environment",

data/modules/mu/cleanup.rb

@@ -157,6 +157,10 @@ module MU
                       rescue MU::MuError, NoMethodError => e
                         MU.log e.message, MU::WARN
                         next
+                      rescue ::Aws::EC2::Errors::AuthFailure => e
+                        # AWS has been having transient auth problems with ap-east-1 lately
+                        MU.log e.message+" in "+r, MU::ERR
+                        next
                       end
 
                       if @mommacat.nil? or @mommacat.numKittens(types: [t]) > 0

data/modules/mu/clouds/aws/container_cluster.rb

@@ -586,6 +586,9 @@ module MU
                 "sgs" => ["container_cluster#{cluster['name']}"],
                 "port_range" => "1-65535"
               ]
+              worker_pool["application_attributes"] ||= {}
+              worker_pool["application_attributes"]["skip_recipes"] ||= []
+              worker_pool["application_attributes"]["skip_recipes"] << "set_local_fw"
             end
             if cluster["vpc"]
               worker_pool["vpc"] = cluster["vpc"].dup

data/modules/mu/clouds/aws/role.rb

@@ -37,6 +37,10 @@ module MU
         def create
           if @config['iam_policies']
             @config['iam_policies'].each { |policy|
+              policy.values.each { |p|
+                p["Version"] ||= "2012-10-17"
+              }
+
               policy_name = @mu_name+"-"+policy.keys.first.upcase
               MU.log "Creating IAM policy #{policy_name}"
               resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).create_policy(
@@ -111,6 +115,9 @@ module MU
 
             if @config['iam_policies']
               @config['iam_policies'].each { |policy|
+                policy.values.each { |p|
+                  p["Version"] ||= "2012-10-17"
+                }
                 policy_name = @mu_name+"-"+policy.keys.first.upcase
 
                 arn = "arn:"+(MU::Cloud::AWS.isGovCloud? ? "aws-us-gov" : "aws")+":iam::"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":policy/#{@deploy.deploy_id}/#{policy_name}"
@@ -207,6 +214,7 @@ module MU
           if !policy.match(/^#{@deploy.deploy_id}/)
             policy = @mu_name+"-"+policy.upcase
           end
+
           my_policies = cloud_desc["policies"]
           my_policies.each { |p|
             if p.policy_name == policy
@@ -224,15 +232,19 @@ module MU
                       type: target["type"]
                     )
                     sibling.cloudobj.arn
-                  else
+                  elsif target.is_a?(Hash)
                     target['identifier']
+                  else
+                    target
                   end
-                  if sibling and !s["Resource"].include?(targetstr)
+
+                  if targetstr and !s["Resource"].include?(targetstr)
                     s["Resource"] << targetstr
                     need_update = true
                   end
                 }
               }
+
               if need_update
                 MU.log "Updating IAM policy #{policy} to grant permissions on #{targets.to_s}", details: doc
                 update_policy(p.arn, doc)

data/modules/mu/clouds/aws/userdata/linux.erb

@@ -66,7 +66,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
         version=6
       fi
     fi
-    if [ $version -eq 7 ];then
+    if [ "$version" == "7" ];then
       userdata_dir="/var/lib/cloud/instances/$instance_id"
     else
       userdata_dir="/var/lib/cloud/instance"
@@ -79,7 +79,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
 
     sed -i 's/^Defaults.*requiretty$/Defaults   !requiretty/' /etc/sudoers
 
-    if [ $version == 7 ];then
+    if [ "$version" == "7" ];then
       chmod 755 /etc/rc.d/rc.local
       systemctl reset-failed sshd.service
     fi
@@ -161,7 +161,6 @@ if [ "$need_reboot" == "1" ];then
   shutdown -r now "Applying new kernel"
 fi
 <% end %>
-fi
 
 if [ "$AWSCLI" != "" ];then
   $AWSCLI --region="$region" s3 cp s3://<%= MU.adminBucketName %>/<%= $mu.muID %>-secret .

data/modules/mu/clouds/aws.rb

@@ -142,6 +142,13 @@ module MU
         [:arn]
       end
 
+      # Given an AWS region, check the API to make sure it's a valid one
+      # @param r [String]
+      # @return [String]
+      def self.validate_region(r)
+        MU::Cloud::AWS.ec2(region: r).describe_availability_zones.availability_zones.first.region_name
+      end
+
       # If we've configured AWS as a provider, or are simply hosted in AWS, 
       # decide what our default region is.
       def self.myRegion
@@ -151,12 +158,6 @@ module MU
           return nil
         end
 
-        # Given an AWS region, check the API to make sure it's a valid one
-        # @param r [String]
-        # @return [String]
-        def self.validate_region(r)
-          MU::Cloud::AWS.ec2(region: r).describe_availability_zones.availability_zones.first.region_name
-        end
 
         if $MU_CFG and $MU_CFG['aws']
           $MU_CFG['aws'].each_pair { |credset, cfg|
@@ -537,6 +538,7 @@ module MU
       end
 
       @@regions = {}
+      @@regions_semaphore = Mutex.new
       # List the Amazon Web Services region names available to this account. The
       # region that is local to this Mu server will be listed first.
       # @param us_only [Boolean]: Restrict results to United States only
@@ -547,13 +549,20 @@ module MU
           return [] if credConfig.nil?
           result = MU::Cloud::AWS.ec2(region: myRegion, credentials: credentials).describe_regions.regions
           regions = []
-          result.each { |r|
-            @@regions[r.region_name] = Proc.new {
-              listAZs(region: r.region_name, credentials: credentials)
-            }
+          @@regions_semaphore.synchronize {
+            begin
+              result.each { |r|
+                @@regions[r.region_name] = Proc.new {
+                  listAZs(region: r.region_name, credentials: credentials)
+                }
+              }
+            rescue ::Aws::EC2::Errors::AuthFailure => e
+              MU.log "Region #{r.region_name} throws #{e.message}, ignoring it", MU::ERR
+            end
           }
         end
 
+
         regions = if us_only
           @@regions.keys.delete_if { |r| !r.match(/^us\-/) }.uniq
         else
@@ -589,10 +598,17 @@ module MU
         if !MU::Cloud::CloudFormation.emitCloudFormation
           MU::Cloud::AWS.listRegions.each { |region|
             MU.log "Replicating #{keyname} to EC2 in #{region}", MU::DEBUG, details: @ssh_public_key
-            MU::Cloud::AWS.ec2(region: region, credentials: credentials).import_key_pair(
-              key_name: keyname,
-              public_key_material: public_key
-            )
+            begin
+              MU::Cloud::AWS.ec2(region: region, credentials: credentials).import_key_pair(
+                key_name: keyname,
+                public_key_material: public_key
+              )
+            rescue ::Aws::EC2::Errors::AuthFailure => e
+              @@regions_semaphore.synchronize {
+                @@regions.delete(region)
+              }
+              MU.log "#{region} threw #{e.message}, skipping", MU::ERR
+            end
           }
         end
       end

data/modules/mu.rb

@@ -80,6 +80,10 @@ module MU
     @@myRoot
   end
 
+  # Front our global $MU_CFG hash with a read-only copy
+  def self.muCfg
+    Marshal.load(Marshal.dump($MU_CFG)).freeze
+  end
 
   # The main (root) Mu user's data directory.
   @@mainDataDir = File.expand_path(@@myRoot+"/../var")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloud-mu
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.2
 platform: ruby
 authors:
 - John Stange
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-01 00:00:00.000000000 Z
+date: 2019-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubis
@@ -399,6 +399,34 @@ files:
 - cookbooks/awscli/metadata.rb
 - cookbooks/awscli/recipes/default.rb
 - cookbooks/awscli/templates/default/config.erb
+- cookbooks/firewall/CHANGELOG.md
+- cookbooks/firewall/CONTRIBUTING.md
+- cookbooks/firewall/MAINTAINERS.md
+- cookbooks/firewall/README.md
+- cookbooks/firewall/attributes/default.rb
+- cookbooks/firewall/attributes/firewalld.rb
+- cookbooks/firewall/attributes/iptables.rb
+- cookbooks/firewall/attributes/ufw.rb
+- cookbooks/firewall/attributes/windows.rb
+- cookbooks/firewall/libraries/helpers.rb
+- cookbooks/firewall/libraries/helpers_firewalld.rb
+- cookbooks/firewall/libraries/helpers_iptables.rb
+- cookbooks/firewall/libraries/helpers_ufw.rb
+- cookbooks/firewall/libraries/helpers_windows.rb
+- cookbooks/firewall/libraries/matchers.rb
+- cookbooks/firewall/libraries/provider_firewall_firewalld.rb
+- cookbooks/firewall/libraries/provider_firewall_iptables.rb
+- cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu.rb
+- cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu1404.rb
+- cookbooks/firewall/libraries/provider_firewall_rule.rb
+- cookbooks/firewall/libraries/provider_firewall_ufw.rb
+- cookbooks/firewall/libraries/provider_firewall_windows.rb
+- cookbooks/firewall/libraries/resource_firewall.rb
+- cookbooks/firewall/libraries/resource_firewall_rule.rb
+- cookbooks/firewall/metadata.json
+- cookbooks/firewall/recipes/default.rb
+- cookbooks/firewall/recipes/disable_firewall.rb
+- cookbooks/firewall/templates/default/ufw/default.erb
 - cookbooks/mu-activedirectory/Berksfile
 - cookbooks/mu-activedirectory/CHANGELOG.md
 - cookbooks/mu-activedirectory/LICENSE