cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 99522ede9ca4a27289aa29774c90fb8d4ddced64040b62c916a0fc0859530a4a
-  data.tar.gz: c0a6ef2bcbc026bb4ae35a2661fe1b4d3fd18698aef3822949d14f29ef1ca0d3
+  metadata.gz: f1059c013e5f5e7c2ece20c2aa9ffcb60facab001cc864c0c26f60baf384d7c4
+  data.tar.gz: ebca2309c37e8a8038df432be18bb2b4f54e44b52da42029a79461515603f899
 SHA512:
-  metadata.gz: 8cd48976fc196d3da6c7d6abfd8b1eaeadcfcf88ae4f338513aaa9ce27b8602d4f9912f2a05352b28e4f219915c118c89b8efffebeed8299778e3ad9062efbd9
-  data.tar.gz: 02ad48db967c76bdadb9c82516c3f0b4ec8926201ff0fb84c3ea8045e72639838b4f97e2e0c92ec21817b94cfb903d21a704adb5d02146981bcff9c901d701b4
+  metadata.gz: d73ffd1304213dc06b04f0f350ef9d70cf99e48a3dd1c0ea7db0424606437b83e710bd87c0b9df471ca0ed0998991cf4e0e7a7e482d859ce647d0e0e09f84f6e
+  data.tar.gz: ad2503195884645217ee178aa1a9d94fb12520c83132f355cc1042535059d3b8d89cd8ce0066f1eae3d4e665aa6b678599ba2d174acbd7f9701430a1c242b464

data/Berksfile

@@ -1,56 +1,18 @@
 source "https://supermarket.chef.io"
-cookbook_path = "cookbooks"
+source chef_repo: "cookbooks/"
 
-# Platform Cookbooks
-cookbook 'mu-activedirectory', path: "#{cookbook_path}/mu-activedirectory"
-cookbook 'mu-splunk', path: "#{cookbook_path}/mu-splunk"
-cookbook 'mu-firewall', path: "#{cookbook_path}/mu-firewall"
-cookbook 'mu-glusterfs', path: "#{cookbook_path}/mu-glusterfs"
-cookbook 'mu-jenkins', path: "#{cookbook_path}/mu-jenkins"
-cookbook 'mu-master', path: "#{cookbook_path}/mu-master"
-cookbook 'mu-mongo', path: "#{cookbook_path}/mu-mongo"
-cookbook 'mu-openvpn', path: "#{cookbook_path}/mu-openvpn"
-cookbook 'mu-php54', path: "#{cookbook_path}/mu-php54"
-cookbook 'mu-tools', path: "#{cookbook_path}/mu-tools"
-cookbook 'mu-utility', path: "#{cookbook_path}/mu-utility"
-
-# Forked Cookbooks
-# cookbook 'awscli', path: "#{cookbook_path}/awscli"
-cookbook 'cloudcli', '~> 1.2.0'
-# cookbook 's3fs', path: "#{cookbook_path}/s3fs"
-cookbook 's3fs', '~> 3.0.1'
-
-# Nagios cookbook is borked, and dragging these cookbooks down with it...
-cookbook 'nagios', path: "#{cookbook_path}/nagios"
-cookbook 'apache2', '< 4.0'
-
-# Supermarket Cookbooks that are using latest as of 09/07/18
-cookbook 'chef-vault', '~> 3.1.1'
-cookbook 'windows', '~> 5.1.1'
-cookbook 'aws', '~> 2.9.3'
-cookbook 'build-essential', '~> 8.2.1'
-cookbook 'chef_nginx', '~> 6.2.0'
-cookbook 'freebsd', '~> 1.0.2'
-cookbook 'jenkins', '~> 6.2.0'
-cookbook 'logrotate', '~> 2.2.0'
-cookbook 'memcached', '~> 5.1.1'
-cookbook 'mongodb', '~> 0.16.2'
-cookbook 'runit', '~> 4.3.0'
-cookbook 'zipfile', '~> 0.2.0'
-cookbook 'yum', '~> 5.1.0'
-cookbook 'bind', '~> 2.2.0'
-cookbook 'bind9-ng', '~> 0.1.0'
-cookbook 'consul-cluster', '~> 2.0.0'
-cookbook 'database', '~> 6.1.1'
-cookbook 'firewall', '~> 2.6.5'
-cookbook 'hostsfile', '~> 3.0.1'
-cookbook 'java', '~> 2.2.0'
-cookbook 'mysql', '~> 8.5.1'
-cookbook 'nrpe', '~> 2.0.3'
-cookbook 'oracle-instantclient', '~> 1.1.0'
-cookbook 'poise-python', '~> 1.7.0'
-cookbook 'postfix', '~> 5.3.1'
-cookbook 'postgresql', '~> 7.1.0'
-cookbook 'simple_iptables', '~> 0.8.0'
-cookbook 'vault-cluster', '~> 2.1.0'
-cookbook 'yum-epel', '~> 3.2.0'
+# Mu Platform Cookbooks
+cookbook 'awscli', path: 'cookbooks/awscli'
+cookbook 'mu-activedirectory'
+cookbook 'mu-splunk'
+cookbook 'mu-firewall'
+cookbook 'mu-glusterfs'
+cookbook 'mu-jenkins'
+cookbook 'mu-master'
+cookbook 'mu-mongo'
+cookbook 'mu-openvpn'
+cookbook 'mu-php54'
+cookbook 'mu-tools'
+cookbook 'mu-utility'
+cookbook 'nagios', path: 'cookbooks/nagios'
+#cookbook 's3fs', path: 'cookbooks/s3fs'

data/Berksfile.lock

@@ -1,72 +1,30 @@
 DEPENDENCIES
-  apache2 (< 4.0.0)
-  aws (~> 2.9.3)
-  bind (~> 2.2.0)
-  bind9-ng (~> 0.1.0)
-  build-essential (~> 8.2.1)
-  chef-vault (~> 3.1.1)
-  chef_nginx (~> 6.2.0)
-  cloudcli (~> 1.2.0)
-  consul-cluster (~> 2.0.0)
-  database (~> 6.1.1)
-  firewall (~> 2.6.5)
-  freebsd (~> 1.0.2)
-  hostsfile (~> 3.0.1)
-  java (~> 2.2.0)
-  jenkins (~> 6.2.0)
-  logrotate (~> 2.2.0)
-  memcached (~> 5.1.1)
-  mongodb (~> 0.16.2)
+  awscli
+    path: cookbooks/awscli
   mu-activedirectory
-    path: cookbooks/mu-activedirectory
   mu-firewall
-    path: cookbooks/mu-firewall
   mu-glusterfs
-    path: cookbooks/mu-glusterfs
   mu-jenkins
-    path: cookbooks/mu-jenkins
   mu-master
-    path: cookbooks/mu-master
   mu-mongo
-    path: cookbooks/mu-mongo
   mu-openvpn
-    path: cookbooks/mu-openvpn
   mu-php54
-    path: cookbooks/mu-php54
   mu-splunk
-    path: cookbooks/mu-splunk
   mu-tools
-    path: cookbooks/mu-tools
   mu-utility
-    path: cookbooks/mu-utility
-  mysql (~> 8.5.1)
   nagios
     path: cookbooks/nagios
-  nrpe (~> 2.0.3)
-  oracle-instantclient (~> 1.1.0)
-  poise-python (~> 1.7.0)
-  postfix (~> 5.3.1)
-  postgresql (~> 7.1.0)
-  runit (~> 4.3.0)
-  s3fs (~> 3.0.1)
-  simple_iptables (~> 0.8.0)
-  vault-cluster (~> 2.1.0)
-  windows (~> 5.1.1)
-  yum (~> 5.1.0)
-  yum-epel (~> 3.2.0)
-  zipfile (~> 0.2.0)
 
 GRAPH
   apache2 (3.3.1)
   apt (7.1.1)
-  aws (2.9.3)
-    ohai (>= 2.1.0)
+  awscli (0.2.1)
   bind (2.2.1)
   bind9-ng (0.1.0)
   build-essential (8.2.1)
     mingw (>= 1.1)
     seven_zip (>= 0.0.0)
-  chef-sugar (4.1.0)
+  chef-sugar (5.0.0)
   chef-vault (3.1.1)
   chef_nginx (6.2.0)
     build-essential (>= 0.0.0)
@@ -74,8 +32,6 @@ GRAPH
     ohai (>= 4.1.0)
     yum-epel (>= 0.0.0)
     zypper (>= 0.0.0)
-  cloudcli (1.2.0)
-    poise-python (~> 1.6)
   compat_resource (12.19.1)
   consul (2.3.0)
     build-essential (>= 0.0.0)
@@ -92,9 +48,8 @@ GRAPH
   database (6.1.1)
     postgresql (>= 1.0.0)
   dpkg_autostart (0.2.0)
-  firewall (2.6.5)
+  firewall (2.7.0)
     chef-sugar (>= 0.0.0)
-  freebsd (1.0.2)
   golang (1.7.0)
   hashicorp-vault (2.5.0)
     build-essential (>= 0.0.0)
@@ -107,12 +62,9 @@ GRAPH
   java (2.2.1)
     homebrew (>= 0.0.0)
     windows (>= 0.0.0)
-  jenkins (6.2.0)
+  jenkins (6.2.1)
     dpkg_autostart (>= 0.0.0)
     runit (>= 1.7)
-  logrotate (2.2.0)
-  memcached (5.1.1)
-    runit (>= 1.2.0)
   mingw (2.1.0)
     seven_zip (>= 0.0.0)
   mongodb (0.16.2)
@@ -125,7 +77,7 @@ GRAPH
     windows (~> 5.1.1)
     yum-epel (~> 3.2.0)
   mu-firewall (0.1.0)
-    firewall (~> 2.6.5)
+    firewall (~> 2.7.0)
   mu-glusterfs (0.1.0)
     mu-firewall (>= 0.0.0)
     yum (~> 5.1.0)
@@ -136,7 +88,7 @@ GRAPH
     mu-master (>= 0.0.0)
     mu-tools (>= 0.0.0)
     mu-utility (>= 0.0.0)
-  mu-master (0.9.0)
+  mu-master (0.9.2)
     apache2 (< 4.0)
     bind (~> 2.2.0)
     bind9-ng (~> 0.1.0)
@@ -150,7 +102,7 @@ GRAPH
     nagios (>= 0.0.0)
     nrpe (~> 2.0.3)
     postfix (~> 5.3.1)
-    s3fs (~> 3.0.1)
+    s3fs (>= 0.0.0)
     vault-cluster (~> 2.1.0)
   mu-mongo (0.5.0)
     chef-vault (~> 3.1.1)
@@ -182,6 +134,7 @@ GRAPH
     windows (~> 5.1.1)
     yum-epel (~> 3.2.0)
   mu-utility (0.6.0)
+    mu-firewall (>= 0.0.0)
     windows (~> 5.1.1)
   mysql (8.5.1)
   nagios (7.2.7)
@@ -197,7 +150,7 @@ GRAPH
     perl (>= 0.0.0)
     runit (>= 0.0.0)
     yum-epel (>= 0.0.0)
-  nrpe (2.0.3)
+  nrpe (2.0.5)
     build-essential (>= 0.0.0)
     yum-epel (>= 0.0.0)
   nssm (4.0.1)
@@ -224,13 +177,13 @@ GRAPH
   poise-service (1.5.2)
     poise (~> 2.0)
   postfix (5.3.1)
-  postgresql (7.1.1)
+  postgresql (7.1.3)
   python (1.4.6)
     build-essential (>= 0.0.0)
     yum-epel (>= 0.0.0)
   rubyzip (1.3.1)
     poise (~> 2.2)
-  runit (4.3.0)
+  runit (4.3.1)
     packagecloud (>= 0.0.0)
     yum-epel (>= 0.0.0)
   s3fs (3.0.1)
@@ -242,9 +195,8 @@ GRAPH
     consul-cluster (~> 2.0)
     hashicorp-vault (~> 2.1)
     ssl_certificate (~> 1.11)
-  windows (5.1.5)
+  windows (5.1.6)
   yum (5.1.0)
   yum-epel (3.2.0)
   zap (1.1.0)
-  zipfile (0.2.0)
   zypper (0.4.0)

data/bin/mu-aws-setup

@@ -46,6 +46,20 @@ Usage:
   opt :ephemeral, "Make sure all of our instance store (ephemeral) block devices are mapped and available.", :require => false, :default => false, :type => :boolean
 end
 
+if MU::Cloud::AWS.hosted? and !$MU_CFG['aws']
+  new_cfg = $MU_CFG.dup
+  cfg_blob = MU::Cloud::AWS.hosted_config
+  if cfg_blob
+    cfg_blob['log_bucket_name'] ||= $MU_CFG['hostname']
+    new_cfg["aws"] = { "default" => cfg_blob }
+    MU.log "Adding auto-detected AWS stanza to #{cfgPath}", MU::NOTICE
+    if new_cfg != $MU_CFG or !cfgExists?
+      MU.log "Generating #{cfgPath}"
+      saveMuConfig(new_cfg)
+      $MU_CFG = new_cfg
+    end
+  end
+end
 
 my_instance_id = MU::Cloud::AWS.getAWSMetaData("instance-id")
 
@@ -169,137 +183,146 @@ end
 $bucketname = MU.adminBucketName
 
 if $opts[:logs]
-  exists = false
+  MU::Cloud::AWS.listCredentials.each { |credset|
+    bucketname = MU::Cloud::AWS.adminBucketName(credset)
 
-  MU.log "Configuring log and secret Amazon S3 bucket '#{$bucketname}'"
+    exists = false
 
-  resp = MU::Cloud::AWS.s3.list_buckets
-  resp.buckets.each { |bucket|
-    exists = true if bucket['name'] == $bucketname
-  }
-  if !exists
-    MU.log "Creating #{$bucketname} bucket"
-    begin
-      resp = MU::Cloud::AWS.s3.create_bucket(bucket: $bucketname, acl: "private")
-    rescue Aws::S3::Errors::BucketAlreadyExists => e
-      MU.log "#{e.inspect}", MU::NOTICE
+    MU.log "Configuring log and secret Amazon S3 bucket '#{bucketname}' for credential set #{credset}"
+
+    resp = MU::Cloud::AWS.s3(credentials: credset).list_buckets
+    resp.buckets.each { |bucket|
+      exists = true if bucket['name'] == bucketname
+    }
+    if !exists
+      MU.log "Creating #{bucketname} bucket"
+      begin
+        resp = MU::Cloud::AWS.s3(credentials: credset).create_bucket(bucket: bucketname, acl: "private")
+      rescue Aws::S3::Errors::BucketAlreadyExists => e
+        MU.log "#{e.inspect}", MU::NOTICE
+      end
     end
-  end
 
-  resp = MU::Cloud::AWS.s3.list_objects(
-    bucket: $bucketname,
-    prefix: "log_vol_ebs_key"
-  )
-  found = false
-  resp.contents.each { |object|
-    found = true if object.key == "log_vol_ebs_key"
-  }
-  if !found
-    MU.log "Creating new key for encrypted EBS log volume"
-    key = SecureRandom.random_bytes(32)
-    MU::Cloud::AWS.s3.put_object(
-      bucket: $bucketname,
-      key: "log_vol_ebs_key",
-      body: "#{key}"
-    )
-  end
-  if File.exists?("#{MU.mySSLDir}/Mu_CA.pem")
-    MU.log "Putting the Mu Master's public SSL certificate into #{$bucketname}/Mu_CA.pem"
-    MU::Cloud::AWS.s3.put_object(
-      bucket: $bucketname,
-      key: "Mu_CA.pem",
-      body: File.read("#{MU.mySSLDir}/Mu_CA.pem"),
-  		acl: "public-read",
+    resp = MU::Cloud::AWS.s3(credentials: credset).list_objects(
+      bucket: bucketname,
+      prefix: "log_vol_ebs_key"
     )
-  end
+    found = false
+    resp.contents.each { |object|
+      found = true if object.key == "log_vol_ebs_key"
+    }
+    if !found
+      MU.log "Creating new key for encrypted EBS log volume"
+      key = SecureRandom.random_bytes(32)
+      MU::Cloud::AWS.s3(credentials: credset).put_object(
+        bucket: bucketname,
+        key: "log_vol_ebs_key",
+        body: "#{key}"
+      )
+    end
+    if File.exists?("#{MU.mySSLDir}/Mu_CA.pem")
+      MU.log "Putting the Mu Master's public SSL certificate into #{bucketname}/Mu_CA.pem"
+      MU::Cloud::AWS.s3(credentials: credset).put_object(
+        bucket: bucketname,
+        key: "Mu_CA.pem",
+        body: File.read("#{MU.mySSLDir}/Mu_CA.pem"),
+    		acl: "public-read",
+      )
+    end
 
-  MU::Master.disk("/dev/xvdl", "/Mu_Logs", 50, "log_vol_ebs_key", "ram7")
+    MU::Master.disk("/dev/xvdl", "/Mu_Logs", 50, "log_vol_ebs_key", "ram7")
 
-#	MU.log "Uploading Mu_CA.pem to #{$bucketname}"
+#	MU.log "Uploading Mu_CA.pem to #{bucketname}"
 #	MU::Cloud::AWS.s3.put_object(
-#		bucket: $bucketname,
+#		bucket: bucketname,
 #		acl: "public-read",
 #		key: "Mu_CA.pem",
 #		body: File.read("#{ENV['MU_DATADIR']}/ssl/Mu_CA.pem")
 #	)
 
-  resp = MU::Cloud::AWS.s3.list_objects(
-    bucket: $bucketname,
-    prefix: "log_vol_ebs_key"
-  )
-  owner = MU.structToHash(resp.contents.first.owner)
-
-  MU::Cloud::AWS.s3.put_bucket_acl(
-      bucket: $bucketname,
-      acl: "log-delivery-write"
-  )
-
-  MU::Cloud::AWS.s3.put_bucket_versioning(
-      bucket: $bucketname,
-      versioning_configuration: {
-          status: "Enabled"
-      }
-  )
-
-  MU::Cloud::AWS.s3.put_bucket_lifecycle(
-      bucket: $bucketname,
-      lifecycle_configuration: {
-          rules: [
-              {
-                  expiration: {
-                      days: 180
-                  },
-                  prefix: "master.log/",
-                  status: "Enabled"
-              },
-              {
-                  expiration: {
-                      days: 180
-                  },
-                  prefix: "nodes.log/",
-                  status: "Enabled"
-              },
-              {
-                  expiration: {
-                      days: 180
-                  },
-                  prefix: "AWSLogs/",
-                  status: "Enabled"
-              }
-          ]
-      }
-  )
+    resp = MU::Cloud::AWS.s3(credentials: credset).list_objects(
+      bucket: bucketname,
+      prefix: "log_vol_ebs_key"
+    )
+    owner = MU.structToHash(resp.contents.first.owner)
 
-  MU::Cloud::AWS.s3.put_bucket_policy(
-      bucket: $bucketname,
-      policy: ERB.new(MU::CLOUDTRAIL_BUCKET_POLICY).result
-  )
+    MU::Cloud::AWS.s3(credentials: credset).put_bucket_acl(
+        bucket: bucketname,
+        acl: "log-delivery-write"
+    )
 
+    MU::Cloud::AWS.s3(credentials: credset).put_bucket_versioning(
+        bucket: bucketname,
+        versioning_configuration: {
+            status: "Enabled"
+        }
+    )
 
-  begin
-    resp = MU::Cloud::AWS.cloudtrail.describe_trails.trail_list
-  rescue Aws::CloudTrail::Errors::AccessDeniedException => e
-    MU.log e.inspect, MU::WARN
-  end
-  if resp.empty?
-    MU.log "Enabling Cloud Trails, logged to bucket #{$bucketname}"
+    MU::Cloud::AWS.s3(credentials: credset).put_bucket_lifecycle(
+        bucket: bucketname,
+        lifecycle_configuration: {
+            rules: [
+                {
+                    expiration: {
+                        days: 180
+                    },
+                    prefix: "master.log/",
+                    status: "Enabled"
+                },
+                {
+                    expiration: {
+                        days: 180
+                    },
+                    prefix: "nodes.log/",
+                    status: "Enabled"
+                },
+                {
+                    expiration: {
+                        days: 180
+                    },
+                    prefix: "AWSLogs/",
+                    status: "Enabled"
+                }
+            ]
+        }
+    )
 
     begin
-      MU::Cloud::AWS.cloudtrail.create_trail(
-          name: "cloudtrail",
-          s3_bucket_name: $bucketname,
-          include_global_service_events: true
+      MU::Cloud::AWS.s3(credentials: credset).put_bucket_policy(
+        bucket: bucketname,
+        policy: MU::Cloud::AWS.cloudtrailBucketPolicy(credset)
       )
-    rescue Aws::CloudTrail::Errors::MaximumNumberOfTrailsExceededException, Aws::CloudTrail::Errors::AccessDeniedException => e
+    rescue Aws::S3::Errors::MalformedPolicy => e
+      MU.log e.message, MU::ERR, details: MU::Cloud::AWS.cloudtrailBucketPolicy(credset)
+      next
+    end
+
+
+    begin
+      resp = MU::Cloud::AWS.cloudtrail(credentials: credset).describe_trails.trail_list
+    rescue Aws::CloudTrail::Errors::AccessDeniedException => e
       MU.log e.inspect, MU::WARN
     end
+    if resp.empty?
+      MU.log "Enabling Cloud Trails, logged to bucket #{bucketname}"
 
-  # Make sure we actually enable cloudtrail logging
-  MU::Cloud::AWS.cloudtrail.start_logging(
-    name: "cloudtrail"
-  )
-  end
+      begin
+        MU::Cloud::AWS.cloudtrail(credentials: credset).create_trail(
+            name: "cloudtrail",
+            s3_bucket_name: bucketname,
+            include_global_service_events: true
+        )
+      rescue Aws::CloudTrail::Errors::MaximumNumberOfTrailsExceededException, Aws::CloudTrail::Errors::AccessDeniedException => e
+        MU.log e.inspect, MU::WARN
+      end
 
+    # Make sure we actually enable cloudtrail logging
+    MU::Cloud::AWS.cloudtrail(credentials: credset).start_logging(
+      name: "cloudtrail"
+    )
+    end
+
+  }
   # Now that we've got S3 logging, let's also create an Mu_Logs stack in
   # CloudWatch logs.
   # For instances to log to this, they need to invoke the Chef recipe