cloud-mu 1.9.0.pre.beta

data/bin/mu-aws-setup

@@ -0,0 +1,376 @@
+#!/usr/local/ruby-current/bin/ruby
+#
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Perform initial Mu setup tasks:
+# 1. Set up an appropriate Security Group
+# 2. Associate a specific Elastic IP address to this MU server, if required.
+# 3. Create an S3 bucket for Mu logs.
+
+require 'etc'
+require 'securerandom'
+
+require File.expand_path(File.dirname(__FILE__))+"/mu-load-config.rb"
+
+require 'rubygems'
+require 'bundler/setup'
+require 'json'
+require 'erb'
+require 'optimist'
+require 'json-schema'
+require 'mu'
+Dir.chdir(MU.installDir)
+
+$opts = Optimist::options do
+  banner <<-EOS
+Usage:
+#{$0} [-i] [-s] [-l] [-u] [-d]
+  EOS
+  opt :ip, "Attempt to configure the IP requested in the CHEF_PUBLIC_IP environment variable, or if none is set, to associate an arbitrary Elastic IP.", :require => false, :default => false, :type => :boolean
+  opt :sg, "Attempt to configure a Security Group with appropriate permissions.", :require => false, :default => false, :type => :boolean
+  opt :logs, "Ensure the presence of a cloud storage bucket for use with CloudTrails, syslog, deploy secrets, node SSL certificates, etc.", :require => false, :default => false, :type => :boolean
+  opt :dns, "Ensure the presence of a private DNS Zone called for internal amongst Mu resources.", :require => false, :default => false, :type => :boolean
+  opt :uploadlogs, "Push today's log files to the S3 bucket created by the -l option.", :require => false, :default => false, :type => :boolean
+  opt :ephemeral, "Make sure all of our instance store (ephemeral) block devices are mapped and available.", :require => false, :default => false, :type => :boolean
+end
+
+
+my_instance_id = MU::Cloud::AWS.getAWSMetaData("instance-id")
+
+resp = MU::Cloud::AWS.ec2.describe_instances(instance_ids: [my_instance_id])
+instance = resp.reservations.first.instances.first
+
+preferred_ip = MU.mu_public_ip
+
+if $opts[:ephemeral] and !MU::Cloud::AWS.isGovCloud?
+  instancetypes = MU::Cloud::AWS.listInstanceTypes
+  if instancetypes[MU.myRegion][instance.instance_type]["storage"] == "EBS only"
+    MU.log "#{instance.instance_type} instance types do not have ephemeral volumes, skipping ephemeral device setup", MU::NOTICE
+  else
+#    instance.block_device_mappings.each { |dev|
+#      next if dev.ebs
+#    }
+    MU::Cloud::AWS.ec2.modify_instance_attribute(
+      instance_id: instance.instance_id,
+      block_device_mappings: MU::Cloud::AWS::Server.ephemeral_mappings
+    )
+  end
+end
+
+# Create a security group, or manipulate an existing one, so that we have all
+# of the appropriate network holes.
+if $opts[:sg]
+  open_ports = [80, 443, 2260, 7443, 8443, 9443, 8200]
+
+  # This doesn't make sense. we can have multiple security groups in our account with a name tag of "Mu Master". This will then find and modify a security group that has nothing to do with us.
+  # found = MU::MommaCat.findStray("AWS", "firewall_rule", region: MU.myRegion, dummy_ok: true, tag_key: "Name", tag_value: "Mu Master")
+  found = nil
+
+  if found.nil? or found.size < 1 and instance.security_groups.size > 0
+    # maybe we should make sure we don't use the "Mu Client Rules for" security group for this.
+    found = MU::MommaCat.findStray("AWS", "firewall_rule", region: MU.myRegion, dummy_ok: true, cloud_id: instance.security_groups.first.group_id)
+  end
+  admin_sg = found.first if !found.nil? and found.size > 0
+
+  if !admin_sg.nil?
+    MU.log "Using an existing Security Group, #{admin_sg}, already associated with this Mu server."
+    open_ports.each { |port|
+      admin_sg.addRule(["0.0.0.0/0"], port: port)
+    }
+    admin_sg.addRule(["#{preferred_ip}/32"], port: 22)
+  else
+    rules = Array.new
+    open_ports.each { |port|
+      rules << {
+        "port" => port,
+        "hosts" => ["0.0.0.0/0"]
+      }
+    }
+    rules << {
+      "port" => 22,
+      "hosts" => ["#{preferred_ip}/32"]
+    }
+    cfg = {
+      "name" => "Mu Master",
+      "cloud" => "AWS",
+      "region" => MU.myRegion,
+      "rules" => rules
+    }
+
+    if !instance.vpc_id.nil?
+      cfg["vpc"] = {"vpc_id" => instance.vpc_id}
+    end
+    admin_sg = MU::Cloud::FirewallRule.new(kitten_cfg: cfg, mu_name: "Mu Master")
+    admin_sg.create
+    admin_sg.groom
+  end
+end
+
+# Muddle with our IP address
+if instance.public_ip_address != preferred_ip and !preferred_ip.nil? and !preferred_ip.empty? and $opts[:ip]
+
+  has_elastic_ip = false
+  if !instance.public_ip_address.nil?
+    filters = Array.new
+    filters << {name: "domain", values: ["vpc"]} if !instance.vpc_id.nil?
+    filters << {name: "public-ip", values: [instance.public_ip_address]}
+    resp = MU::Cloud::AWS.ec2.describe_addresses(filters: filters)
+    pp resp
+    if resp.addresses.size > 0
+      has_elastic_ip
+    end
+  end
+
+  if has_elastic_ip
+    MU.log "Public IP address is #{instance.public_ip_address}"
+  else
+    is_private = false
+    if !instance.vpc_id.nil?
+      # Fix this to actually verify the subnet is private
+      is_private = true if instance.public_ip_address.nil? && instance.public_dns_name.empty?
+      # is_private = MU::VPC.isSubnetPrivate?(instance.subnet_id)
+      public_ip = MU::Cloud::AWS::Server.findFreeElasticIp if !is_private
+    else
+      public_ip = MU::Cloud::AWS::Server.findFreeElasticIp(classic: true)
+    end
+
+    if !is_private
+      if public_ip.nil?
+        MU.log "Warning: Could not find a free Elastic IP to associate, continuing to use #{instance.public_ip_address} for now", MU::NOTICE
+      else
+        MU.log "Warning: About to associate the IP address #{public_ip} with this instance. This will disconnect your session. It is safe to reconnect and restart configuration.", MU::NOTICE
+        sleep 5
+        if !instance.vpc_id.nil?
+          MU::Cloud::AWS::Server.associateElasticIp(my_instance_id, ip: public_ip)
+        else
+          MU::Cloud::AWS::Server.associateElasticIp(my_instance_id, classic: true, ip: public_ip)
+        end
+      end
+    else
+      MU.log "We are in a private subnet, will not attempt to assign a public IP."
+    end
+  end
+elsif $opts[:ip]
+  MU.log "Currently assigned IP address is #{instance.public_ip_address}"
+end
+
+$bucketname = MU.adminBucketName
+
+if $opts[:logs]
+  exists = false
+
+  MU.log "Configuring log and secret Amazon S3 bucket '#{$bucketname}'"
+
+  resp = MU::Cloud::AWS.s3.list_buckets
+  resp.buckets.each { |bucket|
+    exists = true if bucket['name'] == $bucketname
+  }
+  if !exists
+    MU.log "Creating #{$bucketname} bucket"
+    begin
+      resp = MU::Cloud::AWS.s3.create_bucket(bucket: $bucketname, acl: "private")
+    rescue Aws::S3::Errors::BucketAlreadyExists => e
+      MU.log "#{e.inspect}", MU::NOTICE
+    end
+  end
+
+  resp = MU::Cloud::AWS.s3.list_objects(
+    bucket: $bucketname,
+    prefix: "log_vol_ebs_key"
+  )
+  found = false
+  resp.contents.each { |object|
+    found = true if object.key == "log_vol_ebs_key"
+  }
+  if !found
+    MU.log "Creating new key for encrypted EBS log volume"
+    key = SecureRandom.random_bytes(32)
+    MU::Cloud::AWS.s3.put_object(
+      bucket: $bucketname,
+      key: "log_vol_ebs_key",
+      body: "#{key}"
+    )
+  end
+  if File.exists?("#{MU.mySSLDir}/Mu_CA.pem")
+    MU.log "Putting the Mu Master's public SSL certificate into #{$bucketname}/Mu_CA.pem"
+    MU::Cloud::AWS.s3.put_object(
+      bucket: $bucketname,
+      key: "Mu_CA.pem",
+      body: File.read("#{MU.mySSLDir}/Mu_CA.pem"),
+  		acl: "public-read",
+    )
+  end
+
+  MU::Master.disk("/dev/xvdl", "/Mu_Logs", 50, "log_vol_ebs_key", "ram7")
+
+#	MU.log "Uploading Mu_CA.pem to #{$bucketname}"
+#	MU::Cloud::AWS.s3.put_object(
+#		bucket: $bucketname,
+#		acl: "public-read",
+#		key: "Mu_CA.pem",
+#		body: File.read("#{ENV['MU_DATADIR']}/ssl/Mu_CA.pem")
+#	)
+
+  resp = MU::Cloud::AWS.s3.list_objects(
+    bucket: $bucketname,
+    prefix: "log_vol_ebs_key"
+  )
+  owner = MU.structToHash(resp.contents.first.owner)
+
+  MU::Cloud::AWS.s3.put_bucket_acl(
+      bucket: $bucketname,
+      acl: "log-delivery-write"
+  )
+
+  MU::Cloud::AWS.s3.put_bucket_versioning(
+      bucket: $bucketname,
+      versioning_configuration: {
+          status: "Enabled"
+      }
+  )
+
+  MU::Cloud::AWS.s3.put_bucket_lifecycle(
+      bucket: $bucketname,
+      lifecycle_configuration: {
+          rules: [
+              {
+                  expiration: {
+                      days: 180
+                  },
+                  prefix: "master.log/",
+                  status: "Enabled"
+              },
+              {
+                  expiration: {
+                      days: 180
+                  },
+                  prefix: "nodes.log/",
+                  status: "Enabled"
+              },
+              {
+                  expiration: {
+                      days: 180
+                  },
+                  prefix: "AWSLogs/",
+                  status: "Enabled"
+              }
+          ]
+      }
+  )
+
+  MU::Cloud::AWS.s3.put_bucket_policy(
+      bucket: $bucketname,
+      policy: ERB.new(MU::CLOUDTRAIL_BUCKET_POLICY).result
+  )
+
+
+  begin
+    resp = MU::Cloud::AWS.cloudtrail.describe_trails.trail_list
+  rescue Aws::CloudTrail::Errors::AccessDeniedException => e
+    MU.log e.inspect, MU::WARN
+  end
+  if resp.empty?
+    MU.log "Enabling Cloud Trails, logged to bucket #{$bucketname}"
+
+    begin
+      MU::Cloud::AWS.cloudtrail.create_trail(
+          name: "cloudtrail",
+          s3_bucket_name: $bucketname,
+          include_global_service_events: true
+      )
+    rescue Aws::CloudTrail::Errors::MaximumNumberOfTrailsExceededException, Aws::CloudTrail::Errors::AccessDeniedException => e
+      MU.log e.inspect, MU::WARN
+    end
+
+  # Make sure we actually enable cloudtrail logging
+  MU::Cloud::AWS.cloudtrail.start_logging(
+    name: "cloudtrail"
+  )
+  end
+
+  # Now that we've got S3 logging, let's also create an Mu_Logs stack in
+  # CloudWatch logs.
+  # For instances to log to this, they need to invoke the Chef recipe
+  # aws-cloudwatch-logs.
+  # XXX this isn't supported on CentOS yet, ostensibly. Bother later.
+
+end
+
+if $opts[:dns] and !MU::Cloud::AWS.isGovCloud?
+  if instance.vpc_id.nil? or instance.vpc_id.empty?
+    MU.log "This Mu master appears to be in EC2 Classic. Route53 private DNS zones are not supported. Falling back to old /etc/hosts chicanery.", MU::ERR
+  else
+    ext_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu")
+
+    if ext_zone.nil? or ext_zone.size == 0
+      params = {
+          :name => "platform-mu",
+          :vpc => {
+              :vpc_region => MU.myRegion,
+              :vpc_id => instance.vpc_id
+          },
+          :hosted_zone_config => {
+              :comment => $bucketname,
+          },
+          :caller_reference => $bucketname
+      }
+
+      begin
+        resp = MU::Cloud::AWS.route53.create_hosted_zone(params)
+      rescue Aws::Route53::Errors::HostedZoneAlreadyExists => e
+        MU.log "#{e.inspect}, appending some gibberish...", MU::WARN
+        params[:caller_reference] = params[:caller_reference]+(0...2).map { ('a'..'z').to_a[rand(26)] }.join
+        retry
+      end
+      MU.log ".platform-mu private domain created"
+    else
+      ext_zone = ext_zone.values.first
+      begin
+        MU::Cloud::AWS.route53.associate_vpc_with_hosted_zone(
+            hosted_zone_id: ext_zone.id,
+            vpc: {
+                vpc_region: MU.myRegion,
+                vpc_id: instance.vpc_id
+            }
+        )
+      rescue Aws::Route53::Errors::ConflictingDomainExists
+      end
+    end
+    resolver = Resolv::DNS.new
+    my_ip = ""
+    begin
+      my_ip = resolver.getaddress($MU_CFG['hostname']).to_s
+    end rescue Resolv::ResolvError
+    if my_ip != MU.mu_public_ip
+      MU::Cloud::AWS::DNSZone.manageRecord(ext_zone.id, $MU_CFG['hostname'], "A", targets: [MU.mu_public_ip], sync_wait: false)
+   end
+  end
+end
+
+if $opts[:uploadlogs]
+  today = Time.new.strftime("%Y%m%d").to_s
+  ["master.log", "nodes.log"].each { |log|
+    if File.exists?("/Mu_Logs/#{log}-#{today}")
+      MU.log "Uploading /Mu_Logs/#{log}-#{today} to bucket #{$bucketname}"
+      MU::Cloud::AWS.s3.put_object(
+          bucket: $bucketname,
+          key: "#{log}/#{today}",
+          body: File.read("/Mu_Logs/#{log}-#{today}")
+      )
+    else
+      MU.log "No log /Mu_Logs/#{log}-#{today} was found", MU::WARN
+    end
+  }
+end

data/bin/mu-cleanup

@@ -0,0 +1,68 @@
+#!/usr/local/ruby-current/bin/ruby
+#
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+# now we have our global config available as the read-only hash $MU_CFG
+
+require 'rubygems'
+require 'bundler/setup'
+require 'optimist'
+require 'mu'
+Dir.chdir(MU.installDir)
+
+$opts = Optimist::options do
+  banner <<-EOS
+Usage:
+#{$0} [-i] [-n] [-s] [-o] [-w] [-v] <deploy_id>
+  EOS
+  opt :ignoremaster, "Ignore the MU-MASTER-IP tag when identifying resources to terminate. Caution- this can result in accidentally terminating someone else's resources.", :default => false, :type => :boolean
+  opt :noop, "Don't actually do anything, just show what we would do. Equivalent to specifying -o and -k together.", :default => false, :type => :boolean
+  opt :skipsnapshots, "Skip creation of final snapshots for EBS volumes and RDS instances.", :default => false, :type => :boolean
+  opt :onlycloud, "Only purge cloud resources, and skip all Mu master deployment metadata cleanup.", :require => false, :default => false, :type => :boolean
+  opt :skipcloud, "Only purge Mu master deployment metadata, and skip all cloud resources.", :require => false, :default => false, :type => :boolean
+  opt :web, "Generate web-friendly (HTML) output.", :require => false, :default => false, :type => :boolean
+  opt :verbose, "Display debugging output.", :require => false, :default => false, :type => :boolean
+  opt :quiet, "Display minimal output.", :require => false, :default => false, :type => :boolean
+end
+verbosity = MU::Logger::NORMAL
+verbosity = MU::Logger::LOUD if $opts[:verbose]
+verbosity = MU::Logger::QUIET if $opts[:quiet]
+
+if $opts[:verbose] and $opts[:quiet]
+  MU.log "Cannot set both --verbose and --quiet", MU::ERR
+  exit 1
+end
+
+MU.setLogging(verbosity, $opts[:web])
+
+if (!ARGV[0] or ARGV[0].empty?) and !$opts[:deploy]
+  MU.log("You must specify a deploy id!", MU::ERR, html: $opts[:web])
+  exit 1
+else
+  $opts[:deploy] = ARGV[0]
+end
+
+
+MU::Cleanup.run(
+    $opts[:deploy],
+    noop: $opts[:noop],
+    skipsnapshots: $opts[:skipsnapshots],
+    onlycloud: $opts[:onlycloud],
+    verbosity: verbosity,
+    web: $opts[:web],
+    skipcloud: $opts[:skipcloud],
+    ignoremaster: $opts[:ignoremaster]
+)