cumulus-aws 0.11.1

data/lib/route53/models/ZoneDiff.rb

@@ -0,0 +1,118 @@
+require "conf/Configuration"
+require "common/models/Diff"
+require "route53/models/RecordDiff"
+require "util/Colors"
+
+module Cumulus
+  module Route53
+    # Public: The types of changes that can be made to zones
+    module ZoneChange
+      include Common::DiffChange
+
+      COMMENT = Common::DiffChange::next_change_id
+      DOMAIN = Common::DiffChange::next_change_id
+      PRIVATE = Common::DiffChange::next_change_id
+      RECORD = Common::DiffChange::next_change_id
+      VPC = Common::DiffChange::next_change_id
+    end
+
+    # Public: Represents a single difference between local configuration and AWS
+    # configuration of zones.
+    class ZoneDiff < Common::Diff
+      include ZoneChange
+
+      attr_accessor :changed_records
+
+      # Public: Static method that produces a diff representing changes in records
+      #
+      # changed_records - the RecordDiffs
+      # local           - the local configuration for the zone
+      #
+      # Returns the diff
+      def self.records(changed_records, local)
+        diff = ZoneDiff.new(RECORD, nil, local)
+        diff.changed_records = changed_records
+        diff
+      end
+
+      def asset_type
+        "Zone"
+      end
+
+      def aws_name
+        access = if @aws.config.private_zone then "private" else "public" end
+        "#{@aws.name} (#{access})"
+      end
+
+      def add_string
+        "has been added locally, but must be created in AWS manually."
+      end
+
+      def diff_string
+        case @type
+        when COMMENT
+          [
+            "Comment:",
+            Colors.aws_changes("\tAWS - #{@aws.config.comment}"),
+            Colors.local_changes("\tLocal - #{@local.comment}")
+          ].join("\n")
+        when DOMAIN
+          [
+            "Domain: AWS - #{Colors.aws_changes(@aws.name)}, Local - #{Colors.local_changes(@local.domain)}",
+            "\tAWS doesn't allow you to change the domain for a zone."
+          ].join("\n")
+        when PRIVATE
+          [
+            "Private: AWS - #{Colors.aws_changes(@aws.config.private_zone)}, Local - #{Colors.local_changes(@local.private)}",
+            "\tAWS doesn't allow you to change whether a zone is private."
+          ].join("\n")
+        when RECORD
+          if Configuration.instance.route53.print_all_ignored
+            [
+              "Records:",
+              @changed_records.map { |r| "\t#{r}" }
+            ].flatten.join("\n")
+          else
+            [
+              "Records:",
+              @changed_records.reject { |r| r.type == RecordChange::IGNORED }.map { |r| "\t#{r}" },
+              "\tYour blacklist ignored #{@changed_records.select { |r| r.type == RecordChange::IGNORED }.size} records."
+            ].flatten.join("\n")
+          end
+        when VPC
+          [
+            "VPCs:",
+            added_vpc_ids.map { |vpc| Colors.added("\t#{vpc["id"]} | #{vpc["region"]}") },
+            removed_vpc_ids.map { |vpc| Colors.removed("\t#{vpc["id"]} | #{vpc["region"]}") }
+          ].flatten.join("\n")
+        end
+      end
+
+      # Public: Get the VPCs that have been added locally.
+      #
+      # Returns an array of vpc ids
+      def added_vpc_ids
+        @local.vpc - @aws.vpc
+      end
+
+      # Public: Get the VPCs that have been removed locally.
+      #
+      # Returns an array of vpc ids
+      def removed_vpc_ids
+        @aws.vpc - @local.vpc
+      end
+
+      # Public: Override the info_only attribute such that if this diff is a record diff and it
+      # contains only ignored record diffs, we return true.
+      #
+      # Returns whether or not this is an info only diff
+      def info_only
+        if @type == RECORD
+          @changed_records.all? { |r| r.type == RecordChange::IGNORED }
+        else
+          false
+        end
+      end
+    end
+  end
+end

data/lib/s3/S3.rb

@@ -0,0 +1,89 @@
+require "conf/Configuration"
+
+require "aws-sdk"
+
+module Cumulus
+  module S3
+    class << self
+      def client(region = nil)
+        @clients ||= {}
+        if !region then region = "us-east-1" end
+        @clients[region] ||= Aws::S3::Client.new(Configuration.instance.client.merge({:force_path_style => true, :region => region}))
+      end
+
+      @@zone_ids = {
+        "ap-northeast-1" => "Z2M4EHUR26P7ZW",
+        "ap-southeast-1" => "Z3O0J2DXBE1FTB",
+        "ap-southeast-2" => "Z1WCIGYICN2BYD",
+        "eu-central-1" => "Z21DNDUVLTQW6Q",
+        "eu-west-1" => "Z1BKCTXD74EZPE",
+        "sa-east-1" => "Z7KQH4QJS55SO",
+        "us-east-1" => "Z3AQBSTGFYJSTF",
+        "us-gov-west-1" => "Z31GFT0UA1I2HV",
+        "us-west-1" => "Z2F56UZL2M1ACD",
+        "us-west-2" => "Z3BJ6K6RIION7M",
+      }
+
+      # Public: A mapping of region name to its hosted zone id. This mapping is needed because
+      # the S3 API doesn't expose the hosted ids, you have to get them at
+      # http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region.
+      def zone_ids
+        @@zone_ids
+      end
+
+      # Public: Static method that will get an S3 bucket from AWS by its name
+      #
+      # name - the name of the bucket to get
+      #
+      # Returns the Aws::S3::Types::Bucket by that name
+      def get_aws(name)
+        buckets.fetch(name)
+      rescue KeyError
+        puts "No S3 bucket named #{name}"
+        exit
+      end
+
+      # Public: Get the full data for a bucket. Lazily loads resources only once.
+      #
+      # bucket_name - the name of the bucket to get
+      #
+      # Returns the full bucket
+      def full_bucket(bucket_name)
+        @monkey_patched ||= monkey_patch_bucket
+        @full_buckets ||= Hash.new
+
+        bucket = buckets[bucket_name]
+        @full_buckets[bucket_name] ||= Aws::S3::Bucket.new(name: bucket_name, client: client(bucket.location))
+      end
+
+      # Public: Provide a mapping of S3 buckets to their names. Lazily loads resources.
+      #
+      # Returns the buckets mapped to their names
+      def buckets
+        @buckets ||= init_buckets
+      end
+
+      # Public: Refereshes the bucket list and full_buckets map
+      def refresh!
+        @buckets = init_buckets
+        @full_buckets = Hash.new
+      end
+
+      private
+
+      # Internal: Monkey patch Bucket so it can get its location
+      def monkey_patch_bucket
+        require "aws_extensions/s3/Bucket"
+        Aws::S3::Types::Bucket.send(:include, AwsExtensions::S3::Types::Bucket)
+        true
+      end
+
+      # Internal: Load the buckets and map them to their names.
+      #
+      # Returns the buckets mapped to their names
+      def init_buckets
+        Hash[client.list_buckets.buckets.map { |bucket| [bucket.name, bucket] }]
+      end
+    end
+  end
+end

data/lib/s3/loader/Loader.rb

@@ -0,0 +1,66 @@
+require "common/BaseLoader"
+require "conf/Configuration"
+require "s3/models/BucketConfig"
+
+require "aws-sdk"
+
+module Cumulus
+  module S3
+    module Loader
+      include Common::BaseLoader
+
+      @@buckets_dir = Configuration.instance.s3.buckets_directory
+      @@cors_dir = Configuration.instance.s3.cors_directory
+      @@policies_dir = Configuration.instance.s3.policies_directory
+
+      # Public: Load all the bucket configurations a BucketConfig objects
+      #
+      # Returns an array of BucketConfigs
+      def self.buckets
+        Common::BaseLoader.resources(@@buckets_dir, &BucketConfig.method(:new))
+      end
+
+      # Public: Load a specific CORS policy by name, applying any variables.
+      #
+      # name - the name of the file to load
+      # vars - the variables to apply to the template
+      #
+      # Returns the CORS policy as a string
+      def self.cors_policy(name, vars)
+        Common::BaseLoader.template(
+          name,
+          @@cors_dir,
+          vars,
+          &proc do |n, json| json.map do |rule|
+              Aws::S3::Types::CORSRule.new({
+                allowed_headers: rule.fetch("headers"),
+                allowed_methods: rule.fetch("methods"),
+                allowed_origins: rule.fetch("origins"),
+                expose_headers: rule.fetch("exposed-headers"),
+                max_age_seconds: rule.fetch("max-age-seconds")
+              })
+            end
+          end
+        )
+      rescue KeyError
+        puts "CORS configuration #{name} does not contain all required keys."
+        exit
+      end
+
+      # Public: Load a specific bucket policy by name, applying any variables
+      #
+      # name - the name of the file to load
+      # vars - the variables to apply to the template
+      #
+      # Returns the bucket policy as a string
+      def self.bucket_policy(name, vars)
+        Common::BaseLoader.template(
+          name,
+          @@policies_dir,
+          vars,
+          &proc { |n, json| json.to_json }
+        )
+      end
+    end
+  end
+end

data/lib/s3/manager/Manager.rb

@@ -0,0 +1,296 @@
+require "common/manager/Manager"
+require "conf/Configuration"
+require "s3/loader/Loader"
+require "s3/models/BucketConfig"
+require "s3/models/BucketDiff"
+require "s3/S3"
+require "util/Colors"
+
+require "aws-sdk"
+
+module Cumulus
+  module S3
+    class Manager < Common::Manager
+      def migrate
+        buckets_dir = "#{@migration_root}/buckets"
+        cors_dir = "#{@migration_root}/cors"
+        policy_dir = "#{@migration_root}/policies"
+
+        [@migration_root, buckets_dir, cors_dir, policy_dir].each do |dir|
+          if !Dir.exists?(dir)
+            Dir.mkdir(dir)
+          end
+        end
+
+        cors = {}
+        policies = {}
+
+        aws_resources.each_value do |resource|
+          puts "Processing #{resource.name}..."
+          full_aws = S3.full_bucket(resource.name)
+          config = BucketConfig.new(resource.name)
+          new_policy_key, new_cors_key = config.populate!(full_aws, cors, policies)
+
+          puts "Writing #{resource.name} configuration to file..."
+          if new_policy_key
+            File.open("#{policy_dir}/#{new_policy_key}.json", "w") { |f| f.write(policies[new_policy_key]) }
+          end
+          if new_cors_key
+            File.open("#{cors_dir}/#{new_cors_key}.json", "w") { |f| f.write(cors[new_cors_key]) }
+          end
+          File.open("#{buckets_dir}/#{resource.name}.json", "w") { |f| f.write(config.pretty_json) }
+        end
+      end
+
+      def resource_name
+        "Bucket"
+      end
+
+      def local_resources
+        Hash[Loader.buckets.map { |bucket| [bucket.name, bucket] }]
+      end
+
+      def aws_resources
+        S3.buckets
+      end
+
+      def unmanaged_diff(aws)
+        BucketDiff.unmanaged(aws)
+      end
+
+      def added_diff(local)
+        BucketDiff.added(local)
+      end
+
+      def diff_resource(local, aws)
+        if Configuration.instance.s3.print_progress
+          puts "checking for differences in #{local.name}"
+        end
+        full_aws = S3.full_bucket(aws.name)
+        local.diff(full_aws)
+      end
+
+      def create(local)
+        S3.client(local.region).create_bucket({
+          bucket: local.name,
+          create_bucket_configuration: if local.region != "us-east-1" then {
+            location_constraint: local.region
+          } end
+        })
+        S3.refresh!
+        update_policy(local.region, local.name, local.policy)
+        update_cors(local.region, local.name, local.cors)
+        update_grants(local.region, local.name, local.grants)
+        update_versioning(local.region, local.name, local.versioning)
+        update_logging(local.region, local.name, local.logging)
+        update_website(local.region, local.name, local.website)
+        update_lifecycle(local.region, local.name, local.lifecycle)
+        update_notifications(local.region, local.name, local.notifications)
+        update_replication(local.region, local.name, local.replication)
+        update_tags(local.region, local.name, local.tags) if !local.tags.empty?
+      end
+
+      def update(local, diffs)
+        diffs.each do |diff|
+          if diff.type == BucketChange::TAGS
+            puts Colors.blue("\tupdating tags...")
+            update_tags(diff.local.region, diff.local.name, diff.local.tags)
+          elsif diff.type == BucketChange::POLICY
+            puts Colors.blue("\tupdating policy...")
+            update_policy(diff.local.region, diff.local.name, diff.local.policy)
+          elsif diff.type == BucketChange::CORS
+            puts Colors.blue("\tupdating CORS rules...")
+            update_cors(diff.local.region, diff.local.name, diff.local.cors)
+          elsif diff.type == BucketChange::WEBSITE
+            puts Colors.blue("\tupdating S3 bucket website...")
+            update_website(diff.local.region, diff.local.name, diff.local.website)
+          elsif diff.type == BucketChange::LOGGING
+            puts Colors.blue("\tupdating S3 bucket logging..")
+            update_logging(diff.local.region, diff.local.name, diff.local.logging)
+          elsif diff.type == BucketChange::VERSIONING
+            puts Colors.blue("\tupdating versioning status...")
+            update_versioning(diff.local.region, diff.local.name, diff.local.versioning)
+          elsif diff.type == BucketChange::GRANTS
+            puts Colors.blue("\tupdating grants...")
+            update_grants(diff.local.region, diff.local.name, diff.local.grants)
+          elsif diff.type == BucketChange::LIFECYCLE
+            puts Colors.blue("\tupdating lifecycle...")
+            update_lifecycle(diff.local.region, diff.local.name, diff.local.lifecycle)
+          elsif diff.type == BucketChange::NOTIFICATIONS
+            puts Colors.blue("\tupdating notifications...")
+            update_notifications(diff.local.region, diff.local.name, diff.local.notifications)
+          elsif diff.type == BucketChange::REPLICATION
+            puts Colors.blue("\tupdating replication...")
+            update_replication(diff.local.region, diff.local.name, diff.local.replication)
+          end
+        end
+      end
+
+      private
+
+      # Internal: Update the tags for a bucket.
+      #
+      # region - the region of the bucket
+      # bucket_name - the name of the bucket
+      # tags        - the tags that belong to the bucket
+      def update_tags(region, bucket_name, tags)
+        S3.client(region).put_bucket_tagging({
+          bucket: bucket_name,
+          tagging: {
+            tag_set: tags.map { |k, v| { key: k, value: v } }
+          }
+        })
+      end
+
+      # Internal: Update the policy for a bucket.
+      #
+      # region - the region of the bucket
+      # bucket_name - the name of the bucket
+      # policy      - the policy to apply to the bucket
+      def update_policy(region, bucket_name, policy)
+        if policy
+          S3.client(region).put_bucket_policy({
+            bucket: bucket_name,
+            policy: policy
+          })
+        else
+          S3.client(region).delete_bucket_policy({
+            bucket: bucket_name
+          })
+        end
+      end
+
+      # Internal: Update the CORS rules for a bucket.
+      #
+      # region - the region of the bucket
+      # bucket_name - the name of the bucket
+      # cors        - the cors rules for the bucket
+      def update_cors(region, bucket_name, cors)
+        if cors
+          S3.client(region).put_bucket_cors({
+            bucket: bucket_name,
+            cors_configuration: {
+              cors_rules: cors
+            }
+          })
+        else
+          S3.client(region).delete_bucket_cors({
+            bucket: bucket_name
+          })
+        end
+      end
+
+      # Internal: Update the bucket website configuration for the bucket.
+      #
+      # region - the region of the bucket
+      # bucket_name - the name of the bucket
+      # website     - the website configuration for the bucket
+      def update_website(region, bucket_name, website)
+        if website
+          S3.client(region).put_bucket_website({
+            bucket: bucket_name,
+            website_configuration: website.to_aws
+          })
+        else
+          S3.client(region).delete_bucket_website({ bucket: bucket_name })
+        end
+      end
+
+      # Internal: Update the bucket logging configuration for the bucket.
+      #
+      # region - the region of the bucket
+      # bucket_name - the name of the bucket
+      # logging     - the logging configuration for the bucket
+      def update_logging(region, bucket_name, logging)
+        S3.client(region).put_bucket_logging({
+          bucket: bucket_name,
+          bucket_logging_status: {
+            logging_enabled: (logging.to_aws rescue nil)
+          }
+        })
+      end
+
+      # Internal: Update the bucket versioning for the bucket.
+      #
+      # region - the region of the bucket
+      # bucket_name - the name of the bucket
+      # enabled     - whether versioning should be enabled or not
+      def update_versioning(region, bucket_name, enabled)
+        S3.client(region).put_bucket_versioning({
+          bucket: bucket_name,
+          versioning_configuration: {
+            status: if enabled then "Enabled" else "Suspended" end
+          }
+        })
+      end
+
+      # Internal: Update the permission grants for the bucket.
+      #
+      # region - the region of the bucket
+      # bucket_name - the name of the bucket
+      # grants      - the grants for the bucket
+      def update_grants(region, bucket_name, grants)
+        S3.client(region).put_bucket_acl({
+          bucket: bucket_name,
+          access_control_policy: {
+            grants: grants.values.map(&:to_aws).flatten,
+            owner: S3.full_bucket(bucket_name).acl.owner
+          }
+        })
+      end
+
+      # Internal: Update the lifecycle rules for the bucket.
+      #
+      # region - the region of the bucket
+      # bucket_name - the name of the bucket
+      # lifecycle   - the lifecycle rules for the bucket
+      def update_lifecycle(region, bucket_name, lifecycle)
+        if lifecycle.empty?
+          S3.client(region).delete_bucket_lifecycle({
+            bucket: bucket_name
+          })
+        else
+          S3.client(region).put_bucket_lifecycle({
+            bucket: bucket_name,
+            lifecycle_configuration: {
+              rules: lifecycle.values.map(&:to_aws)
+            }
+          })
+        end
+      end
+
+      # Internal: Update the notification rules for the bucket.
+      #
+      # bucket_name   - the name of the bucket
+      # notifications - the notification rules for the bucket
+      def update_notifications(region, bucket_name, notifications)
+        S3.client(region).put_bucket_notification_configuration({
+          bucket: bucket_name,
+          notification_configuration: {
+            topic_configurations: notifications.values.select { |n| n.type == "sns" }.map(&:to_aws),
+            queue_configurations: notifications.values.select { |n| n.type == "sqs" }.map(&:to_aws),
+            lambda_function_configurations: notifications.values.select { |n| n.type == "lambda" }.map(&:to_aws)
+          }
+        })
+      end
+
+      # Internal: Update the replication rules for the bucket.
+      #
+      # region - the region of the bucket
+      # bucket_name - the name of the bucket
+      # replication - the replication rules for the bucket
+      def update_replication(region, bucket_name, replication)
+        if replication
+          S3.client(region).put_bucket_replication({
+            bucket: bucket_name,
+            replication_configuration: replication.to_aws
+          })
+        else
+          S3.client(region).delete_bucket_replication({
+            bucket: bucket_name
+          })
+        end
+      end
+    end
+  end
+end