cumulus-aws 0.11.1

data/lib/cloudfront/loader/Loader.rb

@@ -0,0 +1,31 @@
+require "common/BaseLoader"
+require "conf/Configuration"
+require "cloudfront/models/DistributionConfig"
+require "cloudfront/models/InvalidationConfig"
+
+# Public: Load CloudFront assets
+module Cumulus
+  module CloudFront
+    module Loader
+      include Common::BaseLoader
+
+      @@distributions_dir = Configuration.instance.cloudfront.distributions_directory
+      @@invalidations_dir = Configuration.instance.cloudfront.invalidations_directory
+
+      # Public: Load all the distribution configurations as DistributionConfig objects
+      #
+      # Returns an array of DistributionConfig
+      def self.distributions
+        Common::BaseLoader::resources(@@distributions_dir, &DistributionConfig.method(:new))
+      end
+
+      # Public loads all of the invalidation configurations as InvalidationConfig objects
+      #
+      # Returns an array of InvalidationConfig
+      def self.invalidations
+        Common::BaseLoader::resources(@@invalidations_dir, &InvalidationConfig.method(:new))
+      end
+
+    end
+  end
+end

data/lib/cloudfront/manager/Manager.rb

@@ -0,0 +1,183 @@
+require "common/manager/Manager"
+require "conf/Configuration"
+require "cloudfront/CloudFront"
+require "cloudfront/loader/Loader"
+require "cloudfront/models/DistributionDiff"
+require "util/Colors"
+require "util/StatusCodes"
+
+require "aws-sdk"
+
+module Cumulus
+  module CloudFront
+    class Manager < Common::Manager
+      def initialize
+        super()
+        @cloudfront = Aws::CloudFront::Client.new(Configuration.instance.client)
+      end
+
+      def resource_name
+        "CloudFront Distribution"
+      end
+
+      def local_resources
+        @local_resources ||= Hash[Loader.distributions.map { |local| [local.id, local] }]
+      end
+
+      def aws_resources
+        @aws_resources ||= CloudFront::id_distributions
+      end
+
+      def full_distribution(distribution_id)
+        @full_aws_configs ||= Hash.new
+
+        @full_aws_configs[distribution_id] ||= CloudFront::load_distribution_config(distribution_id)
+      end
+
+      def unmanaged_diff(aws)
+        DistributionDiff.unmanaged(aws)
+      end
+
+      def added_diff(local)
+        DistributionDiff.added(local)
+      end
+
+      def diff_resource(local, aws)
+        local.diff(full_distribution(aws.id).distribution_config)
+      end
+
+      # Migrate AWS CloudFront distributions to local config
+      def migrate
+        distributions_dir = "#{@migration_root}/distributions"
+
+        if !Dir.exists?(@migration_root)
+          Dir.mkdir(@migration_root)
+        end
+        if !Dir.exists?(distributions_dir)
+          Dir.mkdir(distributions_dir)
+        end
+
+        aws_resources.each_key do |dist_id|
+          puts "Processing #{dist_id}..."
+          full_config = full_distribution(dist_id).distribution_config
+
+          config = DistributionConfig.new(dist_id)
+          config.populate!(dist_id, full_config)
+
+          puts "Writing #{dist_id} configuration to file"
+          File.open("#{distributions_dir}/#{dist_id}.json", "w") { |f| f.write(config.pretty_json) }
+        end
+      end
+
+      def update(local, diffs)
+        if !diffs.empty?
+          full_aws_response = full_distribution(local.id)
+
+          aws_config = full_aws_response.distribution_config
+
+          updated_config = {
+            aliases: {
+              quantity: local.aliases.size,
+              items: if local.aliases.empty? then nil else local.aliases end
+            },
+            origins: {
+              quantity: local.origins.size,
+              items: if local.origins.empty? then nil else local.origins.map(&:to_aws) end
+            },
+            default_cache_behavior: local.default_cache_behavior.to_aws,
+            cache_behaviors: {
+              quantity: local.cache_behaviors.size,
+              items: if local.cache_behaviors.empty? then nil else local.cache_behaviors.map(&:to_aws) end
+            },
+            comment: local.comment,
+            enabled: local.enabled
+          }
+
+          update_params = {
+            id: local.id,
+            if_match: full_aws_response.etag,
+            distribution_config: aws_config.to_h.merge(updated_config)
+          }
+
+          begin
+            @cloudfront.update_distribution(update_params)
+          rescue Aws::CloudFront::Errors::InvalidArgument => e
+            if e.message =~ /OriginSslProtocols is required/
+              puts Colors.red("Distribution #{local.name} must specify $.custom-origin-config.origin-ssl-protocols when \"protocol-policy\" is \"https-only\". Distribution not updated")
+              StatusCodes.set_status(StatusCodes::EXCEPTION)
+            end
+          end
+        end
+
+      end
+
+      def create(local)
+        create_config = {
+          distribution_config: {
+            caller_reference: local.name,
+            aliases: {
+              quantity: local.aliases.size,
+              items: if local.aliases.empty? then nil else local.aliases end
+            },
+            origins: {
+              quantity: local.origins.size,
+              items: if local.origins.empty? then nil else local.origins.map(&:to_aws) end
+            },
+            default_cache_behavior: local.default_cache_behavior.to_aws,
+            cache_behaviors: {
+              quantity: local.cache_behaviors.size,
+              items: if local.cache_behaviors.empty? then nil else local.cache_behaviors.map(&:to_aws) end
+            },
+            comment: local.comment,
+            enabled: local.enabled
+          }
+        }
+
+        local.id = @cloudfront.create_distribution(create_config).distribution.id
+
+        # Save the updated local config with id
+        File.open("#{Configuration.instance.cloudfront.distributions_directory}/#{local.name}.json", "w") { |f| f.write(local.pretty_json) }
+        puts "Distribution #{local.name} created with id #{local.id}"
+
+      rescue Aws::CloudFront::Errors::InvalidArgument => e
+        if e.message =~ /OriginSslProtocols is required/
+          puts Colors.red("Distribution #{local.name} must specify $.custom-origin-config.origin-ssl-protocols when \"protocol-policy\" is \"https-only\". Distribution not created")
+          StatusCodes.set_status(StatusCodes::EXCEPTION)
+        end
+      rescue => e
+        puts "Failed to create distribution #{local.name}\n#{e}"
+      end
+
+      def invalidations
+        @invalidations ||= Hash[Loader.invalidations.map { |local| [local.name, local] }]
+      end
+
+      def list_invalidations
+        puts invalidations.keys.join(" ")
+      end
+
+      def invalidate(invalidation_name)
+
+        invalidation = invalidations[invalidation_name]
+
+        # Use a combination of the current time and md5 of paths to prevent
+        # identical invalidations from being ran too often
+        time_throttle = (Time.now.to_i / 60 / 5)
+        md5 = Digest::MD5.hexdigest(invalidation.paths.join)[0..5]
+
+        @cloudfront.create_invalidation({
+          distribution_id: invalidation.distribution_id,
+          invalidation_batch: {
+            paths: {
+              quantity: invalidation.paths.size,
+              items: if !invalidation.paths.empty? then invalidation.paths end
+            },
+            caller_reference: "#{invalidation_name}-#{md5}-#{time_throttle}"
+          }
+        })
+
+      end
+
+    end
+  end
+end

data/lib/cloudfront/models/CacheBehaviorConfig.rb

@@ -0,0 +1,237 @@
+require "conf/Configuration"
+require "cloudfront/models/CacheBehaviorDiff"
+
+require "json"
+
+module Cumulus
+  module CloudFront
+    # Public: An object representing configuration for a distribution cache behavior
+    class CacheBehaviorConfig
+      attr_reader :default
+      attr_reader :path_pattern
+      attr_reader :target_origin_id
+      attr_reader :forward_query_strings
+      attr_reader :forwarded_cookies
+      attr_reader :forwarded_cookies_whitelist
+      attr_reader :forward_headers
+      attr_reader :allow_blank_referer
+      attr_reader :referer_checks
+      attr_reader :referer_whitelist
+      attr_reader :trusted_signers
+      attr_reader :viewer_protocol_policy
+      attr_reader :min_ttl
+      attr_reader :max_ttl
+      attr_reader :default_ttl
+      attr_reader :smooth_streaming
+      attr_reader :allowed_methods
+      attr_reader :cached_methods
+      attr_reader :compress
+
+      # Public: Constructor
+      #
+      # json - a hash containing the JSON configuration for the distribution cache behavior
+      # default - indicates if the cache configuration is the default config (ignore path_pattern if so)
+      def initialize(json = nil, default = false)
+        if !json.nil?
+          @default = default
+          @path_pattern = json["path-pattern"] if !default
+          @target_origin_id = json["target-origin-id"]
+          @forward_query_strings = json["forward-query-strings"]
+          @forwarded_cookies = json["forwarded-cookies"]
+          @forwarded_cookies_whitelist = json["forwarded-cookies-whitelist"] || []
+          @forward_headers = json["forward-headers"] || []
+          @trusted_signers = json["trusted-signers"] || []
+          @viewer_protocol_policy = json["viewer-protocol-policy"]
+          @min_ttl = json["min-ttl"]
+          @max_ttl = json["max-ttl"]
+          @default_ttl = json["default-ttl"]
+          @smooth_streaming = json["smooth-streaming"]
+          @allowed_methods = json["allowed-methods"] || []
+          @cached_methods = json["cached-methods"] || []
+          @compress = json["compress"] || false
+        end
+      end
+
+      def populate!(aws, default = false)
+        @default = default
+        @path_pattern = aws.path_pattern if !default
+        @target_origin_id = aws.target_origin_id
+        @forward_query_strings = aws.forwarded_values.query_string
+        @forwarded_cookies = aws.forwarded_values.cookies.forward
+        @forwarded_cookies_whitelist = if aws.forwarded_values.cookies.whitelisted_names.nil? then [] else aws.forwarded_values.cookies.whitelisted_names.items end
+        @forward_headers = if aws.forwarded_values.headers.nil? then [] else aws.forwarded_values.headers.items end
+        @trusted_signers = if aws.trusted_signers.enabled then aws.trusted_signers.items else [] end
+        @viewer_protocol_policy = aws.viewer_protocol_policy
+        @min_ttl = aws.min_ttl
+        @max_ttl = aws.max_ttl
+        @default_ttl = aws.default_ttl
+        @smooth_streaming = aws.smooth_streaming
+        @allowed_methods = aws.allowed_methods.items
+        @cached_methods = aws.allowed_methods.cached_methods.items
+        @compress = aws.compress
+      end
+
+      # Public: Get the config as a hash
+      #
+      # Returns the hash
+      def to_local
+        {
+          "path-pattern" => @path_pattern,
+          "target-origin-id" => @target_origin_id,
+          "forward-query-strings" => @forward_query_strings,
+          "forwarded-cookies" => @forwarded_cookies,
+          "forwarded-cookies-whitelist" => @forwarded_cookies_whitelist,
+          "forward-headers" => @forward_headers,
+          "trusted-signers" => @trusted_signers,
+          "viewer-protocol-policy" => @viewer_protocol_policy,
+          "min-ttl" => @min_ttl,
+          "max-ttl" => @max_ttl,
+          "default-ttl" => @default_ttl,
+          "smooth-streaming" => @smooth_streaming,
+          "allowed-methods" => @allowed_methods,
+          "cached-methods" => @cached_methods,
+          "compress" => @compress
+        }.reject { |k, v| v.nil? }
+      end
+
+      # Public: Get the config in the format needed for AWS
+      #
+      # Returns the hash
+      def to_aws
+        {
+          path_pattern: @path_pattern,
+          target_origin_id: @target_origin_id,
+          forwarded_values: {
+            query_string: @forward_query_strings,
+            cookies: {
+              forward: @forwarded_cookies,
+              whitelisted_names: {
+                quantity: @forwarded_cookies_whitelist.size,
+                items: if @forwarded_cookies_whitelist.empty? then nil else @forwarded_cookies_whitelist end
+              }
+            },
+            headers: {
+              quantity: @forward_headers.size,
+              items: if @forward_headers.empty? then nil else @forward_headers end
+            }
+          },
+          trusted_signers: {
+            enabled: !@trusted_signers.empty?,
+            quantity: @trusted_signers.size,
+            items: if @trusted_signers.empty? then nil else @trusted_signers end
+          },
+          viewer_protocol_policy: @viewer_protocol_policy,
+          min_ttl: @min_ttl,
+          max_ttl: @max_ttl,
+          default_ttl: @default_ttl,
+          smooth_streaming: @smooth_streaming,
+          allowed_methods: {
+            quantity: @allowed_methods.size,
+            items: if @allowed_methods.empty? then nil else @allowed_methods end,
+            cached_methods: {
+              quantity: @cached_methods.size,
+              items: if @cached_methods.empty? then nil else @cached_methods end
+            }
+          },
+          compress: @compress
+        }
+      end
+
+      def name
+        if @default
+          "Default Cache"
+        else
+          "#{target_origin_id}/#{path_pattern}"
+        end
+      end
+
+      # Public: Produce an array of differences between this local configuration and the
+      # configuration in AWS
+      #
+      # aws - the AWS resource
+      #
+      # Returns an array of the CacheBehaviorDiffs that were found
+      def diff(aws)
+        diffs = []
+
+        if !default and @path_pattern != aws.path_pattern
+          diffs << CacheBehaviorDiff.new(CacheBehaviorChange::PATH, aws, self)
+        end
+
+        if @target_origin_id != aws.target_origin_id
+          diffs << CacheBehaviorDiff.new(CacheBehaviorChange::TARGET, aws, self)
+        end
+
+        if @forward_query_strings != aws.forwarded_values.query_string
+          diffs << CacheBehaviorDiff.new(CacheBehaviorChange::QUERY, aws, self)
+        end
+
+        if @forwarded_cookies != aws.forwarded_values.cookies.forward
+          diffs << CacheBehaviorDiff.new(CacheBehaviorChange::COOKIES, aws, self)
+        end
+
+        aws_whitelist_cookies = if aws.forwarded_values.cookies.whitelisted_names.nil? then [] else aws.forwarded_values.cookies.whitelisted_names.items end
+        added_cookies = (@forwarded_cookies_whitelist - aws_whitelist_cookies)
+        removed_cookies = (aws_whitelist_cookies - @forwarded_cookies_whitelist)
+        if !added_cookies.empty? or !removed_cookies.empty?
+          diffs << CacheBehaviorDiff.cookies_whitelist(added_cookies, removed_cookies, self)
+        end
+
+        aws_headers = if aws.forwarded_values.headers.nil? then [] else aws.forwarded_values.headers.items end
+        added_headers = (@forward_headers - aws_headers)
+        removed_headers = (aws_headers - @forward_headers)
+        if !added_headers.empty? or !removed_headers.empty?
+          diffs << CacheBehaviorDiff.headers(added_headers, removed_headers, self)
+        end
+
+        aws_signers = if !aws.trusted_signers.enabled then [] else aws.trusted_signers.items end
+        added_signers = (@trusted_signers - aws_signers)
+        removed_signers = (aws_signers - @trusted_signers)
+        if !added_signers.empty? or !removed_signers.empty?
+          diffs << CacheBehaviorDiff.signers(added_signers, removed_signers, self)
+        end
+
+        if @viewer_protocol_policy != aws.viewer_protocol_policy
+          diffs << CacheBehaviorDiff.new(CacheBehaviorChange::VIEWER_PROTOCOL, aws, self)
+        end
+
+        if @min_ttl != aws.min_ttl
+          diffs << CacheBehaviorDiff.new(CacheBehaviorChange::MINTTL, aws, self)
+        end
+
+        if @max_ttl != aws.max_ttl
+          diffs << CacheBehaviorDiff.new(CacheBehaviorChange::MAXTTL, aws, self)
+        end
+
+        if @default_ttl != aws.default_ttl
+          diffs << CacheBehaviorDiff.new(CacheBehaviorChange::DEFTTL, aws, self)
+        end
+
+        if @smooth_streaming != aws.smooth_streaming
+          diffs << CacheBehaviorDiff.new(CacheBehaviorChange::STREAMING, aws, self)
+        end
+
+        aws_allowed_methods = if aws.allowed_methods.nil? then [] else aws.allowed_methods.items end
+        added_allowed_methods = (@allowed_methods - aws_allowed_methods)
+        removed_allowed_methods = (aws_allowed_methods - @allowed_methods)
+        if !added_allowed_methods.empty? or !removed_allowed_methods.empty?
+          diffs << CacheBehaviorDiff.allowed_methods(added_allowed_methods, removed_allowed_methods, self)
+        end
+
+        aws_cached_methods = if aws.allowed_methods.nil? or aws.allowed_methods.cached_methods.nil? then [] else aws.allowed_methods.cached_methods.items end
+        added_cached_methods = (@cached_methods - aws_cached_methods)
+        removed_cached_methods = (aws_cached_methods - @cached_methods)
+        if !added_cached_methods.empty? or !removed_cached_methods.empty?
+          diffs << CacheBehaviorDiff.cached_methods(added_cached_methods, removed_cached_methods, self)
+        end
+
+        if @compress != aws.compress
+          diffs << CacheBehaviorDiff.new(CacheBehaviorChange::COMPRESS, aws, self)
+        end
+
+        diffs
+      end
+
+    end
+  end
+end