cumulus-aws 0.11.1

data/lib/kinesis/loader/Loader.rb

@@ -0,0 +1,19 @@
+require "common/BaseLoader"
+require "conf/Configuration"
+require "kinesis/models/StreamConfig"
+
+module Cumulus
+  module Kinesis
+    module Loader
+
+      include Common::BaseLoader
+
+      @@streams_dir = Configuration.instance.kinesis.directory
+
+      def self.streams
+        Common::BaseLoader::resources(@@streams_dir, &StreamConfig.method(:new))
+      end
+
+    end
+  end
+end

data/lib/kinesis/manager/Manager.rb

@@ -0,0 +1,206 @@
+require "common/manager/Manager"
+require "conf/Configuration"
+require "kinesis/Kinesis"
+require "kinesis/loader/Loader"
+require "kinesis/models/StreamConfig"
+require "kinesis/models/StreamDiff"
+
+require "aws-sdk"
+
+module Cumulus
+  module Kinesis
+    class Manager < Common::Manager
+
+      def initialize
+        super()
+        @create_asset = true
+        @client = Aws::Kinesis::Client.new(Configuration.instance.client)
+      end
+
+      def resource_name
+        "Kinesis Stream"
+      end
+
+      def local_resources
+        @local_resources ||= Hash[Loader.streams.map { |local| [local.name, local] }]
+      end
+
+      def aws_resources
+        @aws_resources ||= Kinesis::named_streams
+      end
+
+      def unmanaged_diff(aws)
+        StreamDiff.unmanaged(aws)
+      end
+
+      def added_diff(local)
+        StreamDiff.added(local)
+      end
+
+      def diff_resource(local, aws)
+        local.diff(aws)
+      end
+
+      def migrate
+        puts Colors.blue("Migrating Kinesis Streams...")
+
+        # Create the directories
+        streams_dir = "#{@migration_root}/kinesis"
+
+        if !Dir.exists?(@migration_root)
+          Dir.mkdir(@migration_root)
+        end
+        if !Dir.exists?(streams_dir)
+          Dir.mkdir(streams_dir)
+        end
+
+        Kinesis::named_streams.each do |name, stream|
+          puts "Migrating stream #{name}"
+
+          cumulus_stream = StreamConfig.new(name).populate!(stream)
+          json = JSON.pretty_generate(cumulus_stream.to_hash)
+          File.open("#{streams_dir}/#{name}.json", "w") { |f| f.write(json) }
+        end
+
+      end
+
+      def create(local)
+        @client.create_stream({
+          stream_name: local.name,
+          shard_count: local.shards
+        })
+
+        @client.wait_until(:stream_exists, {
+          stream_name: local.name
+        })
+
+        # Describe the newly created stream
+        created_stream = Kinesis::describe_stream(local.name)
+
+        # If the stream retention period is different, then update it
+        if created_stream.retention_period_hours > local.retention_period
+          @client.decrease_stream_retention_period({
+            stream_name: local.name,
+            retention_period_hours: local.retention_period
+          })
+        elsif created_stream.retention_period_hours < local.retention_period
+          @client.increase_stream_retention_period({
+            stream_name: local.name,
+            retention_period_hours: local.retention_period
+          })
+        end
+
+        # If the created stream has tags, add them
+        if !local.tags.empty?
+          @client.add_tags_to_stream({
+            stream_name: local.name,
+            tags: local.tags
+          })
+        end
+
+      end
+
+      def update(local, diffs)
+        diffs.each do |diff|
+          case diff.type
+          when StreamChange::SHARDS
+
+            # See if we are splitting or merging and make sure it is a multiple of 2
+            if diff.aws < diff.local
+              if diff.aws != diff.local / 2.0
+                puts Colors.red("Can only increase the number of shards by a factor of 2")
+              else
+                aws_stream = Kinesis::named_streams[local.name]
+
+                # Split the shards 1 at a time
+                aws_stream.sorted_shards.each do |shard|
+                  puts Colors.blue("Splitting shard #{shard.shard_id}")
+
+                  # The splitting point is halfway between the hash start and end
+                  hash_start = shard.hash_key_range.starting_hash_key.to_i
+                  hash_end = shard.hash_key_range.ending_hash_key.to_i
+                  hash_split = hash_start + ((hash_end - hash_start) / 2)
+
+                  @client.split_shard({
+                    stream_name: local.name,
+                    shard_to_split: shard.shard_id,
+                    new_starting_hash_key: hash_split.to_s
+                  })
+
+                  # After every split we have to wait until the stream is ready
+                  @client.wait_until(:stream_exists, {
+                    stream_name: local.name
+                  })
+                end
+
+              end
+            elsif diff.aws > diff.local
+              aws_stream = Kinesis::named_streams[local.name]
+
+              if aws_stream.sorted_shards.length != local.shards * 2.0
+                puts Colors.red("Can only decrease the number of shards by a factor of 2")
+              else
+                # Merge the sorted shards in groups of 2
+                aws_stream.sorted_shards.each_slice(2) do |slice|
+                  puts Colors.blue("Merging shards #{slice[0].shard_id} and #{slice[1].shard_id}")
+
+                  @client.merge_shards({
+                    stream_name: local.name,
+                    shard_to_merge: slice[0].shard_id,
+                    adjacent_shard_to_merge: slice[1].shard_id
+                  })
+
+                  # After every merge we have to wait until the stream is ready
+                  @client.wait_until(:stream_exists, {
+                    stream_name: local.name
+                  })
+                end
+
+              end
+            end
+
+          when StreamChange::RETENTION
+            puts Colors.blue("Updating retention period...")
+
+
+            if diff.aws > local.retention_period
+              @client.decrease_stream_retention_period({
+                stream_name: local.name,
+                retention_period_hours: local.retention_period
+              })
+            elsif diff.aws < local.retention_period
+              @client.increase_stream_retention_period({
+                stream_name: local.name,
+                retention_period_hours: local.retention_period
+              })
+            end
+
+            # Wait for the stream to be in an active state or shard updates will fail
+            @client.wait_until(:stream_exists, {
+              stream_name: local.name
+            })
+          when StreamChange::TAGS
+            puts Colors.blue("Updating tags...")
+
+            if !diff.tags_to_remove.empty?
+              @client.remove_tags_from_stream({
+                stream_name: local.name,
+                tag_keys: diff.tags_to_remove.keys
+              })
+            end
+
+            if !diff.tags_to_add.empty?
+              @client.add_tags_to_stream({
+                stream_name: local.name,
+                tags: diff.tags_to_add
+              })
+            end
+
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/kinesis/models/StreamConfig.rb

@@ -0,0 +1,75 @@
+require "conf/Configuration"
+require "kinesis/loader/Loader"
+require "kinesis/models/StreamDiff"
+
+require "json"
+
+module Cumulus
+  module Kinesis
+
+    # Public: An object representing configuration for a Kiensis stream
+    class StreamConfig
+      attr_reader :name
+      attr_reader :retention_period
+      attr_reader :shards
+      attr_reader :tags
+
+      # Public: Constructor
+      #
+      # json - a hash containing the JSON configuration for the stream
+      def initialize(name, json = nil)
+        @name = name
+        if !json.nil?
+          @shards = json["shards"]
+          @retention_period = json["retention-period"] || 24
+          @tags = json["tags"] || {}
+        end
+      end
+
+      def to_hash
+        {
+          "retention-period" => @retention_period,
+          "shards" => @shards,
+          "tags" => @tags
+        }
+      end
+
+      # Public: Populate a config object with AWS configuration
+      #
+      # aws - the AWS configuration for the strean
+      def populate!(aws)
+        @retention_period = aws.retention_period_hours
+        @shards = aws.sorted_shards.length
+        @tags = Kinesis::stream_tags[aws.stream_name] || {}
+
+        self
+      end
+
+      # Public: Produce an array of differences between this local configuration and the
+      # configuration in AWS
+      #
+      # aws - the AWS resource
+      #
+      # Returns an array of the StreamDiffs that were found
+      def diff(aws)
+        diffs = []
+
+        if @retention_period != aws.retention_period_hours
+          diffs << StreamDiff.new(StreamChange::RETENTION, aws.retention_period_hours, @retention_period)
+        end
+
+        if @shards != aws.sorted_shards.length
+          diffs << StreamDiff.new(StreamChange::SHARDS, aws.sorted_shards.length, @shards)
+        end
+
+        aws_tags = Kinesis::stream_tags[aws.stream_name]
+        if @tags != aws_tags
+          diffs << StreamDiff.new(StreamChange::TAGS, aws_tags, @tags)
+        end
+
+        diffs
+      end
+
+    end
+  end
+end

data/lib/kinesis/models/StreamDiff.rb

@@ -0,0 +1,58 @@
+require "common/models/Diff"
+require "common/models/ListChange"
+require "common/models/TagsDiff"
+require "util/Colors"
+
+module Cumulus
+  module Kinesis
+    # Public: The types of changes that can be made to a stream
+    module StreamChange
+      include Common::DiffChange
+
+      SHARDS = Common::DiffChange.next_change_id
+      RETENTION = Common::DiffChange.next_change_id
+      TAGS = Common::DiffChange.next_change_id
+    end
+
+    # Public: Represents a single difference between local configuration and AWS configuration
+    class StreamDiff < Common::Diff
+      include StreamChange
+      include Common::TagsDiff
+
+      def local_tags
+        @local
+      end
+
+      def aws_tags
+        @aws
+      end
+
+      def asset_type
+        "Stream"
+      end
+
+      def aws_name
+        @aws.stream_name
+      end
+
+      def diff_string
+        case @type
+        when SHARDS
+          [
+            "Shards:",
+            Colors.aws_changes("\tAWS - #{aws}"),
+            Colors.local_changes("\tLocal - #{local}"),
+          ].join("\n")
+        when RETENTION
+          [
+            "Retention:",
+            Colors.aws_changes("\tAWS - #{aws} hours"),
+            Colors.local_changes("\tLocal - #{local} hours"),
+          ].join("\n")
+        when TAGS
+          tags_diff_string
+        end
+      end
+    end
+  end
+end

data/lib/lambda/Lambda.rb

@@ -0,0 +1,41 @@
+require "conf/Configuration"
+
+require "aws-sdk"
+
+module Cumulus
+  module Lambda
+    class << self
+      @@client = Aws::Lambda::Client.new(Configuration.instance.client)
+
+      # Public: Static method that will get a Lambda function from AWS by its
+      # name
+      #
+      # name - the name of the function to get
+      #
+      # Returns the function
+      def get_aws(name)
+        functions.fetch(name)
+      rescue KeyError
+        puts "No Lambda function named #{name}"
+        exit
+      end
+
+      # Public: Provide a mapping of functions to their names. Lazily loads
+      # resources.
+      #
+      # Returns the functions mapped to their names
+      def functions
+        @functions ||= init_functions
+      end
+
+      private
+
+      # Internal: Load the functions and map them to their names
+      #
+      # Returns the functions mapped to their names
+      def init_functions
+        Hash[@@client.list_functions.functions.map { |f| [f.function_name, f] }]
+      end
+    end
+  end
+end

data/lib/route53/loader/Loader.rb

@@ -0,0 +1,32 @@
+require "common/BaseLoader"
+require "conf/Configuration"
+require "route53/models/ZoneConfig"
+
+# Public: Load Route53 assets
+module Cumulus
+  module Route53
+    module Loader
+      include Common::BaseLoader
+
+      @@zones_dir = Configuration.instance.route53.zones_directory
+      @@includes_dir = Configuration.instance.route53.includes_directory
+
+      # Public: Load all the zone configurations as ZoneConfig objects
+      #
+      # Returns an array of ZoneConfig
+      def self.zones
+        Common::BaseLoader::resources(@@zones_dir, &ZoneConfig.method(:new))
+      end
+
+      # Public: Load a single "includes file" as parsed JSON
+      #
+      # name - the name of the file to include
+      #
+      # Returns an array of parsed JSON
+      def self.includes_file(name)
+        Common::BaseLoader::resource(name, @@includes_dir, &Proc.new { |n, json| json })
+      end
+
+    end
+  end
+end

data/lib/route53/manager/Manager.rb

@@ -0,0 +1,241 @@
+require "common/manager/Manager"
+require "conf/Configuration"
+require "route53/loader/Loader"
+require "route53/models/RecordDiff"
+require "route53/models/Vpc"
+require "route53/models/ZoneDiff"
+require "util/Colors"
+
+require "aws-sdk"
+
+module Cumulus
+  module Route53
+    class Manager < Common::Manager
+      def initialize
+        super()
+        @create_asset = false
+        @route53 = Aws::Route53::Client.new(Configuration.instance.client)
+      end
+
+      # Public: Migrate AWS Route53 configuration to Cumulus configuration.
+      def migrate
+        zones_dir = "#{@migration_root}/zones"
+
+        if !Dir.exists?(@migration_root)
+          Dir.mkdir(@migration_root)
+        end
+        if !Dir.exists?(zones_dir)
+          Dir.mkdir(zones_dir)
+        end
+
+        aws_resources.each_value do |resource|
+          puts "Processing #{resource.name}..."
+          config = ZoneConfig.new(resource.name)
+          config.populate(resource)
+
+          puts "Writing #{resource.name} configuration to file"
+          filename = if config.private then "#{config.name}-private" else config.name end
+          File.open("#{zones_dir}/#{filename.sub(".", "-")}.json", "w") { |f| f.write(config.pretty_json) }
+        end
+      end
+
+      def resource_name
+        "Zone"
+      end
+
+      def local_resources
+        @local_resources ||= Hash[Loader.zones.map { |local| [local.id, local] }]
+      end
+
+      def aws_resources
+        @aws_resources ||= init_aws_resources
+      end
+
+      def unmanaged_diff(aws)
+        ZoneDiff.unmanaged(aws)
+      end
+
+      def added_diff(local)
+        ZoneDiff.added(local)
+      end
+
+      def diff_resource(local, aws)
+        local.diff(aws)
+      end
+
+      def update(local, diffs)
+        diffs.each do |diff|
+          case diff.type
+          when ZoneChange::COMMENT
+            puts Colors.blue("\tupdating comment...")
+            update_comment(local.id, local.comment)
+          when ZoneChange::DOMAIN
+            puts "\tAWS doesn't allow you to change the domain for a zone."
+          when ZoneChange::PRIVATE
+            puts "\tAWS doesn't allow you to change whether a zone is private."
+          when ZoneChange::VPC
+            update_vpc(local.id, diff.added_vpc_ids, diff.removed_vpc_ids)
+          when ZoneChange::RECORD
+            update_records(
+              local.id,
+              diff.changed_records.reject do |r|
+                r.type == RecordChange::IGNORED or r.type == RecordChange::DEFAULT
+              end
+            )
+
+            ignored = diff.changed_records.select { |r| r.type == RecordChange::IGNORED }
+            if Configuration.instance.route53.print_all_ignored
+              ignored.each do |record_diff|
+                puts "\tIgnoring record #{record_diff.aws_name}"
+              end
+            else
+              if ignored.size > 0
+                puts "\tYour blacklist ignored #{ignored.size} records."
+              end
+            end
+          end
+        end
+      end
+
+      private
+
+      # Internal: Update the comment associated with a zone.
+      #
+      # id      - the id of the zone to update
+      # comment - the new comment
+      def update_comment(id, comment)
+        @route53.update_hosted_zone_comment({
+          id: id,
+          comment: comment
+        })
+      end
+
+      # Internal: Update the VPCs associated with a zone.
+      #
+      # id         - the id of the zone to update
+      # associate  - the vpc ids to associate with the zone
+      # dissociate - the vpc ids to dissociate from the zone
+      def update_vpc(id, associate, dissociate)
+        if !associate.empty?
+          puts Colors.blue("\tassociating VPCs...")
+          associate.each do |vpc|
+            @route53.associate_vpc_with_hosted_zone({
+              hosted_zone_id: id,
+              vpc: { vpc_id: vpc.id, vpc_region: vpc.region }
+            })
+          end
+        end
+        if !dissociate.empty?
+          puts Colors.blue("\tdissociating VPCs...")
+          dissociate.each do |vpc|
+            @route53.disassociate_vpc_from_hosted_zone({
+              hosted_zone_id: id,
+              vpc: { vpc_id: vpc.id, vpc_region: vpc.region }
+            })
+          end
+        end
+      end
+
+      # Internal: Update the records associated with a zone.
+      #
+      # id      - the id of the zone to update
+      # records - RecordDiff objects representing the changes
+      def update_records(id, records)
+        puts Colors.blue("\tupdating records...")
+        if !records.empty?
+          changes = records.map do |record|
+            action = nil
+            resource = nil
+
+            case record.type
+            when RecordChange::CHANGED
+              action = "UPSERT"
+              resource = record.local
+            when RecordChange::ADD
+              action = "CREATE"
+              resource = record.local
+            when RecordChange::UNMANAGED
+              action = "DELETE"
+              resource = record.aws
+            end
+
+            {
+              action: action,
+              resource_record_set: {
+                name: resource.name.gsub("@", "\\\\100"),
+                type: resource.type,
+                ttl: resource.ttl,
+                resource_records: resource.resource_records,
+                alias_target: if resource.alias_target.nil? then nil else {
+                  hosted_zone_id: resource.alias_target.hosted_zone_id,
+                  dns_name: resource.alias_target.dns_name,
+                  evaluate_target_health: resource.alias_target.evaluate_target_health
+                } end
+              }
+            }
+          end
+
+          @route53.change_resource_record_sets({
+            hosted_zone_id: id,
+            change_batch: {
+              changes: changes
+            }
+          })
+        end
+      end
+
+      # A struct that combines all the data about a hosted zone in AWS
+      AwsZone = Struct.new(:id, :name, :config, :vpc, :route53) do
+        def records
+          @records ||= get_zone_records
+        end
+
+        private
+
+        # Internal: Get the records for this hosted zone.
+        #
+        # Returns an array of records belonging to the zone
+        def get_zone_records()
+          records = []
+          all_records_retrieved = false
+          next_record_name = nil
+          next_record_type = nil
+          next_record_identifier = nil
+
+          until all_records_retrieved
+            response = route53.list_resource_record_sets({
+              hosted_zone_id: id,
+              start_record_name: next_record_name,
+              start_record_type: next_record_type,
+              start_record_identitifier: next_record_identifier
+            }.reject { |k, v| v.nil? })
+            records << response.resource_record_sets
+            next_record_name = response.next_record_name
+            next_record_type = response.next_record_type
+            next_record_identifier = response.next_record_identifier
+
+            if !response.is_truncated
+              all_records_retrieved = true
+            end
+          end
+
+          records.flatten.map { |r| r.name = r.name.chomp(".").sub(/\\052/, "*"); r }
+        end
+      end
+
+      def init_aws_resources
+        aws = @route53.list_hosted_zones.hosted_zones.map do |zone|
+          vpc = if zone.config.private_zone
+            details = @route53.get_hosted_zone(id: zone.id)
+            details.vp_cs.map { |v| Vpc.new(v.vpc_id, v.vpc_region) }
+          else
+            nil
+          end
+          AwsZone.new(zone.id, zone.name.chomp("."), zone.config, vpc, @route53)
+        end
+        Hash[aws.map { |z| [z.id, z] }]
+      end
+
+    end
+  end
+end