ec2-snapshot-replicator 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b48d31de7f61485b9989ba74e0b87202f35f5fd5
+  data.tar.gz: a9b6f7f55d77bf5b89ebd977cd8784a0ef9d3d1b
+SHA512:
+  metadata.gz: b3e2a3cba19ef7822db45f9610bf2c87718cceb1cffa4f05ec6aad9879141de044f6b5d1ae389b02b8948f270eb2d4917381c3e6dc2c851a383f90e0aabd9476
+  data.tar.gz: 1e1dabbd1780bab1f4b36c6096f5c9f0ccbe264adb7663b1fca10d651d5ea83847cf6dad518bc8afea7933b54c3f01e51b4167d5a5026fdbd00e439ad21ae9ca

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.2.1

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ec2-snapshot-replicator.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Ryota Arai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,42 @@
+# ec2-snapshot-replicator
+
+**THIS IS NOT READY TO USE!!**
+
+Replicate snapshots to another region with delayed deletion.
+
+## Installation
+
+    $ gem install ec2-snapshot-replicator
+
+## Usage
+
+```
+$ ec2-snapshot-replicator start \
+    --delay-deletion-sec=3600 \
+    --interval-sec=600 \
+    --source-region=ap-northeast-1 \
+    --destination-region=us-east-1 \
+    --owner-id=123456789
+```
+
+If the above settings are provided:
+
+- loop the following:
+  - If a snapshot in the source region is not found in the destination region, copy it to the destination region and create `SourceSnapshotId` tag.
+  - If a snapshot in the destination region is not found in the source region, create `DeleteAfter` tag which is (now + 1 day) in seconds since unix epoch.
+  - If `DeleteAfter` time of a snapshot in the destination region is over now, delete it.
+  - Sleep 10 minutes
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/ryotarai/ec2-snapshot-replicator/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "ec2/snapshot/replicator"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/ec2-snapshot-replicator

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'ec2/snapshot/replicator'
+
+EC2::Snapshot::Replicator::CLI.start
+

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/ec2-snapshot-replicator.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ec2/snapshot/replicator/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "ec2-snapshot-replicator"
+  spec.version       = EC2::Snapshot::Replicator::VERSION
+  spec.authors       = ["Ryota Arai"]
+  spec.email         = ["ryota.arai@gmail.com"]
+
+  spec.summary       = %q{Replicate EC2 snapshots to another region.}
+  spec.homepage      = "https://github.com/ryotarai/ec2-snapshot-replicator"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "thor"
+  spec.add_dependency "aws-sdk", "~> 2"
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+end

data/lib/ec2/snapshot/replicator.rb

@@ -0,0 +1,13 @@
+require "ec2/snapshot/replicator/cli"
+require "ec2/snapshot/replicator/config"
+require "ec2/snapshot/replicator/engine"
+require "ec2/snapshot/replicator/logger"
+require "ec2/snapshot/replicator/version"
+
+module EC2
+  module Snapshot
+    module Replicator
+      # Your code goes here...
+    end
+  end
+end

data/lib/ec2/snapshot/replicator/cli.rb

@@ -0,0 +1,34 @@
+require "ec2/snapshot/replicator"
+require 'thor'
+
+module EC2
+  module Snapshot
+    module Replicator
+      class CLI < Thor
+        desc "version", "Show version"
+        def version
+          puts "v#{EC2::Snapshot::Replicator::VERSION}"
+        end
+
+        desc "start", "Start"
+        method_option :source_region, type: :string, required: true
+        method_option :destination_region, type: :string, required: true
+        method_option :interval_sec, type: :numeric, default: 60 * 10
+        method_option :delay_deletion_sec, type: :numeric, default: 60 * 60 * 24 * 7
+        method_option :owner_id, type: :string, required: true
+        method_option :debug, type: :boolean, default: false
+        method_option :once, type: :boolean, default: false
+        def start
+          config = Config.new
+          config.load_options(options)
+
+          if options[:once]
+            Engine.new(config).run_once
+          else
+            Engine.new(config).start
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ec2/snapshot/replicator/config.rb

@@ -0,0 +1,17 @@
+module EC2
+  module Snapshot
+    module Replicator
+      class Config < Struct.new(:source_region, :destination_region, :interval_sec, :delay_deletion_sec, :owner_id, :debug)
+        def load_options(options)
+          self.source_region = options[:source_region]
+          self.destination_region = options[:destination_region]
+          self.interval_sec = options[:interval_sec]
+          self.delay_deletion_sec = options[:delay_deletion_sec]
+          self.owner_id = options[:owner_id]
+          self.debug = options[:debug]
+        end
+      end
+    end
+  end
+end
+

data/lib/ec2/snapshot/replicator/engine.rb

@@ -0,0 +1,147 @@
+require 'aws-sdk'
+
+module EC2
+  module Snapshot
+    module Replicator
+      class Engine
+        SOURCE_SNAPSHOT_ID_TAG_KEY = 'SourceSnapshotId'
+        DELETE_AFTER_TAG_KEY = 'DeleteAfter'
+
+        attr_reader :source_ec2, :destination_ec2
+
+        def initialize(config)
+          @config = config
+
+          @source_ec2 = Aws::EC2::Resource.new(region: @config.source_region)
+          @destination_ec2 = Aws::EC2::Resource.new(region: @config.destination_region)
+        end
+
+        def start
+          Logger.info "start loop"
+          while true
+            run_once
+
+            Logger.info "sleeping for #{@config.interval_sec} sec..."
+            sleep @config.interval_sec
+          end
+        end
+
+        def run_once
+          replicate_snapshots
+          mark_deleted_snapshots
+          delete_snapshots
+        end
+
+        def replicate_snapshots
+          Logger.info ">>> replicating snapshots..."
+
+          source_snapshots = @source_ec2.snapshots(owner_ids: [@config.owner_id])
+          source_snapshot_ids = source_snapshots.map {|s| s.id }
+
+          destination_snapshots = @destination_ec2.snapshots(owner_ids: [@config.owner_id], filters: [{name: "tag:#{SOURCE_SNAPSHOT_ID_TAG_KEY}", values: source_snapshot_ids}])
+          source_snapshots.each do |snapshot|
+            destination_snapshot = destination_snapshots.find do |s|
+              s.tags.find do |t|
+                t.key == SOURCE_SNAPSHOT_ID_TAG_KEY &&
+                  t.value == snapshot.id
+              end
+            end
+
+            if destination_snapshot
+              Logger.debug "[#{snapshot.id}] already replicated"
+            else
+              Logger.info "[#{snapshot.id}] replicating..."
+
+              ask_continue("Copy snapshot.")
+
+              res = @destination_ec2.snapshot(snapshot.id).copy(
+                source_region: @config.source_region,
+                destination_region: @config.destination_region,
+                description: snapshot.description,
+              )
+
+              Logger.debug "[#{res.snapshot_id}] created in #{@config.destination_region}"
+              copied_snapshot = @destination_ec2.snapshot(res.snapshot_id)
+              copied_snapshot.create_tags(
+                tags: [
+                  {key: SOURCE_SNAPSHOT_ID_TAG_KEY, value: snapshot.id},
+                ],
+              )
+            end
+          end
+        end
+
+        def mark_deleted_snapshots
+          Logger.info ">>> marking deleted snapshots..."
+
+          destination_snapshots = @destination_ec2.snapshots(owner_ids: [@config.owner_id]).select do |snapshot|
+            if snapshot.tags.find {|t| t.key == DELETE_AFTER_TAG_KEY }
+              next false
+            end
+
+            unless snapshot.tags.find {|t| t.key == SOURCE_SNAPSHOT_ID_TAG_KEY }
+              Logger.debug "[#{snapshot.id}] tag #{SOURCE_SNAPSHOT_ID_TAG_KEY} is not found."
+              next false
+            end
+
+            true
+          end
+
+          source_snapshot_ids = destination_snapshots.map do |snapshot|
+            snapshot.tags.find {|t| t.key == SOURCE_SNAPSHOT_ID_TAG_KEY }.value
+          end
+
+          source_snapshots = @source_ec2.snapshots(owner_ids: [@config.owner_id], filters: [{name: 'snapshot-id', values: source_snapshot_ids}])
+
+          destination_snapshots.each do |snapshot|
+            source_snapshot_id = snapshot.tags.find {|t| t.key == SOURCE_SNAPSHOT_ID_TAG_KEY }.value
+            if source_snapshots.find {|s| s.id == source_snapshot_id }
+              Logger.debug "[#{snapshot.id}] source snapshot (#{source_snapshot_id}) exists"
+            else
+              Logger.info "[#{snapshot.id}] creating #{DELETE_AFTER_TAG_KEY} tag because source snapshot (#{source_snapshot_id}) is deleted."
+
+              delete_after = (Time.now + @config.delay_deletion_sec).to_i
+              ask_continue("Create a tag #{DELETE_AFTER_TAG_KEY}:#{delete_after}.")
+              snapshot.create_tags(
+                tags: [
+                  {key: DELETE_AFTER_TAG_KEY, value: delete_after.to_s},
+                ],
+              )
+            end
+          end
+        end
+
+        def delete_snapshots
+          Logger.info ">>> deleting snapshots..."
+          @destination_ec2.snapshots(owner_ids: [@config.owner_id]).each do |snapshot|
+            tag = snapshot.tags.find {|t| t.key == DELETE_AFTER_TAG_KEY }
+
+            unless tag
+              Logger.debug "[#{snapshot.id}] tag #{DELETE_AFTER_TAG_KEY} is not found."
+              next
+            end
+
+            delete_after = Time.at(tag.value.to_i)
+            if delete_after < Time.now
+              Logger.info "[#{snapshot.id}] deleting..."
+              ask_continue("Delete #{snapshot.id}.")
+              snapshot.delete
+            end
+          end
+        end
+
+        private
+
+        def ask_continue(msg)
+          return unless @config.debug
+
+          print "#{msg} continue? (y/N): "
+          unless $stdin.gets.downcase =~ /\Ay/
+            abort
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/ec2/snapshot/replicator/logger.rb

@@ -0,0 +1,21 @@
+require 'logger'
+
+module EC2
+  module Snapshot
+    module Replicator
+      class Logger
+        @logger = ::Logger.new($stdout)
+
+        class << self
+          %w!fatal error info debug trace!.each do |meth|
+            meth = meth.to_sym
+            define_method(meth) do |msg|
+              @logger.public_send(meth, msg)
+            end
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/ec2/snapshot/replicator/version.rb

@@ -0,0 +1,7 @@
+module EC2
+  module Snapshot
+    module Replicator
+      VERSION = "0.1.0"
+    end
+  end
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: ec2-snapshot-replicator
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ryota Arai
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-04-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- ryota.arai@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/ec2-snapshot-replicator
+- bin/setup
+- ec2-snapshot-replicator.gemspec
+- lib/ec2/snapshot/replicator.rb
+- lib/ec2/snapshot/replicator/cli.rb
+- lib/ec2/snapshot/replicator/config.rb
+- lib/ec2/snapshot/replicator/engine.rb
+- lib/ec2/snapshot/replicator/logger.rb
+- lib/ec2/snapshot/replicator/version.rb
+homepage: https://github.com/ryotarai/ec2-snapshot-replicator
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Replicate EC2 snapshots to another region.
+test_files: []