aws_one_click_staging 0.0.0 → 0.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e6bd6e2d8174efa8efa1ec17c2d1e12dccc215e0
-  data.tar.gz: bf4f9dc0df6d3f5203b11571a135886dbd38728e
+  metadata.gz: 90ed89f6ad179598fb6989d7655c9524305589b7
+  data.tar.gz: 8a8c004ddeaf405de3670debbe1501cc70321889
 SHA512:
-  metadata.gz: b84fa130a34f92acbc686f85def227427b4a0734a2e701c239106b0ff5fbcde3e01d1d496d2b383f7970c5da800d988debc21a738605bfc11d90a7c2e56680ff
-  data.tar.gz: d08dbed9ca05b3c0a1fd36da0675316e985bc21772c4c90a8211ee610d44d14be018d3a25911f98a5331809dab4a86e32bdc54946a994da08c9c9b34436db31a
+  metadata.gz: 26af7c6c981ed74c0708f87b898fb2dab81a30111e814cf88b1e1ed9b8655be48533d54fce5327c833f25fccdf43bcc1969a68aa3c9f1af87a6dc1d8f1ca26ee
+  data.tar.gz: 6019696f745680f4057122db72c8d99cd6193a1cb0137449773de011668ad3dccf738769a4a0d142e5150c8e821ab50c6b686c65c40dbd863220c379b3dc865c

data/.gitignore

@@ -7,3 +7,5 @@
 /pkg/
 /spec/reports/
 /tmp/
+
+config/aws_actual_fffing_secrets.yml

data/README.md

@@ -16,13 +16,13 @@ Next you'll need to setup the config file:
 
 (~/.config/aws_one_click_staging.yml)
 ```
-aws_access_key_id: ''
-aws_secret_access_key: ''
+aws_access_key_id: ""
+aws_secret_access_key: ""
 aws_region: 'us-west-1'
-aws_bucket: ''
-db_host: ''
-db_port: '5432'
-db_password: ''
+aws_master_username: ""
+aws_master_user_password: ""
+aws_production_bucket: "" # this bucket is read from
+aws_staging_bucket: ""    # this bucket is DELETED and written to!
 ```
 
 
@@ -31,7 +31,7 @@ db_password: ''
 Because amazon services are kind of a mess right now, it's best to run this from on an actual Amazon server you have shell access to (it downloads the bucket files and then re-uploads them, lol).
 
 ```
-aws_one_click_staging stage 
+aws_one_click_staging stage
 ```
 
 After a while, the operation will complete and it will say 'congrats' or something and output the RDS url and bucket name for the staging clone.  Plug those values into your staging server and you should be good to go.  

data/aws_one_click_staging.gemspec

@@ -14,11 +14,13 @@ Gem::Specification.new do |spec|
   # spec.license       = "MIT" # uncomment this line if MIT is the best license for your situation
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
+  spec.bindir        = "bin"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency 'aws-sdk', '~> 2'
   spec.add_dependency "aws-sdk-v1"
+  spec.add_dependency "thor"
 
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"

data/bin/aws_one_click_staging

@@ -1,3 +1,28 @@
 #!/usr/bin/env ruby
 
+require 'thor'
 require "aws_one_click_staging"
+
+class AwsOneClickStagingRunner < Thor
+
+  default_task :start
+
+  desc "list", "Shows help"
+  def list
+    AwsOneClickStaging.list
+  end
+
+  desc "stage", "Makes a copy of the RDS database and the staging server's S3 bucket.  This takes a while."
+  def stage
+    AwsOneClickStaging.stage(421)
+  end
+
+  desc "version", "Prints gem's version"
+  def version
+    AwsOneClickStaging::VERSION
+  end
+  map %w(-v --version) => :version
+
+end
+
+AwsOneClickStagingRunner.start

data/config/aws_one_click_staging.yml

@@ -0,0 +1,9 @@
+# edit these values to match that of your production machine
+# which will be cloned from
+aws_access_key_id: ""
+aws_secret_access_key: ""
+aws_region: 'us-west-1'
+aws_master_username: ""
+aws_master_user_password: ""
+aws_production_bucket: "" # this bucket is read from
+aws_staging_bucket: ""    # this bucket is DELETED and written to!

data/lib/aws_one_click_staging.rb

@@ -1,5 +1,18 @@
+require 'aws_one_click_staging/bucket_sync_service'
+require "aws_one_click_staging/aws_warrior"
 require "aws_one_click_staging/version"
 
+SOURCE_ROOT = File.expand_path("#{File.dirname(__FILE__)}/..")
+
 module AwsOneClickStaging
-  # Your code goes here...
+
+  def self.stage
+    warrior = AwsWarrior.new
+  end
+
+  def self.list
+    warrior = AwsWarrior.new # this makes a config file if needed
+    puts "This is aws_one_click_staging, use stage to set things up"
+  end
+
 end

data/lib/aws_one_click_staging/aws_warrior.rb

@@ -0,0 +1,139 @@
+require 'fileutils'
+require 'aws-sdk'
+require 'yaml'
+
+module AwsOneClickStaging
+
+  class AwsWarrior
+
+    def initialize
+      # read config file
+      @config_dir = "#{ENV['HOME']}/.config"
+      @config_file = File.expand_path("#{@config_dir}/aws_one_click_staging.yml")
+      @config = YAML.load(read_config_file)
+    end
+
+    def clone_rds
+      setup_aws_credentials
+
+      @db_instance_id_production = "actioncenter-staging"
+      @db_instance_id_staging = "actioncenter-staging-test"
+      @db_snapshot_id = "actioncenter-snapshot-for-staging"
+
+      @c = Aws::RDS::Client.new
+
+      delete_snapshot_for_staging!
+      create_new_snapshot_for_staging!
+
+      delete_staging_db_instance!
+      spawn_new_staging_db_instance!
+    end
+
+    def clone_s3_bucket
+      setup_aws_credentials
+
+      from_creds = { aws_access_key_id: @access_key_id,
+        aws_secret_access_key: @secret_access_key,
+        bucket: @aws_production_bucket}
+      to_creds = { aws_access_key_id: @access_key_id,
+        aws_secret_access_key: @secret_access_key,
+        bucket: @aws_staging_bucket}
+
+      bs = BucketSyncService.new(from_creds, to_creds)
+
+      bs.perform
+    end
+
+
+    private
+
+    def setup_aws_credentials
+      @config[""]
+      aws_region = @config["aws_region"]
+      @access_key_id = @config["aws_access_key_id"]
+      @secret_access_key = @config["aws_secret_access_key"]
+      @master_username = @config["aws_master_username"]
+      @master_user_password = @config["aws_master_user_password"]
+      @aws_production_bucket = @config["aws_production_bucket"]
+      @aws_staging_bucket = @config["aws_staging_bucket"]
+
+      Aws.config.update({ region: aws_region,
+        credentials: Aws::Credentials.new(@access_key_id, @secret_access_key) })
+    end
+
+    def delete_snapshot_for_staging!
+      puts "deleting old staging db snapshot"
+      response = @c.delete_db_snapshot(db_snapshot_identifier: @db_snapshot_id)
+
+      sleep 1 while response.percent_progress != 100
+      true
+    rescue
+      false
+    end
+
+    def create_new_snapshot_for_staging!
+      puts "creating new snapshot... this takes like 70 seconds..."
+      response = @c.create_db_snapshot({db_instance_identifier: @db_instance_id_production,
+        db_snapshot_identifier: @db_snapshot_id })
+
+      sleep 10 while get_fresh_db_snapshot_state.status != "available"
+      true
+    rescue
+      false
+    end
+
+
+    def delete_staging_db_instance!
+      response = @c.delete_db_instance(db_instance_identifier: @db_instance_id_staging)
+
+      sleep 2 while response.percent_progress != 100
+    rescue
+      false
+    end
+
+    def spawn_new_staging_db_instance!
+      response = @c.create_db_instance(db_instance_identifier: @db_instance_id_staging,
+        db_instance_class: "db.t1.micro",
+        engine: "postgres",
+        master_username: @master_username,
+        master_user_password: @master_user_password,
+        allocated_storage: "10")
+
+      sleep 10 while get_fresh_db_instance_state.db_instance_status != "available"
+    end
+
+
+    # we use this methods cause amazon lawl-pain
+    def get_fresh_db_snapshot_state
+      @c.describe_db_snapshots(db_snapshot_identifier: @db_snapshot_id).db_snapshots.first
+    end
+
+    def get_fresh_db_instance_state
+      @c.describe_db_instances(db_instance_identifier: @db_instance_id_staging).db_instances.first
+    end
+
+
+    def read_config_file
+      return if create_config_file_if_needed!
+      config = File.read(@config_file)
+    end
+
+    def create_config_file_if_needed!
+      return false if File.exists?(@config_file)
+      msg = ""
+      msg += "Config file not found, creating...\n\n"
+
+      # copy example config file to config file path
+      FileUtils.mkdir_p @config_dir
+      FileUtils.cp("#{SOURCE_ROOT}/config/aws_one_click_staging.yml", @config_file)
+
+      msg += "An empty config file was created for you in #{@config_file}\n"
+      msg += "Please populate it with the correct information and run this \n"
+      msg += "command again.  \n"
+
+      true
+    end
+
+  end
+
+end

data/lib/aws_one_click_staging/bucket_sync_service.rb

@@ -0,0 +1,85 @@
+# props to bantic
+# https://gist.github.com/bantic/4080793
+require 'aws/s3' # gem name is 'aws-sdk'
+
+class BucketSyncService
+  attr_reader :from_bucket, :to_bucket, :logger
+  attr_accessor :debug
+
+  DEFAULT_ACL = :public_read
+
+  # from_credentials and to_credentials are both hashes with these keys:
+  #  * :aws_access_key_id
+  #  * :aws_secret_access_key
+  #  * :bucket
+  def initialize(from_credentials, to_credentials)
+    @from_bucket = bucket_from_credentials(from_credentials)
+    @to_bucket   = bucket_from_credentials(to_credentials)
+  end
+
+  def perform(output=STDOUT)
+    object_counts = {sync:0, skip:0}
+    create_logger(output)
+
+    logger.info "Starting sync."
+    from_bucket.objects.each do |object|
+      if object_needs_syncing?(object)
+        sync(object)
+        object_counts[:sync] += 1
+      else
+        logger.debug "Skipped #{pp object}"
+        object_counts[:skip] += 1
+      end
+    end
+    logger.info "Done. Synced #{object_counts[:sync]}, " +
+      "skipped #{object_counts[:skip]}."
+  end
+
+  private
+
+  def create_logger(output)
+    @logger = Logger.new(output).tap do |l|
+      l.level = debug ? Logger::DEBUG : Logger::INFO
+    end
+  end
+
+  def sync(object)
+    logger.debug "Syncing #{pp object}"
+    object.copy_to( to_bucket.objects[object.key], acl:DEFAULT_ACL)
+  end
+
+  def pp(object)
+    content_length_in_kb = object.content_length / 1024
+    "#{object.key} #{content_length_in_kb}k " +
+      "#{object.last_modified.strftime("%b %d %Y %H:%M")}"
+  end
+
+  def object_needs_syncing?(object)
+    to_object = to_bucket.objects[object.key]
+    return true if !to_object.exists?
+
+    return to_object.etag != object.etag
+  end
+
+  def bucket_from_credentials(credentials)
+    s3 = AWS::S3.new(access_key_id:      credentials[:aws_access_key_id],
+                     secret_access_key:  credentials[:aws_secret_access_key])
+
+    bucket = s3.buckets[ credentials[:bucket] ]
+    if !bucket.exists?
+      bucket = s3.buckets.create( credentials[:bucket] )
+    end
+    bucket
+  end
+end
+
+
+
+=begin
+Example usage:
+ from_creds = {aws_access_key_id:"XXX", aws_secret_access_key:"YYY", bucket:"first-bucket"}
+ to_creds = {aws_access_key_id:"ZZZ", aws_secret_access_key:"AAA", bucket:"second-bucket"}
+ syncer = BucketSyncService.new(from_creds, to_creds)
+ syncer.debug = true # log each object
+ syncer.perform
+=end

data/lib/aws_one_click_staging/version.rb

@@ -1,3 +1,3 @@
 module AwsOneClickStaging
-  VERSION = "0.0.0"
+  VERSION = "0.0.1"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: aws_one_click_staging
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
 platform: ruby
 authors:
 - TheNotary
 autorequire: 
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2015-11-23 00:00:00.000000000 Z
+date: 2015-11-26 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-v1
   requirement: !ruby/object:Gem::Requirement
@@ -24,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -96,9 +124,11 @@ files:
 - Rakefile
 - aws_one_click_staging.gemspec
 - bin/aws_one_click_staging
-- bin/console
 - changelog
+- config/aws_one_click_staging.yml
 - lib/aws_one_click_staging.rb
+- lib/aws_one_click_staging/aws_warrior.rb
+- lib/aws_one_click_staging/bucket_sync_service.rb
 - lib/aws_one_click_staging/version.rb
 - rspec
 homepage: https://github.com/TheNotary/aws_one_click_staging

data/bin/console

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "aws_one_click_staging"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start