synchronize 0.2.0

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in backups.gemspec
+gemspec

data/README.md

@@ -0,0 +1,65 @@
+# DESCRIPTION
+
+Synchronize is designed to automate the ugly tasks of restoring database dumps and assets from s3 to our different environments.
+
+# INSTALL
+
+## Gemfile
+Synchronize will be available to download as a gem in the [downloads](https://github.com/bebanjo/synchronize/downloads) section.
+
+Get Latest: https://github.com/downloads/bebanjo/synchronize/synchronize-0.1.1.gem
+
+## From source
+
+    git clone git@github.com:bebanjo/synchronize.git
+    cd synchronize
+    gem build synchronize.gemspec 
+    gem install synchronize-X.Y.Z.gem
+
+## Dependencies
+
+  * Synchronize requires duplicity and python boto to restore files from s3, you can install both running `synchronize dependencies`. It only does work on mac and requires brew.
+
+# USAGE
+
+## Subcommands
+
+Run synchronize followed by one of these sub-commands:
+
+  * `assets`               Synchronizes the assets to the system directory                
+  * `check`                Perform a health check of system dependencies and environment variables required in order to run synchronize           
+  * `db`                   Synchronizes a database with the latest s3 dbdump.             
+  * `db2disk`              Downloads the latest backup from s3 to local disk.             
+  * `dependencies`         Installs the command line tools required for synchronize to work. Requires brew.               
+  * `help`                 Display global or [command] help documentation.        
+
+## Environment variables, options and paths.
+
+There are three of ways in which synchronize is able to get the information it needs to perform its tasks: trough command options, environment variables and knife. Command options has preference over knife, environment variables are always a last resource. Not all the information can be gathered trough knife (see below) so in these cases they should be present as options or environment variables. In any case the command helps you to find what you are missing. Run synchronize help to see every available option.
+
+  * Synchronize assumes that you are sitting it in your `RAILS_ROOT` directory when you run it (`if RAILS_ENV ==  development`). In other `RAILS_ENV` situations (`production`, `staging`, `preproduction`) Synchronize assumes that the directory structure where the app is sitting is `/home/APP_NAME/app/current`. The command also accepts a `RAILS_ROOT` env variable. Alternatively you need to provide mysql credentials through command line options.
+  * `RAILS_ENV` variable must be defined or `--rails_env` paramenter has to be used.
+  * When using `--knife` parameter, synchronize assumes that there is chef data bag (already created in our chef server) set up with the following structure:
+
+````
+    [
+     "buckets" : {
+     "id": "buckets",
+     "s3_backup_bucket": "VALUE"
+     },
+     "keys": {
+     "aws_access_key_id": "VALUE",
+     "id": "keys",
+     "aws_secret_access_key": "VALUE"
+     }
+    ]
+````
+  * Alternatively you can set the following environment variables: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_BUCKET` use the related command line options.
+  * You can run synchronize check to know what's missing to run synchronize. 
+
+
+# TODO / IMPROVEMENTS
+
+  * Refactor code, at this moment synchronize does not use any classes.
+  * Error parsing, output of some commands need to be parsed and synchronize the error output, also the verbosity of output on errors should be improved.
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/synchronize

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+require 'synchronize'

data/lib/synchronize.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'yaml'
+require 'fog'
+require 'commander/import'
+require 'synchronize/cli'
+require 'synchronize/version'
+
+program :name, "synchronize"
+program :description, "Performs synchronization of the contents between different Rails environments."
+program :version, VERSION
+
+# Some nice defaults
+RAILS_ROOT = Dir.pwd unless ENV['RAILS_ROOT']
+
+
+

data/lib/synchronize/assets.rb

@@ -0,0 +1,9 @@
+require 'synchronize/tasks'
+
+def duplicity_restore
+  puts "  - Restoring assets from s3"
+  system("duplicity -verror --no-encryption --force restore s3+http://#{@bucket}/projects/#{@app_name}/files/#{@app_name} #{RAILS_ROOT}/public/system/")
+  puts $? != 0 ? "Assets restore unsuccessful." : "Assets restore successful."
+end
+
+

data/lib/synchronize/cli.rb

@@ -0,0 +1,109 @@
+require 'synchronize/tasks'
+require 'synchronize/db'
+require 'synchronize/utils'
+require 'synchronize/assets'
+
+default_command :all
+
+command :all do |c|
+  c.syntax = "#{program(:name)}"
+  c.description = "Synchronizes assets and database."
+  c.option '-K', '--knife PATH', String, 'Path to knife config file'
+  c.option '-k', '--aws_accesskey KEY', String, 'aws secret access key.'
+  c.option '-I', '--aws_accessid KEY', String, 'aws access key id.'
+  c.option '-b', '--aws_bucket BUCKET', String, 'aws bucket name.'
+  c.option '-e', '--rails_env ENVIRONMENT', String, 'rails environment'
+  c.option '-u', '--mysql_user USERNAME (Defaults to root)', String, 'mysql user'
+  c.option '-p', '--mysql_pass PASSWORD (Defaults to empty)', String, 'mysql password'
+  c.option '-h', '--mysql_host HOSTNAME (Defaults to 127.0.0.1)', String, 'mysql hostname'
+  c.option '-d', '--database NAME', String, 'database'
+  c.action do |args, options|
+    @options = options
+    check_environment
+    get_rails_env
+    get_mysql_credentials
+    get_aws_credentials
+    get_app_name
+    get_production_db_name
+    get_latest_s3_dump
+    unzip_database
+    load_database
+    rm_dump
+    duplicity_restore
+  end
+end
+
+command :assets do |c|
+  c.syntax = "#{program(:name)} assets"
+  c.description = "Synchronizes the assets to the system directory"
+  c.option '-K', '--knife PATH', String, 'Path to knife config file'
+  c.option '-k', '--aws_accesskey KEY', String, 'aws secret access key.'
+  c.option '-I', '--aws_accessid KEY', String, 'aws access key id.'
+  c.option '-b', '--aws_bucket BUCKET', String, 'aws bucket name.'
+  c.action do |args, options|
+    @options = options
+    check_environment
+    get_aws_credentials
+    get_app_name
+    duplicity_restore
+  end
+end
+
+command :db do |c|
+  c.syntax = "#{program(:name)} db"
+  c.description = "Synchronizes a database with the latest s3 dbdump."
+  c.option '-K', '--knife PATH', String, 'Path to knife config file'
+  c.option '-k', '--aws_accesskey KEY', String, 'aws secret access key.'
+  c.option '-I', '--aws_accessid KEY', String, 'aws access key id.'
+  c.option '-b', '--aws_bucket BUCKET', String, 'aws bucket name.'
+  c.option '-e', '--rails_env ENVIRONMENT', String, 'rails environment'
+  c.option '-u', '--mysql_user USERNAME (Defaults to root)', String, 'mysql user'
+  c.option '-p', '--mysql_pass PASSWORD (Defaults to empty)', String, 'mysql password'
+  c.option '-h', '--mysql_host HOSTNAME (Defaults to 127.0.0.1)', String, 'mysql hostname'
+  c.option '-d', '--database NAME', String, 'database'
+  c.action do |args, options|
+    @options = options
+    check_environment
+    get_rails_env
+    get_mysql_credentials
+    get_aws_credentials
+    get_app_name
+    get_production_db_name
+    get_latest_s3_dump
+    unzip_database
+    load_database
+    rm_dump
+  end
+end
+
+command :db2disk do |c|
+  c.syntax = "#{program(:name)} db2disk"
+  c.description = "Downloads the latest backup from s3 to local disk."
+  c.action do |args, options|
+    check_environment
+    get_aws_credentials
+    get_app_name
+    get_production_db_name
+    get_latest_s3_dump
+    unzip_database
+  end
+end
+
+command :check do |c|
+  c.syntax = "#{program(:name)} check"
+  c.description = "Perform a health check of system dependencies and environment variables required in order to run synchronize"
+  c.action do |args, options|
+    check_environment
+  end
+end
+
+command :dependencies do |c|
+  c.syntax = "#{program(:name)} dependencies"
+  c.description = "Installs the command line tools required for synchronize to work. Requires brew."
+  c.action do |args, options|
+    install_dependencies
+  end
+end
+
+
+

data/lib/synchronize/db.rb

@@ -0,0 +1,64 @@
+require 'synchronize/tasks'
+
+def get_latest_s3_dump
+  puts "  - Downloading #{@app_name} S3 dump."
+  connection = Fog::Storage.new(
+                                :provider                 => 'AWS',
+                                :aws_secret_access_key    => @aws_secret_access_key,
+                                :aws_access_key_id        => @aws_access_key_id
+                                )
+
+  directory = connection.directories.get(@bucket)
+  s3_file_url = "projects/#{@app_name}/dbdumps/#{@production_db}/latest.gz"
+  s3_file = directory.files.get(s3_file_url)
+
+  File.open("#{RAILS_ROOT}/#{@production_db}.gz", "w+") do |local_file|
+    local_file.write(s3_file.body)
+  end
+end
+
+def get_mysql_credentials
+  if @mysql_option
+    @db_user = @options.mysql_user || 'root'
+    @db_pass = @options.mysql_pass
+    @db_host = @options.mysql_host || '127.0.0.1'
+    @db_name = @options.database
+  else
+    credentials = Utils.load_yaml_file("#{RAILS_ROOT}/config/database.yml")
+    @db_user = credentials[@rails_env]['username'] || 'root'
+    @db_pass = credentials[@rails_env]['password']
+    @db_host = credentials[@rails_env]['host'] || '127.0.0.1'
+    @db_name = credentials[@rails_env]['database']
+  end
+end
+
+def unzip_database
+  puts
+  puts "  - Unzipping #{@app_name} dump."
+  `gunzip -f #{RAILS_ROOT}/#{@production_db}.gz`
+  puts "  - #{@app_name} dump sucessfully unzipped."
+end
+
+def load_database
+  puts "  - Loading #{@app_name} dump into the database."
+
+  command_pre = "mysql -u #{@db_user} -h #{@db_host} "
+  command_in = @db_pass ? "-p#{@db_pass} " : String.new
+
+  command_post = "-e \"DROP DATABASE IF EXISTS #{@db_name}; CREATE DATABASE #{@db_name}\""
+  system(command_pre + command_in + command_post)
+
+  command_post = "#{@db_name} < #{RAILS_ROOT}/#{@production_db}"
+  system(command_pre + command_in + command_post)
+
+  puts $? != 0 ? "  - There was a problem loading database #{@app_name}." : "  - Database #{@app_name} sucessfully loaded."
+end
+
+def get_production_db_name
+  @production_db = "jorgesancha_#{@app_name}_production"
+end
+
+def rm_dump
+  `rm -f #{RAILS_ROOT}/#{@production_db}`
+end
+

data/lib/synchronize/tasks.rb

@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*-
+def get_app_name
+  @app_name = @rails_env == "development" ? RAILS_ROOT.split('/').last : RAILS_ROOT.split('/')[2]
+end
+
+def get_rails_env
+ @rails_env = @options.rails_env ? @options.rails_env : ENV['RAILS_ENV']
+end
+
+def get_knife_config
+  @knife_config =  @options.knife ? @options.knife : ENV['KNIFE_CONFIG']
+end
+
+def get_databag(databag, key)
+  aws_yml = `unset BUNDLE_GEMFILE; cd /; knife data bag show #{databag} #{key} -F yml -c #{@knife_config}`
+  YAML.load(aws_yml)
+end
+
+def get_aws_credentials
+  if @knife_option
+    puts "  - Obtaining AWS credentials from knife."
+
+    aws_data = get_databag('aws','keys')
+    @aws_access_key_id = aws_data['aws_access_key_id']
+    @aws_secret_access_key = aws_data['aws_secret_access_key']
+
+    aws_data = get_databag('aws','buckets')
+    @bucket = aws_data['s3_backup_bucket']
+
+    ENV['AWS_ACCESS_KEY_ID'] = @aws_access_key_id
+    ENV['AWS_SECRET_ACCESS_KEY'] = @aws_secret_access_key
+  elsif @aws_option
+    @bucket = @options.aws_bucket
+    @aws_access_key_id = @options.aws_accessid
+    @aws_secret_access_key = @options.aws_accesskey
+    ENV['AWS_ACCESS_KEY_ID'] = @aws_access_key_id
+    ENV['AWS_SECRET_ACCESS_KEY'] = @aws_secret_access_key
+  else
+    @bucket = ENV['AWS_BUCKET']
+    @aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
+    @aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+  end
+end
+
+# Installs the command line tools required for synchronize.
+def install_dependencies
+  puts "  - Installing dependencies"
+  `brew install python`
+  `/usr/local/share/python/easy_install pip`
+  `/usr/local/share/python/pip install boto`
+  `brew install duplicity`
+  puts "  - Synchronize dependencies installed"
+end
+
+def check_duplicity
+  unless `duplicity -V`
+    raise "  - duplicity not installed or not in PATH. You can install it manually or run synchronize dependencies to install through brew"
+  else
+    puts "  - Found duplicity command."
+  end
+end
+
+def check_knife_env
+  if ! ENV['KNIFE_CONFIG']
+    @knife_option = false
+    puts "  - KNIFE_CONFIG variable or --knife parameter has not been set, falling back to AWS command line options or environment variables."
+  elsif ENV['KNIFE_CONFIG'] && ! File.exists?(ENV['KNIFE_CONFIG'])
+    raise "  - KNIFE_CONFIG env variable exists but is not pointing to a proper knife config file, please point it to your knife config file."
+    else
+    puts "  - knife properly configured."
+    @knife_option = true
+  end
+end
+
+def check_knife_option
+  @knife_option = true
+  if ! @options.knife
+    @knife_option = false
+  elsif @options.knife  && ! File.exists?(@options.knife)
+    raise "  - --knife is not pointing to a proper knife config file, please point it to your knife config file."
+    else
+    puts "  - knife properly configured."
+    @knife_option = true
+  end
+end
+
+def check_aws_env
+  unless ENV['AWS_ACCESS_KEY_ID']
+    raise "  - Environment variable AWS_ACCESS_KEY_ID not set."
+  end
+
+  unless ENV['AWS_SECRET_ACCESS_KEY']
+    raise "  - Environment variable AWS_SECRET_ACCESS_KEY not set."
+  end
+
+  unless ENV['AWS_BUCKET']
+    raise "  - Enviroment variable AWS_BUCKET not set."
+  end
+
+  puts " - AWS env variables present."
+end
+
+def check_aws_option
+  @aws_option = true if (@options.aws_accesskey && @options.aws_accessid && @options.aws_bucket)
+end
+
+def check_aws_knife
+  aws_data = get_databag('aws','buckets')
+  unless aws_data['s3_backup_bucket']
+    raise "  - Could not get s3_backup_bucket parameter from chef databag aws"
+  end
+
+  aws_data = get_databag('aws','keys')
+  unless aws_data['aws_access_key_id']
+    raise "  - Could not get aws_access_key_id parameter from chef databag aws"
+  end
+
+  unless aws_data['aws_secret_access_key']
+    raise "  - Could not get aws_secret_access_key  parameter from chef databag aws"
+  end
+end
+
+def check_rails_option
+  @rails_option = true if @options.rails_env
+end
+
+def check_rails_env
+  unless ENV['RAILS_ENV']
+    raise "  - Neither environment variable RAILS_ENV nor --rails_env parameter has been set."
+  end
+  puts "  - RAILS_ENV variable present."
+end
+
+def check_database_yml
+  unless File.exists?("#{RAILS_ROOT}/config/database.yml")
+    raise "database.yml not found, are you running synchronize on your RAILS_ROOT directory?"
+  end
+  puts  "  - Found database.yml config file"
+end
+
+def check_mysql_option
+  @mysql_option = true if @options.database
+end
+
+# "Tests the existence of the environment variables required for backups tasks"
+def check_environment
+  puts "  - Checking environment."
+  check_duplicity
+
+  check_knife_option
+  check_knife_env unless @knife_option
+  check_aws_option unless @knife_option
+
+  if (@knife_option && ! @aws_option)
+    get_knife_config
+    check_aws_knife
+  end
+
+  check_aws_env if (! @knife_option && ! @aws_option)
+
+  check_rails_option
+  check_rails_env  unless @rails_option
+
+  check_mysql_option
+  check_database_yml unless @mysql_option
+
+  puts "  - Environment check successful, you should be able to run synchronize without trouble."
+end
+
+

data/lib/synchronize/utils.rb

@@ -0,0 +1,10 @@
+class Utils
+  class << self
+    def load_yaml_file(filename)
+      require 'erb'
+      
+      template = ERB.new(File.read(filename))
+      YAML.load(template.result)
+    end
+  end
+end

data/lib/synchronize/version.rb

@@ -0,0 +1,3 @@
+module Synchronize
+  VERSION = "0.2.0"
+end

data/synchronize.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "synchronize/version"
+
+Gem::Specification.new do |s|
+  s.name        = "synchronize"
+  s.version     = Synchronize::VERSION
+  s.authors     = ["bebanjo support team"]
+  s.email       = ["support@bebanjo.com"]
+  s.homepage    = "https://github.com/bebanjo/synchronize"
+  s.summary     = %q{ Performs synchronization of the contents between different Rails environments. }
+  s.description = %q{ Synchronize is designed to automate the ugly tasks of restoring database dumps and assets from s3 to our different environments. }
+
+  s.rubyforge_project = "synchronize"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency("fog", " >= 0.10.0" )
+  s.add_dependency("commander", " >= 4.0.6" )
+  s.add_dependency("chef", ">= 0.10")
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification 
+name: synchronize
+version: !ruby/object:Gem::Version 
+  hash: 23
+  prerelease: 
+  segments: 
+  - 0
+  - 2
+  - 0
+  version: 0.2.0
+platform: ruby
+authors: 
+- bebanjo support team
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-04-18 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fog
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 55
+        segments: 
+        - 0
+        - 10
+        - 0
+        version: 0.10.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: commander
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 51
+        segments: 
+        - 4
+        - 0
+        - 6
+        version: 4.0.6
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: chef
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 31
+        segments: 
+        - 0
+        - 10
+        version: "0.10"
+  type: :runtime
+  version_requirements: *id003
+description: " Synchronize is designed to automate the ugly tasks of restoring database dumps and assets from s3 to our different environments. "
+email: 
+- support@bebanjo.com
+executables: 
+- synchronize
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- bin/synchronize
+- lib/synchronize.rb
+- lib/synchronize/assets.rb
+- lib/synchronize/cli.rb
+- lib/synchronize/db.rb
+- lib/synchronize/tasks.rb
+- lib/synchronize/utils.rb
+- lib/synchronize/version.rb
+- synchronize.gemspec
+homepage: https://github.com/bebanjo/synchronize
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: synchronize
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Performs synchronization of the contents between different Rails environments.
+test_files: []
+