syncoku 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f4fb8a94d545cfbd3ceca353015892221e9cb754
+  data.tar.gz: b2d8fd4509402e592ae9f70ce314402018b530c2
+SHA512:
+  metadata.gz: 0e6aa7c4c8d0c6152069c95f517a954f7d5af33a7cf674220ab575998c82c6d285fd4e99b854bfdf54ed0e8d0db73cdca01082a35c5ec523e0b94abc59196896
+  data.tar.gz: e02b9ab5796c8b1abd3fcb8c03dc754b4d9b70677291cbf9c3854a2cdc56e69b4c00148142a3ecaf583589dc3902b2f5227e191195dc773db282b36f612511cc

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+
+.ruby-version

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in syncoku.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Bill Horsman
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,80 @@
+# Syncoku
+
+Copies a production Heroku Postgresql database to the local development database or a staging Heroku database. Optionally syncs the production S3 bucket with another bucket.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'syncoku'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install syncoku
+
+## Usage
+
+To copy production to your local development database:
+```
+syncoku
+```
+
+If you have S3 configured it will do the database and S3. To choose just one of those:
+```
+syncoku db
+syncoku s3
+```
+
+To target the staging environment:
+```
+syncoku staging
+```
+
+or more simply:
+```
+syncoku s
+```
+
+## Downloading the database (locally)
+
+It will capture a backup of the database and download it to a local file called `.syncoku.dump`. If you run Syncoku a second time and it discovers this file then it will give you the option of reusing it or downloading a new one. Reusing the existing one comes in useful if you have messed around with the local database and want to clean it up.
+
+## Hooks
+
+If you define a rake task called `syncoku:after_sync` then it will automatically be run after the database has been restored and migrated. This is a good place to put anonymization tasks, for instance.
+
+## S3
+
+If you add a file called `syncoku.yml` with the following information, it can sync between S3 buckets too:
+
+```
+# syncoku.yml
+s3:
+  access_key_id: "ABCDEFGH123456789"
+  secret_access_key: "a1secret2key3to4access5s3"
+
+  development:
+    bucket: "my-bucket-development"
+
+  staging:
+    bucket: "my-bucket-staging"
+
+  production:
+    bucket: "my-bucket-production"
+```
+
+*Note:* a limitation is that that the buckets must use the same credentials.
+
+## Contributing
+
+1. Fork it ( https://github.com/billhorsman/syncoku/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/syncoku

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../lib') unless $LOAD_PATH.include?(File.dirname(__FILE__) + '/../lib')
+require 'bundler'
+require 'syncoku'
+
+Syncoku::Control.run ARGV.dup

data/lib/syncoku/capture_backup.rb

@@ -0,0 +1,21 @@
+module Syncoku
+  module CaptureBackup
+    include Runnable
+
+    def capture
+      puts "Capturing #{production_app_name} backup..."
+      run_on_production("pgbackups:capture --expire")
+      run_on_production("pgbackups:url")
+    end
+
+    def run_on_production(command)
+      run_command "heroku #{command} --app #{production_app_name}"
+    end
+
+
+    def production_app_name
+      @production_app_name ||= run_command("git remote -v | grep production | grep push").match(/heroku[^:]*:(.*)\.git/)[1]
+    end
+
+  end
+end

data/lib/syncoku/control.rb

@@ -0,0 +1,52 @@
+module Syncoku
+
+  module Control
+    extend Syncoku::Runnable
+
+    def self.run(args)
+      matching_remotes = remotes & args
+      if matching_remotes.size == 0 && remote_index_uniq?
+        if key = (remote_index.keys & args)[0]
+          matching_remotes = [remote_index[key]]
+          args.delete key
+        end
+      end
+      target = case matching_remotes.compact.size
+      when 0
+        Syncoku::Local.new
+      when 1
+        remote = matching_remotes[0]
+        args.delete remote
+        Syncoku::Remote.new(remote)
+      else
+        puts "Please choose just one remote out of #{remotes.join(" or ")}"
+        exit 1
+      end
+      commands = %w[both db s3 rebuild] & args
+      commands << "both" if commands.size == 0
+      if commands.size > 1
+        puts "Choose just one command"
+        exit 1
+      else
+        args.delete commands[0]
+        target.send(commands[0], args)
+      end
+    end
+
+    def self.remotes
+      @remotes ||= run_command("git remote -v | grep heroku | grep push").split("\n").map {|line|
+        line.match(/^(.*)\t/)[1]
+      }.reject {|r| r == "production" || r == "heroku" }
+    end
+
+    def self.remote_index_uniq?
+      remote_index.size == remotes.size
+    end
+
+    def self.remote_index
+      @remote_index ||= Hash[remotes.map{|r| [r.slice(0, 1), r] }]
+    end
+
+  end
+
+end

data/lib/syncoku/local.rb

@@ -0,0 +1,26 @@
+module Syncoku
+
+  # Responsible for syncing to a local app
+  class Local
+    include Runnable
+    include CaptureBackup
+
+    def both(args)
+      db(args)
+      s3(args) if S3.config?
+    end
+
+    def db(args)
+      Syncoku::LocalDb.new.sync
+    end
+
+    def s3(args)
+      Syncoku::S3.new(:development).sync
+    end
+
+    def rebuild(args)
+      Syncoku::LocalDb.new.rebuild
+    end
+
+  end
+end

data/lib/syncoku/local_db.rb

@@ -0,0 +1,112 @@
+module Syncoku
+
+  class LocalDb
+    include Runnable
+    include CaptureBackup
+
+    def sync
+      if File.exist?("#{dump_filename}")
+        ask_to_download
+      else
+        download
+      end
+      drop_and_create
+      pg_restore
+      migrate
+      run_hook 'after_sync'
+      `touch tmp/restart.txt`
+    end
+
+    def rebuild
+      kill_connections
+      puts "Rebuilding database"
+      run_command "bundle exec rake db:drop db:create db:migrate"
+      puts "Seeding"
+      run_command "bundle exec rake db:seed"
+    end
+
+    private
+
+    def run_hook(name)
+      if test_command "rake syncoku:#{name}"
+        puts "Running #{name} hook"
+        run_command "bundle exec rake syncoku:#{name}"
+      else
+        puts "Skipping #{name} hook. Define a Rake task called syncoku:#{name} to activate."
+      end
+    end
+
+    def ask_to_download
+      print "Found existing #{dump_filename}. Choose:\n  D = download new backup, or\n  R = reuse existing backup\nPress D or R or anything else to abort: "
+      proceed = STDIN.getch.downcase
+      puts proceed
+      if proceed == 'd'
+        download
+      elsif proceed == 'r'
+        puts "OK, reusing backup"
+      else
+        exit 1
+      end
+    end
+
+    def download
+      run_command "curl -o #{dump_filename} \"#{capture}\""
+    end
+
+    def kill_connections
+      pids = `ps x|grep postgres|grep #{database_config["database"]} | grep -v 'grep' | cut -b 1,2,3,4,5,6`.gsub(/[^0-9]/, ' ').split(' ')
+      if pids.any?
+        puts "Killing #{pids.size} Postgres connection(s) (#{pids.join(", ")})"
+        pids.each do |pid|
+          `kill #{pid}`
+        end
+      else
+        puts "No connections to kill"
+      end
+    end
+
+    def drop_and_create
+      kill_connections
+      puts "Dropping and recreating #{database_name} database"
+      run_command "bundle exec rake db:drop db:create"
+    end
+
+    def migrate
+      run_command "bundle exec rake db:migrate"
+    end
+
+    def pg_restore
+      puts "Restoring database from #{dump_filename}"
+      options = []
+      options << "--verbose"
+      options << "--clean"
+      options << "--no-acl"
+      options << "--no-owner"
+      options << "--username=#{database_config["user"]}" if database_config["user"]
+      options << "--password=#{database_config["password"]}" if database_config["password"]
+      options << "--dbname=#{database_name}"
+      options << "--port=#{database_config["port"] || "5432"}"
+      output = `pg_restore #{options.join(' ')} #{dump_filename} 2> /dev/null`
+      if output =~ /a transfer is currently in progress/
+        puts "It looks like a backup is already in progress (or possibly stuck):"
+        puts output
+        `heroku pgbackups --app #{production_app}`
+        puts "Use pgbackups:destroy to remove the offending backup (or wait a bit to see if it fixes itself)"
+        exit 1
+      end
+    end
+
+    def database_name
+      database_config["database"]
+    end
+
+    def database_config
+      YAML.load(File.read("config/database.yml"))["development"]
+    end
+
+    def dump_filename
+      ".syncoku.dump"
+    end
+
+  end
+end

data/lib/syncoku/remote.rb

@@ -0,0 +1,35 @@
+module Syncoku
+
+  # Responsible for syncing to a remote app
+  class Remote
+    include Runnable
+
+    attr_reader :remote
+
+    def initialize(remote)
+      @remote = remote
+    end
+
+    def both(args)
+      db(args)
+      s3(args) if S3.config?
+    end
+
+    def db(args)
+      Syncoku::RemoteDb.new(app_name).sync
+    end
+
+    def s3(args)
+      Syncoku::S3.new(remote).sync
+    end
+
+    def rebuild(args)
+      puts "Rebuild not implemented"
+    end
+
+    def app_name
+      @app_name ||= run_command("git remote -v | grep #{remote} | grep push").match(/heroku\.com:(.*)\.git/)[1]
+    end
+
+  end
+end

data/lib/syncoku/remote_db.rb

@@ -0,0 +1,31 @@
+module Syncoku
+
+  class RemoteDb
+    include Runnable
+    include CaptureBackup
+
+    attr_reader :app_name
+
+    def initialize(app_name)
+      @app_name = app_name
+    end
+
+    def sync
+      puts "Switch on maintenance mode"
+      run_remotely "maintenance:on"
+      puts "Restoring database"
+      run_remotely "pg:reset DATABASE_URL --confirm #{app_name}"
+      run_remotely "pgbackups:restore DATABASE_URL '#{capture}' --confirm #{app_name}"
+      run_remotely "run rake db:migrate"
+      run_remotely "run rake syncoku:after_sync"
+      run_remotely "restart"
+      puts "Switch off maintenance mode"
+      run_remotely "maintenance:off"
+    end
+
+    def run_remotely(command)
+      run_command "heroku #{command} --app #{app_name}"
+    end
+
+  end
+end

data/lib/syncoku/runnable.rb

@@ -0,0 +1,26 @@
+module Syncoku
+  module Runnable
+
+    def run_command(command)
+      Bundler.with_clean_env {
+        out = `#{command}`
+        if $?.success?
+          out
+        else
+          puts "Error running command:"
+          puts command
+          puts out
+          exit $?.exitstatus
+        end
+      }
+    end
+
+    def test_command(command)
+      Bundler.with_clean_env {
+        `#{command} 2> /dev/null`
+        $?.success?
+      }
+    end
+
+  end
+end

data/lib/syncoku/s3.rb

@@ -0,0 +1,93 @@
+module Syncoku
+
+  class S3
+
+    attr_reader :to_env, :from_name, :to_name, :from_bucket, :to_bucket, :from_keys, :to_keys
+
+    def initialize(to_env)
+      @to_env = to_env
+    end
+
+    def sync
+      @missing = []
+      @from_name = config_value "production.bucket"
+      @to_name = config_value "#{to_env}.bucket"
+      access_key_id = config_value "access_key_id"
+      secret_access_key = config_value "secret_access_key"
+      if @missing.any?
+        puts "Missing syncoku.yml values prevented S3 sync"
+        @missing.each do |path|
+          puts "  s3.#{path}"
+        end
+        return
+      end
+      puts "Syncing S3 from #{from_name} to #{to_name}..."
+      AWS.config access_key_id: access_key_id, secret_access_key: secret_access_key
+      @from_bucket = AWS::S3.new.buckets[from_name]
+      @to_bucket = AWS::S3.new.buckets[to_name]
+      return unless get_keys
+      if !simple_sync & !remove_spare
+        puts "S3 is in sync. Nothing to do :)"
+      end
+    end
+
+    def get_keys
+      @from_keys = from_bucket.objects.map(&:key)
+      @to_keys = to_bucket.objects.map(&:key)
+      true
+    rescue AWS::S3::Errors::SignatureDoesNotMatch => e
+      puts "Can't sync S3 because #{e.message.sub(/^T/, 't')}"
+      false
+    end
+
+    def simple_sync
+      missing = from_keys - to_keys
+      return false if missing.empty?
+      puts "On #{from_name} but not on #{to_name}: #{"%7d" % missing.size}"
+      puts "Copying to #{to_name}"
+      missing.each do |key|
+        to_bucket.objects[key].copy_from key, { bucket: from_bucket, acl: :public_read}
+        print "."
+        STDOUT.flush
+      end
+      puts " done"
+      true
+    end
+
+    def remove_spare
+      spare = to_keys - from_keys
+      return false if spare.empty?
+      puts "On #{to_name} but not on #{from_name}:   #{"%7d" % spare.size}"
+      puts "Deleting from #{to_name}"
+      spare.each do |key|
+        to_bucket.objects[key].delete
+        print "."
+        STDOUT.flush
+      end
+      puts " done"
+      true
+    end
+
+    def config_value(path)
+      value = config.dup["s3"]
+      path.split(".").each do |name|
+        value = value[name]
+        if value.nil?
+          @missing << path
+          return nil
+        end
+      end
+      value
+    end
+
+    def config
+      @config ||= YAML.load(File.read("syncoku.yml"))
+    end
+
+    def self.config?
+      File.exist?("syncoku.yml")
+    end
+
+  end
+
+end

data/lib/syncoku/version.rb

@@ -0,0 +1,3 @@
+module Syncoku
+  VERSION = "0.0.1"
+end

data/lib/syncoku.rb

@@ -0,0 +1,14 @@
+require 'aws-sdk-v1'
+require 'ostruct'
+require 'rake'
+require 'yaml'
+require 'io/console'
+require "syncoku/runnable"
+require "syncoku/capture_backup"
+require "syncoku/control"
+require "syncoku/local"
+require "syncoku/local_db"
+require "syncoku/remote"
+require "syncoku/remote_db"
+require "syncoku/s3"
+require "syncoku/version"

data/syncoku.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'syncoku/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "syncoku"
+  spec.version       = Syncoku::VERSION
+  spec.authors       = ["Bill Horsman"]
+  spec.email         = ["bill@logicalcobwebs.com"]
+  spec.summary       = %q{Convenient way of syncing data from and to Heroku}
+  spec.homepage      = "https://github.com/billhorsman/syncoku"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "aws-sdk-v1", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: syncoku
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Bill Horsman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-v1
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 
+email:
+- bill@logicalcobwebs.com
+executables:
+- syncoku
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/syncoku
+- lib/syncoku.rb
+- lib/syncoku/capture_backup.rb
+- lib/syncoku/control.rb
+- lib/syncoku/local.rb
+- lib/syncoku/local_db.rb
+- lib/syncoku/remote.rb
+- lib/syncoku/remote_db.rb
+- lib/syncoku/runnable.rb
+- lib/syncoku/s3.rb
+- lib/syncoku/version.rb
+- syncoku.gemspec
+homepage: https://github.com/billhorsman/syncoku
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Convenient way of syncing data from and to Heroku
+test_files: []