themigrator 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b1f67f74c4b64f07f670d8878a8854e64b6de64e
+  data.tar.gz: 45b335ce112247d11d2adda886f635b512cb3f80
+SHA512:
+  metadata.gz: a1764195422d095d79055d1e84cd3257672e493fbf1658e8874843ffd4ca7b4f94b0141cc486bbc909a75499bf4e3cc10db62589a715fc27f44c92a6e672ba32
+  data.tar.gz: 50a6430052769b67d0e3bb28a400514f427c7ed282c6b7a0ff3cceb541ba6ed659da1a39e33a3c476062e07a685fedb78c3bb5551108869000a82dc11cede8b6

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+test/fixtures/sample_project/logs/
+.byebug_history

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.12.5

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in themigrator.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,29 @@
+Copyright (c) 2016, Wooga GmbH
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+

data/README.md

@@ -0,0 +1,64 @@
+# Themigrator
+
+The migrator is a program that eases and automates data migrations. 
+
+## Installation
+
+
+Install it as a normal gem
+
+    $ gem install themigrator
+
+## Usage
+
+
+`themigrator` helps a data migration by automating a pre-defined process.
+
+It uses the concepts of _roles_(folders), and _actions_(scripts).
+
+Each _role_ can contain any file, but `themigrator` will treat some of them in
+a special way.
+
+This is the list of actions and its intended behaviour:
+
+  * `setup` Provision machines with tooling or opening tunnels.
+  * `pre-migrate` Put the site in maintenance mode.
+  * `migrate` Copy data and check integrity.
+  * `cleanup` In case `migrate` works, it will cleanup, or enable a site.
+  * `rollback` In case any previous command fails, `rollback` is called.
+
+There are some constrains:
+
+  * All the _actions_ are called at the same time for all the _roles_.
+  * Unless all the _actions_ return a 0, the next _action_ is not executed.
+  * In case an _action_ return a non zero code, a SIGINT is sent to the rest of
+the actions still alive.  
+  * If case an _action_ return a non zero code, the `rollback` script will be
+called only for the roles that run any previous action.
+
+
+## Example
+
+For example. We have three roles: _mariadb_, _redis_, and _app_.
+
+    /mariadb/setup         # Check maria is still app and running.
+    /mariadb/pre-migrate   # Disable inserts.
+    /mariadb/migrate       # Run `ssh oldmaria mysqldump | mysql newmaria`.
+    /mariadb/cleanup       # Stop mariadb.
+    /mariadb/rollback      # Enable inserts.
+    /redis/migrate         # Run `redis-cli oldredis dump | redis-cli newredis bullread`.
+    /app/pre-migrate       # Set maintenance mode and stop the app servers.
+    /app/rollback          # Enable the old site
+
+Will do the following:
+
+   1. Run the `mariadb/setup`. If fails `mariadb/rollback`
+   2. Run the `mariadb/pre-migrate` and `app/pre-migrate`. If fails `mariadb/rollback` and `app/rollback`.
+   3. Run the `mariadb/migrate` and `redis/migrate`. If fails `mariadb/rollback` and `app/rollback`.
+   4. Run the `mariadb/cleanup`. Will output the error.
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/wooga/themigrator.
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "themigrator"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/themigrator

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require "themigrator/cli"
+
+Themigrator::CLI.start(ARGV)
+
+
+
+
+

data/lib/themigrator.rb

@@ -0,0 +1,9 @@
+require "themigrator/version"
+require "themigrator/logger"
+require "themigrator/runner"
+require "themigrator/migration"
+
+module Themigrator
+
+
+end

data/lib/themigrator/cli.rb

@@ -0,0 +1,21 @@
+require 'thor'
+require 'themigrator/migrator'
+
+
+module Themigrator
+  class CLI < Thor
+
+    desc "migrate", "Migrate the current project"
+    def migrate
+      migrator = Themigrator::Migrator.new(Dir.pwd)
+      migrator.migrate!
+    end
+
+    desc "roles", "show the list of roles"
+    def roles
+      migration = Themigrator::Migration.new(Dir.pwd)
+      migration.analyze_project!
+      puts migration.roles.join("\n")
+    end
+  end
+end

data/lib/themigrator/logger.rb

@@ -0,0 +1,21 @@
+require 'fileutils'
+
+module Themigrator
+  module Logger
+    include FileUtils
+
+
+    def log_path(base_dir, run_id, role, action)
+      filename = "#{role}-#{action}.log"
+      directory = log_dir(base_dir, run_id)
+      File.join(directory,filename)
+    end
+
+    def log_dir(base_dir, run_id)
+      dir = File.join(base_dir,"logs",run_id)
+      mkdir_p(dir, mode: 0700)
+      dir
+    end
+
+  end
+end

data/lib/themigrator/migration.rb

@@ -0,0 +1,47 @@
+
+
+module Themigrator
+  
+  # Holds the plan of the migration
+  class Migration
+
+    ACTIONS = %w(setup pre-migrate migrate cleanup rollback)
+    SCRIPTS = ACTIONS | %w(rollback)
+
+    attr_reader :roles, :actions
+
+    def initialize(dir)
+      @dir = dir
+      @action_and_roles = Hash.new{|hash,key| hash[key] = [] }
+      @roles = []
+      @actions = []
+    end
+
+    def analyze_project!
+      roles = Set.new
+      used_actions = [] # Should be in order
+
+      SCRIPTS.each do |action|
+	path_match = File.join(@dir,"*/#{action}")
+	Dir[path_match].select { |f|
+	  File.executable?(f)
+	}.each {|f|
+	  role = File.basename(File.dirname(f))
+	  roles.add(role)
+	  used_actions << action
+	  @action_and_roles[action] << role
+
+	}
+
+      end
+      @roles = roles.to_a.sort
+      @actions = used_actions.uniq
+      @actions.delete("rollback")
+    end
+
+    def roles_for_action(action)
+      @action_and_roles[action]
+    end
+
+  end
+end

data/lib/themigrator/migrator.rb

@@ -0,0 +1,71 @@
+require 'themigrator/runner'
+require 'themigrator/logger'
+require 'themigrator/migration'
+
+
+module Themigrator
+  class Migrator
+    include Runner
+    include Logger
+
+
+    def initialize(dir)
+      @dir = dir
+      @runned_roles = []
+    end
+
+    def migrate!
+      @migration = Migration.new(@dir)
+      @migration.analyze_project!
+      
+      Migration::ACTIONS.each do |action|
+	run_action_and_wait(action)
+      end
+
+    rescue ActionFailedException
+      run_rollback_and_wait
+    end
+
+    private
+
+    def run_rollback_and_wait
+      @runned_roles.each do |role|
+	run_role_action(role, "rollback")
+      end
+
+      wait_all_processes
+    end
+
+    def run_id
+      @run_id ||= Time.now.strftime("%Y-%m-%d-%H:%M:%S")
+    end
+
+    def run_action_and_wait(action)
+      run_action(action)
+      wait
+    end
+
+    def run_action(action)
+      @migration.roles_for_action(action).each do |role|
+	@runned_roles << role # To prevent calling rollback on roles that
+	                      # have not being executed
+
+	start_role_action(role, action)
+	
+      end
+      @runned_roles.uniq!
+    end
+
+    def start_role_action(role, action)
+      script_path = find_script(role, action)
+      log_path    = log_path(@dir,run_id,role, action)
+
+      run(script_path,log_path)
+    end
+
+    def find_script(role, action)
+      File.join(@dir, role, action)
+    end
+  end
+
+end

data/lib/themigrator/runner.rb

@@ -0,0 +1,53 @@
+require 'themigrator/logger'
+
+
+module Themigrator
+  module Runner
+    include Logger
+
+    class ActionFailedException < Exception; end
+
+
+    class Script
+      def initialize(script, log_file)
+	@script = script
+	@log_file = File.open(log_file, "w", 0600)
+      end
+
+      def close
+	@log_file.close
+      end
+    end
+
+
+    def run(script, log_file)
+      wd_dir = File.dirname(script)
+      log_fd = File.open(log_file, "w", 0600)
+      pid = Process.spawn(script, err: log_fd, out: log_fd, chdir: wd_dir)
+      add_process(pid)
+    end
+
+    def wait
+      @processes.each do |pid|
+	pid, status = Process.wait2(pid)
+	return_code = status.exitstatus
+	@processes.delete(pid)
+
+	if return_code != 0
+	  raise ActionFailedException.new(pid)
+	end
+      end
+    end
+
+    private
+
+    def processes
+      @processes || []
+    end
+    def add_process(pid)
+      @processes ||= []
+      @processes << pid 
+    end
+
+  end
+end

data/lib/themigrator/version.rb

@@ -0,0 +1,3 @@
+module Themigrator
+  VERSION = "0.1.0"
+end

data/test.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+
+set -eu
+
+
+cat <<-EOF > remote_file
+#!/bin/bash
+
+redis-cli redis03 dump everything NOW > my file
+process my file
+
+
+cat thecooloutput
+
+EOF
+
+chmod +x remote_file
+
+scp remote_file redis:
+
+ssh redis remote_file | redis-cli pipe asdf
+
+redis-cli PIPE << asdf

data/themigrator.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'themigrator/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "themigrator"
+  spec.version       = Themigrator::VERSION
+  spec.authors       = ["Guillermo Álvarez"]
+  spec.email         = ["guillermo@cientifico.net"]
+
+  spec.summary       = %q{tool to ease migration a data migration process.}
+  spec.description   = %q{The migrator is a program that eases and automates data migrations.}
+  spec.homepage      = "https://github.com/wooga/themigrator"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+
+  spec.add_dependency "thor", "~> 0.19"
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: themigrator
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Guillermo Álvarez
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-08-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+description: The migrator is a program that eases and automates data migrations.
+email:
+- guillermo@cientifico.net
+executables:
+- themigrator
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/themigrator
+- lib/themigrator.rb
+- lib/themigrator/cli.rb
+- lib/themigrator/helper.rb
+- lib/themigrator/logger.rb
+- lib/themigrator/migration.rb
+- lib/themigrator/migrator.rb
+- lib/themigrator/runner.rb
+- lib/themigrator/version.rb
+- test.sh
+- themigrator.gemspec
+homepage: https://github.com/wooga/themigrator
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: tool to ease migration a data migration process.
+test_files: []