migraine 0.0.1

data/examples/generation.rb

@@ -0,0 +1,9 @@
+require 'migraine'
+
+migration = Migraine::Migration.new(
+  from: 'mysql://root:root@localhost/migraine_from',
+  to: 'mysql://root:root@localhost/migraine_to'
+)
+
+migration.prefix 'spree_'
+migration.generate 'generated.rb'

data/examples/migration.rb

@@ -0,0 +1,15 @@
+require 'migraine'
+
+migration = Migraine::Migration.new(
+  to: 'mysql://root:root@localhost/spree_old',
+  from: 'mysql://root:root@localhost/spree_new'
+)
+
+migration.map 'users' => 'spree_users' do
+  map 'email'
+  # ...
+  map 'crypted_password' => 'encrypted_password'
+  # ...
+end
+
+migration.run

data/lib/migraine.rb

@@ -0,0 +1,5 @@
+require 'sequel'
+require 'migraine/version'
+require 'migraine/generator'
+require 'migraine/map'
+require 'migraine/migration'

data/lib/migraine/generator.rb

@@ -0,0 +1,103 @@
+module Migraine
+  # Module containing schema generators for different database
+  # adapters. To add support for a new adapter, simply add a
+  # method such as `database_schema_for_<adapter>`. The method
+  # should return a Hash containing table and column hierarchy,
+  # as follows:
+  #
+  # { 
+  #   'table' => ['one_column', 'another_column'],
+  #   'another_table' => ['one_column']
+  # }
+  module Generator
+    # Generates a new migration file template by analyzing the
+    # source and destination connections, saving it to the
+    # location specified in the argument. This makes it easy to
+    # create migration files where you can fill in destination
+    # tables/columns instead of writing it all by hand.
+    #
+    # TODO: Factor out any dependencies on instance variables
+    # such as @source_connection and @destination_connection.
+    #
+    # @param [String] Relative path to file.
+    def generate(file)
+      connect
+      src = @source_connection
+      dest = @destination_connection
+      path = File.join(Dir.pwd, File.dirname($0), file)
+
+      File.open(path, 'w') do |file|
+        file.puts "require 'migraine'\n\n"
+        file.puts "migration = Migraine::Migration.new("
+        file.puts "  from: '#{source}',"
+        file.puts "  to: '#{destination}'"
+        file.puts ")\n\n"
+        
+        dest_schema = database_schema_for(@destination)
+        database_schema_for(@source).each do |table, columns|
+          if dest_schema.has_key? table
+            file.puts "migration.map '#{table}' do"
+          elsif dest_schema.has_key? prefix + table
+            file.puts "migration.map '#{table}' => '#{prefix + table}'"
+          else
+            file.puts "migration.map '#{table}' => ''"
+          end
+          
+          columns.each do |column|
+            if !dest_schema[table].nil? && dest_schema[table].include?(column)
+              file.puts "  map '#{column}'"
+            elsif !dest_schema[prefix + table].nil? &&
+                   dest_schema[prefix + table].include?(column)
+              file.puts "  map '#{column}'"
+            else
+              file.puts "  map '#{column}' => ''"
+            end
+          end
+
+          file.puts "end\n\n"
+        end
+
+        file.puts "migration.run"
+      end
+    end
+
+    # Fetches names of all tables and columns at specified URI
+    # (`@source` or `@destination`) and returns them neatly
+    # organized hierarchically in an array.
+    #
+    # @param [String] Database connection string, e.g.
+    # 'mysql://root:root@localhost/myproj'.
+    def database_schema_for(uri)
+      adapter = uri.split(':')[0]
+      send("database_schema_for_#{adapter}", uri)
+    end
+    
+    # Generates a database schema for a MySQL database using the
+    # available connections/instance variables. We do this by
+    # peeking into the information_schema database.
+    #
+    # @param [String] Database connection string, e.g.
+    # 'mysql://root:root@localhost/myproj'.
+    def database_schema_for_mysql(uri)
+      user_and_db = uri.split('://')[1]
+      user_and_host = user_and_db.split('/')[0]
+      source_db = user_and_db.split('/')[1]
+      schema = {}
+
+      db = Sequel.connect('mysql://' + user_and_host + '/information_schema')
+      db[:tables].
+        filter(table_schema: source_db).select(:table_name).
+        all.each do |record|
+          table = record[:table_name]
+
+          columns = db[:columns].
+            filter(table_schema: source_db, table_name: table).
+            select(:column_name).all.map { |record| record[:column_name] }
+
+          schema.merge!({ table => columns })
+        end
+      
+      schema
+    end
+  end
+end

data/lib/migraine/map.rb

@@ -0,0 +1,85 @@
+module Migraine
+  class Map
+    attr_accessor :source
+    attr_accessor :destination
+    attr_accessor :maps
+
+    # Sets the source and destination to use for mapping. Takes a
+    # Hash containing a single element as argument, where key
+    # specifies source and value specifies destination.
+    #
+    # This constructor should not be used publicly, but rather
+    # through the `Migraine::Map#map` DSL method which adds
+    # instances to the @maps array.
+    #
+    #     Migraine::Map.new 'source_table' => 'destinatin_table'
+    #
+    # @param [Hash] Hash containing source and destination.
+    def initialize(source_and_destination)
+      begin
+        set_source_and_destination_from(source_and_destination)
+      rescue ArgumentError => e
+        puts e.message
+        exit
+      end
+    end
+
+    # Adds a new map to the @maps array. In other words, maps an
+    # old location to a new location one level below the current,
+    # be it at database, table or column level. This lets us have
+    # nested Maps, using the same class for any 'level'.
+    #
+    # For example, if the current source and destination are
+    # databases, this will map a source table to a destination
+    # table within that database. If current is a table, it will
+    # map column names.
+    #
+    # It accepts a block as optional argument, which will be
+    # evaluated against the new Map instance so that `#map` calls
+    # can be nested using a more convenient DSL.
+    #
+    # Usage in migration/mapping DSL:
+    #
+    #     # Map#map at database-level, mapping table names
+    #     @migration.map 'source_table' => 'destination_table' do
+    #       # Map#map at table-level, mapping column names
+    #       map 'source_column' => 'destination_column'
+    #     end
+    def map(source_and_destination, &block)
+      @maps ||= []
+      map = Migraine::Map.new(source_and_destination)
+      
+      if block_given?
+        map.instance_eval(&block)
+      end
+
+      @maps << map
+    end
+
+    private
+    
+    # Sets the `@source` and `@destination` instance variables
+    # from argument. Argument can be either a String (when source
+    # has the same name as destination) or a Hash (where both
+    # source and destination need to be specified.)
+    #
+    # Raises ArgumentError unless source and destination is
+    # properly defined in a Hash argument.
+    #
+    # @param [Hash] Hash containing source and destination.
+    def set_source_and_destination_from(s_and_d)
+      # If String is provided, convert it to Hash
+      s_and_d = { s_and_d => s_and_d } if s_and_d.is_a? String
+
+      unless s_and_d.is_a?(Hash) && s_and_d.length === 1
+        raise ArgumentError, "Argument must be a Hash containing one element;
+          source as key, destination as value."
+      end
+
+      s_and_d.each do |source, destination|
+        @source = source
+        @destination = destination
+      end
+    end
+  end
+end

data/lib/migraine/migration.rb

@@ -0,0 +1,108 @@
+module Migraine
+  # This is a special case of a `Migraine::Map` as it specifies
+  # source and destination *databases* and provides methods to,
+  # for example, run the migration. It also overrides
+  # Migraine::Map#set_source_and_destination_from(s_and_d) to
+  # accept surce and destination in a more elegant way.
+  class Migration < Map
+    include Migraine::Generator
+
+    attr_accessor :prefix
+
+    # Runs the migration using the mappings that have been set
+    # up. Walks through the nested Map objects, copying database
+    # records from source to destination.
+    def run
+      connect
+      src = @source_connection
+      dest = @destination_connection
+
+      log "BEGINNING MIGRATION\n" +
+          "-------------------"
+
+      # Iterate through table mappings
+      maps.each do |table|
+        log "MIGRATING TABLE #{table.source}"
+
+        # Fetch all rows from source table
+        rows = src[table.source.to_sym].all
+        
+        # Iterate through the records and migrate each one
+        rows.each do |row|
+          new_record = {}
+
+          # Identify the columns we need to grab data from
+          table.maps.each do |column|
+            new_record.merge!(
+              column.destination.to_sym => row[column.source.to_sym]
+            )
+          end
+
+          # Insert new record to destination table
+          log " -> Inserting record into #{table.destination}"
+          dest[table.destination.to_sym].insert(new_record)
+        end
+      end
+
+      log "-------------------\n" +
+          "MIGRATION COMPLETED"
+    end
+
+    # DSL convenience method for skipping the assignment operator
+    # when specifying prefix.
+    #
+    # @param [String] Prefix to use in generations.
+    def prefix(new_prefix = nil)
+      return @prefix if new_prefix.nil?
+      @prefix = new_prefix
+    end
+
+    private
+    
+    # Sets the `@source` and `@destination` instance variables
+    # from a Hash containing :to and :from keys. Raises
+    # ArgumentError unless a proper Hash is provided.
+    #
+    # @param [Hash] Hash containing source and destination.
+    def set_source_and_destination_from(s_and_d)
+      if !s_and_d.is_a? Hash || s_and_d.length != 2
+        raise ArgumentError, "Argument must be a Hash containing two elements;
+          source (:from) and destination (:to)."
+      elsif !s_and_d[:from] || !s_and_d[:to]
+        raise ArgumentError, "Both source (:from) and destination (:to)
+          need to be specified."
+      end
+
+      @source = s_and_d[:from]
+      @destination = s_and_d[:to]
+    end
+
+    # Uses `@source` and `@destination` variables to initialize
+    # database connections. If source or destination is specified
+    # using a 'test://' URI, a `Sequel.sqlite` test database is
+    # used as connection.
+    def connect
+      unless @source && @destination
+        raise RuntimeError, 'Source and destination are not specified.'
+      end
+
+      return if @source_connection && @destination_connection
+      
+      if @source =~ /^test:\/\//
+        @source_connection = Sequel.sqlite
+      else
+        @source_connection = Sequel.connect(@source)
+      end
+      
+      if @destination =~ /^test:\/\//
+        @destination_connection = Sequel.sqlite
+      else
+        @destination_connection = Sequel.connect(@destination)
+      end
+    end
+
+    def log(message)
+      print "\n#{message}\n"
+    end
+  end
+end

data/lib/migraine/version.rb

@@ -0,0 +1,3 @@
+module Migraine
+  VERSION = "0.0.1"
+end

data/migraine.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "migraine/version"
+
+Gem::Specification.new do |s|
+  s.name        = "migraine"
+  s.version     = Migraine::VERSION
+  s.authors     = ["Daniel Nordstrom"]
+  s.email       = ["d@nintera.com"]
+  s.homepage    = ""
+  s.summary     = %q{Simple data migration scripts in Ruby.}
+  s.description = %q{Avoid a migraine when migrating your data from one database to another. Simply specify which, and where, tables and columns should be migrated.}
+
+  s.rubyforge_project = "migraine"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_development_dependency "rake"
+  s.add_runtime_dependency "sequel"
+end

data/spec/map_spec.rb

@@ -0,0 +1,55 @@
+require 'spec_helper'
+
+describe Migraine::Map do
+  before do
+    @map = Migraine::Map.new('old_location' => 'new_location')
+  end
+
+  describe "#new" do
+    it 'should set the source to map from' do
+      @map.source.must_equal 'old_location'
+    end
+
+    it 'should set the destination to map to' do
+      @map.destination.must_equal 'new_location'
+    end
+
+  end
+
+  describe '#set_source_and_destination_from' do
+    it 'should raise ArgumentError on invalid argument' do
+      begin
+        @map.send(:set_source_and_destination_from,
+          this: 'is not', a: 'valid argument'
+        )
+      rescue ArgumentError
+        raised = true
+      end
+
+      raised.wont_be_nil
+    end
+  end
+
+  describe '#map' do
+    it "should store new Migraine::Map instance" do
+      @map.map 'old_location' => 'new_location'
+      @map.maps.length.must_equal 1
+    end
+
+    it "should accept a block for use as DSL" do
+      @map.maps = [] # Clear maps added by other tests
+
+      @map.map 'old_table' => 'new_table' do
+        map 'old_olumn' => 'new_column'
+      end
+
+      @map.maps[0].maps.length.must_equal 1
+    end
+
+    it "should accept string argument" do
+      @map.maps = []
+      @map.map 'unchanged'
+      @map.maps[0].destination.must_equal 'unchanged'
+    end
+  end
+end

data/spec/migration_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+include SpecHelper
+
+describe Migraine::Migration do
+  before do
+    @migration = Migraine::Migration.new(
+      from: 'test://myproj_old.db',
+      to: 'test://myproj.db'
+    )
+  end
+
+  describe "#new" do
+    it 'should set the source database' do
+      @migration.source.must_equal 'test://myproj_old.db'
+    end
+
+    it 'should set the destination to map to' do
+      @migration.destination.must_equal 'test://myproj.db'
+    end
+  end
+
+  describe '#set_source_and_destination_from' do
+    it 'should raise ArgumentError on invalid argument' do
+      begin
+        @migration.send(:set_source_and_destination_from,
+          'this is a Map argument' => 'Migration uses :to and :from'
+        )
+      rescue ArgumentError
+        raised = true
+      end
+
+      raised.wont_be_nil
+    end
+  end
+
+  describe '#connect' do
+    it "should initialize a database connections" do
+      @migration.send(:connect)
+      @migration.instance_variable_get(:@source_connection).
+        wont_be_nil
+    end
+  end
+
+  describe '#run' do
+    it "should run the migration" do
+      @migration = Migraine::Migration.new(from: 'test', to: 'test')
+      @migration.map 'old_table' => 'new_table' do
+        map 'old_column' => 'new_column'
+      end
+
+      create_test_connections_for(@migration)
+
+      @migration.run
+
+      # Check that new column now has one record, which was
+      # migrated from the source (inserted into source by
+      # `create_test_connections_for` helper).
+      @migration.instance_variable_get(
+        :@destination_connection)[:new_table].
+        count.must_equal 1
+    end
+  end
+end