data-import 0.0.1

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+reports/*
+bin/*

data/.rvmrc

@@ -0,0 +1 @@
+rvm --create ruby-1.9.2@data-import

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in data-import.gemspec
+gemspec

data/README.md

@@ -0,0 +1,157 @@
+# DataImport
+
+data-import is a data-migration framework. The goal of the project is to provide a simple api to migrate data from a legacy schema into a new one. It's based on jeremyevans/sequel.
+
+## Installation
+
+```ruby
+gem 'data-import'
+```
+
+you can put your migration configuration in any file you like. We suggest something like `mapping.rb`
+
+```ruby
+source :sequel, 'sqlite:/'
+target :sequel, 'sqlite:/'
+
+import 'Animals' do
+  from 'tblAnimal', :primary_key => 'sAnimalID'
+  to 'animals'
+
+  mapping 'sAnimalID' => 'id'
+  mapping 'strAnimalTitleText' => 'name'
+  mapping 'sAnimalAge' => 'age'
+  mapping 'strThreat' do |context, threat|
+    rating = ['none', 'medium', 'big'].index(threat) + 1
+    {:danger_rating => rating}
+  end
+end
+```
+
+to run the import just execute:
+
+```ruby
+  mapping_path = Rails.root + 'mapping.rb'
+  DataImport.run_config! mapping_path
+```
+
+if you execute the import frequently you can create a Rake-Task:
+
+```ruby
+desc "Imports the date from the source database"
+task :import do
+  mapping_path = Rails.root + 'mapping.rb'
+  options = {}
+  options[:only] = ENV['RUN_ONLY'].split(',') if ENV['RUN_ONLY'].present?
+
+  DataImport.run_config! mapping_path, options
+end
+```
+
+## Configuration
+
+data-import provides a clean dsl to define your mappings from the legacy schema to the new one.
+
+### Before Filter ###
+
+data-import allows you to definie a global filter. This filter can be used to make global transformations like encoding fixes. You can define a filter, which downcases every string like so:
+
+```ruby
+before_filter do |row|
+  row.each do |k, v|
+    row[k] = v.downcase if v.respond_to?(:downcase)
+  end
+end
+```
+
+### Simple Mappings
+
+You've already seen a very basic example of the dsl in the Installation-Section. This part shows off the features of the mapping-DSL.
+
+#### Structure ####
+
+every mapping starts with a call to `import` followed by the name of the mapping. You can name mappings however you like. The block passed to import contains the mapping itself. You can supply the source-table with `from` and the target-table with `to`. Make sure that you set the primary-key on the source-table otherwhise pagination is not working properly and the migration will fill up your RAM.
+
+```ruby
+import 'Users' do
+  from 'tblUser', :primary_key => 'sUserID'
+  to 'users'
+```
+
+#### Column-Mappings ####
+
+You can create simple name-mappings with a call to `mapping`:
+
+```ruby
+mapping 'sUserID' => 'id'
+mapping 'strEmail' => 'email'
+mapping 'strUsername' => 'username'
+```
+
+If you need to process a column you can add a block. This will pass in the values of the columns you specified after mapping. The return value of the block should be a hash or nil. Nil means no mapping at all and in case of a hash you have to use the column-names of the target-table as keys.
+
+```ruby
+mapping 'strThreat' do |context, threat|
+  rating = ['none', 'medium', 'big'].index(threat) + 1
+  {:danger_rating => rating}
+end
+```
+
+### Dependencies
+
+You can specify dependencies between definitions. Dependencies are always run before a given definition will be executed. Adding all necessary dependencies also allows you to run a set of definitions instead of everything.
+
+```ruby
+import 'Roles' do
+  from 'tblRole', :primary_key => 'sRoleID'
+  to 'roles'
+end
+
+import 'SubscriptionPlans' do
+  from 'tblSubcriptionCat', :primary_key => 'sSubscriptionCatID'
+  to 'subscription_plans'
+end
+
+import 'Users' do
+  from 'tblUser', :primary_key => 'sUserID'
+  to 'users'
+  dependencies 'SubscriptionPlans'
+end
+
+import 'Permissions' do
+  from 'tblUserRoles'
+  to 'permissions'
+  dependencies 'Users', 'Roles'
+end
+```
+
+you can now run parts of your mappings using the :only option:
+
+```ruby
+DataImport.run_config! 'mappings.rb', :only => ['Users'] # => imports SubscriptionPlans then Users
+DataImport.run_config! 'mappings.rb', :only => ['Roles'] # => imports Roles only
+DataImport.run_config! 'mappings.rb', :only => ['Permissions'] # => imports Roles, SubscriptionPlans, Users and then Permissions
+```
+
+## Examples
+
+you can learn a lot from the [integration specs](https://github.com/garaio/data-import/tree/master/spec/integration).
+
+## Community
+
+### Got a question?
+
+Just send me a message and I'll try to get to you as soon as possible.
+
+### Found a bug?
+
+Please submit a new issue.
+
+### Fixed something?
+
+1. Fork data-import
+2. Create a topic branch - `git checkout -b my_branch`
+3. Make your changes and update the History.txt file
+4. Push to your branch - `git push origin my_branch`
+5. Send me a pull-request for your topic branch
+6. That's it!

data/Rakefile

@@ -0,0 +1,20 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+namespace :ci do
+  task :setup do
+    include FileUtils
+
+    rm_rf 'reports'
+    mkdir_p 'reports/rspec'
+  end
+
+  RSpec::Core::RakeTask.new(:rspec => :setup) do |t|
+    t.rspec_opts = ['--no-color',
+                    '-r ./spec/junit_formatter.rb',
+                    '-f "JUnitFormatter"',
+                    '-o reports/rspec/junit.xml']
+    t.pattern    = "spec/**/*_spec.rb"
+  end
+end

data/data-import.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "data-import/version"
+
+Gem::Specification.new do |s|
+  s.name        = "data-import"
+  s.version     = DataImport::VERSION
+  s.authors     = ['Michael Stämpfli', 'Yves Senn']
+  s.email       = ['michael.staempfli@garaio.com', 'yves.senn@garaio.com']
+  s.homepage    = ""
+  s.summary     = %q{migrate your data to a better palce}
+  s.description = %q{sequel based dsl to migrate data from a legacy database to a new home}
+
+  s.rubyforge_project = "data-import"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "sqlite3"
+
+  s.add_runtime_dependency "sequel"
+  s.add_runtime_dependency "progress"
+  s.add_runtime_dependency "activesupport"
+  s.add_runtime_dependency "i18n"
+end

data/lib/data-import.rb

@@ -0,0 +1,35 @@
+
+require 'yaml'
+require 'progress'
+require 'active_support/all'
+
+require "data-import/version"
+require 'data-import/runner'
+require 'data-import/execution_plan'
+require 'data-import/dsl'
+require 'data-import/database'
+require 'data-import/definition'
+require 'data-import/importer'
+
+# Monkeypatch for active support (see https://github.com/rails/rails/pull/2801)
+class Time
+  class << self
+    def ===(other)
+      super || (self == Time && other.is_a?(ActiveSupport::TimeWithZone))
+    end
+  end
+end
+
+module DataImport
+
+  def self.run_config!(config_path, options = {})
+    plan = DataImport::Dsl.evaluate_import_config(config_path)
+    run_plan!(plan, options)
+  end
+
+  def self.run_plan!(plan, options = {})
+    runner = Runner.new(plan)
+    runner.run(options)
+  end
+
+end

data/lib/data-import/adapters/sequel.rb

@@ -0,0 +1,96 @@
+require 'sequel'
+require 'iconv'
+
+module DataImport
+  module Adapters
+    class Sequel
+
+      attr_reader :db
+
+      def self.connect(options = {})
+        ::Sequel.identifier_output_method = :to_s
+        self.new ::Sequel.connect(options)
+      end
+
+      def initialize(db)
+        @db = db
+      end
+
+      def truncate(table)
+        @db.from(table).delete
+      end
+
+      def transaction(&block)
+        @db.transaction do
+          yield block
+        end
+      end
+
+      def each_row(table, options = {}, &block)
+        if options[:primary_key].nil? || !numeric_column?(table, options[:primary_key])
+          each_row_without_batches table, options, &block
+        else
+          each_row_in_batches table, options, &block
+        end
+      end
+
+      def each_row_without_batches(table, options = {}, &block)
+        sql = @db.from(table)
+        sql = sql.select(*options[:columns]) unless options[:columns].nil?
+        sql = sql.distinct if options[:distinct]
+        sql = sql.order(*options[:order]) unless options[:order].nil?
+        sql.each do |row|
+          yield row if block_given?
+        end
+      end
+
+      def each_row_in_batches(table, options = {}, &block)
+        personen = @db.from(table)
+        max = maximum_value(table, options[:primary_key]) || 0
+        lower_bound = 0
+        batch_size = 1000
+        while (lower_bound <= max) do
+          upper_bound = lower_bound + batch_size - 1
+          sql = personen.filter(options[:primary_key] => lower_bound..upper_bound)
+          sql = sql.select(*options[:columns]) unless options[:columns].nil?
+          sql = sql.distinct if options[:distinct]
+          sql = sql.order(*options[:order]) unless options[:order].nil?
+          sql.each do |result|
+            yield result if block_given?
+          end unless sql.nil?
+          lower_bound += batch_size
+        end
+      end
+
+      def maximum_value(table, column)
+        @db.from(table).max(column)
+      end
+
+      def count(table, options = {})
+        sql = @db.from(table)
+        sql = sql.select(*options[:columns]) unless options[:columns].nil?
+        sql = sql.distinct if options[:distinct]
+        sql.count
+      end
+
+      def insert_row(table, row)
+        @db.from(table).insert(row)
+      end
+
+      def update_row(table, row)
+        id = row.delete(:id) || row.delete('id')
+        @db.from(table).filter(:id => id).update(row)
+      end
+
+      def numeric_column?(table, column)
+        column_definition = @db.schema(table).select{|c| c.first == column}.first
+        column_definition[1][:type] == :integer unless column_definition.nil?
+      end
+
+      def unique_row(table, key)
+        @db.from(table)[:id => key]
+      end
+
+    end
+  end
+end

data/lib/data-import/database.rb

@@ -0,0 +1,28 @@
+module DataImport
+  class Database
+
+    def self.connect(name, options = {})
+      adapter = find_adapter(name)
+      unless adapter.nil?
+        adapter.connect options
+      end
+    end
+
+    private
+
+    SUPPORTED_ADAPTERS = [:sequel]
+
+    def self.find_adapter(name)
+      @@loaded_adapters ||= {}
+      if SUPPORTED_ADAPTERS.include? name.to_sym
+        if @@loaded_adapters[name.to_sym].nil?
+          require "data-import/adapters/#{name.to_s}"
+          class_name = name.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+          @@loaded_adapters[name.to_sym] = DataImport::Adapters.const_get(class_name)
+        end
+        @@loaded_adapters[name.to_sym]
+      end
+    end
+
+  end
+end

data/lib/data-import/definition.rb

@@ -0,0 +1,20 @@
+require 'data-import/definition/simple'
+
+module DataImport
+  class Definition
+    attr_reader :name
+    attr_reader :source_database, :target_database
+    attr_reader :dependencies
+
+    def initialize(name, source_database, target_database)
+      @name = name
+      @source_database = source_database
+      @target_database = target_database
+      @dependencies = []
+    end
+
+    def add_dependency(dependency)
+      @dependencies << dependency
+    end
+  end
+end

data/lib/data-import/definition/simple.rb

@@ -0,0 +1,59 @@
+module DataImport
+  class Definition
+    class Simple < Definition
+      attr_reader :id_mappings
+      attr_reader :source_primary_key
+      attr_accessor :source_table_name, :source_columns, :source_distinct_columns, :source_order_columns
+      attr_accessor :target_table_name
+      attr_accessor :after_blocks, :after_row_blocks
+      attr_reader :mode
+
+      def initialize(name, source_database, target_database)
+        super
+        @mode = :insert
+        @id_mappings = {}
+        @after_blocks = []
+        @after_row_blocks = []
+        @source_columns = []
+        @source_order_columns = []
+      end
+
+      def mappings
+        @mappings ||= {}
+      end
+
+      def source_primary_key=(value)
+        @source_primary_key = value.to_sym unless value.nil?
+      end
+
+      def add_id_mapping(mapping)
+        @id_mappings.merge! mapping
+      end
+
+      def new_id_of(value)
+        @id_mappings[value]
+      end
+
+      def definition(name = nil)
+        if name.nil?
+          self
+        else
+          DataImport.definitions[name] or raise ArgumentError
+        end
+      end
+
+      def use_mode(mode)
+        @mode = mode
+      end
+
+      def run(context)
+        options = {:columns => source_columns, :distinct => source_distinct_columns}
+        Progress.start("Importing #{name}", source_database.count(source_table_name, options)) do
+          Importer.new(context, self).run do
+            Progress.step
+          end
+        end
+      end
+    end
+  end
+end