data-import 0.0.1

data/lib/data-import/dsl.rb

@@ -0,0 +1,54 @@
+require 'data-import/dsl/import'
+require 'data-import/dsl/import/from'
+
+module DataImport
+  class Dsl
+    class << self
+
+      def evaluate_import_config(file)
+        plan = DataImport::ExecutionPlan.new
+        context = new(plan)
+        context.instance_eval read_import_config(file), file
+        plan
+      end
+
+      def define(&block)
+        plan = DataImport::ExecutionPlan.new
+        context = new(plan)
+        context.instance_eval &block
+        plan
+      end
+
+      def read_import_config(file)
+        File.read(file)
+      end
+      private :read_import_config
+
+    end
+
+    attr_reader :source_database, :target_database
+
+    def initialize(plan)
+      @plan = plan
+    end
+
+    def source(*args)
+      @source_database = DataImport::Database.connect(*args)
+    end
+
+    def target(*args)
+      @target_database = DataImport::Database.connect(*args)
+    end
+
+    def import(name, &block)
+      definition = DataImport::Definition::Simple.new(name, source_database, target_database)
+      @plan.add_definition(definition)
+
+      Import.new(definition).instance_eval &block if block_given?
+    end
+
+    def before_filter(&block)
+      @plan.before_filter = block
+    end
+  end
+end

data/lib/data-import/dsl/import.rb

@@ -0,0 +1,48 @@
+module DataImport
+  class Dsl
+    class Import
+      attr_reader :definition
+
+      def initialize(definition)
+        @definition = definition
+      end
+
+      def from(name = nil, options = {}, &block)
+        definition.source_table_name = name
+        definition.source_primary_key = options[:primary_key]
+
+        From.new(definition).instance_eval &block if block_given?
+      end
+
+      def to(name, options = {})
+        definition.target_table_name = name
+        definition.use_mode(:update) if options[:mode] == :update
+      end
+
+      def mapping(*hash_or_symbols, &block)
+        if hash_or_symbols.first.is_a? Hash
+          definition.mappings.merge! hash_or_symbols.first
+        else
+          symbols = hash_or_symbols
+          symbols = symbols.first if symbols.count == 1
+          definition.mappings[symbols] = block
+        end
+      end
+
+      def after(&block)
+        definition.after_blocks << block
+      end
+
+      def after_row(&block)
+        definition.after_row_blocks << block
+      end
+
+      def dependencies(*dependencies)
+        dependencies.each do |dependency|
+          definition.add_dependency(dependency)
+        end
+      end
+
+    end
+  end
+end

data/lib/data-import/dsl/import/from.rb

@@ -0,0 +1,35 @@
+module DataImport
+  class Dsl
+    class Import
+      class From
+
+        attr_reader :definition
+
+        def initialize(definition)
+          @definition = definition
+        end
+
+        def table(name)
+          definition.source_table_name = name
+        end
+
+        def primary_key(name)
+          definition.source_primary_key = name
+        end
+
+        def columns(*args)
+          if args.last.is_a? Hash
+            options = args.pop
+            definition.source_distinct_columns = options[:distinct]
+          end
+          definition.source_columns |= args
+        end
+
+        def order(*args)
+          definition.source_order_columns |= args
+        end
+
+      end
+    end
+  end
+end

data/lib/data-import/execution_plan.rb

@@ -0,0 +1,16 @@
+module DataImport
+  class ExecutionPlan
+
+    attr_reader :definitions
+    attr_accessor :before_filter
+
+    def initialize(definitions = [])
+      @definitions = definitions
+    end
+
+    def add_definition(definition)
+      @definitions << definition
+    end
+
+  end
+end

data/lib/data-import/importer.rb

@@ -0,0 +1,55 @@
+module DataImport
+  class Importer
+
+    def initialize(context, definition)
+      @context = context
+      @definition = definition
+    end
+
+    def run
+      @definition.target_database.transaction do
+        options = {}
+        options[:primary_key] = @definition.source_primary_key
+        options[:columns] = @definition.source_columns
+        options[:distinct] = @definition.source_distinct_columns
+        options[:order] = @definition.source_order_columns
+        @definition.source_database.each_row(@definition.source_table_name, options) do |row|
+          @context.before_filter.call(row) if @context.before_filter
+          import_row row
+          yield if block_given?
+        end
+      end
+      @definition.after_blocks.each do |block|
+        @definition.instance_exec(@context, &block)
+      end
+    end
+
+    def import_row(row)
+      mapped_row = {}
+      @definition.mappings.each do |old_key, new_key|
+        if new_key.respond_to?(:call)
+          keys = old_key
+          keys = [keys] unless keys.is_a? Array
+          params = [@context] + keys.map{|key| row[key.to_sym]}
+          mapped_values = @definition.instance_exec(*params, &new_key)
+          mapped_row.merge! mapped_values if mapped_values.present?
+        else
+          mapped_row[new_key] = row[old_key.to_sym]
+        end
+      end
+
+      case @definition.mode
+      when :insert
+        new_id = @definition.target_database.insert_row @definition.target_table_name, mapped_row
+        @definition.add_id_mapping row[@definition.source_primary_key] => new_id
+      when :update
+        @definition.target_database.update_row(@definition.target_table_name, mapped_row)
+      end
+
+      @definition.after_row_blocks.each do |block|
+        @definition.instance_exec(@context, row, mapped_row, &block)
+      end
+    end
+    private :import_row
+  end
+end

data/lib/data-import/runner.rb

@@ -0,0 +1,62 @@
+module DataImport
+  class Runner
+
+    def initialize(plan)
+      @plan = plan
+      @definitions = Hash[@plan.definitions.map do |definition|
+                            [definition.name, definition]
+                          end]
+
+      @executed_definitions = []
+    end
+
+    def run(options = {})
+      definitions_to_execute = definitions_for_execution(options[:only])
+      while @executed_definitions.count < definitions_to_execute.count
+        did_execute = false
+        definitions_to_execute.each do |name|
+          candidate = definition(name)
+          next if @executed_definitions.include?(name)
+          if (candidate.dependencies - @executed_definitions).blank?
+            candidate.run(self)
+            @executed_definitions << name
+            did_execute = true
+          end
+        end
+
+        unless did_execute
+          raise "something went wrong! Could not execute all necessary definitions: #{candidate.dependencies - @@executed_definitions}"
+        end
+      end
+    end
+
+    def definition(name)
+      raise "no definition found for '#{name}'" unless @definitions[name].present?
+      @definitions[name]
+    end
+
+    def before_filter
+      @plan.before_filter
+    end
+
+    def definitions_for_execution(run_only = nil)
+      (run_only || @definitions.keys).map do |name|
+        [name] + dependencies(name)
+      end.flatten.uniq
+    end
+    private :definitions_for_execution
+
+    def dependencies(name, visited_definitions = [])
+      definition = definition(name)
+      direct_dependencies = definition.dependencies
+      indirect_dependencies = direct_dependencies.map do |dep|
+        raise RuntimeError, "ciruclar dependencies: '#{name}' <-> '#{dep}'" if visited_definitions.include?(dep)
+        dependencies(dep, visited_definitions + [name])
+      end.flatten
+      direct_dependencies + indirect_dependencies
+    end
+    private :dependencies
+
+
+  end
+end

data/lib/data-import/version.rb

@@ -0,0 +1,3 @@
+module DataImport
+  VERSION = "0.0.1"
+end

data/scripts/ci.sh

@@ -0,0 +1,12 @@
+if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
+  source "$HOME/.rvm/scripts/rvm"
+elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then
+  source "/usr/local/rvm/scripts/rvm"
+else
+  printf "ERROR: An RVM installation was not found.\n"
+fi
+
+`cat .rvmrc`
+gem install bundler --conservative --no-ri --no-rdoc
+bundle install
+rake ci:rspec

data/spec/data-import/adapters/sequel_spec.rb

@@ -0,0 +1,159 @@
+require 'spec_helper'
+
+require 'data-import/adapters/sequel'
+
+describe DataImport::Adapters::Sequel do
+
+  let(:table) { Object.new }
+  let(:dummy_db) do
+    class DummyDB
+    end
+    DummyDB.any_instance.stub(:from).and_return { table }
+    DummyDB
+  end
+  let(:db) { dummy_db.new }
+  subject { DataImport::Adapters::Sequel.new db }
+
+  describe ".connect" do
+    subject { DataImport::Adapters::Sequel }
+
+    it "connects to the database" do
+      Sequel.should_receive(:connect).with(:adapter => :test)
+      subject.connect :adapter => :test
+    end
+
+    it "returns an instance of DataImport::Adapters::Sequel" do
+      Sequel.stub(:connect)
+      subject.connect.should be_a(DataImport::Adapters::Sequel)
+    end
+  end
+
+  describe "#truncate" do
+    it "deletes all rows from a table" do
+      table.should_receive(:delete)
+      subject.truncate('svp')
+    end
+  end
+
+  describe "#transaction" do
+    let(:block) { Proc.new {} }
+
+    it "runs the block in a transaction" do
+      db.should_receive(:transaction)
+      subject.transaction &block
+    end
+  end
+
+  describe ".each_row" do
+    it "delegates to .each_row_in_batches if there is a numeric primary key" do
+      subject.stub(:numeric_column?).and_return { true }
+      subject.should_receive(:each_row_in_batches).with(:abc, :primary_key => :PersonenID)
+      subject.each_row(:abc, :primary_key => :PersonenID)
+    end
+
+    it "delegates to .each_row_without_batches if there is a primary key other than numeric" do
+      subject.stub(:numeric_column?).and_return { false }
+      subject.should_receive(:each_row_without_batches).with(:abc, :primary_key => :PersonenID)
+      subject.each_row(:abc, :primary_key => :PersonenID)
+    end
+
+    it "delegates to .each_row_without_batches if there is no primary key" do
+      subject.should_receive(:each_row_without_batches).with(:abc, {})
+      subject.each_row(:abc)
+    end
+  end
+
+  describe "#each_row_without_batches" do
+    let(:resultset) { [:id => 2, :id => 4] }
+    let(:proxy) { Object.new }
+
+    it "selects all rows from the database" do
+      db.stub_chain(:from, :each).and_return { proxy }
+      db.should_receive(:from).with('Personen')
+      subject.each_row_without_batches('Personen')
+    end
+
+    it "yields each item of the resultset" do
+      db.stub_chain(:from, :each).and_yield(resultset[0])
+        .and_yield(resultset[1])
+      left_results = resultset.clone
+      subject.each_row_without_batches('Personen') do |row|
+        left_results.delete(row)
+      end
+      left_results.should be_empty
+    end
+  end
+
+  describe "#each_row_in_batches" do
+    it "gets the maximum id" do
+      subject.should_receive(:maximum_value)
+      table.stub(:filter)
+      subject.each_row_in_batches('abc', :primary_key => :PersonenID)
+    end
+
+    it "selects batches of a fixed size" do
+      subject.stub(:maximum_value).and_return { 2000 }
+      table.stub(:filter)
+      table.should_receive(:filter).with(:PersonenID => 0..999)
+      table.should_receive(:filter).with(:PersonenID => 1000..1999)
+      subject.each_row_in_batches('abc', :primary_key => :PersonenID)
+    end
+  end
+
+  describe "#maximum_value" do
+    it "selects the maximum value of a column" do
+      table.should_receive(:max).with(:PersonenID)
+      subject.maximum_value('abc', :PersonenID)
+    end
+  end
+
+  describe "#count" do
+    it "returns the amount of rows of a table" do
+      table.should_receive(:count)
+      subject.count('abc')
+    end
+  end
+
+  describe "#insert_row" do
+    it "inserts a single row into the database" do
+      table.should_receive(:insert).with(:id => 29)
+      subject.insert_row(:abc, :id => 29)
+    end
+  end
+
+  describe "#update_row" do
+    it 'updates the row with the given id' do
+      filtered_records = stub
+      table.should_receive(:filter).with(:id => 9).and_return(filtered_records)
+      filtered_records.should_receive(:update).with(:name => 'Hans', :alter => 17)
+      subject.update_row(:abc, {:id => 9, :name => 'Hans', :alter => 17})
+    end
+
+    it 'works with string keys' do
+      filtered_records = stub
+      table.should_receive(:filter).with(:id => 11).and_return(filtered_records)
+      filtered_records.should_receive(:update).with('name' => 'Hans', 'alter' => 17)
+      subject.update_row(:abc, {'id' => 11, 'name' => 'Hans', 'alter' => 17})
+    end
+  end
+
+  describe "numeric_column?" do
+    it "returns true for numbers" do
+      db.stub(:schema).and_return { [[:id, {:type => :integer}]] }
+      subject.numeric_column?(:table, :id).should be_true
+    end
+
+    it "returns false for other column types" do
+      db.stub(:schema).and_return { [[:id, {:type => :string}]] }
+      subject.numeric_column?(:table, :id).should be_false
+    end
+  end
+
+  describe "#unique_row" do
+    it "returns a row by its key" do
+      db.stub_chain(:from, :[]).and_return { {:id => 5, :a => 7} }
+      subject.unique_row(:table, 5).should == {:id => 5, :a => 7}
+    end
+  end
+
+end