data-import 0.0.1

data/spec/data-import/runner_spec.rb

@@ -0,0 +1,136 @@
+require 'spec_helper'
+
+describe DataImport::Runner do
+
+  context 'with simple definitions' do
+    let(:people) { DataImport::Definition.new('People', 'tblPerson', 'people') }
+    let(:animals) { DataImport::Definition.new('Animals', 'tblAnimal', 'animals') }
+    let(:articles) { DataImport::Definition.new('Articles', 'tblNewsMessage', 'articles') }
+    let(:plan) { DataImport::ExecutionPlan.new }
+    before do
+      plan.add_definition(articles)
+      plan.add_definition(people)
+      plan.add_definition(animals)
+    end
+
+    subject { DataImport::Runner.new(plan) }
+
+    it 'runs a set of definitions' do
+      articles.should_receive(:run)
+      people.should_receive(:run)
+      animals.should_receive(:run)
+
+      subject.run
+    end
+
+    it ":only limits the definitions, which will be run" do
+      people.should_receive(:run)
+      articles.should_receive(:run)
+
+      subject.run :only => ['People', 'Articles']
+    end
+  end
+
+  context "dependent definitions" do
+    it 'executes leaf-definitions first and works to the top' do
+      a = DataImport::Definition.new 'A', :source, :target
+      b = DataImport::Definition.new 'B', :source, :target
+
+      a_1 = DataImport::Definition.new 'A1', :source, :target
+      a_1.add_dependency('A')
+
+      ab_1 = DataImport::Definition.new 'A-B-1', :source, :target
+      ab_1.add_dependency('A')
+      ab_1.add_dependency('B')
+
+      ab_a1_1 = DataImport::Definition.new 'AB-A1-1', :source, :target
+      ab_a1_1.add_dependency('A-B-1')
+      ab_a1_1.add_dependency('A1')
+
+      plan = DataImport::ExecutionPlan.new([ab_a1_1, ab_1, b, a, a_1])
+      importer = DataImport::Runner.new(plan)
+
+      call_order = []
+
+      a.should_receive(:run) { call_order << :a }
+      b.should_receive(:run) { call_order << :b }
+      ab_1.should_receive(:run) { call_order << :ab_1 }
+      a_1.should_receive(:run) { call_order << :a_1 }
+      ab_a1_1.should_receive(:run) { call_order << :ab_a1_1}
+
+      importer.run
+
+      call_order.should == [:a, :b, :ab_1, :a_1, :ab_a1_1]
+    end
+
+    it 'handles dependencies correctly when :only is present' do
+      a = DataImport::Definition.new 'A', :source, :target
+      ab = DataImport::Definition.new 'AB', :source, :target
+      ab.add_dependency('A')
+      abc = DataImport::Definition.new 'ABC', :source, :target
+      abc.add_dependency('AB')
+
+      plan = DataImport::ExecutionPlan.new([abc, a, ab])
+      importer = DataImport::Runner.new(plan)
+
+      call_order = []
+
+      a.should_receive(:run) { call_order << :a }
+      ab.should_receive(:run) { call_order << :ab }
+      abc.should_receive(:run) { call_order << :abc }
+
+      importer.run :only => ['ABC']
+
+      call_order.should == [:a, :ab, :abc]
+    end
+
+    it "raises an exception when the dependencies can't be resolved" do
+      a = DataImport::Definition.new 'A', :source, :target
+      b = DataImport::Definition.new 'B', :source, :target
+      a.add_dependency('B')
+      b.add_dependency('A')
+
+      plan = DataImport::ExecutionPlan.new([a, b])
+      importer = DataImport::Runner.new(plan)
+
+      lambda do
+        importer.run
+      end.should raise_error(RuntimeError, "ciruclar dependencies: 'B' <-> 'A'")
+    end
+
+    it 'raises an error when invalid dependencies are found' do
+      a = DataImport::Definition.new 'A', :source, :target
+      a.add_dependency('NOT_PRESENT')
+
+      plan = DataImport::ExecutionPlan.new([a])
+      importer = DataImport::Runner.new(plan)
+
+      lambda do
+        importer.run
+      end.should raise_error(RuntimeError, "no definition found for 'NOT_PRESENT'")
+    end
+
+    it 'can resolve dependencies which appear to be circular but are not' do
+      a = DataImport::Definition.new 'A', :source, :target
+      ab = DataImport::Definition.new 'AB', :source, :target
+      ab.add_dependency('A')
+      aba = DataImport::Definition.new 'AB-A', :source, :target
+      aba.add_dependency('AB')
+      aba.add_dependency('A')
+
+      plan = DataImport::ExecutionPlan.new([a, aba, ab])
+      importer = DataImport::Runner.new(plan)
+
+      call_order = []
+
+      a.should_receive(:run) { call_order << :a }
+      ab.should_receive(:run) { call_order << :ab }
+      aba.should_receive(:run) { call_order << :aba }
+
+      importer.run
+
+      call_order.should == [:a, :ab, :aba]
+    end
+  end
+
+end

data/spec/data-import_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+describe DataImport do
+
+  subject { DataImport }
+
+  describe ".run_definitions!" do
+    let(:runner) { stub }
+    let(:plan) { DataImport::ExecutionPlan.new(definitions) }
+    let(:definitions) { [stub, stub] }
+
+    it "can execute a configuration file" do
+      DataImport::Dsl.should_receive(:evaluate_import_config).with('my_file').and_return(plan)
+      DataImport::Runner.should_receive(:new).with(plan).and_return(runner)
+      runner.should_receive(:run).with(:only => ['C'])
+
+      subject.run_config! 'my_file', :only => ['C']
+    end
+
+    it "uses the DataImport::Runner to execute the plan" do
+      DataImport::Runner.should_receive(:new).with(plan).and_return(runner)
+      runner.should_receive(:run)
+
+      subject.run_plan!(plan)
+    end
+
+    it "passes options to the runner" do
+      DataImport::Runner.should_receive(:new).with(plan).and_return(runner)
+      runner.should_receive(:run).with(:only => ['A', 'B'])
+
+      subject.run_plan!(plan, :only => ['A', 'B'])
+    end
+  end
+end

data/spec/integration/before_block_spec.rb

@@ -0,0 +1,59 @@
+require 'data-import'
+
+describe "before filter" do
+
+  let(:plan) do
+    DataImport::Dsl.define do
+      source :sequel, 'sqlite:/'
+      target :sequel, 'sqlite:/'
+
+      before_filter do |row|
+        row.each do |k, v|
+          row[k] = v.downcase if v.respond_to?(:downcase)
+        end
+      end
+
+      source_database.db.create_table :tblUser do
+        primary_key :sUserID
+        String :strEmail
+        String :strUsername
+      end
+
+      target_database.db.create_table :users do
+        primary_key :id
+        String :email
+        String :username
+      end
+
+      import 'Users' do
+        from 'tblUser', :primary_key => 'sUserID'
+        to 'users'
+
+        mapping 'sUserID' => 'id'
+        mapping 'strEmail' => 'email'
+        mapping 'strUsername' => 'username'
+      end
+    end
+  end
+
+  let(:source) { plan.definitions.first.source_database.db[:tblUser] }
+  let(:target) { plan.definitions.first.target_database.db[:users] }
+
+  before do
+    source.insert(:sUserID => 1,
+                  :strEmail => 'JANE.MEIERS@GMX.NET',
+                  :strUsername => 'JANEMRS')
+
+    source.insert(:sUserID => 2,
+                  :strEmail => 'JOHN.DOE@GMAIL.COM',
+                  :strUsername => 'JOHN_DOE')
+  end
+
+
+  it 'mapps columns to the new schema' do
+    DataImport.run_plan!(plan)
+    target.to_a.should == [{:id => 1, :email => "jane.meiers@gmx.net", :username => 'janemrs'},
+                           {:id => 2, :email => "john.doe@gmail.com", :username => 'john_doe'}]
+  end
+
+end

data/spec/integration/simple_mappings_spec.rb

@@ -0,0 +1,68 @@
+require 'data-import'
+
+describe "simple mappings" do
+
+  let(:plan) do
+    DataImport::Dsl.define do
+      source :sequel, 'sqlite:/'
+      target :sequel, 'sqlite:/'
+
+      source_database.db.create_table :tblAnimal do
+        primary_key :sAnimalID
+        String :strAnimalTitleText
+        Integer :sAnimalAge
+        Integer :sDangerRating
+        String :strThreat
+      end
+
+      target_database.db.create_table :animals do
+        primary_key :id
+        String :name
+        Integer :age
+        Integer :danger_rating
+      end
+
+      import 'Animals' do
+        from 'tblAnimal', :primary_key => 'sAnimalID'
+        to 'animals'
+
+        mapping 'sAnimalID' => 'id'
+        mapping 'strAnimalTitleText' => 'name'
+        mapping 'sAnimalAge' => 'age'
+        mapping 'strThreat' do |context, threat|
+          rating = ['none', 'medium', 'big'].index(threat) + 1
+          {:danger_rating => rating}
+        end
+      end
+    end
+  end
+
+  let(:source) { plan.definitions.first.source_database.db[:tblAnimal] }
+  let(:target) { plan.definitions.first.target_database.db[:animals] }
+
+  before do
+    source.insert(:sAnimalID => 1,
+                  :strAnimalTitleText => 'Tiger',
+                  :sAnimalAge => 23,
+                  :strThreat => 'big')
+
+    source.insert(:sAnimalID => 1293,
+                  :strAnimalTitleText => 'Horse',
+                  :sAnimalAge => 11,
+                  :strThreat => 'medium')
+
+    source.insert(:sAnimalID => 99,
+                  :strAnimalTitleText => 'Cat',
+                  :sAnimalAge => 5,
+                  :strThreat => 'none')
+  end
+
+
+  it 'mapps columns to the new schema' do
+    DataImport.run_plan!(plan)
+    target.to_a.should == [{:id => 1, :name => "Tiger", :age => 23, :danger_rating => 3},
+                           {:id => 99, :name => "Cat", :age => 5, :danger_rating => 1},
+                           {:id => 1293, :name => "Horse", :age => 11, :danger_rating => 2}]
+  end
+
+end

data/spec/integration/update_records_spec.rb

@@ -0,0 +1,57 @@
+require 'data-import'
+
+describe "update existing records" do
+
+  let(:plan) do
+    DataImport::Dsl.define do
+      source :sequel, 'sqlite:/'
+      target :sequel, 'sqlite:/'
+
+      source_database.db.create_table :tblArticleAbout do
+        primary_key :sID
+        Integer :lArticleId
+        String :strWho
+      end
+
+      target_database.db.create_table :articles do
+        primary_key :id
+        String :title
+        String :author
+      end
+
+      import 'Article Authors' do
+        from 'tblArticleAbout', :primary_key => 'sID'
+        to 'articles', :mode => :update
+
+        mapping 'lArticleId' => 'id'
+        mapping 'strWho' => 'author'
+      end
+    end
+  end
+
+  let(:source) { plan.definitions.first.source_database.db[:tblArticleAbout] }
+  let(:target) { plan.definitions.first.target_database.db[:articles] }
+
+  before do
+    source.insert(:sID => 1,
+                  :lArticleId => 12,
+                  :strWho => 'Adam K.')
+    source.insert(:sID => 2,
+                  :lArticleId => 145,
+                  :strWho => 'James G.')
+
+    target.insert(:id => 12,
+                  :title => 'The Book!')
+    target.insert(:id => 145,
+                  :title => 'The other Book.')
+
+  end
+
+
+  it 'mapps columns to the new schema' do
+    DataImport.run_plan!(plan)
+    target.to_a.should == [{:id => 12, :title => "The Book!", :author => 'Adam K.'},
+                           {:id => 145, :title => "The other Book.", :author => 'James G.'}]
+  end
+
+end

data/spec/junit_formatter.rb

@@ -0,0 +1,106 @@
+# -*- encoding : utf-8 -*-
+#
+# Copyright (c) 2011, Diego Souza
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#   * Redistributions of source code must retain the above copyright notice,
+#     this list of conditions and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#   * Neither the name of the <ORGANIZATION> nor the names of its contributors
+#     may be used to endorse or promote products derived from this software
+#     without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+require "time"
+require "rspec/core/formatters/base_formatter"
+
+class JUnitFormatter < RSpec::Core::Formatters::BaseFormatter
+
+  attr_reader :test_results
+
+  def initialize(output)
+    super(output)
+    @test_results = { :failures => [], :successes => [] }
+  end
+
+  def example_passed(example)
+    super(example)
+    @test_results[:successes].push(example)
+  end
+
+  def example_pending(example)
+    self.example_failed(example)
+  end
+
+  def example_failed(example)
+    super(example)
+    @test_results[:failures].push(example)
+  end
+
+  def read_failure(t)
+    exception = t.metadata[:execution_result][:exception_encountered] || t.metadata[:execution_result][:exception]
+    message = ""
+    unless (exception.nil?)
+      message  = exception.message
+      message += "\n"
+      message += format_backtrace(exception.backtrace, t).join("\n")
+    end
+    return(message)
+  end
+
+  def dump_summary(duration, example_count, failure_count, pending_count)
+    super(duration, example_count, failure_count, pending_count)
+    output.puts("<?xml version=\"1.0\" encoding=\"utf-8\" ?>")
+    output.puts("<testsuite errors=\"0\" failures=\"#{failure_count+pending_count}\" tests=\"#{example_count}\" time=\"#{duration}\" timestamp=\"#{Time.now.iso8601}\">")
+    output.puts("  <properties />")
+    @test_results[:successes].each do |t|
+      md      = t.metadata
+      runtime = md[:execution_result][:run_time]
+      output.puts("  <testcase classname=\"#{md[:file_path]}\" name=\"#{xml_escape(md[:full_description])}\" time=\"#{runtime}\" />")
+    end
+    @test_results[:failures].each do |t|
+      md      = t.metadata
+      runtime = md[:execution_result][:run_time]
+      output.puts("  <testcase classname=\"#{md[:file_path]}\" name=\"#{xml_escape(md[:full_description])}\" time=\"#{runtime}\">")
+      output.puts("    <failure message=\"failure\" type=\"failure\">")
+      output.puts("<![CDATA[ #{read_failure(t)} ]]>")
+      output.puts("    </failure>")
+      output.puts("  </testcase>")
+    end
+    output.puts("</testsuite>")
+  end
+
+  def xml_escape(input)
+    result = input.dup
+
+    result.gsub!(/[&<>'"]/) do | match |
+      case match
+      when '&' then return '&amp;'
+      when '<' then return '&lt;'
+      when '>' then return '&gt;'
+      when "'" then return '&apos;'
+      when '"' then return '&quote;'
+      end
+    end
+
+    return result
+  end
+  private :xml_escape
+
+
+end