activewarehouse-etl-sgonyea 0.9.6

data/test/delimited_with_bulk_load.ctl

@@ -0,0 +1,34 @@
+infile = 'data/people.txt'
+outfile = 'output/people.txt'
+
+source :in, {
+  :file => infile,
+  :parser => {
+    :name => :csv
+  }
+}, 
+[ 
+  :first_name,
+  :last_name,
+]
+
+before_write :surrogate_key, :target => :data_warehouse, :table => 'person_dimension', :column => 'id'
+before_write :check_exist, {
+  :target => :data_warehouse, 
+  :table => 'person_dimension', 
+  :columns => [:first_name, :last_name]
+}
+
+destination :out, {
+  :file => outfile
+},
+{
+  :order => [:id, :first_name, :last_name]
+}
+
+post_process :bulk_import, {
+  :file => outfile,
+  :target => :data_warehouse,
+  :table => 'person_dimension',
+  :order => [:id, :first_name, :last_name]
+}

data/test/destination_test.rb

@@ -0,0 +1,275 @@
+require 'rubygems'
+require 'spreadsheet'
+require File.dirname(__FILE__) + '/test_helper'
+
+class Person < ActiveRecord::Base
+end
+
+class BadDestination < ETL::Control::Destination
+  def initialize(control, configuration, mapping)
+    super
+  end
+end
+
+# Test the functionality of destinations
+class DestinationTest < Test::Unit::TestCase
+  # Test a file destination
+  def test_file_destination
+    outfile = 'output/test_file_destination.txt'
+    outfile2 = 'output/test_file_destination.2.txt'
+    row = ETL::Row[ :address => '123 SW 1st Street', :city => 'Melbourne', 
+      :state => 'Florida', :country => 'United States' ]
+    row_needs_escape = ETL::Row[ :address => "Allen's Way", 
+      :city => 'Some City', :state => 'Some State', :country => 'Mexico' ]
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+      '/delimited.ctl')
+
+    # First define a basic configuration to check defaults
+    configuration = { 
+      :file => outfile, 
+      :buffer_size => 0,
+    }
+    mapping = { 
+      :order => [:address, :city, :state, :country, :country_code],
+      :virtual => {
+        :country_code => Proc.new do |r|
+          {
+            'United States' => 'US',
+            'Mexico' => 'MX'
+          }[r[:country]]
+        end
+      }
+    }
+    
+    dest = ETL::Control::FileDestination.new(control, configuration, mapping)    
+    dest.write(row)
+    dest.write(row_needs_escape)
+    dest.close
+    
+    configuration[:file] = outfile2
+    configuration[:separator] = '|'
+    configuration[:eol] = "[EOL]\n"
+    
+    dest = ETL::Control::FileDestination.new(control, configuration, mapping)
+    dest.write(row)
+    dest.write(row_needs_escape)
+    dest.close
+    
+    # Read back the resulting
+    lines = open(File.join(File.dirname(__FILE__), outfile), 'r').readlines
+    assert_equal "123 SW 1st Street,Melbourne,Florida,United States,US\n", lines[0]
+    assert_equal "Allen's Way,Some City,Some State,Mexico,MX\n", lines[1]
+    
+    lines = open(File.join(File.dirname(__FILE__), outfile2), 'r').readlines
+    assert_equal "123 SW 1st Street|Melbourne|Florida|United States|US[EOL]\n", lines[0]
+    assert_equal "Allen's Way|Some City|Some State|Mexico|MX[EOL]\n", lines[1]
+  end
+  
+  # Test a database destination
+  def test_database_destination
+    row = ETL::Row[:id => 1, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
+    row_needs_escape = ETL::Row[:id => 2, :first_name => "Foo's", :last_name => "Bar", :ssn => '000000000' ]
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+      '/delimited.ctl')
+    
+    Person.delete_all
+    assert_equal 0, Person.count
+    
+    # First define a basic configuration to check defaults
+    configuration = { 
+      :target => :data_warehouse,
+      :database => 'etl_unittest',
+      :table => 'people',
+      :buffer_size => 0 
+    }
+    mapping = { :order => [:id, :first_name, :last_name, :ssn] }
+    dest = ETL::Control::DatabaseDestination.new(control, configuration, mapping)
+    dest.write(row)
+    dest.close
+    
+    assert_equal 1, Person.find(:all).length
+  end
+  
+  def test_database_destination_with_control
+    row = ETL::Row[:id => 1, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+      '/delimited_destination_db.ctl')
+    Person.delete_all
+    assert_equal 0, Person.count
+    d = control.destinations.first
+    dest = ETL::Control::DatabaseDestination.new(control, d.configuration, d.mapping)
+    dest.write(row)
+    dest.close
+    assert_equal 1, Person.count
+  end
+  
+  def test_unique
+    row1 = ETL::Row[:id => 1, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
+    row2 = ETL::Row[:id => 2, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
+    row3 = ETL::Row[:id => 3, :first_name => 'John', :last_name => 'Smith', :ssn => '000112222']
+         
+    outfile = 'output/test_unique.txt'
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+
+    # First define a basic configuration to check defaults
+    configuration = { :file => outfile, :buffer_size => 0, :unique => [:ssn]}
+    mapping = { 
+      :order => [:first_name, :last_name, :ssn]
+    }
+    dest = ETL::Control::FileDestination.new(control, configuration, mapping)    
+    dest.write(row1)
+    dest.write(row2)
+    dest.write(row3)
+    
+    # Close (and flush) the destination
+    dest.close
+    
+    # Read back the resulting
+    lines = open(File.join(File.dirname(__FILE__), outfile), 'r').readlines
+    assert_equal "Bob,Smith,111234444\n", lines[0]
+    assert_equal "John,Smith,000112222\n", lines[1]
+  end
+  
+  def test_multiple_unique
+    row1 = ETL::Row[:id => 1, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
+    row2 = ETL::Row[:id => 2, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
+    row3 = ETL::Row[:id => 3, :first_name => 'Bob', :last_name => 'Smith', :ssn => '000112222']
+    
+    outfile = 'output/test_multiple_unique.txt'
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+
+    # First define a basic configuration to check defaults
+    configuration = { :file => outfile, :buffer_size => 0, :unique => [:last_name,:first_name]}
+    mapping = { 
+      :order => [:first_name, :last_name, :ssn]
+    }
+    dest = ETL::Control::FileDestination.new(control, configuration, mapping)    
+    dest.write(row1)
+    dest.write(row2)
+    dest.write(row3)
+    
+    # Close (and flush) the destination
+    dest.close
+    
+    # Read back the resulting
+    lines = open(File.join(File.dirname(__FILE__), outfile), 'r').readlines
+    assert_equal "Bob,Smith,111234444\n", lines[0]
+    assert_equal 1, lines.length
+  end
+
+  def test_bad_destination
+    control = ETL::Control::Control.parse_text('')
+    configuration = {}
+    mapping = {}
+    dest = BadDestination.new(control, configuration, mapping)
+    dest.write(nil)
+    assert_raise NotImplementedError do
+      dest.flush
+    end
+    assert_raise NotImplementedError do
+      dest.close
+    end
+  end
+
+  def test_excel_destination
+    outfile = 'output/test_excel_destination.xls'
+    row = ETL::Row[ :address => '123 SW 1st Street', :city => 'Melbourne', 
+      :state => 'Florida', :country => 'United States' ]
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+      '/delimited_excel.ctl')
+
+    # First define a basic configuration to check defaults
+    configuration = { 
+      :file => outfile, 
+      :buffer_size => 0,
+    }
+    mapping = { 
+      :order => [:address, :city, :state, :country, :country_code],
+      :virtual => {
+        :country_code => Proc.new do |r|
+          {
+            'United States' => 'US',
+            'Mexico' => 'MX'
+          }[r[:country]]
+        end
+      }
+    }
+    
+    dest = ETL::Control::ExcelDestination.new(control, configuration, mapping)    
+    dest.write(row)
+    dest.close
+    
+    # Read back the resulting
+    book = Spreadsheet.open File.join(File.dirname(__FILE__), outfile)
+    sheet = book.worksheet(0)
+
+    assert_equal "123 SW 1st Street", sheet[0, 0]
+    assert_equal "Melbourne", sheet[0, 1]
+    assert_equal "Florida", sheet[0, 2]
+    assert_equal "United States", sheet[0, 3]
+    assert_equal "US", sheet[0, 4]
+  end
+
+  # Test a update database destination
+  def test_update_database_destination
+    row = ETL::Row[:id => 1, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+      '/delimited_update.ctl')
+    
+    Person.delete_all
+    assert_equal 0, Person.count
+    test_database_destination
+
+    # First define a basic configuration to check defaults
+    configuration = { 
+      :type => :update_database,
+      :target => :data_warehouse,
+      :database => 'etl_unittest',
+      :table => 'people',
+      :buffer_size => 0 
+    }
+    mapping = {
+      :conditions => [{:field => "\#{conn.quote_column_name(:id)}", :value => "\#{conn.quote(row[:id])}", :comp => "="}],
+      :order => [:id, :first_name, :last_name, :ssn]
+    }
+    dest = ETL::Control::UpdateDatabaseDestination.new(control, configuration, mapping)
+    dest.write(row)
+    dest.close
+    
+    assert_equal 1, Person.find(:all).length
+
+  end
+
+  # Test a insert update database destination
+  def test_insert_update_database_destination
+    row = ETL::Row[:id => 1, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
+    row_needs_escape = ETL::Row[:id => 2, :first_name => "Foo's", :last_name => "Bar", :ssn => '000000000' ]
+    row_needs_update = ETL::Row[:id => 1, :first_name => "Sean", :last_name => "Toon", :ssn => '000000000' ]
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+      '/delimited_insert_update.ctl')
+    
+    Person.delete_all
+    assert_equal 0, Person.count
+    
+    # First define a basic configuration to check defaults
+    configuration = { 
+      :type => :insert_update_database,
+      :target => :data_warehouse,
+      :database => 'etl_unittest',
+      :table => 'people',
+      :buffer_size => 0 
+    }
+    mapping = {
+      :primarykey => [:id],
+      :order => [:id, :first_name, :last_name, :ssn]
+    }
+    dest = ETL::Control::InsertUpdateDatabaseDestination.new(control, configuration, mapping)
+    dest.write(row)
+    dest.write(row_needs_escape)
+    dest.write(row_needs_update)
+    dest.close
+    
+    assert_equal 2, Person.find(:all).length
+  end
+  
+end

data/test/directive_test.rb

@@ -0,0 +1,23 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class BadDirective < ETL::Batch::Directive
+  
+end
+
+class BatchTest < Test::Unit::TestCase
+  
+  attr_reader :file
+  attr_reader :engine
+  def setup
+    @file =  File.dirname(__FILE__) + '/all.ebf'
+    @engine = ETL::Engine.new
+  end
+  
+  def test_directive_without_implementation_should_fail
+    batch = ETL::Batch::Batch.resolve(file, engine)
+    assert_raise RuntimeError do
+      d = BadDirective.new(batch)
+      d.execute
+    end
+  end
+end

data/test/encode_processor_test.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+require File.dirname(__FILE__) + '/test_helper'
+
+class EncodeProcessorTest < Test::Unit::TestCase
+
+  SOURCE = 'data/encode_source_latin1.txt'
+  TARGET = 'output/encode_destination_utf-8.txt'
+
+  def setup
+    @control = flexmock("control")
+    @control.should_receive(:file).twice.and_return(File.dirname(__FILE__) + '/fake-control.ctl')
+  end
+  
+  def test_should_transform_a_latin1_file_to_utf8_with_grace
+    configuration = { :source_file => SOURCE, :source_encoding => 'latin1', :target_file => TARGET, :target_encoding => 'utf-8' }
+    ETL::Processor::EncodeProcessor.new(@control, configuration).process
+    assert_equal "éphémère has accents.\nlet's encode them.", IO.read(File.join(File.dirname(__FILE__),TARGET))
+  end
+  
+  def test_should_throw_exception_on_unsupported_encoding
+    configuration = { :source_file => SOURCE, :source_encoding => 'acme-encoding', :target_file => TARGET, :target_encoding => 'utf-8' }
+    error = assert_raise(ETL::ControlError) { ETL::Processor::EncodeProcessor.new(@control, configuration) }
+    assert_equal "Either the source encoding 'acme-encoding' or the target encoding 'utf-8' is not supported", error.message
+  end
+  
+  def test_should_throw_exception_when_target_and_source_are_the_same
+    configuration = { :source_file => SOURCE, :source_encoding => 'latin1', :target_file => SOURCE, :target_encoding => 'utf-8' }
+    error = assert_raise(ETL::ControlError) { ETL::Processor::EncodeProcessor.new(@control, configuration) }
+    assert_equal "Source and target file cannot currently point to the same file", error.message
+  end
+  
+end

data/test/engine_test.rb

@@ -0,0 +1,78 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class EngineTest < Test::Unit::TestCase
+
+  context 'process' do
+  
+    should 'raise an error when a file which does not exist is given' do
+      error = assert_raise(Errno::ENOENT) do
+        ETL::Engine.process('foo-bar.ctl')
+      end
+      
+      assert_equal "No such file or directory - foo-bar.ctl", error.message
+    end
+    
+    should 'raise an error when an unknown file type is given' do
+      error = assert_raise(RuntimeError) do
+        ETL::Engine.process(__FILE__)
+      end
+      
+      assert_match /Unsupported file type/, error.message
+    end
+    
+    should_eventually 'stop as soon as the error threshold is reached' do
+      engine = ETL::Engine.new
+
+      assert_equal 0, engine.errors.size
+      
+      engine.process ETL::Control::Control.parse_text <<CTL
+        set_error_threshold 1
+        source :in, { :type => :enumerable, :enumerable => (1..100) }
+        after_read { |row| raise "Failure" }
+CTL
+      
+      assert_equal 1, engine.errors.size
+    end
+    
+  end
+  
+  context 'connection' do
+  
+    should 'return an ActiveRecord configuration by name' do
+      assert_not_nil ETL::Engine.connection(:data_warehouse)
+    end
+    
+    should 'raise an error on non existent connection' do
+      error = assert_raise(ETL::ETLError) do
+        ETL::Engine.connection(:does_not_exist)
+      end
+      assert_equal "Cannot find connection named :does_not_exist", error.message
+    end
+    
+    should 'raise an error when requesting a connection with no name' do
+      error = assert_raise(ETL::ETLError) do
+        ETL::Engine.connection("  ")
+      end
+      assert_equal "Connection with no name requested. Is there a missing :target parameter somewhere?", error.message
+    end
+  end
+  
+  context 'temp tables' do
+    attr_reader :connection
+    
+    setup do
+      @connection = ETL::Engine.connection(:data_warehouse)
+    end
+    
+    should 'return unmodified table name when temp tables are disabled' do
+      assert_equal 'foo', ETL::Engine.table('foo', ETL::Engine.connection(:data_warehouse))
+    end
+    
+    should 'return temp table name instead of table name when temp tables are enabled' do
+      ETL::Engine.use_temp_tables = true
+      assert_equal 'tmp_people', ETL::Engine.table('people', connection)
+      ETL::Engine.use_temp_tables = false
+    end
+  end
+  
+end

data/test/ensure_fields_presence_processor_test.rb

@@ -0,0 +1,28 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class EnsureFieldsPresenceProcessorTest < Test::Unit::TestCase
+
+  def new_processor(options)
+    ETL::Processor::EnsureFieldsPresenceProcessor.new(nil, options)
+  end
+  
+  should 'raise an error unless :fields is specified' do
+    error = assert_raises(ETL::ControlError) { new_processor({}) }
+    assert_equal ":fields must be specified", error.message
+  end
+
+  should 'raise an error if a field is missing in the row' do
+    error = assert_raise(ETL::ControlError) do
+      processor = new_processor(:fields => [:key])
+      processor.process(ETL::Row[])
+    end
+    
+    assert_match /missing required field\(s\)/, error.message
+  end
+  
+  should 'return the row if the required fields are in the row' do
+    row = ETL::Row[:first => nil, :second => "Barry"]
+    assert_equal row, new_processor(:fields => [:first, :second]).process(row)
+  end
+  
+end