activewarehouse-etl 0.9.5.rc1 → 1.0.0.rc1

data/lib/etl/transform/md5_transform.rb

@@ -0,0 +1,13 @@
+require 'digest/md5'
+
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which hashes the original value with a MD5 hash algorithm
+    class Md5Transform < ETL::Transform::Transform
+      # Transform the value with a MD5 digest algorithm.
+      def transform(name, value, row)
+        Digest::MD5.hexdigest(value)
+      end
+    end
+  end
+end

data/lib/etl/version.rb

@@ -1,3 +1,3 @@
 module ETL#:nodoc:
-  VERSION = "0.9.5.rc1"
+  VERSION = "1.0.0.rc1"
 end

data/test/.gitignore

@@ -1 +0,0 @@
-database*.yml

data/test/check_exist_processor_test.rb

@@ -0,0 +1,89 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class CheckExistProcessorTest < Test::Unit::TestCase
+
+  context 'CheckExistProcessor' do
+
+    setup do
+      @config = {
+        :target => :data_warehouse,
+        :table => 'people',
+        :columns => [:first_name, :last_name]
+      }
+    end
+    
+    should_eventually "compare based on all columns if no columns are provided" do
+      # TBI
+    end
+    
+    should_eventually "compare based on all columns except skipped ones if columns to skip are provided" do
+      # TBI
+    end
+    
+    should "raise an error if no table is provided" do
+      error = assert_raises(ETL::ControlError) do
+        ETL::Processor::CheckExistProcessor.new(nil, @config.except(:table))
+      end
+      # bug #2413 on assert_raises won't let me check error message above
+      assert_equal 'table must be specified', error.message 
+    end
+    
+    should "raise an error if no target is provided" do
+      error = assert_raises(ETL::ControlError) do
+        ETL::Processor::CheckExistProcessor.new(nil, @config.except(:target))
+      end
+      
+      assert_equal 'target must be specified', error.message
+    end
+    
+    should "bypass checking if the table has no rows" do
+      Person.delete_all
+      
+      processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
+      assert_equal false, processor.should_check?
+    end
+    
+    should "raise an error if one of the keys used for checking existence is not available in a row" do
+      Person.delete_all
+      # we need at least one record to avoid automatic skipping
+      # this should be mocked instead, probably
+      Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
+      
+      error = assert_raise(ETL::ControlError) do
+        row = ETL::Row[:first_name => 'John']
+        processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
+
+        # guard against bypassing
+        assert_equal true, processor.should_check?
+        
+        processor.process(row)
+      end
+      
+      assert_equal "Row missing required field :last_name for existence check", error.message
+    end
+    
+    should "return nil if the same row is found in database" do
+      Person.delete_all
+      Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
+
+      row = ETL::Row[:first_name => 'John', :last_name => 'Barry']
+      processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
+      assert_equal true, processor.should_check? # guard against bypassing
+        
+      assert_equal nil, processor.process(row)
+    end
+
+    should "return the row if no same row is found in database" do
+      Person.delete_all
+      Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
+
+      row = ETL::Row[:first_name => 'John', :last_name => 'OtherName']
+      processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
+      assert_equal true, processor.should_check? # guard against bypassing
+        
+      assert_equal row, processor.process(row)
+    end
+  
+  end
+
+end

data/test/check_unique_processor_test.rb

@@ -0,0 +1,40 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class CheckUniqueProcessorTest < Test::Unit::TestCase
+
+  context 'CheckUniqueProcessor' do
+    attr_reader :processor
+    
+    setup do
+      @processor = ETL::Processor::CheckUniqueProcessor.new(nil,
+        :keys => [:first, :second])
+    end
+
+    should "keep a row whose keys didn't already appear in the pipeline" do
+      row = ETL::Row[:first => 'A', :second => 'B']
+
+      assert_equal row, processor.process(row)
+      
+      assert_equal({ 'A|B' => 1 }, processor.compound_key_constraints)
+    end
+    
+    should "remove a row whose keys already appeared in the pipeline" do
+      row = ETL::Row[:first => 'A', :second => 'B']
+
+      assert_equal row, processor.process(row)
+      assert_equal nil, processor.process(row)
+    end
+    
+    should "raise an error if a row lacks one of the keys specified" do
+      row = ETL::Row[:first => 'A']
+      
+      error = assert_raises(ETL::ControlError) do
+        processor.process(row)
+      end
+      
+      assert_equal "Row missing required field :second for unicity check", error.message
+    end
+    
+  end
+  
+end

data/test/config/.gitignore

@@ -0,0 +1 @@
+*.lock

data/test/config/database.yml

@@ -0,0 +1,28 @@
+<% raise "ENV['DB'] not specified!" unless ENV['DB'] %>
+
+# a bit hackish - tests would require a refactoring instead
+
+mysql2: &mysql2
+  adapter: mysql2
+  database: activewarehouse_etl_test
+  username: root
+  encoding: utf8
+  local_infile: true
+  # the tests would require a rework: disabling casting for now
+  cast: false
+
+postgresql: &postgresql
+  adapter: postgresql
+  database: activewarehouse_etl_test
+  username: postgres
+
+# TODO - refactor test to avoid using 2 databases maybe?
+operational_database:
+  <<: *<%= ENV['DB'] %>
+
+data_warehouse:
+  <<: *<%= ENV['DB'] %>
+
+etl_execution:
+  adapter: mysql2
+  database: etl_execution

data/test/config/{Gemfile.rails-3.0.x → gemfiles/Gemfile.rails-3.0.x}

@@ -1,3 +1,3 @@
 require File.dirname(__FILE__) + '/common'
 
-common_gemfile('3.0.7')
+declare_gems '3.0.11'

data/test/config/{Gemfile.rails-2.3.x → gemfiles/Gemfile.rails-3.1.x}

@@ -1,3 +1,3 @@
 require File.dirname(__FILE__) + '/common'
 
-common_gemfile('2.3.11')
+declare_gems '3.1.3'

data/test/config/gemfiles/Gemfile.rails-3.2.x

@@ -0,0 +1,3 @@
+require File.dirname(__FILE__) + '/common'
+
+declare_gems '3.2.1'

data/test/config/gemfiles/common.rb

@@ -0,0 +1,29 @@
+def declare_gems(activerecord_version)
+  source :rubygems
+
+  gem 'activerecord', activerecord_version
+  gem 'adapter_extensions', :git => 'https://github.com/activewarehouse/adapter_extensions.git', :branch => 'rails-3'
+
+  if activerecord_version < '3.1'
+    gem 'mysql2', '< 0.3'
+  else
+    # use our own fork for bulk load support until issue fixed:
+    # https://github.com/brianmario/mysql2/pull/242
+    gem 'mysql2', :git => 'https://github.com/activewarehouse/mysql2.git'
+  end
+
+  gem 'pg'
+  gem 'activerecord-sqlserver-adapter'
+
+  gem 'awesome_print'
+  gem 'rake'
+  gem 'flexmock'
+  gem 'shoulda', '3.0.1'
+  gem 'sqlite3'
+
+  gem 'spreadsheet'
+  gem 'nokogiri'
+  gem 'fastercsv'
+
+  gem 'standalone_migrations'
+end

data/test/control_test.rb

@@ -4,8 +4,8 @@ class ControlTest < Test::Unit::TestCase
   # Test the ability to parse control files.
   def test_parse
     assert_nothing_raised do
-      Dir.glob('*.ctl') do |f|
-        ETL::Control::Control.parse(File.join(File.dirname(__FILE__), f))
+      Dir.glob(File.join(File.dirname(__FILE__), '*.ctl')) do |f|
+        ETL::Control::Control.parse(f)
       end
     end
   end

data/test/data/nokogiri.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="US-ASCII" standalone="yes"?>
+<!DOCTYPE people >
+<people>
+  <person ssn="123456789" type="employee">
+    <first_name>Bob</first_name>
+    <last_name>Smith</last_name>
+    <email>bsmith@foo.com</email>
+    <colours>
+      <eyes>brown</eyes>
+      <hair>black</hair>
+      <skin>fair</skin>
+    </colours>
+    <age>24</age>
+  </person>
+  <person ssn="111223333" type="client">
+    <first_name>Jane</first_name>
+    <last_name>Doe</last_name>
+    <email>jdoe@bar.com</email>
+    <colours>
+      <eyes>blue</eyes>
+      <hair>blond</hair>
+      <skin>medium</skin>
+    </colours>
+    <age>45</age>
+  </person>
+  <person ssn="133244566" type="client">
+    <first_name>Jake</first_name>
+    <last_name>Smithsonian</last_name>
+    <email>jake@example.com</email>
+    <colours>
+      <eyes>brown</eyes>
+      <hair>black</hair>
+      <skin>dark</skin>
+    </colours>
+    <age>37</age>
+  </person>
+</people>
+

data/test/database_join_processor_test.rb

@@ -0,0 +1,43 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class DatabaseJoinProcessorTest < Test::Unit::TestCase
+
+  def new_processor(options)
+    ETL::Processor::DatabaseJoinProcessor.new(nil, options)
+  end
+  
+  should 'raise an error unless :fields is specified' do
+    error = assert_raises(ETL::ControlError) { new_processor({}) }
+    assert_equal ":target must be specified", error.message
+  end
+
+  should 'return the row and the database result' do
+    row = ETL::Row[:id => 1, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+      '/delimited.ctl')
+
+    Person.delete_all
+    assert_equal 0, Person.count
+    
+    # First define a basic configuration to check defaults
+    configuration = { 
+      :target => :data_warehouse,
+      :database => 'etl_unittest',
+      :table => 'people',
+      :buffer_size => 0 
+    }
+    mapping = { :order => [:id, :first_name, :last_name, :ssn] }
+    dest = ETL::Control::DatabaseDestination.new(control, configuration, mapping)
+    dest.write(row)
+    dest.close
+    
+    assert_equal 1, Person.find(:all).length
+
+    row = ETL::Row[:last_name => "Smith"]
+    processor = new_processor(:target => :data_warehouse,
+                              :query => "SELECT first_name FROM people WHERE last_name = \#{connection.quote(row[:last_name])}",
+                              :fields => ["first_name"]).process(row)
+    assert_equal row[:first_name], "Bob"
+  end
+  
+end

data/test/delimited.ctl

@@ -1,7 +1,7 @@
 source :in, {
   :file => 'data/delimited.txt',
   :parser => {
-    :name => :delimited
+    :name => :csv
   }
 }, 
 [ 

data/test/delimited_absolute.ctl

@@ -1,9 +1,7 @@
-# puts "executing delimited.ctl"
-
 source :in, {
   :file => '/tmp/delimited_abs.txt',
   :parser => {
-    :name => :delimited
+    :name => :csv
   }
 }, 
 [ 

data/test/delimited_destination_db.ctl

@@ -1,8 +1,6 @@
-# puts "executing delimited.ctl"
-
 source :in, {
   :file => 'data/delimited.txt',
-  :parser => :delimited
+  :parser => :csv
 }, 
 [ 
   :id,

data/test/delimited_excel.ctl

@@ -1,7 +1,7 @@
 source :in, {
   :file => 'data/delimited.txt',
   :parser => {
-    :name => :delimited
+    :name => :csv
   }
 }, 
 [ 

data/test/delimited_insert_update.ctl

@@ -1,7 +1,7 @@
 source :in, {
   :file => 'data/delimited.txt',
   :parser => {
-    :name => :delimited
+    :name => :csv
   }
 }, 
 [ 

data/test/delimited_update.ctl

@@ -1,7 +1,7 @@
 source :in, {
   :file => 'data/delimited.txt',
   :parser => {
-    :name => :delimited
+    :name => :csv
   }
 }, 
 [ 

data/test/delimited_with_bulk_load.ctl

@@ -4,7 +4,7 @@ outfile = 'output/people.txt'
 source :in, {
   :file => infile,
   :parser => {
-    :name => :delimited
+    :name => :csv
   }
 }, 
 [ 
@@ -13,7 +13,7 @@ source :in, {
 ]
 
 before_write :surrogate_key, :target => :data_warehouse, :table => 'person_dimension', :column => 'id'
-before_write :check_exists, {
+before_write :check_exist, {
   :target => :data_warehouse, 
   :table => 'person_dimension', 
   :columns => [:first_name, :last_name]

data/test/destination_test.rb

@@ -1,10 +1,6 @@
-require 'rubygems'
 require 'spreadsheet'
 require File.dirname(__FILE__) + '/test_helper'
 
-class Person < ActiveRecord::Base
-end
-
 class BadDestination < ETL::Control::Destination
   def initialize(control, configuration, mapping)
     super

data/test/encode_processor_test.rb

@@ -1,6 +1,8 @@
 # encoding: utf-8
 require File.dirname(__FILE__) + '/test_helper'
 
+require 'iconv'
+
 class EncodeProcessorTest < Test::Unit::TestCase
 
   SOURCE = 'data/encode_source_latin1.txt'

data/test/engine_test.rb

@@ -1,32 +1,78 @@
 require File.dirname(__FILE__) + '/test_helper'
 
 class EngineTest < Test::Unit::TestCase
+
+  context 'process' do
   
-  def test_connections
-    assert_equal 3, ActiveRecord::Base.configurations.length
-    conn = ETL::Engine.connection(:data_warehouse)
-    assert_not_nil conn
-  end
-  
-  def test_non_existent_connection
-    assert_raise ETL::ETLError do
-      conn = ETL::Engine.connection(:does_not_exist)
+    should 'raise an error when a file which does not exist is given' do
+      error = assert_raise(Errno::ENOENT) do
+        ETL::Engine.process('foo-bar.ctl')
+      end
+      
+      assert_equal "No such file or directory - foo-bar.ctl", error.message
+    end
+    
+    should 'raise an error when an unknown file type is given' do
+      error = assert_raise(RuntimeError) do
+        ETL::Engine.process(__FILE__)
+      end
+      
+      assert_match /Unsupported file type/, error.message
     end
+    
+    should_eventually 'stop as soon as the error threshold is reached' do
+      engine = ETL::Engine.new
+
+      assert_equal 0, engine.errors.size
+      
+      engine.process ETL::Control::Control.parse_text <<CTL
+        set_error_threshold 1
+        source :in, { :type => :enumerable, :enumerable => (1..100) }
+        after_read { |row| raise "Failure" }
+CTL
+      
+      assert_equal 1, engine.errors.size
+    end
+    
   end
   
-  def test_engine_table_method_should_return_same_name_when_not_using_temp_tables
-    assert_equal 'foo', ETL::Engine.table('foo', connection)
-  end
+  context 'connection' do
   
-  def test_engine_table_method_should_return_temp_name_when_using_temp_tables
-    ETL::Engine.use_temp_tables = true
-    assert_equal 'tmp_people', ETL::Engine.table('people', connection)
-    ETL::Engine.use_temp_tables = false
+    should 'return an ActiveRecord configuration by name' do
+      assert_not_nil ETL::Engine.connection(:data_warehouse)
+    end
+    
+    should 'raise an error on non existent connection' do
+      error = assert_raise(ETL::ETLError) do
+        ETL::Engine.connection(:does_not_exist)
+      end
+      assert_equal "Cannot find connection named :does_not_exist", error.message
+    end
+    
+    should 'raise an error when requesting a connection with no name' do
+      error = assert_raise(ETL::ETLError) do
+        ETL::Engine.connection("  ")
+      end
+      assert_equal "Connection with no name requested. Is there a missing :target parameter somewhere?", error.message
+    end
   end
   
-  protected
-  def connection
-    ETL::Engine.connection(:data_warehouse)
+  context 'temp tables' do
+    attr_reader :connection
+    
+    setup do
+      @connection = ETL::Engine.connection(:data_warehouse)
+    end
+    
+    should 'return unmodified table name when temp tables are disabled' do
+      assert_equal 'foo', ETL::Engine.table('foo', ETL::Engine.connection(:data_warehouse))
+    end
+    
+    should 'return temp table name instead of table name when temp tables are enabled' do
+      ETL::Engine.use_temp_tables = true
+      assert_equal 'tmp_people', ETL::Engine.table('people', connection)
+      ETL::Engine.use_temp_tables = false
+    end
   end
   
 end