etl 0.9.5.rc1

data/test/block_processor_test.rb

@@ -0,0 +1,38 @@
+require File.dirname(__FILE__) + '/test_helper'
+include ETL
+include ETL::Control
+
+class TestWitness
+end
+
+class BlockProcessorTest < Test::Unit::TestCase
+  
+  def test_block_processor_should_work_as_both_after_read_and_before_write_row_processor
+    MockSource[:block_processed_input] = [{ :first_name => 'John'},{:first_name => 'Gary'}]
+    process 'block_processor.ctl'
+    assert_equal 4, MockDestination[:block_processed_output].size
+    assert_equal({ :first_name => 'John', :added_by_after_read => 'after-John', :added_by_before_write => "Row 1" }, MockDestination[:block_processed_output][0])
+    assert_equal({ :new_row => 'added by post_processor' }, MockDestination[:block_processed_output][1])
+    assert_equal({ :first_name => 'Gary', :added_by_after_read => 'after-Gary', :added_by_before_write => "Row 2" }, MockDestination[:block_processed_output][2])
+    assert_equal({ :new_row => 'added by post_processor' }, MockDestination[:block_processed_output][3])
+  end
+  
+  def test_block_processor_should_let_rows_be_removed_by_setting_it_to_nil
+    MockSource[:block_input] = [{ :obsolete => true, :name => 'John'},{ :obsolete => false, :name => 'Gary'}]
+    process 'block_processor_remove_rows.ctl'
+    assert_equal([{ :obsolete => false, :name => 'Gary' }], MockDestination[:block_output]) # only one record should be kept
+  end
+  
+  def test_block_processor_should_work_as_pre_or_post_processor
+    flexmock(TestWitness).should_receive(:call).with("I'm called from pre_process")
+    flexmock(TestWitness).should_receive(:call).with("I'm called from post_process")
+    MockSource[:another_input] = [{ :obsolete => true, :name => 'John'},{ :obsolete => false, :name => 'Gary'}]
+    process 'block_processor_pre_post_process.ctl'
+    assert_equal(MockSource[:another_input], MockDestination[:another_output])
+  end
+  
+  def test_block_error_should_be_propagated
+    assert_raise(ControlError) { process 'block_processor_error.ctl' }
+  end
+  
+end

data/test/check_exist_processor_test.rb

@@ -0,0 +1,92 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class Person < ActiveRecord::Base
+end
+
+class CheckExistProcessorTest < Test::Unit::TestCase
+
+  context 'CheckExistProcessor' do
+
+    setup do
+      @config = {
+        :target => :data_warehouse,
+        :table => 'people',
+        :columns => [:first_name, :last_name]
+      }
+    end
+    
+    should_eventually "compare based on all columns if no columns are provided" do
+      # TBI
+    end
+    
+    should_eventually "compare based on all columns except skipped ones if columns to skip are provided" do
+      # TBI
+    end
+    
+    should "raise an error if no table is provided" do
+      error = assert_raises(ETL::ControlError) do
+        ETL::Processor::CheckExistProcessor.new(nil, @config.except(:table))
+      end
+      # bug #2413 on assert_raises won't let me check error message above
+      assert_equal 'table must be specified', error.message 
+    end
+    
+    should "raise an error if no target is provided" do
+      error = assert_raises(ETL::ControlError) do
+        ETL::Processor::CheckExistProcessor.new(nil, @config.except(:target))
+      end
+      
+      assert_equal 'target must be specified', error.message
+    end
+    
+    should "bypass checking if the table has no rows" do
+      Person.delete_all
+      
+      processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
+      assert_equal false, processor.should_check?
+    end
+    
+    should "raise an error if one of the keys used for checking existence is not available in a row" do
+      Person.delete_all
+      # we need at least one record to avoid automatic skipping
+      # this should be mocked instead, probably
+      Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
+      
+      error = assert_raise(ETL::ControlError) do
+        row = ETL::Row[:first_name => 'John']
+        processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
+
+        # guard against bypassing
+        assert_equal true, processor.should_check?
+        
+        processor.process(row)
+      end
+      
+      assert_equal "Row missing required field :last_name for existence check", error.message
+    end
+    
+    should "return nil if the same row is found in database" do
+      Person.delete_all
+      Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
+
+      row = ETL::Row[:first_name => 'John', :last_name => 'Barry']
+      processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
+      assert_equal true, processor.should_check? # guard against bypassing
+        
+      assert_equal nil, processor.process(row)
+    end
+
+    should "return the row if no same row is found in database" do
+      Person.delete_all
+      Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '1234')
+
+      row = ETL::Row[:first_name => 'John', :last_name => 'OtherName']
+      processor = ETL::Processor::CheckExistProcessor.new(nil, @config)
+      assert_equal true, processor.should_check? # guard against bypassing
+        
+      assert_equal row, processor.process(row)
+    end
+  
+  end
+
+end

data/test/check_unique_processor_test.rb

@@ -0,0 +1,40 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class CheckUniqueProcessorTest < Test::Unit::TestCase
+
+  context 'CheckUniqueProcessor' do
+    attr_reader :processor
+    
+    setup do
+      @processor = ETL::Processor::CheckUniqueProcessor.new(nil,
+        :keys => [:first, :second])
+    end
+
+    should "keep a row whose keys didn't already appear in the pipeline" do
+      row = ETL::Row[:first => 'A', :second => 'B']
+
+      assert_equal row, processor.process(row)
+      
+      assert_equal({ 'A|B' => 1 }, processor.compound_key_constraints)
+    end
+    
+    should "remove a row whose keys already appeared in the pipeline" do
+      row = ETL::Row[:first => 'A', :second => 'B']
+
+      assert_equal row, processor.process(row)
+      assert_equal nil, processor.process(row)
+    end
+    
+    should "raise an error if a row lacks one of the keys specified" do
+      row = ETL::Row[:first => 'A']
+      
+      error = assert_raises(ETL::ControlError) do
+        processor.process(row)
+      end
+      
+      assert_equal "Row missing required field :second for unicity check", error.message
+    end
+    
+  end
+  
+end

data/test/config/Gemfile.rails-2.3.x

@@ -0,0 +1,3 @@
+require File.dirname(__FILE__) + '/common'
+
+common_gemfile('2.3.11')

data/test/config/Gemfile.rails-2.3.x.lock

@@ -0,0 +1,53 @@
+GIT
+  remote: git@github.com:activewarehouse/adapter_extensions.git
+  revision: 76e7aa2560e1b799c58e3d75ee0b0257a9a33850
+  specs:
+    adapter_extensions (0.9.5.rc1)
+      activerecord (>= 2.1.0)
+      activesupport (>= 2.1.0)
+      rake (>= 0.8.3)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activerecord (2.3.11)
+      activesupport (= 2.3.11)
+    activesupport (2.3.11)
+    fastercsv (1.5.4)
+    flexmock (0.9.0)
+    mysql (2.8.1)
+    net-sftp (2.0.5)
+      net-ssh (>= 2.0.9)
+    net-ssh (2.1.4)
+    nokogiri (1.4.4)
+    nokogiri (1.4.4-java)
+      weakling (>= 0.0.3)
+    pg (0.11.0)
+    rdoc (3.6.1)
+    ruby-ole (1.2.11.1)
+    shoulda (2.11.3)
+    spreadsheet (0.6.5.4)
+      ruby-ole (>= 1.0)
+    tmail (1.2.7.1)
+    weakling (0.0.4-java)
+    zip (2.0.2)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  activerecord (= 2.3.11)
+  activesupport (= 2.3.11)
+  adapter_extensions!
+  fastercsv (= 1.5.4)
+  flexmock (= 0.9.0)
+  mysql (= 2.8.1)
+  net-sftp (= 2.0.5)
+  nokogiri (= 1.4.4)
+  pg (= 0.11.0)
+  rdoc
+  shoulda (= 2.11.3)
+  spreadsheet (= 0.6.5.4)
+  tmail (= 1.2.7.1)
+  zip (= 2.0.2)

data/test/config/Gemfile.rails-3.0.x

@@ -0,0 +1,3 @@
+require File.dirname(__FILE__) + '/common'
+
+common_gemfile('3.0.7')

data/test/config/Gemfile.rails-3.0.x.lock

@@ -0,0 +1,61 @@
+GIT
+  remote: git@github.com:activewarehouse/adapter_extensions.git
+  revision: 76e7aa2560e1b799c58e3d75ee0b0257a9a33850
+  specs:
+    adapter_extensions (0.9.5.rc1)
+      activerecord (>= 2.1.0)
+      activesupport (>= 2.1.0)
+      rake (>= 0.8.3)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activemodel (3.0.7)
+      activesupport (= 3.0.7)
+      builder (~> 2.1.2)
+      i18n (~> 0.5.0)
+    activerecord (3.0.7)
+      activemodel (= 3.0.7)
+      activesupport (= 3.0.7)
+      arel (~> 2.0.2)
+      tzinfo (~> 0.3.23)
+    activesupport (3.0.7)
+    arel (2.0.10)
+    builder (2.1.2)
+    fastercsv (1.5.4)
+    flexmock (0.9.0)
+    i18n (0.5.0)
+    mysql (2.8.1)
+    net-sftp (2.0.5)
+      net-ssh (>= 2.0.9)
+    net-ssh (2.1.4)
+    nokogiri (1.4.4)
+    pg (0.11.0)
+    rdoc (3.6.1)
+    ruby-ole (1.2.11.1)
+    shoulda (2.11.3)
+    spreadsheet (0.6.5.4)
+      ruby-ole (>= 1.0)
+    tmail (1.2.7.1)
+    tzinfo (0.3.27)
+    zip (2.0.2)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  activerecord (= 3.0.7)
+  activesupport (= 3.0.7)
+  adapter_extensions!
+  fastercsv (= 1.5.4)
+  flexmock (= 0.9.0)
+  mysql (= 2.8.1)
+  net-sftp (= 2.0.5)
+  nokogiri (= 1.4.4)
+  pg (= 0.11.0)
+  rdoc
+  shoulda (= 2.11.3)
+  spreadsheet (= 0.6.5.4)
+  tmail (= 1.2.7.1)
+  zip (= 2.0.2)

data/test/config/common.rb

@@ -0,0 +1,29 @@
+require 'rbconfig'
+
+def common_gemfile(rails_version)
+  source :rubygems
+
+  # using explicit versions for the gems to avoid any weirdness later on
+  gem "activesupport", rails_version
+  gem "activerecord", rails_version
+
+  gem "fastercsv", "1.5.4"
+  gem "spreadsheet", "0.6.5.4"
+  gem "tmail", "1.2.7.1"
+  gem "net-sftp", "2.0.5"
+  gem "zip", "2.0.2"
+
+  gem "shoulda", "2.11.3"
+  gem "flexmock", "0.9.0"
+
+  gem "mysql", "2.8.1"
+  gem "pg", "0.11.0"
+  
+  gem "nokogiri", "1.4.4"
+  
+  gem "rdoc"
+  
+  gem "adapter_extensions", :git => 'git@github.com:activewarehouse/adapter_extensions.git'
+  
+  gem "jruby-openssl" if RUBY_PLATFORM == "java"
+end

data/test/connection/mysql/connection.rb

@@ -0,0 +1,9 @@
+print "Using native MySQL\n"
+
+puts "Resetting database"
+conn = ETL::Engine.connection(:data_warehouse)
+conn.recreate_database(conn.current_database)
+conn.reconnect!
+lines = open(File.join(File.dirname(__FILE__), 'schema.sql')).readlines
+lines.join.split(';').each { |line| conn.execute(line) }
+conn.disconnect!

data/test/connection/mysql/schema.sql

@@ -0,0 +1,37 @@
+drop table if exists people;
+create table people (
+  id int not null primary key,
+  first_name char(255) not null,
+  last_name char(255) not null,
+  ssn char(64) not null
+);
+drop table if exists places;
+create table places (
+	address text,
+	city char(255),
+	state char(255),
+	country char(2)
+);
+
+drop table if exists person_dimension;
+create table person_dimension (
+  id int not null primary key,
+  first_name char(50),
+  last_name char(50),
+  address char(100),
+  city char(50),
+  state char(50),
+  zip_code char(20),
+  effective_date datetime,
+  end_date datetime,
+  latest_version boolean not null
+);
+
+drop table if exists truncate_test;
+create table truncate_test (
+  id int not null primary key auto_increment,
+	x char(4)
+);
+insert into truncate_test (x) values ('a');
+insert into truncate_test (x) values ('b');
+insert into truncate_test (x) values ('c');

data/test/connection/postgresql/connection.rb

@@ -0,0 +1,13 @@
+print "Using PostgreSQL\n"
+
+puts "Resetting database"
+conn = ETL::Engine.connection(:data_warehouse)
+
+lines = open(File.join(File.dirname(__FILE__), 'schema.sql')).readlines
+lines.join.split(';').each_with_index do |line, index|
+  begin
+    conn.execute(line)
+  rescue => e
+    puts "failed to load line #{index}: #{e}"
+  end
+end

data/test/connection/postgresql/schema.sql

@@ -0,0 +1,40 @@
+drop table people;
+create table people (
+  id SERIAL PRIMARY KEY,
+  first_name character varying(255) not null,
+  /* null below allowed for bulk_import_with_empties.txt test */
+  last_name character varying(255) null,
+  ssn character varying(64) not null
+);
+
+drop table places;
+create table places (
+  id SERIAL PRIMARY KEY,
+	address text,
+	city character varying(255),
+	state character varying(255),
+	country character varying(2)
+);
+
+drop table person_dimension;
+create table person_dimension (
+  id SERIAL PRIMARY KEY,
+  first_name character varying(50),
+  last_name character varying(50),
+  address character varying(100),
+  city character varying(50),
+  state character varying(50),
+  zip_code character varying(20),
+  effective_date timestamp without time zone,
+  end_date timestamp without time zone,
+  latest_version boolean not null
+);
+
+drop table truncate_test;
+create table truncate_test (
+  id SERIAL PRIMARY KEY,
+	x character varying(4)
+);
+insert into truncate_test (x) values ('a');
+insert into truncate_test (x) values ('b');
+insert into truncate_test (x) values ('c');

data/test/control_test.rb

@@ -0,0 +1,43 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class ControlTest < Test::Unit::TestCase
+  # Test the ability to parse control files.
+  def test_parse
+    assert_nothing_raised do
+      Dir.glob(File.join(File.dirname(__FILE__), '*.ctl')) do |f|
+        ETL::Control::Control.parse(f)
+      end
+    end
+  end
+  
+  def test_bad_control_raises_error
+    assert_raise ETL::ControlError do
+      ETL::Control::Control.resolve(0)
+    end
+  end
+  
+  def test_resolve_control_object
+    assert_nothing_raised do
+      ETL::Control::Control.resolve(ETL::Control::Control.parse(File.join(File.dirname(__FILE__), 'delimited.ctl')))
+    end
+  end
+  
+  def test_set_error_threshold
+    assert_nothing_raised do
+      ETL::Engine.process(File.join(File.dirname(__FILE__), 'errors.ctl'))
+    end
+  end
+  
+  def test_bad_processor_name
+    assert_raise ETL::ControlError do
+      s = "before_write :chunky_monkey"
+      ETL::Control::Control.parse_text(s)
+    end
+  end
+  
+  def test_dependencies
+    s = "depends_on 'foo', 'bar'"
+    control = ETL::Control::Control.parse_text(s)
+    assert_equal control.dependencies, ['foo','bar']
+  end
+end