activewarehouse-etl-sgonyea 0.9.6

data/test/scd_test_type_1.ctl

@@ -0,0 +1,43 @@
+source :in, {
+  :file => "scd/#{ENV['run_number']}.txt",
+  :parser => :csv
+},
+[
+  :first_name,
+  :last_name,
+  :address,
+  :city,
+  :state,
+  :zip_code
+]
+
+# NOTE: These are not usually required for a type 1 SCD dimension, but since
+# we're sharing this table with the type 2 tests, they're necessary.
+transform :effective_date, :default, :default_value => Time.now.to_s(:db)
+transform :end_date, :default, :default_value => '9999-12-31 00:00:00'
+transform :latest_version, :default, :default_value => true
+
+destination :out, {
+  :file => 'output/scd_test_type_1.txt',
+  :natural_key => [:first_name, :last_name],
+  :scd => {
+    :type => 1,
+    :dimension_target => :data_warehouse,
+    :dimension_table => 'person_dimension'
+  },
+  :scd_fields => [:address, :city, :state, :zip_code]
+}, 
+{
+  :order => [
+    :id, :first_name, :last_name, :address, :city, :state, :zip_code, :effective_date, :end_date, :latest_version
+  ],
+  :virtual => {
+    :id => ETL::Generator::SurrogateKeyGenerator.new(:target => :data_warehouse, :table => 'person_dimension')
+  }
+}
+
+post_process :bulk_import, {
+  :file => 'output/scd_test_type_1.txt',
+  :target => :data_warehouse,
+  :table => 'person_dimension'
+}

data/test/scd_test_type_2.ctl

@@ -0,0 +1,34 @@
+source :in, {
+  :file => "scd/#{ENV['run_number']}.txt",
+  :parser => :csv
+},
+[
+  :first_name,
+  :last_name,
+  :address,
+  :city,
+  :state,
+  :zip_code
+]
+
+destination :out, {
+  :type => :database,
+  :target => :data_warehouse,
+  :database => 'etl_unittest',
+  :table => 'person_dimension',
+  :natural_key => [:first_name, :last_name],
+  :scd => {
+    :type => 2,
+    :dimension_target => :data_warehouse,
+    :dimension_table => 'person_dimension'
+  },
+  :scd_fields => ENV['type_2_scd_fields'] ? Marshal.load(ENV['type_2_scd_fields']) : [:address, :city, :state, :zip_code]
+}, 
+{
+  :order => [
+    :id, :first_name, :last_name, :address, :city, :state, :zip_code, :effective_date, :end_date, :latest_version
+  ],
+  :virtual => {
+    :id => ETL::Generator::SurrogateKeyGenerator.new(:target => :data_warehouse, :table => 'person_dimension')
+  }
+}

data/test/screen_test.rb

@@ -0,0 +1,9 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class ScreenTest < Test::Unit::TestCase
+  def test_screen
+    assert_raises(SystemExit) do
+      ETL::Engine.process(File.dirname(__FILE__) + '/screen_test_fatal.ctl')
+    end
+  end
+end

data/test/screen_test_error.ctl

@@ -0,0 +1,3 @@
+screen(:error){
+  ETL::Screen::RowCountScreen.new(self, :rows => 1)
+}

data/test/screen_test_fatal.ctl

@@ -0,0 +1,3 @@
+screen(:fatal){
+  ETL::Screen::RowCountScreen.new(self, :rows => 1)
+}

data/test/source_test.rb

@@ -0,0 +1,154 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class Person < ActiveRecord::Base
+end
+class SourceTest < Test::Unit::TestCase
+  
+  context "source" do
+    should "set store_locally to true by default" do
+      assert_equal true, Source.new(nil, { :store_locally => true }, nil).store_locally
+    end
+    
+    should "let the user set store_locally to true" do
+      assert_equal true, Source.new(nil, { :store_locally => true }, nil).store_locally
+    end
+
+    should "let the user set store_locally to false" do
+      assert_equal false, Source.new(nil, { :store_locally => false }, nil).store_locally
+    end
+  end
+  
+  context "a file source" do
+    context "with delimited data" do
+      setup do
+        control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+        configuration = {
+          :file => 'data/delimited.txt',
+          :parser => :csv
+        }
+        definition = self.definition + [:sex]
+    
+        source = ETL::Control::FileSource.new(control, configuration, definition)
+        @rows = source.collect { |row| row }
+      end
+      should "find 3 rows in the delimited file" do
+        assert_equal 3, @rows.length
+      end
+    end
+  end
+  
+  context "a file source with a glob" do
+    setup do
+      control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/multiple_delimited.ctl')
+      configuration = {
+        :file => 'data/multiple_delimited_*.txt',
+        :parser => :csv
+      }
+
+      source = ETL::Control::FileSource.new(control, configuration, definition)
+      @rows = source.collect { |row| row }
+    end
+    should "find 6 rows in total" do
+      assert_equal 6, @rows.length
+    end
+  end
+  
+  context "a file source with an absolute path" do
+    setup do
+      FileUtils.cp(File.dirname(__FILE__) + '/data/delimited.txt', '/tmp/delimited_abs.txt')
+
+      control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+        '/delimited_absolute.ctl')
+      configuration = {
+        :file => '/tmp/delimited_abs.txt',
+        :parser => :csv
+      }
+      definition = self.definition + [:sex]
+
+      source = ETL::Control::FileSource.new(control, configuration, definition)
+      @rows = source.collect { |row| row }
+    end
+    should "find 3 rows" do
+      assert_equal 3, @rows.length
+    end
+  end
+  
+  context "multiple sources" do
+    setup do
+      control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+        '/multiple_source_delimited.ctl')
+      @rows = control.sources.collect { |source| source.collect { |row| row }}.flatten!
+    end
+    should "find 12 rows" do
+      assert_equal 12, @rows.length
+    end
+  end
+  
+  context "a database source" do
+    setup do
+      control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+      configuration = {
+        :database => 'etl_unittest',
+        :target => :operational_database,
+        :table => 'people',
+      }
+      definition = [ 
+        :first_name,
+        :last_name,
+        :ssn,
+      ]
+      @source = ETL::Control::DatabaseSource.new(control, configuration, definition)
+    end
+    should "set the local file for extraction storage" do
+      assert_match %r{source_data/localhost/etl_unittest/people/\d+.csv}, @source.local_file.to_s
+    end
+    should "find 1 row" do
+      Person.delete_all
+      assert_equal 0, Person.count
+      Person.create!(:first_name => 'Bob', :last_name => 'Smith', :ssn => '123456789')
+      assert_equal 1, Person.count
+      rows = @source.collect { |row| row }
+      assert_equal 1, rows.length
+    end
+  end
+  
+  context "a file source with an xml parser" do
+    setup do
+      control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+        '/xml.ctl')
+      @rows = control.sources.collect{ |source| source.collect { |row| row }}.flatten!
+    end
+    should "find 2 rows" do
+      assert_equal 2, @rows.length
+    end
+  end
+
+  context "a model source" do
+    setup do
+      control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/model_source.ctl')
+      configuration = {
+
+      }
+      definition = [
+        :first_name,
+        :last_name,
+        :ssn
+      ]
+    end
+    should_eventually "find n rows" do
+      
+    end
+  end
+  
+  def definition
+    [ 
+      :first_name,
+      :last_name,
+      :ssn,
+      {
+        :name => :age,
+        :type => :integer
+      }
+    ]
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,37 @@
+require 'rubygems'
+require 'bundler/setup'
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+$:.unshift(File.dirname(__FILE__))
+
+require 'test/unit'
+require 'pp'
+require 'etl'
+require 'shoulda'
+require 'flexmock/test_unit'
+
+database_yml = File.dirname(__FILE__) + '/config/database.yml'
+ETL::Engine.init(:config => database_yml)
+ETL::Engine.logger = Logger.new(STDOUT)
+# ETL::Engine.logger.level = Logger::DEBUG
+ETL::Engine.logger.level = Logger::FATAL
+
+db = YAML::load(IO.read(database_yml))['operational_database']['adapter']
+# allow both mysql2 and mysql adapters
+db = db.gsub('mysql2', 'mysql')
+raise "Unsupported test db '#{db}'" unless ['mysql', 'postgresql'].include?(db)
+
+require "connection/#{db}/connection"
+ActiveRecord::Base.establish_connection :operational_database
+ETL::Execution::Job.delete_all
+
+require 'mocks/mock_source'
+require 'mocks/mock_destination'
+
+# shortcut to launch a ctl file
+def process(file)
+  Engine.process(File.join(File.dirname(__FILE__), file))
+end
+
+puts "ActiveRecord::VERSION = #{ActiveRecord::VERSION::STRING}"
+

data/test/transform_test.rb

@@ -0,0 +1,101 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class MyResolver
+  def resolve(value)
+    4
+  end
+end
+
+class TransformTest < Test::Unit::TestCase
+  def test_sha1_transform
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+    digest_value = 'a9993e364706816aba3e25717850c26c9cd0d89d'
+    assert_equal digest_value, ETL::Transform::Sha1Transform.new(
+      control, nil
+    ).transform('test', 'abc', [])
+  end
+  def test_block_transform
+    #transforms = [Proc.new(){|name, value, row| value[0,2]}]
+    #assert_equal '11', ETL::Transform::Transform.transform(:ssn, '1111223333', [], transforms)
+  end
+  def test_decode_transform
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+    configuration = {:decode_table_path => 'data/decode.txt'}
+    
+    t = ETL::Transform::DecodeTransform.new(control, nil, configuration)
+    
+    assert_equal 'Male', t.transform(nil, 'M', [])
+    assert_equal 'Female', t.transform(nil, 'F', [])
+    assert_equal 'Unknown', t.transform(nil, '', [])
+    assert_equal 'Unknown', t.transform(nil, 'blah', [])
+  end
+  def test_string_to_date_transform
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+    t = ETL::Transform::StringToDateTransform.new(control, nil)
+    
+    assert_equal Date.parse('2005-01-01'), t.transform(nil, '2005-01-01', [])
+    assert_equal Date.parse('2004-10-20 20:30:00'), t.transform(nil, '2004-10-20', [])
+  end
+  def test_date_to_string_transform
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+    t = ETL::Transform::DateToStringTransform.new(control, nil)
+    
+    d1 = Date.parse('2005-01-01')
+    t1 = Time.parse('2004-10-20 23:03:23')
+    assert_equal '2005-01-01', t.transform(nil, d1, [])
+    assert_equal '2004-10-20', t.transform(nil, t1, [])
+    
+    t = ETL::Transform::DateToStringTransform.new(control, nil, {:format => '%m/%d/%Y'})
+    
+    assert_equal '01/01/2005', t.transform(nil, d1, [])
+    assert_equal '10/20/2004', t.transform(nil, t1, [])
+  end
+  def test_string_to_datetime_transform
+    v = '1/1/1900 04:34:30'
+    t = ETL::Transform::StringToDateTimeTransform.new(flexmock(:control), nil)
+    assert_equal DateTime.parse(v), t.transform(nil, v, nil)
+  end
+  def test_foreign_key_lookup_transform
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+    configuration = {:collection => {'foo' => 1, 'bar' => 2, 'baz' => 3}}
+    t = ETL::Transform::ForeignKeyLookupTransform.new(control, nil, configuration)
+    
+    assert_equal 1, t.transform(nil, 'foo', nil)
+    assert_equal 2, t.transform(nil, 'bar', nil)
+    assert_equal 3, t.transform(nil, 'baz', nil)
+    assert_raises(ETL::ResolverError, 'Foreign key for bing not found and no resolver specified') do
+      assert_equal 4, t.transform(nil, 'bing', nil)
+    end
+    
+    configuration = {:collection => {'foo' => 1, 'bar' => 2, 'baz' => 3}, :resolver => MyResolver}
+    t = ETL::Transform::ForeignKeyLookupTransform.new(control, nil, configuration)
+    assert_equal 1, t.transform(nil, 'foo', nil)
+    assert_equal 2, t.transform(nil, 'bar', nil)
+    assert_equal 3, t.transform(nil, 'baz', nil)
+    assert_equal 4, t.transform(nil, 'bing', nil)
+  end
+  def test_type_transform
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+    assert_equal 10, ETL::Transform::TypeTransform.new(control, nil, {:type => :number}).transform(nil, '10', nil)
+    
+    assert_equal BigDecimal::ROUND_HALF_UP, BigDecimal.mode(BigDecimal::ROUND_MODE)
+    decimal_transformed = ETL::Transform::TypeTransform.new(
+      control, nil, {:type => :decimal, :scale => 4}
+    ).transform(nil, '10.0000000000000000000000000000000001', nil)
+    assert_equal '10.0000000000000000000000000000000001', decimal_transformed.to_s('F')
+  end
+  def test_non_existent_transformer
+    
+  end
+  def test_default_transform
+    t = ETL::Transform::DefaultTransform.new(flexmock('control'), nil, {:default_value => 'foo'})
+    assert_equal 'foo', t.transform(nil, '', nil)
+    assert_equal 'foo', t.transform(nil, nil, nil)
+    assert_equal 'bar', t.transform(nil, 'bar', nil)
+  end
+  def test_ordinalize_transform
+    t = ETL::Transform::OrdinalizeTransform.new(flexmock('control'), nil, {})
+    assert_equal '1st', t.transform(nil, 1, nil)
+    assert_equal '10th', t.transform(nil, 10, nil)
+  end
+end

data/test/truncate_processor_test.rb

@@ -0,0 +1,37 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+include ETL::Processor
+
+class TruncateTest < ActiveRecord::Base
+  set_table_name 'truncate_test'
+end
+
+class TruncateProcessorTest < Test::Unit::TestCase
+
+  def create_item!
+    TruncateTest.create!(:x => 'ABC')
+  end
+  
+  def truncate!(options=nil)
+    TruncateProcessor.new(nil,
+      :target => :data_warehouse,
+      :table => TruncateTest.table_name,
+      :options => options
+      ).process
+  end
+  
+  should 'reset ids by default' do
+    create_item!
+    truncate!
+    assert_equal 1, create_item!.id
+  end
+
+  if ETL::Engine.connection(:data_warehouse).class.name =~ /postgres/i
+    should 'allow disabling id reset for postgres' do
+      truncate!
+      create_item!
+      truncate!('CONTINUE IDENTITY')
+      assert_equal 2, create_item!.id
+    end
+  end
+end

data/test/xml.ctl

@@ -0,0 +1,31 @@
+# puts "executing fixed_width.ctl"
+
+source :in, {
+  :file => 'data/xml.xml',
+  :parser => :xml
+}, 
+{
+  :collection => 'people/person',
+  :fields => [
+    :first_name,
+    :last_name,
+    {
+      :name => :ssn,
+      :xpath => 'social_security_number'
+    },
+    {
+      :name => :age,
+      :type => :integer
+    }
+  ]
+}
+
+destination :out, {
+  :file => 'output/xml.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }