activewarehouse-etl-sgonyea 0.9.6

data/test/errors.ctl

@@ -0,0 +1,24 @@
+class ErrorProcessor < ETL::Processor::RowProcessor
+  def initialize(control, configuration)
+    super
+  end
+  def process(row)
+    raise RuntimeError, "Generated error"
+  end
+end
+
+set_error_threshold 1
+
+source :in, {
+  :type => :enumerable,
+  :enumerable => [
+    {:first_name => 'Bob',:last_name => 'Smith'},
+    {:first_name => 'Joe', :last_name => 'Thompson'}
+  ]
+},
+[
+  :first_name,
+  :last_name
+]
+
+after_read ErrorProcessor

data/test/etl_test.rb

@@ -0,0 +1,42 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+# This is an integration test
+class ETLTest < Test::Unit::TestCase
+  # Test end-to-end integration of ETL engine processing for the delimited.ctl control file
+  def test_delimited_single_file_load
+    #ETL::Engine.logger = Logger.new(STDOUT)
+    #ETL::Engine.logger.level = Logger::DEBUG
+    
+    ETL::Engine.init(:config => File.dirname(__FILE__) + '/database.yml')
+    ETL::Engine.process(File.dirname(__FILE__) + '/delimited.ctl')
+    lines = open(File.dirname(__FILE__) + '/output/delimited.txt').readlines
+    assert_equal 3, lines.length
+    
+    data = lines[0].split(',')
+    assert_equal '1', data[0]
+    assert_equal 'Chris', data[1]
+    assert_equal 'Smith', data[2]
+    assert_equal '23cc5914d48b146f0fbb73c4', data[3]
+    assert_equal '24', data[4]
+    assert_equal 'Male', data[5]
+    assert_equal 'test!', data[6]
+    assert_nothing_raised { Time.parse(data[7]) }
+    
+    data = lines[1].split(',')
+    assert_equal '2', data[0]
+    assert_equal 'Jim', data[1]
+    assert_equal 'Foxworthy', data[2]
+    assert_equal '596e3534978b8c2b47851e37', data[3]
+    assert_equal '51', data[4]
+    assert_equal 'Male', data[5]
+    assert_equal 'test!', data[6]
+    assert_nothing_raised { Time.parse(data[7]) }
+  end
+  
+  # Test end-to-end integration of ETL engine processing for the fixed_width.ctl control file
+  def test_fixed_width_single_file_load
+    ETL::Engine.process(File.dirname(__FILE__) + '/fixed_width.ctl')
+    lines = open(File.dirname(__FILE__) + '/output/delimited.txt').readlines
+    assert_equal 3, lines.length
+  end
+end

data/test/excel.ctl

@@ -0,0 +1,24 @@
+source :in, {
+  :file => 'data/excel.xls',
+  :parser => :excel
+}, 
+{
+  :ignore_blank_line => false,
+  :fields => [ 
+      :first_name,
+      :last_name,
+      :ssn,
+      :age
+  ]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |n, v, r| v[0,24] }
+
+
+destination :out, {
+  :file => 'output/excel.out.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/test/excel2.ctl

@@ -0,0 +1,25 @@
+source :in, {
+  :file => 'data/excel2.xls',
+  :parser => :excel
+}, 
+{
+  :ignore_blank_line => true,
+  :worksheets => [ 1 ],
+  :fields => [ 
+      :first_name,
+      :last_name,
+      :ssn,
+      :age
+  ]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |n, v, r| v[0,24] }
+
+
+destination :out, {
+  :file => 'output/excel2.out.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/test/fixed_width.ctl

@@ -0,0 +1,35 @@
+# puts "executing fixed_width.ctl"
+
+source :in, {
+  :file => 'data/fixed_width.txt',
+  :parser => :fixed_width
+}, 
+{
+  :first_name => {
+    :start => 1,
+    :length => 9
+  },
+  :last_name => {
+    :start => 10,
+    :length => 12
+  },
+  :ssn => {
+    :start => 22,
+    :length => 9
+  },
+  :age => {
+    :start => 31,
+    :length => 2,
+    :type => :integer
+  }
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |n, v, r| v[0,24] }
+
+destination :out, {
+  :file => 'output/fixed_width.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/test/foreign_key_lookup_transform_test.rb

@@ -0,0 +1,50 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+# TODO - use flexmock instead, but I'm not sure how to handle the respond_to part yet
+class TestResolver
+  attr_accessor :cache_loaded
+  
+  def initialize
+    @cache_loaded = false
+  end
+  
+  def load_cache
+    @cache_loaded = true
+  end
+end
+
+class ForeignKeyLookupTransformTest < Test::Unit::TestCase
+
+  context 'configuration' do
+
+    should 'enable cache by default' do
+      resolver = TestResolver.new
+
+      transform = ETL::Transform::ForeignKeyLookupTransform.new(nil, 'name',
+        {:resolver => resolver})
+
+      assert_equal true, resolver.cache_loaded
+    end
+    
+    should 'allow to disable cache' do
+      resolver = TestResolver.new
+
+      transform = ETL::Transform::ForeignKeyLookupTransform.new(nil, 'name',
+        {:resolver => resolver, :cache => false})
+        
+      assert_equal false, resolver.cache_loaded
+    end
+
+    should 'allow to enable cache' do
+      resolver = TestResolver.new
+
+      transform = ETL::Transform::ForeignKeyLookupTransform.new(nil, 'name',
+        {:resolver => resolver, :cache => true})
+        
+      assert_equal true, resolver.cache_loaded
+    end
+    
+  end
+  
+
+end

data/test/generator_test.rb

@@ -0,0 +1,14 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+# Test generators
+class GeneratorTest < Test::Unit::TestCase
+  # Test the surrogate key generator
+  def test_surrogate_key_generator
+    generator_class = ETL::Generator::Generator.class_for_name(:surrogate_key)
+    assert_equal ETL::Generator::SurrogateKeyGenerator, generator_class
+    generator = generator_class.new
+    1.upto(10) do |i|
+      assert_equal i, generator.next
+    end
+  end
+end

data/test/inline_parser.ctl

@@ -0,0 +1,17 @@
+class MyParser < ETL::Parser::Parser
+  def each
+    [{:name => 'foo'},{:name => 'bar'},{:name => 'baz'}].each do |row|
+      yield row
+    end
+  end
+end
+
+source :in, {
+  :file => '',
+  :parser => MyParser
+}, 
+[ 
+  :name
+]
+
+destination :out, {:file => 'output/inline_parser.txt'},{:order => [:name]}

data/test/mocks/mock_destination.rb

@@ -0,0 +1,26 @@
+module ETL
+  module Control
+    # Usage:
+    # - declare in the ctl file:
+    #   destination :out, { :type => :mock, :name => :my_mock_output  }
+    # - run the .ctl from your test
+    # - then assert the content of the rows
+    #   assert_equal [{:name => 'John Barry'},{:name => 'Gary Moore'}], MockDestination[:my_mock_output]
+    class MockDestination < Destination
+      def initialize(control, configuration, mapping={})
+        super
+        @mock_destination_name = configuration[:name] || 'mock_destination'
+        @@registry ||= {}
+        @@registry[@mock_destination_name] ||= []
+      end
+      def self.[](mock_destination_name)
+        @@registry[mock_destination_name]
+      end
+      def write(row)
+        @@registry[@mock_destination_name] << row
+      end
+      # the presence of close is asserted - just do nothing
+      def close; end
+    end
+  end
+end

data/test/mocks/mock_source.rb

@@ -0,0 +1,25 @@
+module ETL
+  module Control
+    # Usage:
+    # - first set the data in your test setup
+    #   MockSource[:my_input] = [ { :first_name => 'John', :last_name => 'Barry' }, { ...} ]
+    # - then declare in the ctl file:
+    #   source :in, { :type => :mock, :name => :my_input } 
+    class MockSource < EnumerableSource
+      def initialize(control, configuration, definition)
+        super
+        mock_source_name = configuration[:name] || 'mock_source'
+        throw "No mock source data set for mock source '#{mock_source_name}'" if @@registry[mock_source_name].nil?
+        configuration[:enumerable] = @@registry[mock_source_name]
+      end
+      def self.[]=(mock_source_name,mock_source_data)
+        @@registry ||= {}
+        @@registry[mock_source_name] = mock_source_data
+      end
+      def self.[](mock_source_name)
+        @@registry[mock_source_name]
+      end
+    end
+  end
+end
+

data/test/model_source.ctl

@@ -0,0 +1,14 @@
+source :in, {
+  :type => :model
+}, 
+[ 
+  :first_name,
+  :last_name
+]
+
+destination :out, {
+  :file => 'data/model_out.txt'
+},
+{
+  :order => [:first_name, :last_name],
+}

data/test/multiple_delimited.ctl

@@ -0,0 +1,22 @@
+# puts "executing delimited.ctl"
+
+source :in, {
+  :file => 'data/multiple_delimited_*.txt',
+  :parser => :csv
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  {
+    :name => :age,
+    :type => :integer
+  }
+]
+
+destination :out, {
+  :file => 'output/multiple_delimited.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/test/multiple_source_delimited.ctl

@@ -0,0 +1,39 @@
+# puts "executing delimited.ctl"
+
+source :source1, {
+  :file => 'data/multiple_delimited_*.txt',
+  :parser => :csv
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  {
+    :name => :age,
+    :type => :integer
+  }
+]
+
+source :source2, {
+  :file => 'data/multiple_delimited_*.txt',
+  :parser => :csv
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  {
+    :name => :age,
+    :type => :integer
+  }
+]
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }
+
+destination :out, {
+  :file => 'output/multiple_source_delimited.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/test/nokogiri_all.ctl

@@ -0,0 +1,35 @@
+# puts "executing nokogiri_all.ctl"
+
+source :in, {
+  :file => 'data/nokogiri.xml',
+  :parser => :nokogiri_xml
+}, 
+{
+  :collection => 'people/person',
+  :fields => [
+    :first_name,
+    :last_name,
+    {
+      :name => :ssn,
+      :xpath => '@ssn'
+    },
+    {
+      :name => :age,
+      :type => :integer
+    },
+    {
+      :name => :hair_colour,
+      :xpath => 'colours/hair'
+    }
+  ]
+}
+
+destination :out, {
+  :file => 'output/xml.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }

data/test/nokogiri_select.ctl

@@ -0,0 +1,35 @@
+# puts "executing nokogiri_select.ctl"
+
+source :in, {
+  :file => 'data/nokogiri.xml',
+  :parser => :nokogiri_xml
+}, 
+{
+  :collection => 'people/person[@type="client"]',
+  :fields => [
+    :first_name,
+    :last_name,
+    {
+      :name => :ssn,
+      :xpath => '@ssn'
+    },
+    {
+      :name => :age,
+      :type => :integer
+    },
+    {
+      :name => :hair_colour,
+      :xpath => 'colours/hair'
+    }
+  ]
+}
+
+destination :out, {
+  :file => 'output/xml.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }