activewarehouse-etl 0.9.1 → 0.9.5.rc1

data/test/directive_test.rb

@@ -0,0 +1,23 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class BadDirective < ETL::Batch::Directive
+  
+end
+
+class BatchTest < Test::Unit::TestCase
+  
+  attr_reader :file
+  attr_reader :engine
+  def setup
+    @file =  File.dirname(__FILE__) + '/all.ebf'
+    @engine = ETL::Engine.new
+  end
+  
+  def test_directive_without_implementation_should_fail
+    batch = ETL::Batch::Batch.resolve(file, engine)
+    assert_raise RuntimeError do
+      d = BadDirective.new(batch)
+      d.execute
+    end
+  end
+end

data/test/encode_processor_test.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+require File.dirname(__FILE__) + '/test_helper'
+
+class EncodeProcessorTest < Test::Unit::TestCase
+
+  SOURCE = 'data/encode_source_latin1.txt'
+  TARGET = 'output/encode_destination_utf-8.txt'
+
+  def setup
+    @control = flexmock("control")
+    @control.should_receive(:file).twice.and_return(File.dirname(__FILE__) + '/fake-control.ctl')
+  end
+  
+  def test_should_transform_a_latin1_file_to_utf8_with_grace
+    configuration = { :source_file => SOURCE, :source_encoding => 'latin1', :target_file => TARGET, :target_encoding => 'utf-8' }
+    ETL::Processor::EncodeProcessor.new(@control, configuration).process
+    assert_equal "éphémère has accents.\nlet's encode them.", IO.read(File.join(File.dirname(__FILE__),TARGET))
+  end
+  
+  def test_should_throw_exception_on_unsupported_encoding
+    configuration = { :source_file => SOURCE, :source_encoding => 'acme-encoding', :target_file => TARGET, :target_encoding => 'utf-8' }
+    error = assert_raise(ETL::ControlError) { ETL::Processor::EncodeProcessor.new(@control, configuration) }
+    assert_equal "Either the source encoding 'acme-encoding' or the target encoding 'utf-8' is not supported", error.message
+  end
+  
+  def test_should_throw_exception_when_target_and_source_are_the_same
+    configuration = { :source_file => SOURCE, :source_encoding => 'latin1', :target_file => SOURCE, :target_encoding => 'utf-8' }
+    error = assert_raise(ETL::ControlError) { ETL::Processor::EncodeProcessor.new(@control, configuration) }
+    assert_equal "Source and target file cannot currently point to the same file", error.message
+  end
+  
+end

data/test/engine_test.rb

@@ -0,0 +1,32 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class EngineTest < Test::Unit::TestCase
+  
+  def test_connections
+    assert_equal 3, ActiveRecord::Base.configurations.length
+    conn = ETL::Engine.connection(:data_warehouse)
+    assert_not_nil conn
+  end
+  
+  def test_non_existent_connection
+    assert_raise ETL::ETLError do
+      conn = ETL::Engine.connection(:does_not_exist)
+    end
+  end
+  
+  def test_engine_table_method_should_return_same_name_when_not_using_temp_tables
+    assert_equal 'foo', ETL::Engine.table('foo', connection)
+  end
+  
+  def test_engine_table_method_should_return_temp_name_when_using_temp_tables
+    ETL::Engine.use_temp_tables = true
+    assert_equal 'tmp_people', ETL::Engine.table('people', connection)
+    ETL::Engine.use_temp_tables = false
+  end
+  
+  protected
+  def connection
+    ETL::Engine.connection(:data_warehouse)
+  end
+  
+end

data/test/errors.ctl

@@ -0,0 +1,24 @@
+class ErrorProcessor < ETL::Processor::RowProcessor
+  def initialize(control, configuration)
+    super
+  end
+  def process(row)
+    raise RuntimeError, "Generated error"
+  end
+end
+
+set_error_threshold 1
+
+source :in, {
+  :type => :enumerable,
+  :enumerable => [
+    {:first_name => 'Bob',:last_name => 'Smith'},
+    {:first_name => 'Joe', :last_name => 'Thompson'}
+  ]
+},
+[
+  :first_name,
+  :last_name
+]
+
+after_read ErrorProcessor

data/test/etl_test.rb

@@ -0,0 +1,42 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+# This is an integration test
+class ETLTest < Test::Unit::TestCase
+  # Test end-to-end integration of ETL engine processing for the delimited.ctl control file
+  def test_delimited_single_file_load
+    #ETL::Engine.logger = Logger.new(STDOUT)
+    #ETL::Engine.logger.level = Logger::DEBUG
+    
+    ETL::Engine.init(:config => File.dirname(__FILE__) + '/database.yml')
+    ETL::Engine.process(File.dirname(__FILE__) + '/delimited.ctl')
+    lines = open(File.dirname(__FILE__) + '/output/delimited.txt').readlines
+    assert_equal 3, lines.length
+    
+    data = lines[0].split(',')
+    assert_equal '1', data[0]
+    assert_equal 'Chris', data[1]
+    assert_equal 'Smith', data[2]
+    assert_equal '23cc5914d48b146f0fbb73c4', data[3]
+    assert_equal '24', data[4]
+    assert_equal 'Male', data[5]
+    assert_equal 'test!', data[6]
+    assert_nothing_raised { Time.parse(data[7]) }
+    
+    data = lines[1].split(',')
+    assert_equal '2', data[0]
+    assert_equal 'Jim', data[1]
+    assert_equal 'Foxworthy', data[2]
+    assert_equal '596e3534978b8c2b47851e37', data[3]
+    assert_equal '51', data[4]
+    assert_equal 'Male', data[5]
+    assert_equal 'test!', data[6]
+    assert_nothing_raised { Time.parse(data[7]) }
+  end
+  
+  # Test end-to-end integration of ETL engine processing for the fixed_width.ctl control file
+  def test_fixed_width_single_file_load
+    ETL::Engine.process(File.dirname(__FILE__) + '/fixed_width.ctl')
+    lines = open(File.dirname(__FILE__) + '/output/delimited.txt').readlines
+    assert_equal 3, lines.length
+  end
+end

data/test/excel.ctl

@@ -0,0 +1,24 @@
+source :in, {
+  :file => 'data/excel.xls',
+  :parser => :excel
+}, 
+{
+  :ignore_blank_line => false,
+  :fields => [ 
+      :first_name,
+      :last_name,
+      :ssn,
+      :age
+  ]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |n, v, r| v[0,24] }
+
+
+destination :out, {
+  :file => 'output/excel.out.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/test/excel2.ctl

@@ -0,0 +1,25 @@
+source :in, {
+  :file => 'data/excel2.xls',
+  :parser => :excel
+}, 
+{
+  :ignore_blank_line => true,
+  :worksheets => [ 1 ],
+  :fields => [ 
+      :first_name,
+      :last_name,
+      :ssn,
+      :age
+  ]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |n, v, r| v[0,24] }
+
+
+destination :out, {
+  :file => 'output/excel2.out.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/test/fixed_width.ctl

@@ -0,0 +1,35 @@
+# puts "executing fixed_width.ctl"
+
+source :in, {
+  :file => 'data/fixed_width.txt',
+  :parser => :fixed_width
+}, 
+{
+  :first_name => {
+    :start => 1,
+    :length => 9
+  },
+  :last_name => {
+    :start => 10,
+    :length => 12
+  },
+  :ssn => {
+    :start => 22,
+    :length => 9
+  },
+  :age => {
+    :start => 31,
+    :length => 2,
+    :type => :integer
+  }
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |n, v, r| v[0,24] }
+
+destination :out, {
+  :file => 'output/fixed_width.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/test/generator_test.rb

@@ -0,0 +1,14 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+# Test generators
+class GeneratorTest < Test::Unit::TestCase
+  # Test the surrogate key generator
+  def test_surrogate_key_generator
+    generator_class = ETL::Generator::Generator.class_for_name(:surrogate_key)
+    assert_equal ETL::Generator::SurrogateKeyGenerator, generator_class
+    generator = generator_class.new
+    1.upto(10) do |i|
+      assert_equal i, generator.next
+    end
+  end
+end

data/test/inline_parser.ctl

@@ -0,0 +1,17 @@
+class MyParser < ETL::Parser::Parser
+  def each
+    [{:name => 'foo'},{:name => 'bar'},{:name => 'baz'}].each do |row|
+      yield row
+    end
+  end
+end
+
+source :in, {
+  :file => '',
+  :parser => MyParser
+}, 
+[ 
+  :name
+]
+
+destination :out, {:file => 'output/inline_parser.txt'},{:order => [:name]}

data/test/mocks/mock_destination.rb

@@ -0,0 +1,26 @@
+module ETL
+  module Control
+    # Usage:
+    # - declare in the ctl file:
+    #   destination :out, { :type => :mock, :name => :my_mock_output  }
+    # - run the .ctl from your test
+    # - then assert the content of the rows
+    #   assert_equal [{:name => 'John Barry'},{:name => 'Gary Moore'}], MockDestination[:my_mock_output]
+    class MockDestination < Destination
+      def initialize(control, configuration, mapping={})
+        super
+        @mock_destination_name = configuration[:name] || 'mock_destination'
+        @@registry ||= {}
+        @@registry[@mock_destination_name] ||= []
+      end
+      def self.[](mock_destination_name)
+        @@registry[mock_destination_name]
+      end
+      def write(row)
+        @@registry[@mock_destination_name] << row
+      end
+      # the presence of close is asserted - just do nothing
+      def close; end
+    end
+  end
+end

data/test/mocks/mock_source.rb

@@ -0,0 +1,25 @@
+module ETL
+  module Control
+    # Usage:
+    # - first set the data in your test setup
+    #   MockSource[:my_input] = [ { :first_name => 'John', :last_name => 'Barry' }, { ...} ]
+    # - then declare in the ctl file:
+    #   source :in, { :type => :mock, :name => :my_input } 
+    class MockSource < EnumerableSource
+      def initialize(control, configuration, definition)
+        super
+        mock_source_name = configuration[:name] || 'mock_source'
+        throw "No mock source data set for mock source '#{mock_source_name}'" if @@registry[mock_source_name].nil?
+        configuration[:enumerable] = @@registry[mock_source_name]
+      end
+      def self.[]=(mock_source_name,mock_source_data)
+        @@registry ||= {}
+        @@registry[mock_source_name] = mock_source_data
+      end
+      def self.[](mock_source_name)
+        @@registry[mock_source_name]
+      end
+    end
+  end
+end
+

data/test/model_source.ctl

@@ -0,0 +1,14 @@
+source :in, {
+  :type => :model
+}, 
+[ 
+  :first_name,
+  :last_name
+]
+
+destination :out, {
+  :file => 'data/model_out.txt'
+},
+{
+  :order => [:first_name, :last_name],
+}

data/test/multiple_delimited.ctl

@@ -0,0 +1,22 @@
+# puts "executing delimited.ctl"
+
+source :in, {
+  :file => 'data/multiple_delimited_*.txt',
+  :parser => :delimited
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  {
+    :name => :age,
+    :type => :integer
+  }
+]
+
+destination :out, {
+  :file => 'output/multiple_delimited.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/test/multiple_source_delimited.ctl

@@ -0,0 +1,39 @@
+# puts "executing delimited.ctl"
+
+source :source1, {
+  :file => 'data/multiple_delimited_*.txt',
+  :parser => :delimited
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  {
+    :name => :age,
+    :type => :integer
+  }
+]
+
+source :source2, {
+  :file => 'data/multiple_delimited_*.txt',
+  :parser => :delimited
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  {
+    :name => :age,
+    :type => :integer
+  }
+]
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }
+
+destination :out, {
+  :file => 'output/multiple_source_delimited.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn, :age]
+}

data/test/parser_test.rb

@@ -0,0 +1,224 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+# Test the flat text parsers
+class ParserTest < Test::Unit::TestCase
+  # Test parsing delimited data
+  def test_delimited_parser
+    control = ETL::Control::Control.resolve(File.dirname(__FILE__) + '/delimited.ctl')
+    parser = ETL::Parser::DelimitedParser.new(control.sources.first)
+    rows = parser.collect { |row| row }
+    assert_equal 3, rows.length
+    assert_equal({:first_name=>"Chris", :last_name=>"Smith", :ssn=>"111223333", :age=>"24", :sex => 'M'}, rows.first)
+  end
+  
+  # Test parsing fixed-width data
+  def test_fixed_width_parser
+    control = ETL::Control::Control.resolve(File.dirname(__FILE__) + '/fixed_width.ctl')
+    parser = ETL::Parser::FixedWidthParser.new(control.sources.first)
+    rows = parser.collect { |row| row }
+    assert_equal 3, rows.length
+    assert_equal({:first_name=>"Bob", :last_name=>"Smith", :ssn=>"123445555", :age=>"23"}, rows.first)
+  end
+  
+  # Test the DOM-based XML parser. Note that the DOM parser is slow and should
+  # probably be removed.
+  def test_xml_parser
+    control = ETL::Control::Control.resolve(File.dirname(__FILE__) + '/xml.ctl')
+    parser = ETL::Parser::XmlParser.new(control.sources.first)
+    rows = parser.collect { |row| row }
+    assert_equal 2, rows.length
+    assert_equal({:first_name=>"Bob", :last_name=>"Smith", :ssn=>"123456789", :age=>"24"}, rows.first)
+  end
+  
+  # Test an inline parser
+  def test_inline_parser
+    ETL::Engine.process(File.dirname(__FILE__) + '/inline_parser.ctl')
+    lines = open(File.dirname(__FILE__) + '/output/inline_parser.txt').readlines
+    assert_equal 3, lines.length
+  end
+  
+  # Test the SAX parser (preferred for XML parsing)
+  def test_sax_parser
+    control = ETL::Control::Control.resolve(File.dirname(__FILE__) + '/sax.ctl')
+    parser = control.sources.first.parser
+    rows = parser.collect { |row| row }
+    assert_equal 2, rows.length
+    assert_equal({:first_name=>"Bob", :last_name=>"Smith", :ssn=>"123456789", :age=>"24"}, rows.first)
+  end
+
+  # Test the Excel parser
+  def test_excel_parser
+    control = ETL::Control::Control.resolve(File.dirname(__FILE__) + '/excel.ctl')
+    parser = control.sources.first.parser
+    rows = parser.collect { |row| row }
+    assert_equal 2, rows.length
+    assert_equal({:first_name=>"Bob", :last_name=>"Smith", :ssn=>123456789, :age=>24}, rows.first)
+  end
+  
+  # Test 2 the Excel parser
+  def test_excel2_parser
+    control = ETL::Control::Control.resolve(File.dirname(__FILE__) + '/excel2.ctl')
+    parser = control.sources.first.parser
+    rows = parser.collect { |row| row }
+    assert_equal 2, rows.length
+    assert_equal({:first_name=>"John", :last_name=>"Doe", :ssn=>222114545, :age=>31}, rows[1])
+  end
+  
+  # Test the Apache combined log format parser
+  def test_apache_combined_log_parser
+    control = ETL::Control::Control.resolve(File.dirname(__FILE__) + '/apache_combined_log.ctl')
+    parser = ETL::Parser::ApacheCombinedLogParser.new(control.sources.first)
+    # first test the parse method
+    line = %Q(127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)")
+    fields = parser.parse(line)
+    assert_equal '127.0.0.1', fields[:ip_address]
+    assert_equal 'frank', fields[:user]
+    assert_equal Time.mktime(2000, 10, 10, 13, 55, 36), fields[:timestamp]
+    assert_equal 'GET /apache_pb.gif HTTP/1.0', fields[:request]
+    assert_equal '200', fields[:response_code]
+    assert_equal '2326', fields[:bytes]
+    assert_equal 'http://www.example.com/start.html', fields[:referrer]
+    assert_equal 'Mozilla/4.08 [en] (Win98; I ;Nav)', fields[:user_agent]
+    #now test the each method
+    rows = parser.collect { |row| row }
+    assert_equal 3, rows.length
+    assert_equal({
+      :user_agent=>"Mozilla/4.08 [en] (Win98; I ;Nav)",
+      :browser_version_minor=>nil,
+      :timestamp=>Time.mktime(2000, 10, 10, 13, 55, 36),
+      :ostype=>"Windows",
+      :request=>"GET /apache_pb.gif HTTP/1.0",
+      :referrer_host=>"www.example.com",
+      :referrer_domain=>"example.com",
+      :os=>"Win98",
+      :referrer_uri_path=>"/start.html",
+      :method=>"GET",
+      :response_code=>"200",
+      :referrer_port=>80,
+      :os_version=>nil,
+      :ip_address=>"127.0.0.1",
+      :referrer_scheme=>"http",
+      :bytes=>"2326",
+      :browser=>"Mozilla",
+      :identd=>nil,
+      :path=>"/apache_pb.gif",
+      :referrer=>"http://www.example.com/start.html",
+      :browser_version_major=>"4",
+      :user=>"frank"}, rows.first, 'Failed on first row')
+    assert_equal({
+      :user_agent=>"Mozilla/4.08 [en] (Win98; I ;Nav)",
+      :referrer_port=>80,
+      :timestamp=>Time.mktime(2000, 10, 11, 5, 22, 2),
+      :browser_version_minor=>nil,
+      :os_version=>nil,
+      :request=>"GET /apache_pb.gif HTTP/1.1",
+      :ostype=>"Windows",
+      :referrer_scheme=>"http",
+      :response_code=>"200",
+      :referrer_host=>"www.foo.com",
+      :referrer_domain=>"foo.com",
+      :ip_address=>"127.0.0.1",
+      :browser=>"Mozilla",
+      :bytes=>"2326",
+      :os=>"Win98",
+      :identd=>nil,
+      :browser_version_major=>"4",
+      :referrer=>"http://www.foo.com/",
+      :referrer_uri_path=>"/",
+      :method=>"GET",
+      :path=>"/apache_pb.gif",
+      :user=>"bob"}, rows[1], 'Failed on second row')
+    assert_equal({
+      :browser_version_major=>"4",
+      :browser_version_minor=>nil,
+      :referrer_port=>nil,
+      :request=>"GET /apache_pb.gif HTTP/1.1",
+      :ostype=>"Windows",
+      :os_version=>nil,
+      :response_code=>"200",
+      :referrer_host=>nil,
+      :referrer_scheme=>nil,
+      :bytes=>"2326",
+      :ip_address=>"127.0.0.1",
+      :browser=>"Mozilla",
+      :referrer=>nil,
+      :os=>"Win98",
+      :user=>"bob",
+      :user_agent=>"Mozilla/4.08 [en] (Win98; I ;Nav)",
+      :identd=>nil,
+      :referrer_uri_path=>nil,
+      :path=>"/apache_pb.gif",
+      :method=>"GET",
+      :timestamp=>Time.mktime(2000, 10, 11, 5, 52, 31)}, rows[2], 'Failed on third row')
+  end
+  
+  # Test the user agent parser
+  def test_user_agent_parser
+    agents = <<-AGENTS
+      Mozilla/4.7 [en] (WinNT; U)
+      Mozilla/4.0 (compatible; MSIE 5.01; Windows NT)
+      Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461; .NET CLR 1.1.4322)
+      Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 4.0) Opera 5.11 [en]
+      Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.0.2) Gecko/20030208 Netscape/7.02
+      Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040612 Firefox/0.8
+      Mozilla/5.0 (compatible; Konqueror/3.2; Linux) (KHTML, like Gecko)
+      Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6h
+    AGENTS
+    agents = agents.split("\n").collect { |s| s.strip }
+
+    control = ETL::Control::Control.resolve(File.dirname(__FILE__) + '/apache_combined_log.ctl')
+    parser = ETL::Parser::ApacheCombinedLogParser.new(control.sources.first)
+    rows = parser.collect { |row| row }
+    
+    assert_equal({:browser_version_major=>"4",
+      :browser_version_minor=>nil,
+      :ostype=>"Windows",
+      :os=>"WinNT",
+      :os_version=>nil,
+      :browser=>"Mozilla"}, parser.parse_user_agent(agents[0]), 'Agent 0 invalid'
+    )
+    assert_equal({:browser_version_major=>"5",
+      :browser_version_minor=>"01",
+      :ostype=>"Windows",
+      :os=>"Windows NT",
+      :os_version=>nil,
+      :browser=>"MSIE"}, parser.parse_user_agent(agents[1]), 'Agent 1 invalid'
+    )
+    assert_equal({:browser_version_major=>"6",
+      :browser_version_minor=>"0",
+      :ostype=>"Windows",
+      :os=>"Windows NT 5.0",
+      :os_version=>"5.0",
+      :browser=>"MSIE"}, parser.parse_user_agent(agents[2]), 'Agent 2 invalid'
+    )
+    assert_equal({:browser_version_major=>"5",
+      :browser_version_minor=>"0",
+      :ostype=>"Windows",
+      :os=>"Windows NT 4.0",
+      :os_version=>"4.0",
+      :browser=>"MSIE"}, parser.parse_user_agent(agents[3]), 'Agent 3 invalid'
+    )
+    assert_equal({:browser_version_major=>"7",
+      :browser_version_minor=>nil,
+      :ostype=>"Windows",
+      :os=>"Windows NT 5.0",
+      :os_version=>"5.0",
+      :browser=>"Netscape"}, parser.parse_user_agent(agents[4]), 'Agent 4 invalid'
+    )
+    assert_equal({:browser_version_major=>"0.8",
+      :browser_version_minor=>nil,
+      :ostype=>"Linux",
+      :os=>"Linux i686",
+      :os_version=>nil,
+      :browser=>"Firefox"}, parser.parse_user_agent(agents[5]), 'Agent 5 invalid'
+    )
+    # test fails here
+    # assert_equal({:browser_version_major=>"6",
+#       :browser_version_minor=>nil,
+#       :ostype=>"Linux",
+#       :os=>"Linux",
+#       :os_version=>nil,
+#       :browser=>"Konquerer"}, parser.parse_user_agent(agents[6]), 'Agent 6 invalid'
+#     )
+  end
+end