activewarehouse-etl 0.9.1 → 0.9.5.rc1

data/test/block_processor_remove_rows.ctl

@@ -0,0 +1,5 @@
+source :in, { :type => :mock, :name => :block_input } 
+
+before_write { |row| row[:obsolete] == true ? nil : row }
+
+destination :out, { :type => :mock, :name => :block_output  }

data/test/block_processor_test.rb

@@ -0,0 +1,38 @@
+require File.dirname(__FILE__) + '/test_helper'
+include ETL
+include ETL::Control
+
+class TestWitness
+end
+
+class BlockProcessorTest < Test::Unit::TestCase
+  
+  def test_block_processor_should_work_as_both_after_read_and_before_write_row_processor
+    MockSource[:block_processed_input] = [{ :first_name => 'John'},{:first_name => 'Gary'}]
+    process 'block_processor.ctl'
+    assert_equal 4, MockDestination[:block_processed_output].size
+    assert_equal({ :first_name => 'John', :added_by_after_read => 'after-John', :added_by_before_write => "Row 1" }, MockDestination[:block_processed_output][0])
+    assert_equal({ :new_row => 'added by post_processor' }, MockDestination[:block_processed_output][1])
+    assert_equal({ :first_name => 'Gary', :added_by_after_read => 'after-Gary', :added_by_before_write => "Row 2" }, MockDestination[:block_processed_output][2])
+    assert_equal({ :new_row => 'added by post_processor' }, MockDestination[:block_processed_output][3])
+  end
+  
+  def test_block_processor_should_let_rows_be_removed_by_setting_it_to_nil
+    MockSource[:block_input] = [{ :obsolete => true, :name => 'John'},{ :obsolete => false, :name => 'Gary'}]
+    process 'block_processor_remove_rows.ctl'
+    assert_equal([{ :obsolete => false, :name => 'Gary' }], MockDestination[:block_output]) # only one record should be kept
+  end
+  
+  def test_block_processor_should_work_as_pre_or_post_processor
+    flexmock(TestWitness).should_receive(:call).with("I'm called from pre_process")
+    flexmock(TestWitness).should_receive(:call).with("I'm called from post_process")
+    MockSource[:another_input] = [{ :obsolete => true, :name => 'John'},{ :obsolete => false, :name => 'Gary'}]
+    process 'block_processor_pre_post_process.ctl'
+    assert_equal(MockSource[:another_input], MockDestination[:another_output])
+  end
+  
+  def test_block_error_should_be_propagated
+    assert_raise(ControlError) { process 'block_processor_error.ctl' }
+  end
+  
+end

data/test/config/Gemfile.rails-2.3.x

@@ -0,0 +1,3 @@
+require File.dirname(__FILE__) + '/common'
+
+common_gemfile('2.3.11')

data/test/config/Gemfile.rails-2.3.x.lock

@@ -0,0 +1,38 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activerecord (2.3.11)
+      activesupport (= 2.3.11)
+    activesupport (2.3.11)
+    fastercsv (1.5.4)
+    flexmock (0.9.0)
+    mysql (2.8.1)
+    net-sftp (2.0.5)
+      net-ssh (>= 2.0.9)
+    net-ssh (2.1.4)
+    pg (0.11.0)
+    rdoc (3.6.1)
+    ruby-ole (1.2.11.1)
+    shoulda (2.11.3)
+    spreadsheet (0.6.5.4)
+      ruby-ole (>= 1.0)
+    tmail (1.2.7.1)
+    zip (2.0.2)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  activerecord (= 2.3.11)
+  activesupport (= 2.3.11)
+  fastercsv (= 1.5.4)
+  flexmock (= 0.9.0)
+  mysql (= 2.8.1)
+  net-sftp (= 2.0.5)
+  pg (= 0.11.0)
+  rdoc
+  shoulda (= 2.11.3)
+  spreadsheet (= 0.6.5.4)
+  tmail (= 1.2.7.1)
+  zip (= 2.0.2)

data/test/config/Gemfile.rails-3.0.x

@@ -0,0 +1,3 @@
+require File.dirname(__FILE__) + '/common'
+
+common_gemfile('3.0.7')

data/test/config/Gemfile.rails-3.0.x.lock

@@ -0,0 +1,49 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activemodel (3.0.7)
+      activesupport (= 3.0.7)
+      builder (~> 2.1.2)
+      i18n (~> 0.5.0)
+    activerecord (3.0.7)
+      activemodel (= 3.0.7)
+      activesupport (= 3.0.7)
+      arel (~> 2.0.2)
+      tzinfo (~> 0.3.23)
+    activesupport (3.0.7)
+    arel (2.0.10)
+    builder (2.1.2)
+    fastercsv (1.5.4)
+    flexmock (0.9.0)
+    i18n (0.5.0)
+    mysql (2.8.1)
+    net-sftp (2.0.5)
+      net-ssh (>= 2.0.9)
+    net-ssh (2.1.4)
+    pg (0.11.0)
+    rdoc (3.6.1)
+    ruby-ole (1.2.11.1)
+    shoulda (2.11.3)
+    spreadsheet (0.6.5.4)
+      ruby-ole (>= 1.0)
+    tmail (1.2.7.1)
+    tzinfo (0.3.27)
+    zip (2.0.2)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  activerecord (= 3.0.7)
+  activesupport (= 3.0.7)
+  fastercsv (= 1.5.4)
+  flexmock (= 0.9.0)
+  mysql (= 2.8.1)
+  net-sftp (= 2.0.5)
+  pg (= 0.11.0)
+  rdoc
+  shoulda (= 2.11.3)
+  spreadsheet (= 0.6.5.4)
+  tmail (= 1.2.7.1)
+  zip (= 2.0.2)

data/test/config/common.rb

@@ -0,0 +1,21 @@
+def common_gemfile(rails_version)
+  source :rubygems
+
+  # using explicit versions for the gems to avoid any weirdness later on
+  gem "activesupport", rails_version
+  gem "activerecord", rails_version
+
+  gem "fastercsv", "1.5.4"
+  gem "spreadsheet", "0.6.5.4"
+  gem "tmail", "1.2.7.1"
+  gem "net-sftp", "2.0.5"
+  gem "zip", "2.0.2"
+
+  gem "shoulda", "2.11.3"
+  gem "flexmock", "0.9.0"
+
+  gem "mysql", "2.8.1"
+  gem "pg", "0.11.0"
+
+  gem "rdoc"
+end

data/test/connection/mysql/connection.rb

@@ -0,0 +1,9 @@
+print "Using native MySQL\n"
+
+puts "Resetting database"
+conn = ETL::Engine.connection(:data_warehouse)
+conn.recreate_database(conn.current_database)
+conn.reconnect!
+lines = open(File.join(File.dirname(__FILE__), 'schema.sql')).readlines
+lines.join.split(';').each { |line| conn.execute(line) }
+conn.disconnect!

data/test/connection/mysql/schema.sql

@@ -0,0 +1,36 @@
+drop table if exists people;
+create table people (
+  id int not null primary key,
+  first_name char(255) not null,
+  last_name char(255) not null,
+  ssn char(64) not null
+);
+drop table if exists places;
+create table places (
+	address text,
+	city char(255),
+	state char(255),
+	country char(2)
+);
+
+drop table if exists person_dimension;
+create table person_dimension (
+  id int not null primary key,
+  first_name char(50),
+  last_name char(50),
+  address char(100),
+  city char(50),
+  state char(50),
+  zip_code char(20),
+  effective_date datetime,
+  end_date datetime,
+  latest_version boolean not null
+);
+
+drop table if exists truncate_test;
+create table truncate_test (
+	x char(4)
+);
+insert into truncate_test (x) values ('a');
+insert into truncate_test (x) values ('b');
+insert into truncate_test (x) values ('c');

data/test/connection/postgresql/connection.rb

@@ -0,0 +1,13 @@
+print "Using PostgreSQL\n"
+
+puts "Resetting database"
+conn = ETL::Engine.connection(:data_warehouse)
+
+lines = open(File.join(File.dirname(__FILE__), 'schema.sql')).readlines
+lines.join.split(';').each_with_index do |line, index|
+  begin
+    conn.execute(line)
+  rescue => e
+    puts "failed to load line #{index}: #{e}"
+  end
+end

data/test/connection/postgresql/schema.sql

@@ -0,0 +1,39 @@
+drop table people;
+create table people (
+  id SERIAL PRIMARY KEY,
+  first_name character varying(255) not null,
+  /* null below allowed for bulk_import_with_empties.txt test */
+  last_name character varying(255) null,
+  ssn character varying(64) not null
+);
+
+drop table places;
+create table places (
+  id SERIAL PRIMARY KEY,
+	address text,
+	city character varying(255),
+	state character varying(255),
+	country character varying(2)
+);
+
+drop table person_dimension;
+create table person_dimension (
+  id SERIAL PRIMARY KEY,
+  first_name character varying(50),
+  last_name character varying(50),
+  address character varying(100),
+  city character varying(50),
+  state character varying(50),
+  zip_code character varying(20),
+  effective_date timestamp without time zone,
+  end_date timestamp without time zone,
+  latest_version boolean not null
+);
+
+drop table truncate_test;
+create table truncate_test (
+	x character varying(4)
+);
+insert into truncate_test (x) values ('a');
+insert into truncate_test (x) values ('b');
+insert into truncate_test (x) values ('c');

data/test/control_test.rb

@@ -0,0 +1,43 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class ControlTest < Test::Unit::TestCase
+  # Test the ability to parse control files.
+  def test_parse
+    assert_nothing_raised do
+      Dir.glob('*.ctl') do |f|
+        ETL::Control::Control.parse(File.join(File.dirname(__FILE__), f))
+      end
+    end
+  end
+  
+  def test_bad_control_raises_error
+    assert_raise ETL::ControlError do
+      ETL::Control::Control.resolve(0)
+    end
+  end
+  
+  def test_resolve_control_object
+    assert_nothing_raised do
+      ETL::Control::Control.resolve(ETL::Control::Control.parse(File.join(File.dirname(__FILE__), 'delimited.ctl')))
+    end
+  end
+  
+  def test_set_error_threshold
+    assert_nothing_raised do
+      ETL::Engine.process(File.join(File.dirname(__FILE__), 'errors.ctl'))
+    end
+  end
+  
+  def test_bad_processor_name
+    assert_raise ETL::ControlError do
+      s = "before_write :chunky_monkey"
+      ETL::Control::Control.parse_text(s)
+    end
+  end
+  
+  def test_dependencies
+    s = "depends_on 'foo', 'bar'"
+    control = ETL::Control::Control.parse_text(s)
+    assert_equal control.dependencies, ['foo','bar']
+  end
+end

data/test/data/apache_combined_log.txt

@@ -0,0 +1,3 @@
+127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
+127.0.0.1 - bob [11/Oct/2000:05:22:02 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "http://www.foo.com/" "Mozilla/4.08 [en] (Win98; I ;Nav)"
+127.0.0.1 - bob [11/Oct/2000:05:52:31 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "-" "Mozilla/4.08 [en] (Win98; I ;Nav)"

data/test/data/bulk_import.txt

@@ -0,0 +1,3 @@
+1,Chris,Smith,111223333
+2,Jim,Foxworthy,444332222
+3,Brian,Collingsworth,123443435

data/test/data/bulk_import_with_empties.txt

@@ -0,0 +1,3 @@
+1,Chris,Smith,111223333
+2,Jim,,444332222
+3,Brian,Collingsworth,123443435

data/test/data/decode.txt

@@ -0,0 +1,3 @@
+M:Male
+F:Female
+:Unknown

data/test/data/delimited.txt

@@ -0,0 +1,3 @@
+Chris,Smith,111223333,24,M
+Jim,Foxworthy,444332222,51,M
+Brian,Collingsworth,123443435,10,M

data/test/data/encode_source_latin1.txt

@@ -0,0 +1,2 @@
+�ph�m�re has accents.
+let's encode them.

data/test/data/fixed_width.txt

@@ -0,0 +1,3 @@
+Bob      Smith       12344555523
+Jane     Doe         98766211145
+AbcdefghiJklmnopqrstu12345678920

data/test/data/multiple_delimited_1.txt

@@ -0,0 +1,3 @@
+Chris,Smith,111223333,24
+Jim,Foxworthy,444332222,51
+Brian,Collingsworth,123443435,10

data/test/data/multiple_delimited_2.txt

@@ -0,0 +1,3 @@
+Bob,Jones,444223333,28
+Tom,Allen,324001232,33
+Jesse,Baker,555443333,21

data/test/data/people.txt

@@ -0,0 +1,3 @@
+Bob,Smith
+Jane,Doe
+Chris,Cornell

data/test/data/sax.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0"?>
+
+<people>
+  <person age="24">
+    <first_name>Bob</first_name>
+    <last_name>Smith</last_name>
+    <social_security_number>123456789</social_security_number>
+  </person>
+  <person age="31">
+    <first_name>John</first_name>
+    <last_name>Doe</last_name>
+    <social_security_number>222114545</social_security_number>
+  </person>
+</people>

data/test/data/xml.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+
+<people>
+  <person>
+    <first_name>Bob</first_name>
+    <last_name>Smith</last_name>
+    <social_security_number>123456789</social_security_number>
+    <age>24</age>
+  </person>
+  <person>
+    <first_name>John</first_name>
+    <last_name>Doe</last_name>
+    <social_security_number>222114545</social_security_number>
+    <age>31</age>
+  </person>
+</people>

data/test/date_dimension_builder_test.rb

@@ -0,0 +1,96 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class DateDimensionBuilderTest < Test::Unit::TestCase
+  
+  context "the DateDimensionBuilder" do
+    context "when initialized with defaults" do
+      setup do
+        @builder = ETL::Builder::DateDimensionBuilder.new
+      end
+      should "have a start date of 5 years ago" do
+        assert_equal Time.now.years_ago(5).to_date, @builder.start_date.to_date
+      end
+      should "have an end date of now" do
+        assert_equal Time.now.to_date, @builder.end_date.to_date
+      end
+      should "have an empty of array of holiday indicators" do
+        assert_equal [], @builder.holiday_indicators
+      end
+    end
+    context "when initialized with arguments" do
+      setup do 
+        @start_date = Time.now.years_ago(2)
+        @end_date = Time.now.years_ago(1)
+        @builder = ETL::Builder::DateDimensionBuilder.new(@start_date, @end_date)
+      end
+      should "respect a custom start date" do
+        assert_equal @start_date.to_date, @builder.start_date.to_date
+      end
+      should "respect a custom end date" do
+        assert_equal @end_date.to_date, @builder.end_date.to_date
+      end
+    end
+    context "when building a date dimension using the default settings" do
+      setup do
+        # specific dates required when testing, because leap years affect
+        # how many records are built
+        @start_date = Date.parse('2002-05-19').to_time
+        @end_date = Date.parse('2007-05-19').to_time
+        @builder = ETL::Builder::DateDimensionBuilder.new(@start_date, @end_date)
+        @records = @builder.build
+      end
+      should "build a dimension with the correct number of records" do
+        assert_equal 1827, @records.length
+      end
+      should "have the correct first date" do
+        assert_date_dimension_record_equal(@builder.start_date, @records.first)
+      end
+    end
+    context "when building a date dimension with a fiscal year offset month" do
+      should_eventually "respect the fiscal year offset month" do
+        
+      end
+    end
+  end
+  
+  def assert_date_dimension_record_equal(date, record)
+    real_date = date
+    date = date.to_time
+    assert_equal date.strftime("%m/%d/%Y"), record[:date]
+    assert_equal date.strftime("%B %d,%Y"), record[:full_date_description]
+    assert_equal date.strftime("%A"), record[:day_of_week]
+    assert_equal date.day, record[:day_number_in_calendar_month]
+    assert_equal date.yday, record[:day_number_in_calendar_year]
+    assert_equal date.day, record[:day_number_in_fiscal_month]
+    assert_equal date.fiscal_year_yday, record[:day_number_in_fiscal_year]
+    assert_equal "Week #{date.week}", record[:calendar_week]
+    assert_equal date.week, record[:calendar_week_number_in_year]
+    assert_equal date.strftime("%B"), record[:calendar_month_name]
+    assert_equal date.month, record[:calendar_month_number_in_year]
+    assert_equal date.strftime("%Y-%m"), record[:calendar_year_month]
+    assert_equal "Q#{date.quarter}", record[:calendar_quarter]
+    assert_equal date.quarter, record[:calendar_quarter_number_in_year]
+    assert_equal "#{date.strftime('%Y')}-#{record[:calendar_quarter]}", record[:calendar_year_quarter]
+    assert_equal "#{date.year}", record[:calendar_year]
+    assert_equal "FY Week #{date.fiscal_year_week}", record[:fiscal_week]
+    assert_equal date.fiscal_year_week, record[:fiscal_week_number_in_year]
+    assert_equal date.fiscal_year_month, record[:fiscal_month]
+    assert_equal date.fiscal_year_month, record[:fiscal_month_number_in_year]
+    assert_equal "FY#{date.fiscal_year}-" + date.fiscal_year_month.to_s.rjust(2, '0'), record[:fiscal_year_month]
+    assert_equal "FY Q#{date.fiscal_year_quarter}", record[:fiscal_quarter]
+    assert_equal "FY#{date.fiscal_year}-Q#{date.fiscal_year_quarter}", record[:fiscal_year_quarter]
+    assert_equal date.fiscal_year_quarter, record[:fiscal_year_quarter_number]
+    assert_equal "FY#{date.fiscal_year}", record[:fiscal_year]
+    assert_equal date.fiscal_year, record[:fiscal_year_number]
+    assert_equal 'Nonholiday', record[:holiday_indicator]
+    assert_equal weekday_indicators[date.wday], record[:weekday_indicator]
+    assert_equal 'None', record[:selling_season]
+    assert_equal 'None', record[:major_event]
+    assert_equal record[:sql_date_stamp], real_date
+  end
+  
+  private
+  def weekday_indicators
+    ['Weekend','Weekday','Weekday','Weekday','Weekday','Weekday','Weekend']
+  end
+end