activewarehouse-etl-sgonyea 0.9.6

data/test/data/bulk_import_with_empties.txt

@@ -0,0 +1,3 @@
+1,Chris,Smith,111223333
+2,Jim,,444332222
+3,Brian,Collingsworth,123443435

data/test/data/decode.txt

@@ -0,0 +1,3 @@
+M:Male
+F:Female
+:Unknown

data/test/data/delimited.txt

@@ -0,0 +1,3 @@
+Chris,Smith,111223333,24,M
+Jim,Foxworthy,444332222,51,M
+Brian,Collingsworth,123443435,10,M

data/test/data/encode_source_latin1.txt

@@ -0,0 +1,2 @@
+�ph�m�re has accents.
+let's encode them.

data/test/data/fixed_width.txt

@@ -0,0 +1,3 @@
+Bob      Smith       12344555523
+Jane     Doe         98766211145
+AbcdefghiJklmnopqrstu12345678920

data/test/data/multiple_delimited_1.txt

@@ -0,0 +1,3 @@
+Chris,Smith,111223333,24
+Jim,Foxworthy,444332222,51
+Brian,Collingsworth,123443435,10

data/test/data/multiple_delimited_2.txt

@@ -0,0 +1,3 @@
+Bob,Jones,444223333,28
+Tom,Allen,324001232,33
+Jesse,Baker,555443333,21

data/test/data/nokogiri.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="US-ASCII" standalone="yes"?>
+<!DOCTYPE people >
+<people>
+  <person ssn="123456789" type="employee">
+    <first_name>Bob</first_name>
+    <last_name>Smith</last_name>
+    <email>bsmith@foo.com</email>
+    <colours>
+      <eyes>brown</eyes>
+      <hair>black</hair>
+      <skin>fair</skin>
+    </colours>
+    <age>24</age>
+  </person>
+  <person ssn="111223333" type="client">
+    <first_name>Jane</first_name>
+    <last_name>Doe</last_name>
+    <email>jdoe@bar.com</email>
+    <colours>
+      <eyes>blue</eyes>
+      <hair>blond</hair>
+      <skin>medium</skin>
+    </colours>
+    <age>45</age>
+  </person>
+  <person ssn="133244566" type="client">
+    <first_name>Jake</first_name>
+    <last_name>Smithsonian</last_name>
+    <email>jake@example.com</email>
+    <colours>
+      <eyes>brown</eyes>
+      <hair>black</hair>
+      <skin>dark</skin>
+    </colours>
+    <age>37</age>
+  </person>
+</people>
+

data/test/data/people.txt

@@ -0,0 +1,3 @@
+Bob,Smith
+Jane,Doe
+Chris,Cornell

data/test/data/sax.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0"?>
+
+<people>
+  <person age="24">
+    <first_name>Bob</first_name>
+    <last_name>Smith</last_name>
+    <social_security_number>123456789</social_security_number>
+  </person>
+  <person age="31">
+    <first_name>John</first_name>
+    <last_name>Doe</last_name>
+    <social_security_number>222114545</social_security_number>
+  </person>
+</people>

data/test/data/xml.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+
+<people>
+  <person>
+    <first_name>Bob</first_name>
+    <last_name>Smith</last_name>
+    <social_security_number>123456789</social_security_number>
+    <age>24</age>
+  </person>
+  <person>
+    <first_name>John</first_name>
+    <last_name>Doe</last_name>
+    <social_security_number>222114545</social_security_number>
+    <age>31</age>
+  </person>
+</people>

data/test/database_join_processor_test.rb

@@ -0,0 +1,43 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class DatabaseJoinProcessorTest < Test::Unit::TestCase
+
+  def new_processor(options)
+    ETL::Processor::DatabaseJoinProcessor.new(nil, options)
+  end
+  
+  should 'raise an error unless :fields is specified' do
+    error = assert_raises(ETL::ControlError) { new_processor({}) }
+    assert_equal ":target must be specified", error.message
+  end
+
+  should 'return the row and the database result' do
+    row = ETL::Row[:id => 1, :first_name => 'Bob', :last_name => 'Smith', :ssn => '111234444']
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+      '/delimited.ctl')
+
+    Person.delete_all
+    assert_equal 0, Person.count
+    
+    # First define a basic configuration to check defaults
+    configuration = { 
+      :target => :data_warehouse,
+      :database => 'etl_unittest',
+      :table => 'people',
+      :buffer_size => 0 
+    }
+    mapping = { :order => [:id, :first_name, :last_name, :ssn] }
+    dest = ETL::Control::DatabaseDestination.new(control, configuration, mapping)
+    dest.write(row)
+    dest.close
+    
+    assert_equal 1, Person.find(:all).length
+
+    row = ETL::Row[:last_name => "Smith"]
+    processor = new_processor(:target => :data_warehouse,
+                              :query => "SELECT first_name FROM people WHERE last_name = \#{connection.quote(row[:last_name])}",
+                              :fields => ["first_name"]).process(row)
+    assert_equal row[:first_name], "Bob"
+  end
+  
+end

data/test/date_dimension_builder_test.rb

@@ -0,0 +1,96 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class DateDimensionBuilderTest < Test::Unit::TestCase
+  
+  context "the DateDimensionBuilder" do
+    context "when initialized with defaults" do
+      setup do
+        @builder = ETL::Builder::DateDimensionBuilder.new
+      end
+      should "have a start date of 5 years ago" do
+        assert_equal Time.now.years_ago(5).to_date, @builder.start_date.to_date
+      end
+      should "have an end date of now" do
+        assert_equal Time.now.to_date, @builder.end_date.to_date
+      end
+      should "have an empty of array of holiday indicators" do
+        assert_equal [], @builder.holiday_indicators
+      end
+    end
+    context "when initialized with arguments" do
+      setup do 
+        @start_date = Time.now.years_ago(2)
+        @end_date = Time.now.years_ago(1)
+        @builder = ETL::Builder::DateDimensionBuilder.new(@start_date, @end_date)
+      end
+      should "respect a custom start date" do
+        assert_equal @start_date.to_date, @builder.start_date.to_date
+      end
+      should "respect a custom end date" do
+        assert_equal @end_date.to_date, @builder.end_date.to_date
+      end
+    end
+    context "when building a date dimension using the default settings" do
+      setup do
+        # specific dates required when testing, because leap years affect
+        # how many records are built
+        @start_date = Date.parse('2002-05-19').to_time
+        @end_date = Date.parse('2007-05-19').to_time
+        @builder = ETL::Builder::DateDimensionBuilder.new(@start_date, @end_date)
+        @records = @builder.build
+      end
+      should "build a dimension with the correct number of records" do
+        assert_equal 1827, @records.length
+      end
+      should "have the correct first date" do
+        assert_date_dimension_record_equal(@builder.start_date, @records.first)
+      end
+    end
+    context "when building a date dimension with a fiscal year offset month" do
+      should_eventually "respect the fiscal year offset month" do
+        
+      end
+    end
+  end
+  
+  def assert_date_dimension_record_equal(date, record)
+    real_date = date
+    date = date.to_time
+    assert_equal date.strftime("%m/%d/%Y"), record[:date]
+    assert_equal date.strftime("%B %d,%Y"), record[:full_date_description]
+    assert_equal date.strftime("%A"), record[:day_of_week]
+    assert_equal date.day, record[:day_number_in_calendar_month]
+    assert_equal date.yday, record[:day_number_in_calendar_year]
+    assert_equal date.day, record[:day_number_in_fiscal_month]
+    assert_equal date.fiscal_year_yday, record[:day_number_in_fiscal_year]
+    assert_equal "Week #{date.week}", record[:calendar_week]
+    assert_equal date.week, record[:calendar_week_number_in_year]
+    assert_equal date.strftime("%B"), record[:calendar_month_name]
+    assert_equal date.month, record[:calendar_month_number_in_year]
+    assert_equal date.strftime("%Y-%m"), record[:calendar_year_month]
+    assert_equal "Q#{date.quarter}", record[:calendar_quarter]
+    assert_equal date.quarter, record[:calendar_quarter_number_in_year]
+    assert_equal "#{date.strftime('%Y')}-#{record[:calendar_quarter]}", record[:calendar_year_quarter]
+    assert_equal "#{date.year}", record[:calendar_year]
+    assert_equal "FY Week #{date.fiscal_year_week}", record[:fiscal_week]
+    assert_equal date.fiscal_year_week, record[:fiscal_week_number_in_year]
+    assert_equal date.fiscal_year_month, record[:fiscal_month]
+    assert_equal date.fiscal_year_month, record[:fiscal_month_number_in_year]
+    assert_equal "FY#{date.fiscal_year}-" + date.fiscal_year_month.to_s.rjust(2, '0'), record[:fiscal_year_month]
+    assert_equal "FY Q#{date.fiscal_year_quarter}", record[:fiscal_quarter]
+    assert_equal "FY#{date.fiscal_year}-Q#{date.fiscal_year_quarter}", record[:fiscal_year_quarter]
+    assert_equal date.fiscal_year_quarter, record[:fiscal_year_quarter_number]
+    assert_equal "FY#{date.fiscal_year}", record[:fiscal_year]
+    assert_equal date.fiscal_year, record[:fiscal_year_number]
+    assert_equal 'Nonholiday', record[:holiday_indicator]
+    assert_equal weekday_indicators[date.wday], record[:weekday_indicator]
+    assert_equal 'None', record[:selling_season]
+    assert_equal 'None', record[:major_event]
+    assert_equal record[:sql_date_stamp], real_date
+  end
+  
+  private
+  def weekday_indicators
+    ['Weekend','Weekday','Weekday','Weekday','Weekday','Weekday','Weekend']
+  end
+end

data/test/delimited.ctl

@@ -0,0 +1,30 @@
+source :in, {
+  :file => 'data/delimited.txt',
+  :parser => {
+    :name => :csv
+  }
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  :age,
+  :sex
+]
+
+#transform :age, :type, :type => :number
+transform :ssn, :sha1
+transform(:ssn){ |n, v, row| v[0,24] }
+transform :sex, :decode, {:decode_table_path => 'data/decode.txt'}
+
+destination :out, {
+  :file => 'output/delimited.txt'
+},
+{
+  :order => [:id, :first_name, :last_name, :ssn, :age, :sex, :test, :calc_test],
+  :virtual => {
+    :id => :surrogate_key,
+    :test => "test!",
+    :calc_test => Time.now
+  },
+}

data/test/delimited_absolute.ctl

@@ -0,0 +1,31 @@
+source :in, {
+  :file => '/tmp/delimited_abs.txt',
+  :parser => {
+    :name => :csv
+  }
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  {
+    :name => :age,
+    :type => :integer
+  },
+  :sex
+]
+
+transform :ssn, :sha1
+transform(:ssn){ |n, v, row| v[0,24] }
+transform :sex, :decode, {:decode_table_path => 'data/decode.txt'}
+
+destination :out, {
+  :file => 'data/delimited_abs.txt'
+},
+{
+  :order => [:first_name, :last_name, :ssn, :age, :sex, :test, :calc_test],
+  :virtual => {
+    :test => "test!",
+    :calc_test => Time.now
+  }
+}

data/test/delimited_destination_db.ctl

@@ -0,0 +1,23 @@
+source :in, {
+  :file => 'data/delimited.txt',
+  :parser => :csv
+}, 
+[ 
+  :id,
+  :first_name,
+  :last_name,
+  :ssn
+]
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }
+
+destination :out, {
+  :type => :database,
+  :target => :data_warehouse,
+  :database => 'etl_unittest',
+  :table => 'people',
+},
+{
+  :order => [:id, :first_name, :last_name, :ssn]
+}

data/test/delimited_excel.ctl

@@ -0,0 +1,31 @@
+source :in, {
+  :file => 'data/delimited.txt',
+  :parser => {
+    :name => :csv
+  }
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  :age,
+  :sex
+]
+
+#transform :age, :type, :type => :number
+transform :ssn, :sha1
+transform(:ssn){ |n, v, row| v[0,24] }
+transform :sex, :decode, {:decode_table_path => 'data/decode.txt'}
+
+destination :out, {
+  :type => :excel,
+  :file => 'output/delimited_excel.xls'
+},
+{
+  :order => [:id, :first_name, :last_name, :ssn, :age, :sex, :test, :calc_test],
+  :virtual => {
+    :id => :surrogate_key,
+    :test => "test!",
+    :calc_test => Time.now
+  },
+}

data/test/delimited_insert_update.ctl

@@ -0,0 +1,34 @@
+source :in, {
+  :file => 'data/delimited.txt',
+  :parser => {
+    :name => :csv
+  }
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  :age,
+  :sex
+]
+
+#transform :age, :type, :type => :number
+transform :ssn, :sha1
+transform(:ssn){ |n, v, row| v[0,24] }
+transform :sex, :decode, {:decode_table_path => 'data/decode.txt'}
+
+destination :out, {
+  :type => :insert_update_database,
+  :target => :data_warehouse,
+  :database => 'etl_unittest',
+  :table => 'people'
+},
+{
+  :primarykey => [:id],
+  :order => [:id, :first_name, :last_name, :ssn, :age, :sex, :test, :calc_test],
+  :virtual => {
+    :id => :surrogate_key,
+    :test => "test!",
+    :calc_test => Time.now
+  },
+}

data/test/delimited_update.ctl

@@ -0,0 +1,34 @@
+source :in, {
+  :file => 'data/delimited.txt',
+  :parser => {
+    :name => :csv
+  }
+}, 
+[ 
+  :first_name,
+  :last_name,
+  :ssn,
+  :age,
+  :sex
+]
+
+#transform :age, :type, :type => :number
+transform :ssn, :sha1
+transform(:ssn){ |n, v, row| v[0,24] }
+transform :sex, :decode, {:decode_table_path => 'data/decode.txt'}
+
+destination :out, {
+  :type => :update_database,
+  :target => :data_warehouse,
+  :database => 'etl_unittest',
+  :table => 'people'
+},
+{
+  :conditions => [{:field => "\#{conn.quote_column_name(:id)}", :value => "\#{conn.quote(row[:id])}", :comp => "="}],
+  :order => [:id, :first_name, :last_name, :ssn, :age, :sex, :test, :calc_test],
+  :virtual => {
+    :id => :surrogate_key,
+    :test => "test!",
+    :calc_test => Time.now
+  },
+}