activewarehouse-etl 0.9.5.rc1 → 1.0.0.rc1

data/test/ensure_fields_presence_processor_test.rb

@@ -0,0 +1,33 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class EnsureFieldsPresenceProcessorTest < Test::Unit::TestCase
+
+  def new_processor(options)
+    ETL::Processor::EnsureFieldsPresenceProcessor.new(nil, options)
+  end
+  
+  should 'raise an error unless :fields is specified' do
+    error = assert_raises(ETL::ControlError) { new_processor({}) }
+    assert_equal ":fields must be specified", error.message
+  end
+
+  should 'raise an error if a field is missing in the row' do
+    error = assert_raise(ETL::ControlError) do
+      processor = new_processor(:fields => [:key])
+      processor.process(ETL::Row[])
+    end
+    
+    assert_match /missing required field\(s\)/, error.message
+  end
+  
+  should 'return the row if the required fields are in the row' do
+    row = ETL::Row[:first => nil, :second => "Barry"]
+    assert_equal row, new_processor(:fields => [:first, :second]).process(row)
+  end
+
+  should 'accept strings instead of symbols in both places' do
+    row = ETL::Row[:first => nil, 'second' => "Barry"]
+    assert_equal row, new_processor(:fields => ['first', :second]).process(row)
+  end
+  
+end

data/test/foreign_key_lookup_transform_test.rb

@@ -0,0 +1,50 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+# TODO - use flexmock instead, but I'm not sure how to handle the respond_to part yet
+class TestResolver
+  attr_accessor :cache_loaded
+  
+  def initialize
+    @cache_loaded = false
+  end
+  
+  def load_cache
+    @cache_loaded = true
+  end
+end
+
+class ForeignKeyLookupTransformTest < Test::Unit::TestCase
+
+  context 'configuration' do
+
+    should 'enable cache by default' do
+      resolver = TestResolver.new
+
+      transform = ETL::Transform::ForeignKeyLookupTransform.new(nil, 'name',
+        {:resolver => resolver})
+
+      assert_equal true, resolver.cache_loaded
+    end
+    
+    should 'allow to disable cache' do
+      resolver = TestResolver.new
+
+      transform = ETL::Transform::ForeignKeyLookupTransform.new(nil, 'name',
+        {:resolver => resolver, :cache => false})
+        
+      assert_equal false, resolver.cache_loaded
+    end
+
+    should 'allow to enable cache' do
+      resolver = TestResolver.new
+
+      transform = ETL::Transform::ForeignKeyLookupTransform.new(nil, 'name',
+        {:resolver => resolver, :cache => true})
+        
+      assert_equal true, resolver.cache_loaded
+    end
+    
+  end
+  
+
+end

data/test/multiple_delimited.ctl

@@ -2,7 +2,7 @@
 
 source :in, {
   :file => 'data/multiple_delimited_*.txt',
-  :parser => :delimited
+  :parser => :csv
 }, 
 [ 
   :first_name,

data/test/multiple_source_delimited.ctl

@@ -2,7 +2,7 @@
 
 source :source1, {
   :file => 'data/multiple_delimited_*.txt',
-  :parser => :delimited
+  :parser => :csv
 }, 
 [ 
   :first_name,
@@ -16,7 +16,7 @@ source :source1, {
 
 source :source2, {
   :file => 'data/multiple_delimited_*.txt',
-  :parser => :delimited
+  :parser => :csv
 }, 
 [ 
   :first_name,

data/test/nokogiri_all.ctl

@@ -0,0 +1,35 @@
+# puts "executing nokogiri_all.ctl"
+
+source :in, {
+  :file => 'data/nokogiri.xml',
+  :parser => :nokogiri_xml
+}, 
+{
+  :collection => 'people/person',
+  :fields => [
+    :first_name,
+    :last_name,
+    {
+      :name => :ssn,
+      :xpath => '@ssn'
+    },
+    {
+      :name => :age,
+      :type => :integer
+    },
+    {
+      :name => :hair_colour,
+      :xpath => 'colours/hair'
+    }
+  ]
+}
+
+destination :out, {
+  :file => 'output/xml.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }

data/test/nokogiri_select.ctl

@@ -0,0 +1,35 @@
+# puts "executing nokogiri_select.ctl"
+
+source :in, {
+  :file => 'data/nokogiri.xml',
+  :parser => :nokogiri_xml
+}, 
+{
+  :collection => 'people/person[@type="client"]',
+  :fields => [
+    :first_name,
+    :last_name,
+    {
+      :name => :ssn,
+      :xpath => '@ssn'
+    },
+    {
+      :name => :age,
+      :type => :integer
+    },
+    {
+      :name => :hair_colour,
+      :xpath => 'colours/hair'
+    }
+  ]
+}
+
+destination :out, {
+  :file => 'output/xml.txt'
+}, 
+{
+  :order => [:first_name, :last_name, :ssn]
+}
+
+transform :ssn, :sha1
+transform(:ssn){ |v| v[0,24] }

data/test/nokogiri_test.rb

@@ -0,0 +1,35 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+# Test the flat text parsers
+class ParserTest < Test::Unit::TestCase
+
+  # Test the DOM-based Nokogiri XML parser. .
+  def test_nokogiri_xml_parser_for_all_nodes
+    control = ETL::Control::Control.resolve(
+      File.dirname(__FILE__) + '/nokogiri_all.ctl')
+    parser = ETL::Parser::NokogiriXmlParser.new(control.sources.first)
+    rows = parser.collect { |row| row }
+    assert_equal 3, rows.length
+    assert_equal(
+      { :hair_colour=>"black",
+        :first_name=>"Bob", 
+        :last_name=>"Smith", 
+        :ssn=>"123456789", :age=>"24"}, rows.first)
+  end
+
+  # Test the DOM-based Nokogiri XML parser. .
+  def test_nokogiri_xml_parser_for_selected_nodes
+    control = ETL::Control::Control.resolve(
+      File.dirname(__FILE__) + '/nokogiri_select.ctl')
+    parser = ETL::Parser::NokogiriXmlParser.new(control.sources.first)
+    rows = parser.collect { |row| row }
+    assert_equal 2, rows.length
+    assert_equal(
+      { :age=>"37",
+        :hair_colour=>"black",
+        :first_name=>"Jake",
+        :last_name=>"Smithsonian",
+        :ssn=>"133244566"}, rows.last)
+  end
+
+end

data/test/parser_test.rb

@@ -3,9 +3,9 @@ require File.dirname(__FILE__) + '/test_helper'
 # Test the flat text parsers
 class ParserTest < Test::Unit::TestCase
   # Test parsing delimited data
-  def test_delimited_parser
+  def test_csv_parser
     control = ETL::Control::Control.resolve(File.dirname(__FILE__) + '/delimited.ctl')
-    parser = ETL::Parser::DelimitedParser.new(control.sources.first)
+    parser = ETL::Parser::CsvParser.new(control.sources.first)
     rows = parser.collect { |row| row }
     assert_equal 3, rows.length
     assert_equal({:first_name=>"Chris", :last_name=>"Smith", :ssn=>"111223333", :age=>"24", :sex => 'M'}, rows.first)

data/test/performance/delimited.ctl

@@ -2,7 +2,7 @@
 
 source :in, {
   :file => 'delimited.txt',
-  :parser => :delimited
+  :parser => :csv
 }, 
 [ 
   :first_name,

data/test/processor_test.rb

@@ -1,8 +1,5 @@
 require File.dirname(__FILE__) + '/test_helper'
 
-class Person < ActiveRecord::Base
-end
-
 # Test pre- and post-processors
 class ProcessorTest < Test::Unit::TestCase
   # Test bulk import functionality

data/test/scd_test.rb

@@ -72,10 +72,7 @@ class ScdTest < Test::Unit::TestCase
           assert_equal 1, find_bobs.first.id
         end
         should "set the effective date" do
-          # doing comparison on strings, as comparison on objects
-          # doesn't consider things equal for some yet to be understood
-          # reason
-          assert_equal current_datetime.to_s, find_bobs.first.effective_date.to_s
+          assert_in_delta 1, current_datetime.to_time.to_i, find_bobs.first.effective_date.to_time.to_i
         end
         should "set the end date" do
           assert_equal @end_of_time, find_bobs.first.end_date
@@ -115,10 +112,7 @@ class ScdTest < Test::Unit::TestCase
           assert_los_angeles_address(find_bobs.detect { |bob| 2 == bob.id })
         end
         should "activate the new record" do
-          # doing comparison on strings, as comparison on objects
-          # doesn't consider things equal for some yet to be understood
-          # reason
-          assert_equal current_datetime.to_s, find_bobs.detect { |bob| 2 == bob.id }.effective_date.to_s
+          assert_in_delta 1, current_datetime.to_time.to_i, find_bobs.detect { |bob| 2 == bob.id }.effective_date.to_time.to_i
         end
         should "set the end date for the new record" do
           assert_equal @end_of_time, find_bobs.detect { |bob| 2 == bob.id }.end_date

data/test/scd_test_type_1.ctl

@@ -1,6 +1,6 @@
 source :in, {
   :file => "scd/#{ENV['run_number']}.txt",
-  :parser => :delimited
+  :parser => :csv
 },
 [
   :first_name,

data/test/scd_test_type_2.ctl

@@ -1,6 +1,6 @@
 source :in, {
   :file => "scd/#{ENV['run_number']}.txt",
-  :parser => :delimited
+  :parser => :csv
 },
 [
   :first_name,

data/test/screen_test.rb

@@ -2,8 +2,7 @@ require File.dirname(__FILE__) + '/test_helper'
 
 class ScreenTest < Test::Unit::TestCase
   def test_screen
-    assert_raises(SystemExit) do
-      ETL::Engine.process(File.dirname(__FILE__) + '/screen_test_fatal.ctl')
-    end
+    ETL::Engine.process(File.dirname(__FILE__) + '/screen_test_fatal.ctl')
+    assert_equal 2, ETL::Engine.exit_code
   end
 end

data/test/source_test.rb

@@ -1,15 +1,28 @@
 require File.dirname(__FILE__) + '/test_helper'
 
-class Person < ActiveRecord::Base
-end
 class SourceTest < Test::Unit::TestCase
+  
+  context "source" do
+    should "set store_locally to true by default" do
+      assert_equal true, Source.new(nil, { :store_locally => true }, nil).store_locally
+    end
+    
+    should "let the user set store_locally to true" do
+      assert_equal true, Source.new(nil, { :store_locally => true }, nil).store_locally
+    end
+
+    should "let the user set store_locally to false" do
+      assert_equal false, Source.new(nil, { :store_locally => false }, nil).store_locally
+    end
+  end
+  
   context "a file source" do
     context "with delimited data" do
       setup do
         control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
         configuration = {
           :file => 'data/delimited.txt',
-          :parser => :delimited
+          :parser => :csv
         }
         definition = self.definition + [:sex]
     
@@ -27,7 +40,7 @@ class SourceTest < Test::Unit::TestCase
       control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/multiple_delimited.ctl')
       configuration = {
         :file => 'data/multiple_delimited_*.txt',
-        :parser => :delimited
+        :parser => :csv
       }
 
       source = ETL::Control::FileSource.new(control, configuration, definition)
@@ -46,7 +59,7 @@ class SourceTest < Test::Unit::TestCase
         '/delimited_absolute.ctl')
       configuration = {
         :file => '/tmp/delimited_abs.txt',
-        :parser => :delimited
+        :parser => :csv
       }
       definition = self.definition + [:sex]
 
@@ -85,7 +98,7 @@ class SourceTest < Test::Unit::TestCase
       @source = ETL::Control::DatabaseSource.new(control, configuration, definition)
     end
     should "set the local file for extraction storage" do
-      assert_match %r{source_data/localhost/etl_unittest/people/\d+.csv}, @source.local_file.to_s
+      assert_match %r{source_data/localhost/activewarehouse_etl_test/people/\d+.csv}, @source.local_file.to_s
     end
     should "find 1 row" do
       Person.delete_all

data/test/test_helper.rb

@@ -1,24 +1,20 @@
-require 'rubygems'
-require 'bundler/setup'
-
 $:.unshift(File.dirname(__FILE__) + '/../lib')
 $:.unshift(File.dirname(__FILE__))
 
-Dir[File.dirname(__FILE__) + "/vendor/**/lib"].each { |lib| $:.unshift(lib) }
-
 require 'test/unit'
 require 'pp'
 require 'etl'
 require 'shoulda'
 require 'flexmock/test_unit'
 
-ETL::Engine.init(:config => File.dirname(__FILE__) + '/config/database.yml')
+raise "Missing required DB environment variable" unless ENV['DB']
+
+database_yml = File.dirname(__FILE__) + '/config/database.yml'
+ETL::Engine.init(:config => database_yml)
 ETL::Engine.logger = Logger.new(STDOUT)
 # ETL::Engine.logger.level = Logger::DEBUG
 ETL::Engine.logger.level = Logger::FATAL
 
-db = ENV['DB'] ||= 'mysql'
-require "connection/#{db}/connection"
 ActiveRecord::Base.establish_connection :operational_database
 ETL::Execution::Job.delete_all
 
@@ -32,3 +28,5 @@ end
 
 puts "ActiveRecord::VERSION = #{ActiveRecord::VERSION::STRING}"
 
+class Person < ActiveRecord::Base
+end

data/test/truncate_processor_test.rb

@@ -0,0 +1,37 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+include ETL::Processor
+
+class TruncateTest < ActiveRecord::Base
+  set_table_name 'truncate_test'
+end
+
+class TruncateProcessorTest < Test::Unit::TestCase
+
+  def create_item!
+    TruncateTest.create!(:x => 'ABC')
+  end
+  
+  def truncate!(options=nil)
+    TruncateProcessor.new(nil,
+      :target => :data_warehouse,
+      :table => TruncateTest.table_name,
+      :options => options
+      ).process
+  end
+  
+  should 'reset ids by default' do
+    create_item!
+    truncate!
+    assert_equal 1, create_item!.id
+  end
+
+  if ETL::Engine.connection(:data_warehouse).class.name =~ /postgres/i
+    should 'allow disabling id reset for postgres' do
+      truncate!
+      create_item!
+      truncate!('CONTINUE IDENTITY')
+      assert_equal 2, create_item!.id
+    end
+  end
+end

metadata

@@ -1,142 +1,130 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: activewarehouse-etl
-version: !ruby/object:Gem::Version 
-  hash: 15424175
+version: !ruby/object:Gem::Version
+  version: 1.0.0.rc1
   prerelease: 6
-  segments: 
-  - 0
-  - 9
-  - 5
-  - rc
-  - 1
-  version: 0.9.5.rc1
 platform: ruby
-authors: 
+authors:
 - Anthony Eden
-- "Thibaut Barr\xC3\xA8re"
+- Thibaut Barrère
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-06-10 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-03-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rake
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &70307704843740 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 57
-        segments: 
-        - 0
-        - 8
-        - 3
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 0.8.3
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: activesupport
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *70307704843740
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &70307704843000 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 2
-        - 1
-        - 0
-        version: 2.1.0
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: activerecord
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *70307704843000
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: &70307704842100 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 2
-        - 1
-        - 0
-        version: 2.1.0
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
   type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: fastercsv
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: *70307704842100
+- !ruby/object:Gem::Dependency
+  name: fastercsv
+  requirement: &70307704841200 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 31
-        segments: 
-        - 1
-        - 2
-        - 0
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.2.0
   type: :runtime
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: adapter_extensions
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  version_requirements: *70307704841200
+- !ruby/object:Gem::Dependency
+  name: adapter_extensions
+  requirement: &70307704840560 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 0
-        - 5
-        - 0
-        version: 0.5.0
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.5.rc1
   type: :runtime
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: spreadsheet
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  version_requirements: *70307704840560
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: &70307704839660 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  type: :runtime
-  version_requirements: *id006
-description: ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application for loading data into a database.
-email: 
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.11.3
+  type: :development
+  prerelease: false
+  version_requirements: *70307704839660
+- !ruby/object:Gem::Dependency
+  name: flexmock
+  requirement: &70307704839120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :development
+  prerelease: false
+  version_requirements: *70307704839120
+- !ruby/object:Gem::Dependency
+  name: cartesian
+  requirement: &70307704838600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70307704838600
+description: ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application
+  for loading data into a database.
+email:
 - thibaut.barrere@gmail.com
-executables: 
+executables:
 - etl
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
+- .standalone_migrations
+- .travis.yml
 - 0.9-UPGRADE
 - CHANGELOG
 - Gemfile
 - HOW_TO_RELEASE
 - LICENSE
-- README
+- README.textile
 - Rakefile
 - TODO
-- active_support_logger.patch
 - activewarehouse-etl.gemspec
 - bin/etl
 - bin/etl.cmd
+- db/migrate/20120229203554_create_tables.rb
+- db/schema.rb
 - examples/database.example.yml
 - lib/etl.rb
 - lib/etl/batch.rb
@@ -149,16 +137,19 @@ files:
 - lib/etl/control.rb
 - lib/etl/control/control.rb
 - lib/etl/control/destination.rb
+- lib/etl/control/destination/csv_destination.rb
 - lib/etl/control/destination/database_destination.rb
 - lib/etl/control/destination/excel_destination.rb
 - lib/etl/control/destination/file_destination.rb
 - lib/etl/control/destination/insert_update_database_destination.rb
 - lib/etl/control/destination/update_database_destination.rb
+- lib/etl/control/destination/yaml_destination.rb
 - lib/etl/control/source.rb
 - lib/etl/control/source/database_source.rb
 - lib/etl/control/source/enumerable_source.rb
 - lib/etl/control/source/file_source.rb
 - lib/etl/control/source/model_source.rb
+- lib/etl/control/source/mysql_streamer.rb
 - lib/etl/core_ext.rb
 - lib/etl/core_ext/time.rb
 - lib/etl/core_ext/time/calculations.rb
@@ -174,9 +165,10 @@ files:
 - lib/etl/http_tools.rb
 - lib/etl/parser.rb
 - lib/etl/parser/apache_combined_log_parser.rb
-- lib/etl/parser/delimited_parser.rb
+- lib/etl/parser/csv_parser.rb
 - lib/etl/parser/excel_parser.rb
 - lib/etl/parser/fixed_width_parser.rb
+- lib/etl/parser/nokogiri_xml_parser.rb
 - lib/etl/parser/parser.rb
 - lib/etl/parser/sax_parser.rb
 - lib/etl/parser/xml_parser.rb
@@ -188,6 +180,7 @@ files:
 - lib/etl/processor/copy_field_processor.rb
 - lib/etl/processor/database_join_processor.rb
 - lib/etl/processor/encode_processor.rb
+- lib/etl/processor/ensure_fields_presence_processor.rb
 - lib/etl/processor/escape_csv_processor.rb
 - lib/etl/processor/filter_row_processor.rb
 - lib/etl/processor/ftp_downloader_processor.rb
@@ -217,18 +210,18 @@ files:
 - lib/etl/transform/default_transform.rb
 - lib/etl/transform/foreign_key_lookup_transform.rb
 - lib/etl/transform/hierarchy_lookup_transform.rb
+- lib/etl/transform/md5_transform.rb
 - lib/etl/transform/ordinalize_transform.rb
 - lib/etl/transform/sha1_transform.rb
 - lib/etl/transform/split_fields_transform.rb
+- lib/etl/transform/string_to_date_time_transform.rb
 - lib/etl/transform/string_to_date_transform.rb
-- lib/etl/transform/string_to_datetime_transform.rb
 - lib/etl/transform/string_to_time_transform.rb
 - lib/etl/transform/transform.rb
 - lib/etl/transform/trim_transform.rb
 - lib/etl/transform/type_transform.rb
 - lib/etl/util.rb
 - lib/etl/version.rb
-- test-matrix.yml
 - test/.gitignore
 - test/.ignore
 - test/all.ebf
@@ -242,16 +235,14 @@ files:
 - test/block_processor_pre_post_process.ctl
 - test/block_processor_remove_rows.ctl
 - test/block_processor_test.rb
-- test/config/Gemfile.rails-2.3.x
-- test/config/Gemfile.rails-2.3.x.lock
-- test/config/Gemfile.rails-3.0.x
-- test/config/Gemfile.rails-3.0.x.lock
-- test/config/common.rb
-- test/config/database.example.yml
-- test/connection/mysql/connection.rb
-- test/connection/mysql/schema.sql
-- test/connection/postgresql/connection.rb
-- test/connection/postgresql/schema.sql
+- test/check_exist_processor_test.rb
+- test/check_unique_processor_test.rb
+- test/config/.gitignore
+- test/config/database.yml
+- test/config/gemfiles/Gemfile.rails-3.0.x
+- test/config/gemfiles/Gemfile.rails-3.1.x
+- test/config/gemfiles/Gemfile.rails-3.2.x
+- test/config/gemfiles/common.rb
 - test/control_test.rb
 - test/data/apache_combined_log.txt
 - test/data/bulk_import.txt
@@ -264,9 +255,11 @@ files:
 - test/data/fixed_width.txt
 - test/data/multiple_delimited_1.txt
 - test/data/multiple_delimited_2.txt
+- test/data/nokogiri.xml
 - test/data/people.txt
 - test/data/sax.xml
 - test/data/xml.xml
+- test/database_join_processor_test.rb
 - test/date_dimension_builder_test.rb
 - test/delimited.ctl
 - test/delimited_absolute.ctl
@@ -279,11 +272,13 @@ files:
 - test/directive_test.rb
 - test/encode_processor_test.rb
 - test/engine_test.rb
+- test/ensure_fields_presence_processor_test.rb
 - test/errors.ctl
 - test/etl_test.rb
 - test/excel.ctl
 - test/excel2.ctl
 - test/fixed_width.ctl
+- test/foreign_key_lookup_transform_test.rb
 - test/generator_test.rb
 - test/inline_parser.ctl
 - test/mocks/mock_destination.rb
@@ -291,6 +286,9 @@ files:
 - test/model_source.ctl
 - test/multiple_delimited.ctl
 - test/multiple_source_delimited.ctl
+- test/nokogiri_all.ctl
+- test/nokogiri_select.ctl
+- test/nokogiri_test.rb
 - test/output/.ignore
 - test/parser_test.rb
 - test/performance/delimited.ctl
@@ -309,51 +307,30 @@ files:
 - test/source_test.rb
 - test/test_helper.rb
 - test/transform_test.rb
-- test/vendor/adapter_extensions-0.5.0/CHANGELOG
-- test/vendor/adapter_extensions-0.5.0/LICENSE
-- test/vendor/adapter_extensions-0.5.0/README
-- test/vendor/adapter_extensions-0.5.0/Rakefile
-- test/vendor/adapter_extensions-0.5.0/lib/adapter_extensions.rb
-- test/vendor/adapter_extensions-0.5.0/lib/adapter_extensions/connection_adapters/abstract_adapter.rb
-- test/vendor/adapter_extensions-0.5.0/lib/adapter_extensions/connection_adapters/mysql_adapter.rb
-- test/vendor/adapter_extensions-0.5.0/lib/adapter_extensions/connection_adapters/postgresql_adapter.rb
-- test/vendor/adapter_extensions-0.5.0/lib/adapter_extensions/connection_adapters/sqlserver_adapter.rb
-- test/vendor/adapter_extensions-0.5.0/lib/adapter_extensions/version.rb
+- test/truncate_processor_test.rb
 - test/xml.ctl
 homepage: https://github.com/activewarehouse/activewarehouse-etl
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 23
-      segments: 
-      - 1
-      - 3
-      - 6
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
       version: 1.3.6
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.5
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
 summary: Pure Ruby ETL package.
 test_files: []
-