activewarehouse-etl 0.9.1 → 0.9.5.rc1

data/test/screen_test.rb

@@ -0,0 +1,9 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class ScreenTest < Test::Unit::TestCase
+  def test_screen
+    assert_raises(SystemExit) do
+      ETL::Engine.process(File.dirname(__FILE__) + '/screen_test_fatal.ctl')
+    end
+  end
+end

data/test/screen_test_error.ctl

@@ -0,0 +1,3 @@
+screen(:error){
+  ETL::Screen::RowCountScreen.new(self, :rows => 1)
+}

data/test/screen_test_fatal.ctl

@@ -0,0 +1,3 @@
+screen(:fatal){
+  ETL::Screen::RowCountScreen.new(self, :rows => 1)
+}

data/test/source_test.rb

@@ -0,0 +1,139 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class Person < ActiveRecord::Base
+end
+class SourceTest < Test::Unit::TestCase
+  context "a file source" do
+    context "with delimited data" do
+      setup do
+        control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+        configuration = {
+          :file => 'data/delimited.txt',
+          :parser => :delimited
+        }
+        definition = self.definition + [:sex]
+    
+        source = ETL::Control::FileSource.new(control, configuration, definition)
+        @rows = source.collect { |row| row }
+      end
+      should "find 3 rows in the delimited file" do
+        assert_equal 3, @rows.length
+      end
+    end
+  end
+  
+  context "a file source with a glob" do
+    setup do
+      control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/multiple_delimited.ctl')
+      configuration = {
+        :file => 'data/multiple_delimited_*.txt',
+        :parser => :delimited
+      }
+
+      source = ETL::Control::FileSource.new(control, configuration, definition)
+      @rows = source.collect { |row| row }
+    end
+    should "find 6 rows in total" do
+      assert_equal 6, @rows.length
+    end
+  end
+  
+  context "a file source with an absolute path" do
+    setup do
+      FileUtils.cp(File.dirname(__FILE__) + '/data/delimited.txt', '/tmp/delimited_abs.txt')
+
+      control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+        '/delimited_absolute.ctl')
+      configuration = {
+        :file => '/tmp/delimited_abs.txt',
+        :parser => :delimited
+      }
+      definition = self.definition + [:sex]
+
+      source = ETL::Control::FileSource.new(control, configuration, definition)
+      @rows = source.collect { |row| row }
+    end
+    should "find 3 rows" do
+      assert_equal 3, @rows.length
+    end
+  end
+  
+  context "multiple sources" do
+    setup do
+      control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+        '/multiple_source_delimited.ctl')
+      @rows = control.sources.collect { |source| source.collect { |row| row }}.flatten!
+    end
+    should "find 12 rows" do
+      assert_equal 12, @rows.length
+    end
+  end
+  
+  context "a database source" do
+    setup do
+      control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+      configuration = {
+        :database => 'etl_unittest',
+        :target => :operational_database,
+        :table => 'people',
+      }
+      definition = [ 
+        :first_name,
+        :last_name,
+        :ssn,
+      ]
+      @source = ETL::Control::DatabaseSource.new(control, configuration, definition)
+    end
+    should "set the local file for extraction storage" do
+      assert_match %r{source_data/localhost/etl_unittest/people/\d+.csv}, @source.local_file.to_s
+    end
+    should "find 1 row" do
+      Person.delete_all
+      assert_equal 0, Person.count
+      Person.create!(:first_name => 'Bob', :last_name => 'Smith', :ssn => '123456789')
+      assert_equal 1, Person.count
+      rows = @source.collect { |row| row }
+      assert_equal 1, rows.length
+    end
+  end
+  
+  context "a file source with an xml parser" do
+    setup do
+      control = ETL::Control::Control.parse(File.dirname(__FILE__) + 
+        '/xml.ctl')
+      @rows = control.sources.collect{ |source| source.collect { |row| row }}.flatten!
+    end
+    should "find 2 rows" do
+      assert_equal 2, @rows.length
+    end
+  end
+
+  context "a model source" do
+    setup do
+      control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/model_source.ctl')
+      configuration = {
+
+      }
+      definition = [
+        :first_name,
+        :last_name,
+        :ssn
+      ]
+    end
+    should_eventually "find n rows" do
+      
+    end
+  end
+  
+  def definition
+    [ 
+      :first_name,
+      :last_name,
+      :ssn,
+      {
+        :name => :age,
+        :type => :integer
+      }
+    ]
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,34 @@
+require 'rubygems'
+require 'bundler/setup'
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+$:.unshift(File.dirname(__FILE__))
+
+Dir[File.dirname(__FILE__) + "/vendor/**/lib"].each { |lib| $:.unshift(lib) }
+
+require 'test/unit'
+require 'pp'
+require 'etl'
+require 'shoulda'
+require 'flexmock/test_unit'
+
+ETL::Engine.init(:config => File.dirname(__FILE__) + '/config/database.yml')
+ETL::Engine.logger = Logger.new(STDOUT)
+# ETL::Engine.logger.level = Logger::DEBUG
+ETL::Engine.logger.level = Logger::FATAL
+
+db = ENV['DB'] ||= 'mysql'
+require "connection/#{db}/connection"
+ActiveRecord::Base.establish_connection :operational_database
+ETL::Execution::Job.delete_all
+
+require 'mocks/mock_source'
+require 'mocks/mock_destination'
+
+# shortcut to launch a ctl file
+def process(file)
+  Engine.process(File.join(File.dirname(__FILE__), file))
+end
+
+puts "ActiveRecord::VERSION = #{ActiveRecord::VERSION::STRING}"
+

data/test/transform_test.rb

@@ -0,0 +1,101 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class MyResolver
+  def resolve(value)
+    4
+  end
+end
+
+class TransformTest < Test::Unit::TestCase
+  def test_sha1_transform
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+    digest_value = 'a9993e364706816aba3e25717850c26c9cd0d89d'
+    assert_equal digest_value, ETL::Transform::Sha1Transform.new(
+      control, nil
+    ).transform('test', 'abc', [])
+  end
+  def test_block_transform
+    #transforms = [Proc.new(){|name, value, row| value[0,2]}]
+    #assert_equal '11', ETL::Transform::Transform.transform(:ssn, '1111223333', [], transforms)
+  end
+  def test_decode_transform
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+    configuration = {:decode_table_path => 'data/decode.txt'}
+    
+    t = ETL::Transform::DecodeTransform.new(control, nil, configuration)
+    
+    assert_equal 'Male', t.transform(nil, 'M', [])
+    assert_equal 'Female', t.transform(nil, 'F', [])
+    assert_equal 'Unknown', t.transform(nil, '', [])
+    assert_equal 'Unknown', t.transform(nil, 'blah', [])
+  end
+  def test_string_to_date_transform
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+    t = ETL::Transform::StringToDateTransform.new(control, nil)
+    
+    assert_equal Date.parse('2005-01-01'), t.transform(nil, '2005-01-01', [])
+    assert_equal Date.parse('2004-10-20 20:30:00'), t.transform(nil, '2004-10-20', [])
+  end
+  def test_date_to_string_transform
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+    t = ETL::Transform::DateToStringTransform.new(control, nil)
+    
+    d1 = Date.parse('2005-01-01')
+    t1 = Time.parse('2004-10-20 23:03:23')
+    assert_equal '2005-01-01', t.transform(nil, d1, [])
+    assert_equal '2004-10-20', t.transform(nil, t1, [])
+    
+    t = ETL::Transform::DateToStringTransform.new(control, nil, {:format => '%m/%d/%Y'})
+    
+    assert_equal '01/01/2005', t.transform(nil, d1, [])
+    assert_equal '10/20/2004', t.transform(nil, t1, [])
+  end
+  def test_string_to_datetime_transform
+    v = '1/1/1900 04:34:30'
+    t = ETL::Transform::StringToDateTimeTransform.new(flexmock(:control), nil)
+    assert_equal DateTime.parse(v), t.transform(nil, v, nil)
+  end
+  def test_foreign_key_lookup_transform
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+    configuration = {:collection => {'foo' => 1, 'bar' => 2, 'baz' => 3}}
+    t = ETL::Transform::ForeignKeyLookupTransform.new(control, nil, configuration)
+    
+    assert_equal 1, t.transform(nil, 'foo', nil)
+    assert_equal 2, t.transform(nil, 'bar', nil)
+    assert_equal 3, t.transform(nil, 'baz', nil)
+    assert_raises(ETL::ResolverError, 'Foreign key for bing not found and no resolver specified') do
+      assert_equal 4, t.transform(nil, 'bing', nil)
+    end
+    
+    configuration = {:collection => {'foo' => 1, 'bar' => 2, 'baz' => 3}, :resolver => MyResolver}
+    t = ETL::Transform::ForeignKeyLookupTransform.new(control, nil, configuration)
+    assert_equal 1, t.transform(nil, 'foo', nil)
+    assert_equal 2, t.transform(nil, 'bar', nil)
+    assert_equal 3, t.transform(nil, 'baz', nil)
+    assert_equal 4, t.transform(nil, 'bing', nil)
+  end
+  def test_type_transform
+    control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
+    assert_equal 10, ETL::Transform::TypeTransform.new(control, nil, {:type => :number}).transform(nil, '10', nil)
+    
+    assert_equal BigDecimal::ROUND_HALF_UP, BigDecimal.mode(BigDecimal::ROUND_MODE)
+    decimal_transformed = ETL::Transform::TypeTransform.new(
+      control, nil, {:type => :decimal, :scale => 4}
+    ).transform(nil, '10.0000000000000000000000000000000001', nil)
+    assert_equal '10.0000000000000000000000000000000001', decimal_transformed.to_s('F')
+  end
+  def test_non_existent_transformer
+    
+  end
+  def test_default_transform
+    t = ETL::Transform::DefaultTransform.new(flexmock('control'), nil, {:default_value => 'foo'})
+    assert_equal 'foo', t.transform(nil, '', nil)
+    assert_equal 'foo', t.transform(nil, nil, nil)
+    assert_equal 'bar', t.transform(nil, 'bar', nil)
+  end
+  def test_ordinalize_transform
+    t = ETL::Transform::OrdinalizeTransform.new(flexmock('control'), nil, {})
+    assert_equal '1st', t.transform(nil, 1, nil)
+    assert_equal '10th', t.transform(nil, 10, nil)
+  end
+end

data/test/vendor/adapter_extensions-0.5.0/CHANGELOG

@@ -0,0 +1,26 @@
+0.1.0 - March 5, 2007
+* Initial release
+
+0.1.1 - March 5, 2007
+* Bug fixes
+
+0.1.2 - March 5, 2007
+* Bug fixes
+
+0.2.0 - March 6, 2007
+* SQL Server adapter included (Seth Ladd)
+
+0.3.0 - March 8, 2007
+* PostgreSQL adapter included
+* Added tests for bulk loading
+* bulk_load method now handles table missing and file missing as error cases
+
+0.3.1 - May 4, 2007
+* Added support for modifying SELECT statements to add an INSERT INTO.
+
+0.4 - September 17, 2007
+* Added copy_table method that can copy the structure and data from one table to another. Currently implemented in MySQL (tested), PostgreSQL (tested) and SQL Server adapters (untested).
+* Added support for SELECT..INTO for PostgreSQL.
+
+0.5 - 
+* Updated dependencies for gem to current versions of ActiveRecord, ActiveSupport and Rake. May not be compatible with Rails versions less than 2.x.

data/test/vendor/adapter_extensions-0.5.0/LICENSE

@@ -0,0 +1,16 @@
+Copyright (c) 2007 Anthony Eden
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and 
+associated documentation files (the "Software"), to deal in the Software without restriction, including 
+without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 
+copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the 
+following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial 
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT 
+LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN 
+NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/test/vendor/adapter_extensions-0.5.0/README

@@ -0,0 +1,7 @@
+This library provides extensions to Rails' ActiveRecord adapters.
+
+As of version 0.5, adapter_extensions has dependencies on ActiveSupport and ActiveRecord 2.1.x or higher.
+
+To use the MySQL adapter extensions with Rails 2.x, you must patch the mysql_adapter with the mysql_adapter_opt_local_infile.patch.
+
+To execute the unit tests you must first construct a adapter_extensions_unittest database.

data/test/vendor/adapter_extensions-0.5.0/Rakefile

@@ -0,0 +1,158 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+require 'rake/contrib/rubyforgepublisher'
+require 'date'
+
+require File.join(File.dirname(__FILE__), 'lib/adapter_extensions', 'version')
+
+PKG_BUILD       = ENV['PKG_BUILD'] ? '.' + ENV['PKG_BUILD'] : ''
+PKG_NAME        = 'adapter_extensions'
+PKG_VERSION     = AdapterExtensions::VERSION::STRING + PKG_BUILD
+PKG_FILE_NAME   = "#{PKG_NAME}-#{PKG_VERSION}"
+PKG_DESTINATION = ENV["PKG_DESTINATION"] || "../#{PKG_NAME}"
+
+RELEASE_NAME  = "REL #{PKG_VERSION}"
+
+RUBY_FORGE_PROJECT = "activewarehouse"
+RUBY_FORGE_USER    = "aeden"
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the ETL application.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+  # TODO: reset the database
+end
+
+namespace :rcov do
+  desc 'Measures test coverage'
+  task :test do
+    rm_f 'coverage.data'
+    mkdir 'coverage' unless File.exist?('coverage')
+    rcov = "rcov --aggregate coverage.data --text-summary -Ilib"
+    system("#{rcov} test/*_test.rb test/**/*_test.rb")
+    system("open coverage/index.html") if PLATFORM['darwin']
+  end
+end
+
+desc 'Generate documentation for the AdapterExtensions library.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Extensions for Rails adapters'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+PKG_FILES = FileList[
+  'CHANGELOG',
+  'README',
+  'LICENSE',
+  'Rakefile',
+  'doc/**/*',
+  'lib/**/*',
+] - [ 'test' ]
+
+spec = Gem::Specification.new do |s|
+  s.name = 'adapter_extensions'
+  s.version = PKG_VERSION
+  s.summary = "Extensions to Rails ActiveRecord adapters."
+  s.description = <<-EOF
+    Provides various extensions to the Rails ActiveRecord adapters.
+  EOF
+
+  s.add_dependency('rake', '>= 0.8.3')
+  s.add_dependency('activesupport',    '>= 2.1.0')
+  s.add_dependency('activerecord',     '>= 2.1.0')
+  s.add_dependency('fastercsv',        '>= 1.0.0')
+
+  s.rdoc_options << '--exclude' << '.'
+  s.has_rdoc = false
+
+  s.files = PKG_FILES.to_a.delete_if {|f| f.include?('.svn')}
+  s.require_path = 'lib'
+
+  s.author = "Anthony Eden"
+  s.email = "anthonyeden@gmail.com"
+  s.homepage = "http://activewarehouse.rubyforge.org/adapter_extensions"
+  s.rubyforge_project = "activewarehouse"
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+  pkg.need_tar = true
+  pkg.need_zip = true
+end
+
+namespace :github do
+  desc "Update Github Gemspec"
+  task :update_gemspec do
+    File.open(File.join(File.dirname(__FILE__), "#{spec.name}.gemspec"), "w"){|f| f << spec.to_ruby}
+  end
+end
+
+
+
+desc "Generate code statistics"
+task :lines do
+  lines, codelines, total_lines, total_codelines = 0, 0, 0, 0
+
+  for file_name in FileList["lib/**/*.rb"]
+    next if file_name =~ /vendor/
+    f = File.open(file_name)
+
+    while line = f.gets
+      lines += 1
+      next if line =~ /^\s*$/
+      next if line =~ /^\s*#/
+      codelines += 1
+    end
+    puts "L: #{sprintf("%4d", lines)}, LOC #{sprintf("%4d", codelines)} | #{file_name}"
+    
+    total_lines     += lines
+    total_codelines += codelines
+    
+    lines, codelines = 0, 0
+  end
+
+  puts "Total: Lines #{total_lines}, LOC #{total_codelines}"
+end
+
+desc "Publish the release files to RubyForge."
+task :release => [ :package ] do
+  `rubyforge login`
+
+  for ext in %w( gem tgz zip )
+    release_command = "rubyforge add_release activewarehouse #{PKG_NAME} 'REL #{PKG_VERSION}' pkg/#{PKG_NAME}-#{PKG_VERSION}.#{ext}"
+    puts release_command
+    system(release_command)
+  end
+end
+
+desc "Publish the API documentation"
+task :pdoc => [:rdoc] do 
+  Rake::SshDirPublisher.new("aeden@rubyforge.org", "/var/www/gforge-projects/activewarehouse/adapter_extensions/rdoc", "rdoc").upload
+end
+
+desc "Install the gem from a local generated package"
+task :install => [:package] do
+  windows = RUBY_PLATFORM =~ /mswin/
+  sudo = windows ? '' : 'sudo'
+  gem = windows ? 'gem.bat' : 'gem'
+  `#{sudo} #{gem} install pkg/#{PKG_NAME}-#{PKG_VERSION}`
+end
+
+desc "Reinstall the gem from a local package copy"
+task :reinstall => [:package] do
+  windows = RUBY_PLATFORM =~ /mswin/
+  sudo = windows ? '' : 'sudo'
+  gem = windows ? 'gem.bat' : 'gem'
+  `#{sudo} #{gem} uninstall #{PKG_NAME} -x`
+  `#{sudo} #{gem} install pkg/#{PKG_NAME}-#{PKG_VERSION}`
+end