activewarehouse-etl 1.0.0.rc1 → 1.0.0

data/.gitignore

@@ -4,4 +4,5 @@ test/output/*
 rdoc
 .rvmrc
 .bundle
-*.gem
+*.gem
+*.lock

data/CHANGELOG

@@ -1,4 +1,9 @@
-0.9.5 - unreleased
+1.0.0
+* Fixes for Rails 3.2 (pgericson)
+* Allow .ctl.rb and .ebf.rb extensions for easier syntax coloring (chrisgogreen)
+* Support for limit/offset in database source (kennym)
+* Support for error handlers in etl: can be declared with on_error { |error| } (thbar)
+* Support for mysql streaming for better performance on large data sets - use :mysqlstream => true on database source (main code by pdodds, tests and fixes by sgrgic and thbar)
 * EnsureFieldsPresenceProcessor treat symbols and strings equally (thbar)
 * BREAKING CHANGE: you must require 'iconv' if you use the :encode_processor from now on (thbar)
 * A non-zero exit code should be returned on fatal screen or exceeded error threshold (thbar)

data/Guardfile

@@ -0,0 +1,6 @@
+db = ENV['DB'] || 'mysql2'
+gemfile = ENV['GEMFILE'] || 'test/config/gemfiles/Gemfile.rails-3.2.x'
+
+guard 'shell' do
+  watch(/(lib|test)\/\.*/) {|m| `bundle exec rake ci:run_one[#{db},#{gemfile}]` }
+end

data/README.textile

@@ -55,7 +55,7 @@ ActiveWarehouse-ETL is the work of many people since late 2006 - here is a list,
 * Jeremy Lecour
 * Steve Meyfroidt
 * Seth Ladd
-* Thibaut Barrère
+* "Thibaut Barrère":https://github.com/thbar
 * Stephen Touset
 * sasikumargn
 * Andrew Kuklewicz
@@ -64,6 +64,11 @@ ActiveWarehouse-ETL is the work of many people since late 2006 - here is a list,
 * Tyler Kiley
 * Colman Nady
 * Scott Gonyea
+* "Philip Dodds":https://github.com/pdodds
+* "Sinisa Grgic":https://github.com/sgrgic
+* "Kenny Meyer":https://github.com/kennym
+* "Chris":https://github.com/chrisgogreen
+* "Peter Glerup Ericson":https://github.com/pgericson
 
 If your name should be on the list but isn't, please leave a comment!
 

data/activewarehouse-etl.gemspec

@@ -25,6 +25,8 @@ Gem::Specification.new do |s|
   s.add_development_dependency('shoulda', '~>2.11.3')
   s.add_development_dependency('flexmock', '~>0.9.0')
   s.add_development_dependency('cartesian')
+  s.add_development_dependency('guard')
+  s.add_development_dependency('guard-shell')
 
   s.files              = `git ls-files`.split("\n")
   s.test_files         = `git ls-files -- {test}/*`.split("\n")

data/lib/etl/control/control.rb

@@ -41,7 +41,12 @@ module ETL #:nodoc:
       def dependencies
         control.dependencies
       end
-      
+
+      # Register an error handler
+      def on_error(&block)
+        control.error_handlers << block
+      end
+
       # Define a source.
       def source(name, configuration={}, definition={})
         if configuration[:type]
@@ -376,6 +381,11 @@ module ETL #:nodoc:
           :warn => []
         }
       end
+
+      # A array of Procs to be invoked for errors notifications
+      def error_handlers
+        @error_handlers ||= []
+      end
       
       # Get the error threshold. Defaults to 100.
       def error_threshold

data/lib/etl/control/source/database_source.rb

@@ -1,6 +1,8 @@
 require 'fileutils'
 
 module ETL #:nodoc:
+  class NoLimitSpecifiedError < StandardError; end
+  
   class Source < ::ActiveRecord::Base #:nodoc:
     # Connection for database sources
   end
@@ -189,12 +191,13 @@ module ETL #:nodoc:
         
         q << " GROUP BY #{group}" if group
         q << " ORDER BY #{order}" if order
-
-        if ETL::Engine.limit || ETL::Engine.offset
-          options = {}
-          options[:limit] = ETL::Engine.limit if ETL::Engine.limit
-          options[:offset] = ETL::Engine.offset if ETL::Engine.offset
-          connection.add_limit_offset!(q, options)
+        
+        limit = ETL::Engine.limit
+        offset = ETL::Engine.offset
+        if limit || offset
+          raise NoLimitSpecifiedError, "Specifying offset without limit is not allowed" if offset and limit.nil?
+          q << " LIMIT #{limit}"
+          q << " OFFSET #{offset}" if offset
         end
         
         q = q.gsub(/\n/,' ')
@@ -205,7 +208,7 @@ module ETL #:nodoc:
       def query_rows
         return @query_rows if @query_rows
         if (configuration[:mysqlstream] == true)
-          MySqlStreamer.new(query,@target)
+          MySqlStreamer.new(query,@target,connection)
         else
           connection.select_all(query)
         end

data/lib/etl/control/source/mysql_streamer.rb

@@ -1,31 +1,73 @@
 require 'open3'
 
+# Internal: The MySQL streamer is a helper with works with the database_source
+#           in order to allow you to use the --quick option (which stops MySQL)
+#           from building a full result set,  also we don't build a full resultset
+#           in Ruby - instead we yield a row at a time
+#
 class MySqlStreamer
+  # Internal: Creates a MySQL Streamer
+  #
+  # query - the SQL query
+  # target - the name of the ETL configuration (ie. development/production)
+  # connection - the ActiveRecord connection
+  #
+  # Examples
+  #
+  #   MySqlStreamer.new("select * from bob", "development", my_connection)
+  #
+  def initialize(query, target, connection)
+    # Lets just be safe and also make sure there aren't new lines
+    # in the SQL - its bound to cause trouble
+    @query = query.split.join(' ')
+    @name = target
+    @first_row = connection.select_all("#{query} LIMIT 1")
+  end
 
-	def initialize(query, target)
-		@query = query
-		@name = target
-	end
+  # We implement some bits of a hash so that database_source
+  # can use them
+  def any?
+    @first_row.any?
+  end 
 
-	def each
-		puts "Using the Streaming MySQL from the command line"
-		keys = nil
-		connection_configuration = ETL::Base.configurations[@name.to_s]
-		mysql_command = """mysql --quick -h #{connection_configuration["host"]} -u #{connection_configuration["username"]} -e \"#{@query.gsub("\n","")}\" -D #{connection_configuration["database"]} --password=#{connection_configuration["password"]} -B"""
-		Open3.popen3(mysql_command) do |stdin, out, err, external|
-			until (line = out.gets).nil? do
-				line = line.gsub("\n","")
-				if keys.nil?
-					keys = line.split("\t")
-				else
-					hash = Hash[keys.zip(line.split("\t"))]
-					yield hash
-				end
-			end
-			error = err.gets
-			if (!error.nil? && error.strip.length > 0)
-				throw error
-			end
-		end
-	end
+  def first
+    @first_row.first
+  end
+
+  def mandatory_option!(hash, key)
+    value = hash[key]
+    raise "Missing key #{key} in connection configuration #{@name}" if value.blank?
+    value
+  end
+
+  def each
+    keys = nil
+
+    config = ETL::Base.configurations[@name.to_s]
+    host = mandatory_option!(config, 'host')
+    username = mandatory_option!(config, 'username')
+    database = mandatory_option!(config, 'database')
+    password = config['password'] # this one can omitted in some cases
+
+    mysql_command = """mysql --quick -h #{host} -u #{username} -e \"#{@query.gsub("\n","")}\" -D #{database} --password=#{password} -B"""
+    Open3.popen3(mysql_command) do |stdin, out, err, external|
+      until (line = out.gets).nil? do
+        line = line.gsub("\n","")
+        if keys.nil?
+          keys = line.split("\t")
+        else
+          hash = Hash[keys.zip(line.split("\t"))]
+          # map out NULL to nil
+          hash.each do |k, v|
+            hash[k] = nil if v == 'NULL'
+          end
+          yield hash
+        end
+      end
+      error = err.gets
+      if (!error.nil? && error.strip.length > 0)
+        throw error
+      end
+    end
+  end
 end

data/lib/etl/engine.rb

@@ -49,8 +49,8 @@ module ETL #:nodoc:
       # * ETL::Control::Control instance
       # * ETL::Batch::Batch instance
       #
-      # The process command will accept either a .ctl Control file or a .ebf
-      # ETL Batch File.
+      # The process command will accept either a .ctl or .ctl.rb for a Control file or a .ebf
+      #  or .ebf.rb for an ETL Batch File.
       def process(file)
         new().process(file)
       end
@@ -235,6 +235,14 @@ module ETL #:nodoc:
     def errors
       @errors ||= []
     end
+
+    # First attempt at centralizing error notifications
+    def track_error(control, msg)
+      errors << msg
+      control.error_handlers.each do |handler|
+        handler.call(msg)
+      end
+    end
     
     # Get a Hash of benchmark values where each value represents the total
     # amount of time in seconds spent processing in that portion of the ETL
@@ -264,8 +272,8 @@ module ETL #:nodoc:
           process(File.new(file))
         when File
           case file.path
-            when /.ctl$/; process_control(file)
-            when /.ebf$/; process_batch(file)
+            when /\.ctl(\.rb)?$/; process_control(file)
+            when /\.ebf(\.rb)?$/; process_batch(file)
             else raise RuntimeError, "Unsupported file type - #{file.path}"
           end
         when ETL::Control::Control
@@ -352,7 +360,8 @@ module ETL #:nodoc:
               end
             rescue => e
               msg = "Error processing rows after read from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
-              errors << msg
+              # TODO - track more information: row if possible, full exception...
+              track_error(control, msg)
               Engine.logger.error(msg)
               e.backtrace.each { |line| Engine.logger.error(line) }
               exceeded_error_threshold?(control) ? break : next
@@ -374,10 +383,10 @@ module ETL #:nodoc:
               end
             rescue ResolverError => e
               Engine.logger.error(e.message)
-              errors << e.message
+              track_error(control, e.message)
             rescue => e
               msg = "Error transforming from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
-              errors << msg
+              track_error(control, msg)
               Engine.logger.error(msg)
               e.backtrace.each { |line| Engine.logger.error(line) }
             ensure
@@ -403,7 +412,7 @@ module ETL #:nodoc:
               end
             rescue => e
               msg = "Error processing rows before write from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
-              errors << msg
+              track_error(control, msg)
               Engine.logger.error(msg)
               e.backtrace.each { |line| Engine.logger.error(line) }
               exceeded_error_threshold?(control) ? break : next
@@ -423,7 +432,7 @@ module ETL #:nodoc:
               end
             rescue => e
               msg = "Error writing to #{Engine.current_destination}: #{e}"
-              errors << msg
+              track_error(control, msg)
               Engine.logger.error msg
               e.backtrace.each { |line| Engine.logger.error(line) }
               exceeded_error_threshold?(control) ? break : next

data/lib/etl/execution/batch.rb

@@ -5,6 +5,7 @@ module ETL #:nodoc:
       belongs_to :batch
       has_many :batches
       has_many :jobs
+      attr_accessible :batch_file, :status, :completed_at
     end
   end
 end

data/lib/etl/execution/job.rb

@@ -3,6 +3,7 @@ module ETL #:nodoc:
     # Persistent class representing an ETL job
     class Job < Base
       belongs_to :batch
+      attr_accessible :control_file, :status, :batch_id
     end
   end
-end
+end

data/lib/etl/version.rb

@@ -1,3 +1,3 @@
 module ETL#:nodoc:
-  VERSION = "1.0.0.rc1"
+  VERSION = "1.0.0"
 end

data/test/config/database.yml

@@ -3,6 +3,7 @@
 # a bit hackish - tests would require a refactoring instead
 
 mysql2: &mysql2
+  host: localhost
   adapter: mysql2
   database: activewarehouse_etl_test
   username: root

data/test/config/gemfiles/Gemfile.rails-3.2.x

@@ -1,3 +1,3 @@
 require File.dirname(__FILE__) + '/common'
 
-declare_gems '3.2.1'
+declare_gems '3.2.3'

data/test/config/gemfiles/common.rb

@@ -2,7 +2,7 @@ def declare_gems(activerecord_version)
   source :rubygems
 
   gem 'activerecord', activerecord_version
-  gem 'adapter_extensions', :git => 'https://github.com/activewarehouse/adapter_extensions.git', :branch => 'rails-3'
+  gem 'adapter_extensions', :git => 'https://github.com/activewarehouse/adapter_extensions.git'
 
   if activerecord_version < '3.1'
     gem 'mysql2', '< 0.3'

data/test/engine_test.rb

@@ -33,6 +33,19 @@ CTL
       
       assert_equal 1, engine.errors.size
     end
+
+    should 'call error callbacks' do
+      engine = ETL::Engine.new
+
+      $our_errors = []
+      engine.process ETL::Control::Control.parse_text <<CTL
+        source :in, { :type => :enumerable, :enumerable => (1..100) }
+        on_error { |error| $our_errors << error }
+        after_read { |row| raise "Failure" }
+CTL
+      assert_equal 100, $our_errors.size
+      assert_match /on line 100: Failure$/, $our_errors.last
+    end
     
   end
   

data/test/source_test.rb

@@ -1,5 +1,14 @@
 require File.dirname(__FILE__) + '/test_helper'
 
+# TODO - use FactoryGirl or similar
+def build_source(options = {})
+  ETL::Control::DatabaseSource.new(nil, {
+    :target => 'operational_database',
+    :table => 'people',
+    :mysqlstream => true
+  }.merge(options), nil)
+end
+
 class SourceTest < Test::Unit::TestCase
   
   context "source" do
@@ -84,6 +93,8 @@ class SourceTest < Test::Unit::TestCase
   
   context "a database source" do
     setup do
+      @offset = 2
+      @limit = 5
       control = ETL::Control::Control.parse(File.dirname(__FILE__) + '/delimited.ctl')
       configuration = {
         :database => 'etl_unittest',
@@ -97,6 +108,44 @@ class SourceTest < Test::Unit::TestCase
       ]
       @source = ETL::Control::DatabaseSource.new(control, configuration, definition)
     end
+
+    context "with a specified LIMIT `n`" do
+      setup do
+        ETL::Engine.limit = @limit
+        10.times { |i| Person.create!( :first_name => 'Bob',
+                                       :last_name => 'Smith',
+                                       :ssn => i ) }
+      end
+
+      should "only return N rows" do
+        size = build_source(:store_locally => true, :mysqlstream => false).to_a.size
+        assert_equal 5, size
+      end
+
+      teardown do
+        Person.delete_all
+        ETL::Engine.limit = nil
+      end
+    end
+
+    context "with a specified OFFSET `offset`" do
+      setup do
+        ETL::Engine.limit = @limit
+        ETL::Engine.offset = @offset
+      end
+
+      should "raise an exception without LIMIT specified" do
+        ETL::Engine.limit = nil
+        assert_raise (NoLimitSpecifiedError) { build_source(:store_locally => true, :mysqlstream => false).to_a.size }
+      end
+
+      teardown do
+        Person.delete_all
+        ETL::Engine.limit = nil
+        ETL::Engine.offset = nil
+      end
+    end
+
     should "set the local file for extraction storage" do
       assert_match %r{source_data/localhost/activewarehouse_etl_test/people/\d+.csv}, @source.local_file.to_s
     end
@@ -108,6 +157,36 @@ class SourceTest < Test::Unit::TestCase
       rows = @source.collect { |row| row }
       assert_equal 1, rows.length
     end
+    if current_adapter =~ /mysql/
+      context 'with mysqlstream enabled' do
+
+        setup do
+          Person.delete_all
+          Person.create!(:first_name => 'Bob', :last_name => 'Smith', :ssn => '123456789')
+          Person.create!(:first_name => 'John', :last_name => 'Barry', :ssn => '123456790')
+        end
+
+        should 'support store_locally' do
+          assert_equal 2, build_source(:store_locally => true).to_a.size
+        end
+
+        context 'with a NULL value' do
+
+          should 'return nil in row attribute' do
+            Person.create!(:first_name => nil)
+            assert_equal nil, build_source.to_a.last[:first_name]
+          end
+
+          # does not work yet - we probably need a switch on --quick for this
+          should_eventually 'return NULL for string containing NULL' do
+            Person.create!(:first_name => 'NULL', :last_name => 'NULL2')
+            assert_equal 'NULL', build_source.to_a.last[:first_name]
+          end
+
+        end
+
+      end
+    end
   end
   
   context "a file source with an xml parser" do

data/test/test_helper.rb

@@ -30,3 +30,7 @@ puts "ActiveRecord::VERSION = #{ActiveRecord::VERSION::STRING}"
 
 class Person < ActiveRecord::Base
 end
+
+def current_adapter
+  ENV['DB']
+end

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: activewarehouse-etl
 version: !ruby/object:Gem::Version
-  version: 1.0.0.rc1
-  prerelease: 6
+  version: 1.0.0
+  prerelease: 
 platform: ruby
 authors:
 - Anthony Eden
@@ -10,11 +10,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-03 00:00:00.000000000 Z
+date: 2012-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70307704843740 !ruby/object:Gem::Requirement
+  requirement: &70217989126480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,10 +22,10 @@ dependencies:
         version: 0.8.3
   type: :runtime
   prerelease: false
-  version_requirements: *70307704843740
+  version_requirements: *70217989126480
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70307704843000 !ruby/object:Gem::Requirement
+  requirement: &70217989126020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +33,10 @@ dependencies:
         version: 3.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70307704843000
+  version_requirements: *70217989126020
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &70307704842100 !ruby/object:Gem::Requirement
+  requirement: &70217989125560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +44,10 @@ dependencies:
         version: 3.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70307704842100
+  version_requirements: *70217989125560
 - !ruby/object:Gem::Dependency
   name: fastercsv
-  requirement: &70307704841200 !ruby/object:Gem::Requirement
+  requirement: &70217984932020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -55,10 +55,10 @@ dependencies:
         version: 1.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *70307704841200
+  version_requirements: *70217984932020
 - !ruby/object:Gem::Dependency
   name: adapter_extensions
-  requirement: &70307704840560 !ruby/object:Gem::Requirement
+  requirement: &70217984927180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -66,10 +66,10 @@ dependencies:
         version: 0.9.5.rc1
   type: :runtime
   prerelease: false
-  version_requirements: *70307704840560
+  version_requirements: *70217984927180
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &70307704839660 !ruby/object:Gem::Requirement
+  requirement: &70217984925860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -77,10 +77,10 @@ dependencies:
         version: 2.11.3
   type: :development
   prerelease: false
-  version_requirements: *70307704839660
+  version_requirements: *70217984925860
 - !ruby/object:Gem::Dependency
   name: flexmock
-  requirement: &70307704839120 !ruby/object:Gem::Requirement
+  requirement: &70217989152940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -88,10 +88,10 @@ dependencies:
         version: 0.9.0
   type: :development
   prerelease: false
-  version_requirements: *70307704839120
+  version_requirements: *70217989152940
 - !ruby/object:Gem::Dependency
   name: cartesian
-  requirement: &70307704838600 !ruby/object:Gem::Requirement
+  requirement: &70217989152560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -99,7 +99,29 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70307704838600
+  version_requirements: *70217989152560
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: &70217989152100 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70217989152100
+- !ruby/object:Gem::Dependency
+  name: guard-shell
+  requirement: &70217989151680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70217989151680
 description: ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application
   for loading data into a database.
 email:
@@ -115,6 +137,7 @@ files:
 - 0.9-UPGRADE
 - CHANGELOG
 - Gemfile
+- Guardfile
 - HOW_TO_RELEASE
 - LICENSE
 - README.textile