activewarehouse-etl 0.8.4 → 0.9.0

data/lib/etl/control/source/database_source.rb

@@ -8,6 +8,9 @@ module ETL #:nodoc:
   module Control #:nodoc:
     # Source object which extracts data from a database using ActiveRecord.
     class DatabaseSource < Source
+      attr_accessor :target
+      attr_accessor :table
+      
       # Initialize the source.
       #
       # Arguments:
@@ -16,38 +19,39 @@ module ETL #:nodoc:
       # * <tt>definition</tt>: The source definition
       #
       # Required configuration options:
+      # * <tt>:target</tt>: The target connection
       # * <tt>:table</tt>: The source table name
       # * <tt>:database</tt>: The database name
       # 
       # Other options:
-      # * <tt>:adapter</tt>: The adapter to use (defaults to :mysql)
-      # * <tt>:username</tt>: The database username (defaults to 'root')
-      # * <tt>:password</tt>: The password to the database (defaults to 
-      #   nothing)
-      # * <tt>:host</tt>: The host for the database (defaults to 
-      #   'localhost')
       # * <tt>:join</tt>: Optional join part for the query (ignored unless 
       #   specified)
       # * <tt>:select</tt>: Optional select part for the query (defaults to 
       #   '*')
+      # * <tt>:group</tt>: Optional group by part for the query (ignored 
+      #   unless specified)
       # * <tt>:order</tt>: Optional order part for the query (ignored unless 
       #   specified)
+      # * <tt>:new_records_only</tt>: Specify the column to use when comparing
+      #   timestamps against the last successful ETL job execution for the
+      #   current control file.
       # * <tt>:store_locally</tt>: Set to false to not store a copy of the 
       #   source data locally in a flat file (defaults to true)
       def initialize(control, configuration, definition)
         super
-        connect
+        @target = configuration[:target]
+        @table = configuration[:table]
       end
       
       # Get a String identifier for the source
       def to_s
-        "#{host}/#{configuration[:database]}/#{configuration[:table]}"
+        "#{host}/#{database}/#{table}"
       end
       
       # Get the local directory to use, which is a combination of the 
       # local_base, the db hostname the db database name and the db table.
       def local_directory
-        File.join(local_base, host, configuration[:database], configuration[:table])
+        File.join(local_base, host, database, configuration[:table])
       end
       
       # Get the join part of the query, defaults to nil
@@ -202,43 +206,16 @@ module ETL #:nodoc:
       
       # Get the database connection to use
       def connection
-        ETL::Source.connection
-      end
-      
-      # Get the adapter name, defaults to :mysql
-      def adapter
-        configuration[:adapter] || :mysql
+        ETL::Engine.connection(target)
       end
       
       # Get the host, defaults to 'localhost'
       def host
-        configuration[:host] || "localhost"
-      end
-      
-      # Get the username, defaults to 'root'
-      def username
-        configuration[:username] || 'root'
+        ETL::Base.configurations[target.to_s]['host'] || 'localhost'
       end
       
-      # Connect to the database. 
-      #
-      # Required options:
-      # * <tt>:database</tt>: The database name
-      # 
-      # Options:
-      # * <tt>:adapter</tt>: The adapter to use (defaults to :mysql)
-      # * <tt>:username</tt>: The database username (defaults to 'root')
-      # * <tt>:password</tt>: The password to the database (defaults 
-      #   to nothing)
-      # * <tt>:host<tt>: The host for the database (defaults to 'localhost')
-      def connect
-        ETL::Source.establish_connection(
-            :adapter  => adapter,
-            :username => username,
-            :host     => host,
-            :password => configuration[:password],
-            :database => configuration[:database] 
-        )
+      def database
+        ETL::Base.configurations[target.to_s]['database']
       end
     end
   end

data/lib/etl/control/source/file_source.rb

@@ -40,13 +40,16 @@ module ETL #:nodoc:
       
       # Returns each row from the source
       def each
+        count = 0
         copy_sources if store_locally
         @parser.each do |row|
-          # TODO skip rows if offset is defined
-          # TODO stop processing if limit is reached
-          row = ETL::Row[row]
-          row.source = self
-          yield row
+          if ETL::Engine.offset && count < ETL::Engine.offset
+            count += 1
+          else
+            row = ETL::Row[row]
+            row.source = self
+            yield row
+          end
         end
       end
       

data/lib/etl/control/source/model_source.rb

@@ -0,0 +1,39 @@
+#RAILS_ENV = 'development'
+#require '../config/environment'
+
+module ETL #:nodoc:
+  module Control #:nodoc:
+    class ModelSource < Source   
+      
+      def columns
+        case definition
+        when Array
+          definition.collect(&:to_sym)
+        when Hash
+          definition.keys.collect(&:to_sym)
+        else
+          raise "Definition must be either an Array or a Hash"
+        end
+      end
+      
+      def railsmodel
+        configuration[:model]
+      end
+      
+      def order
+        configuration[:order] || "id"
+      end
+         
+      def each(&block)
+          railsmodel.to_s.camelize.constantize.find(:all,:order=>order).each do |row|
+            result_row = ETL::Row.new
+            result_row.source = self
+            columns.each do |column|
+              result_row[column.to_sym] = row.send(column)
+            end
+            yield result_row
+        end
+      end
+    end
+  end
+end

data/lib/etl/core_ext.rb

@@ -0,0 +1 @@
+require 'etl/core_ext/time'

data/lib/etl/core_ext/time.rb

@@ -0,0 +1,5 @@
+require File.dirname(__FILE__) + '/time/calculations'
+
+class Time#:nodoc:
+  include ETL::CoreExtensions::Time::Calculations
+end

data/lib/etl/core_ext/time/calculations.rb

@@ -0,0 +1,40 @@
+module ETL #:nodoc:
+  module CoreExtensions #:nodoc:
+    module Time #:nodoc:
+      # Enables the use of time calculations within Time itself
+      module Calculations
+        def week
+          cyw = ((yday - 1) / 7) + 1
+          cyw = 52 if cyw == 53
+          cyw
+        end
+        def quarter
+          ((month - 1) / 3) + 1
+        end
+        def fiscal_year_week(offset_month=10)
+          fyw = ((fiscal_year_yday(offset_month) - 1) / 7) + 1
+          fyw = 52 if fyw == 53
+          fyw
+        end
+        def fiscal_year_month(offset_month=10)
+          shifted_month = month - (offset_month - 1)
+          shifted_month += 12 if shifted_month < 0
+          shifted_month
+        end
+        def fiscal_year_quarter(offset_month=10)
+          ((fiscal_year_month(offset_month) - 1) / 3) + 1
+        end
+        def fiscal_year(offset_month=10)
+          month >= offset_month ? year + 1 : year
+        end
+        def fiscal_year_yday(offset_month=10)
+          offset_days = 0
+          1.upto(offset_month - 1) { |m| offset_days += ::Time.days_in_month(m, year) }
+          shifted_year_day = yday - offset_days
+          shifted_year_day += 365 if shifted_year_day <= 0
+          shifted_year_day
+        end
+      end
+    end
+  end
+end

data/lib/etl/engine.rb

@@ -1,40 +1,58 @@
 module ETL #:nodoc:
-  module ActiveRecord #:nodoc:
-    # Base class which is used for ActiveRecord connections. This is necessary
-    # since AR connections are tied to the class, and using ActiveRecord::Base
-    # directly can cause problems if the connection is closed.
-    class Base < ::ActiveRecord::Base
-    end
+  
+  class Base < ActiveRecord::Base
   end
   
   # The main ETL engine clas
   class Engine
+    include ETL::Util
+    
     class << self
       # Initialization that is run when a job is executed.
+      # 
+      # Options:
+      # * <tt>:limit</tt>: Limit the number of records returned from sources
+      # * <tt>:offset</tt>: Specify the records for data from sources
+      # * <tt>:log_write_mode</tt>: If true then the log will write, otherwise it will append
+      # * <tt>:skip_bulk_import</tt>: Set to true to skip bulk import
+      # * <tt>:read_locally</tt>: Set to true to read from the local cache
+      # * <tt>:rails_root</tt>: Set to the rails root to boot rails
       def init(options={})
         unless @initialized
+          puts "initializing ETL engine"
           @limit = options[:limit]
           @offset = options[:offset]
           @log_write_mode = 'w' if options[:newlog]
           @skip_bulk_import = options[:skip_bulk_import]
           @read_locally = options[:read_locally]
+          @rails_root = options[:rails_root]
+          
+          require File.join(@rails_root, 'config/environment') if @rails_root
+
           options[:config] ||= 'database.yml'
           database_configuration = YAML::load(ERB.new(IO.read(options[:config])).result + "\n")
-          ETL::ActiveRecord::Base.configurations = database_configuration
-          ActiveRecord::Base.configurations.merge!(ETL::ActiveRecord::Base.configurations)
+          ActiveRecord::Base.configurations.merge!(database_configuration)
+          ETL::Base.configurations = database_configuration
+          #puts "configurations in init: #{ActiveRecord::Base.configurations.inspect}"
+          
           require 'etl/execution'
           ETL::Execution::Base.establish_connection :etl_execution
           ETL::Execution::Execution.migrate
+
           @initialized = true
         end
       end
       
-      # Process the specified control file. Acceptable values for control_file are
+      # Process the specified file. Acceptable values for file are:
       # * Path to a file
       # * File object
       # * ETL::Control::Control instance
-      def process(control_file)
-        new().process(control_file)
+      # * ETL::Batch::Batch instance
+      #
+      # The process command will accept either a .ctl Control file or a .ebf
+      # ETL Batch File.
+      def process(file)
+        new().process(file)
       end
       
       attr_accessor :timestamped_log
@@ -55,7 +73,7 @@ module ETL #:nodoc:
           else
             @logger = Logger.new(File.open('etl.log', log_write_mode))
           end
-          @logger.level = Logger::ERROR
+          @logger.level = Logger::WARN
           @logger.formatter = Logger::Formatter.new
         end
         @logger
@@ -94,6 +112,9 @@ module ETL #:nodoc:
       # Access the current ETL::Execution::Job instance
       attr_accessor :job
       
+      # Access the current ETL::Execution::Batch instance
+      attr_accessor :batch
+      
       # The limit on rows to load from the source, useful for testing the ETL
       # process prior to executing the entire batch. Default value is nil and 
       # indicates that there is no limit
@@ -112,7 +133,80 @@ module ETL #:nodoc:
       
       # Accessor for the average rows per second processed
       attr_accessor :average_rows_per_second
-    end
+      
+      # Get a named connection
+      def connection(name)
+        logger.debug "Retrieving connection #{name}"
+        conn = connections[name] ||= establish_connection(name)
+        #conn.verify!(ActiveRecord::Base.verification_timeout)
+        conn.reconnect! unless conn.active?
+        conn
+      end
+      
+      # Set to true to use temp tables
+      attr_accessor :use_temp_tables
+      
+      # Get a registry of temp tables
+      def temp_tables
+        @temp_tables ||= {}
+      end
+      
+      # Called when a batch job finishes, allowing for cleanup to occur
+      def finish
+        temp_tables.each do |temp_table, mapping|
+          actual_table = mapping[:table]
+          #puts "move #{temp_table} to #{actual_table}"
+          conn = mapping[:connection]
+          conn.transaction do
+            conn.rename_table(actual_table, "#{actual_table}_old")
+            conn.rename_table(temp_table, actual_table)
+            conn.drop_table("#{actual_table}_old")
+          end
+        end
+      end
+      
+      # Return true if using temp tables
+      def use_temp_tables?
+        use_temp_tables ? true : false
+      end
+      
+      # Modify the table name if necessary
+      def table(table_name, connection)
+        if use_temp_tables?
+          returning "tmp_#{table_name}" do |temp_table_name|
+            if temp_tables[temp_table_name].nil?
+              # Create the temp table and add it to the mapping
+              begin connection.drop_table(temp_table_name); rescue; end
+              connection.execute(
+                connection.add_select_into_table(temp_table_name, "SELECT * FROM #{table_name}")
+              )
+              temp_tables[temp_table_name] = {
+                :table => table_name,
+                :connection => connection
+              }
+            end
+          end
+        else
+          table_name
+        end
+      end
+      
+      protected
+      # Hash of database connections that can be used throughout the ETL 
+      # process
+      def connections
+        @connections ||= {}
+      end
+      
+      # Establish the named connection and return the database specific connection
+      def establish_connection(name)
+        logger.debug "Establishing connection to #{name}"
+        conn_config = ETL::Base.configurations[name.to_s]
+        raise ETL::ETLError, "No connection found for #{name}" unless conn_config
+        connection_method = "#{conn_config['adapter']}_connection"
+        ETL::Base.send(connection_method, conn_config)
+      end
+    end # class << self
     
     # Say the specified message, with a newline
     def say(message)
@@ -121,7 +215,7 @@ module ETL #:nodoc:
     
     # Say the specified message without a newline
     def say_without_newline(message)
-      if Engine.realtime_activity
+      if ETL::Engine.realtime_activity
         $stdout.print message
         $stdout.flush
       end
@@ -153,11 +247,48 @@ module ETL #:nodoc:
       }
     end
   
-    # Process a control file or object. Acceptable values for control are:
+    # Process a file, control object or batch object. Acceptable values for 
+    # file are:
     # * Path to a file
     # * File object
     # * ETL::Control::Control instance
-    def process(control)
+    # * ETL::Batch::Batch instance
+    def process(file)
+      case file
+      when String
+        process(File.new(file))
+      when File
+        process_control(file) if file.path =~ /.ctl$/
+        process_batch(file) if file.path =~ /.ebf$/
+      when ETL::Control::Control
+        process_control(file)
+      when ETL::Batch::Batch
+        process_batch(file)
+      else
+        raise RuntimeError, "Process object must be a String, File, Control 
+        instance or Batch instance"
+      end
+    end
+    
+    protected
+    # Process the specified batch file
+    def process_batch(batch)
+      batch = ETL::Batch::Batch.resolve(batch, self)
+    
+      ETL::Engine.batch = ETL::Execution::Batch.create!(
+        :batch_file => batch.file,
+        :status => 'executing'
+      )
+      
+      batch.execute
+      
+      ETL::Engine.batch.completed_at = Time.now
+      ETL::Engine.batch.status = (errors.length > 0 ? 'completed with errors' : 'completed')
+      ETL::Engine.batch.save!
+    end
+    
+    # Process the specified control file
+    def process_control(control)
       control = ETL::Control::Control.resolve(control)
       
       ETL::Engine.job = ETL::Execution::Job.create!(
@@ -168,11 +299,7 @@ module ETL #:nodoc:
       execute_dependencies(control)
       
       start_time = Time.now
-      
-      Engine.logger.debug "Pre-processing #{control.file}"
       pre_process(control)
-      Engine.logger.debug "Pre-processing complete"
-      
       sources = control.sources
       destinations = control.destinations
       
@@ -195,9 +322,7 @@ module ETL #:nodoc:
           Engine.logger.debug "Row #{index}: #{row.inspect}"
           Engine.rows_read += 1
           Engine.current_source_row = index + 1
-          if Engine.realtime_activity && index > 0 && index % 1000 == 0
-            say_without_newline "."
-          end
+          say_without_newline "." if Engine.realtime_activity && index > 0 && index % 1000 == 0
           
           # At this point a single row may be turned into multiple rows via row 
           # processors all code after this line should work with the array of 
@@ -225,7 +350,6 @@ module ETL #:nodoc:
           
           t = Benchmark.realtime do
             begin
-              # execute transforms
               Engine.logger.debug "Executing transforms"
               rows.each do |row|
                 control.transforms.each do |transform|
@@ -253,9 +377,7 @@ module ETL #:nodoc:
               Engine.logger.debug "Processing before write"
               control.before_write_processors.each do |processor|
                 processed_rows = []
-                rows.each do |row|
-                  processed_rows << processor.process(row)
-                end
+                rows.each { |row| processed_rows << processor.process(row) }
                 rows = processed_rows.flatten.compact
               end
             rescue => e
@@ -300,11 +422,20 @@ module ETL #:nodoc:
         destination.close
       end
       
-      say_on_own_line "Executing post processes"
-      Engine.logger.debug "Post-processing #{control.file}"
+      say_on_own_line "Executing screens"
+      begin
+        execute_screens(control)
+      rescue FatalScreenError => e
+        say "Fatal screen error during job execution: #{e.message}"
+        exit
+      rescue ScreenError => e
+        say "Screen error during job execution: #{e.message}"
+        return
+      else
+        say "Screens passed"
+      end
+      
       post_process(control)
-      Engine.logger.debug "Post-processing complete"
-      say "Post-processing complete"
       
       if sources.length > 0
         say_on_own_line "Read #{Engine.rows_read} lines from sources"
@@ -339,16 +470,22 @@ module ETL #:nodoc:
     
     # Execute all preprocessors
     def pre_process(control)
+      Engine.logger.debug "Pre-processing #{control.file}"
       control.pre_processors.each do |processor|
         processor.process
       end
+      Engine.logger.debug "Pre-processing complete"
     end
     
     # Execute all postprocessors
     def post_process(control)
+      say_on_own_line "Executing post processes"
+      Engine.logger.debug "Post-processing #{control.file}"
       control.post_processors.each do |processor|
         processor.process
       end
+      Engine.logger.debug "Post-processing complete"
+      say "Post-processing complete"
     end
     
     # Execute all dependencies
@@ -371,59 +508,23 @@ module ETL #:nodoc:
       end
     end
     
-    # Return the distance of time in words from the given from_time to the specified to_time. If to_time
-    # is not specified then Time.now is used. By default seconds are included...set the include_seconds
-    # argument to false to disable the seconds.
-    def distance_of_time_in_words(from_time, to_time=Time.now)
-      from_time = from_time.to_time if from_time.respond_to?(:to_time)
-      to_time = to_time.to_time if to_time.respond_to?(:to_time)
-      seconds = (to_time - from_time).round
-      distance_in_days = (seconds/(60*60*24)).round
-      seconds = seconds % (60*60*24)
-      distance_in_hours = (seconds/(60*60)).round
-      seconds = seconds % (60*60)
-      distance_in_minutes = (seconds/60).round
-      seconds = seconds % 60
-      distance_in_seconds = seconds
-      
-      s = ''
-      s << "#{distance_in_days} days," if distance_in_days > 0
-      s << "#{distance_in_hours} hours, " if distance_in_hours > 0
-      s << "#{distance_in_minutes} minutes, " if distance_in_minutes > 0
-      s << "#{distance_in_seconds} seconds"
-      s
-    end
-    
-    # Get the approximate disntance of time in words from the given from_time
-    # to the the given to_time. If to_time is not specified then it is set
-    # to Time.now. By default seconds are included...set the include_seconds
-    # argument to false to disable the seconds.
-    def approximate_distance_of_time_in_words(from_time, to_time=Time.now, include_seconds=true)
-      from_time = from_time.to_time if from_time.respond_to?(:to_time)
-      to_time = to_time.to_time if to_time.respond_to?(:to_time)
-      distance_in_minutes = (((to_time - from_time).abs)/60).round
-      distance_in_seconds = ((to_time - from_time).abs).round
-      
-      case distance_in_minutes
-        when 0..1
-          return (distance_in_minutes == 0) ? 'less than a minute' : '1 minute' unless include_seconds
-        case distance_in_seconds
-          when 0..4   then 'less than 5 seconds'
-          when 5..9   then 'less than 10 seconds'
-          when 10..19 then 'less than 20 seconds'
-          when 20..39 then 'half a minute'
-          when 40..59 then 'less than a minute'
-          else             '1 minute'
+    # Execute all screens
+    def execute_screens(control)
+      [:fatal,:error,:warn].each do |type|
+        control.screens[type].each do |block|
+          begin
+            block.call
+          rescue => e
+            case type
+            when :fatal
+              raise FatalScreenError, e
+            when :error
+              raise ScreenError, e
+            when :warn
+              say "Screen warning: #{e}"
+            end
+          end
         end
-        when 2..44           then "#{distance_in_minutes} minutes"
-        when 45..89          then 'about 1 hour'
-        when 90..1439        then "about #{(distance_in_minutes.to_f / 60.0).round} hours"
-        when 1440..2879      then '1 day'
-        when 2880..43199     then "#{(distance_in_minutes / 1440).round} days"
-        when 43200..86399    then 'about 1 month'
-        when 86400..525959   then "#{(distance_in_minutes / 43200).round} months"
-        when 525960..1051919 then 'about 1 year'
-        else                      "over #{(distance_in_minutes / 525960).round} years"
       end
     end
   end