activewarehouse-etl 0.4.0 → 0.5.0

data/lib/etl/control/destination/database_destination.rb

@@ -1,7 +1,34 @@
-module ETL
-  module Control
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # Destination which writes directly to a database. This is useful when you are dealing with
+    # a small amount of data. For larger amounts of data you should probably use the bulk
+    # loader if it is supported with your target database as it will use a much faster load
+    # method.
     class DatabaseDestination < Destination
-      attr_reader :order, :truncate
+      # Specify the order from the source
+      attr_reader :order
+      
+      # Set to true to truncate the destination table first
+      attr_reader :truncate
+      
+      # Initialize the database destination
+      # 
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>mapping</tt>: The mapping
+      #
+      # Configuration options:
+      # * <tt>:database</tt>: The database name (REQUIRED)
+      # * <tt>:table<tt>: The table to write to (REQUIRED)
+      # * <tt>:truncate</tt>: Set to true to truncate before writing (defaults to false)
+      # * <tt>:unique</tt>: Set to true to only insert unique records (defaults to false)
+      # * <tt>:adapter</tt>: The adapter to use (defaults to :mysql)
+      # * <tt>:username</tt>: The database username (defaults to 'root')
+      # * <tt>:password</tt>: The password to the database (defaults to nothing)
+      # * <tt>:host<tt>: The host for the database (defaults to 'localhost')
+      #
+      # Mapping options:
+      # * <tt>:order</tt>: The order of fields to write (REQUIRED)
       def initialize(control, configuration, mapping)
         super
         @truncate = configuration[:truncate] ||= false
@@ -11,8 +38,9 @@ module ETL
         connect
       end
       
+      # Flush the currently buffered data
       def flush
-        conn = ActiveRecord::Base.connection
+        conn = ETL::ActiveRecord::Base.connection
         conn.transaction do
           conn.truncate(configuration[:table]) if truncate
           
@@ -32,7 +60,7 @@ module ETL
             end
             q = "INSERT INTO #{configuration[:table]} (#{names.join(',')}) VALUES (#{values.join(',')})"
             # ETL::Engine.logger.debug("Query: #{q}")
-            conn.execute(q, "Insert row #{current_row}")
+            conn.insert(q, "Insert row #{current_row}")
             @current_row += 1
           end
           buffer.clear
@@ -42,12 +70,22 @@ module ETL
       # Close the connection
       def close
         flush
-        ActiveRecord::Base.connection.disconnect!
+        ETL::ActiveRecord::Base.connection.disconnect!
       end
       
       private
+      # Connect to the database. 
+      #
+      # Required options:
+      # * <tt>:database</tt>: The database name
+      # 
+      # Options:
+      # * <tt>:adapter</tt>: The adapter to use (defaults to :mysql)
+      # * <tt>:username</tt>: The database username (defaults to 'root')
+      # * <tt>:password</tt>: The password to the database (defaults to nothing)
+      # * <tt>:host<tt>: The host for the database (defaults to 'localhost')
       def connect
-        ActiveRecord::Base.establish_connection(
+        ETL::ActiveRecord::Base.establish_connection(
             :adapter  => (configuration[:adapter] || :mysql),
             :username => (configuration[:username] || 'root'),
             :host     => (configuration[:host] || 'localhost'),

data/lib/etl/control/destination/file_destination.rb

@@ -2,13 +2,39 @@ module ETL #:nodoc:
   module Control #:nodoc:
     # File as the final destination.
     class FileDestination < Destination
-      attr_reader :file, :order
-      attr_accessor :append, :separator, :eol, :enclose
+      # The File to write to
+      attr_reader :file
+      
+      # The output order
+      attr_reader :order
+      
+      # Flag which indicates to append (default is to overwrite)
+      attr_accessor :append
+      
+      # The separator
+      attr_accessor :separator
+      
+      # The end of line marker
+      attr_accessor :eol
+      
+      # The enclosure character
+      attr_accessor :enclose
       
       # Initialize the object.
       # * <tt>control</tt>: The Control object
       # * <tt>configuration</tt>: The configuration map
       # * <tt>mapping</tt>: The output mapping
+      # 
+      # Configuration options:
+      # * <tt>:file<tt>: The file to write to (REQUIRED)
+      # * <tt>:append</tt>: Set to true to append to the file (default is to overwrite)
+      # * <tt>:separator</tt>: Record separator (default is a comma)
+      # * <tt>:eol</tt>: End of line marker (default is \n)
+      # * <tt>:enclose</tt>: Enclosure character (default is none)
+      # * <tt>:unique</tt>: Set to true to only write unique records
+      # 
+      # Mapping options:
+      # * <tt>:order</tt>: The order array
       def initialize(control, configuration, mapping)
         super
         @file = File.join(File.dirname(control.file), configuration[:file])
@@ -57,7 +83,6 @@ module ETL #:nodoc:
       end
       
       private
-      
       # Get the open file stream
       def f
         @f ||= open(file, mode)
@@ -67,7 +92,6 @@ module ETL #:nodoc:
       def mode
         append ? 'a' : 'w'
       end
-      
     end
   end
 end

data/lib/etl/control/source.rb

@@ -3,9 +3,20 @@ module ETL #:nodoc:
     # ETL source. Subclasses must implement the <tt>each</tt> method.
     class Source
       include Enumerable
-      attr_accessor :control, :configuration, :definition
+      
+      # The control object
+      attr_accessor :control
+      
+      # The configuration Hash
+      attr_accessor :configuration
+      
+      # The definition Hash
+      attr_accessor :definition
       
       class << self
+        # Convert the name to a Source class.
+        # 
+        # For example if name is :database then this will return a DatabaseSource class
         def class_for_name(name)
           ETL::Control.const_get("#{name.to_s.classify}Source")
         end
@@ -20,6 +31,10 @@ module ETL #:nodoc:
         @configuration = configuration
         @definition = definition
       end
+      
+      def errors
+        @errors ||= []
+      end
     end
   end
 end

data/lib/etl/control/source/database_source.rb

@@ -1,23 +1,53 @@
-module ETL
-  module Control
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # Source object which extracts data from a database using ActiveRecord.
     class DatabaseSource < Source
+      # Initialize the source.
+      #
+      # Arguments:
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>definition</tt>: The source definition
+      #
+      # Required configuration options:
+      # * <tt>:table</tt>: The source table name
+      # * <tt>:database</tt>: The database name
+      # 
+      # Other options:
+      # * <tt>:adapter</tt>: The adapter to use (defaults to :mysql)
+      # * <tt>:username</tt>: The database username (defaults to 'root')
+      # * <tt>:password</tt>: The password to the database (defaults to nothing)
+      # * <tt>:host<tt>: The host for the database (defaults to 'localhost')
       def initialize(control, configuration, definition)
         super
         connect
       end
       
+      def to_s
+        "#{configuration[:host]}/#{configuration[:database]}"
+      end
+      
       # Returns each row from the source
       def each
-        conn = ActiveRecord::Base.connection
+        conn = ETL::ActiveRecord::Base.connection
         conn.select_all("SELECT * FROM #{configuration[:table]}").each do |row|
           yield row
         end
       end
       
       private
+      # Connect to the database. 
+      #
+      # Required options:
+      # * <tt>:database</tt>: The database name
+      # 
+      # Options:
+      # * <tt>:adapter</tt>: The adapter to use (defaults to :mysql)
+      # * <tt>:username</tt>: The database username (defaults to 'root')
+      # * <tt>:password</tt>: The password to the database (defaults to nothing)
+      # * <tt>:host<tt>: The host for the database (defaults to 'localhost')
       def connect
-        # set up the DB connection
-        ActiveRecord::Base.establish_connection(
+        ETL::ActiveRecord::Base.establish_connection(
             :adapter  => (configuration[:adapter] || :mysql),
             :username => (configuration[:username] || 'root'),
             :host     => (configuration[:host] || 'localhost'),

data/lib/etl/control/source/file_source.rb

@@ -1,19 +1,49 @@
-module ETL
-  module Control
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # A File source.
     class FileSource < Source
+      # The number of lines to skip, default is 0
       attr_accessor :skip_lines
+      
+      # Accessor for the underlying parser
+      attr_accessor :parser
+      
+      # Initialize the source
+      #
+      # Configuration options:
+      # * <tt>:parser</tt>: One of the following: a parser name as a String or symbol, a class which extends from Parser, 
+      #   a Hash with :name and optionally an :options key. Whether or not the parser uses the options is dependent on 
+      #   which parser is used. See the documentation for each parser for information on what options it accepts.
+      # * <tt>:skip_lines<tt>: The number of lines to skip (defaults to 0)
       def initialize(control, configuration, definition)
         super
         configure
       end
+      
+      def to_s
+        configuration[:file]
+      end
+      
       # Returns each row from the source
       def each
         @parser.each { |row| yield row }
       end
       
       private
+      # Configure the source
       def configure
-        @parser = ETL::Parser::Parser.class_for_name(@configuration[:parser]).new(self)
+        case @configuration[:parser]
+        when Class
+          @parser = @configuration[:parser].new(self)
+        when String, Symbol
+          @parser = ETL::Parser::Parser.class_for_name(@configuration[:parser]).new(self)
+        when Hash
+          name = @configuration[:parser][:name]
+          options = @configuration[:parser][:options]
+          @parser = ETL::Parser::Parser.class_for_name(name).new(self, options)
+        else
+          raise ControlError, "Configuration option :parser must be a Class, String or Symbol"
+        end
         @skip_lines = @configuration[:skip_lines] ||= 0
       end
     end

data/lib/etl/engine.rb

@@ -1,61 +1,181 @@
-module ETL
+module ETL #:nodoc:
+  module ActiveRecord #:nodoc:
+    # Base class which is used for ActiveRecord connections. This is necessary
+    # since AR connections are tied to the class, and using ActiveRecord::Base
+    # directly can cause problems if the connection is closed.
+    class Base < ::ActiveRecord::Base
+    end
+  end
+  
+  # The main ETL engine clas
   class Engine
   
     class << self
+      # Process the specified control file. Acceptable values for control_file are
+      # * Path to a file
+      # * File object
+      # * ETL::Control::Control instance
       def process(control_file)
         new().process(control_file)
       end
       
+      # A logger for the engine
       attr_accessor :logger
       
-      def logger
+      def logger #:nodoc:
         unless @logger
           @logger = Logger.new('etl.log')
-          @logger.level = Logger::DEBUG
+          @logger.level = Logger::WARN
         end
         @logger
       end
+      
+      attr_accessor :current_source
+      attr_accessor :current_source_row
+      attr_accessor :current_destination
+      
+      attr_accessor :realtime_activity
+    end
+    
+    def say(message)
+      say_without_newline(message + "\n")
+    end
+    
+    def say_without_newline(message)
+      if Engine.realtime_activity
+        $stdout.print message
+        $stdout.flush
+      end
+    end
+    
+    def say_on_own_line(message)
+      say("\n" + message)
     end
   
-    # Process a control file or object.
+    # Process a control file or object. Acceptable values for control are:
+    # * Path to a file
+    # * File object
+    # * ETL::Control::Control instance
     def process(control)
+      start_time = Time.now
       control = ETL::Control::Control.resolve(control)
       
+      Engine.logger.debug "Pre-processing #{control.file}"
       pre_process(control)
+      Engine.logger.debug "Pre-processing complete"
       
       sources = control.sources
       destinations = control.destinations
       
       sources.each do |source|
+        Engine.current_source = source
+        Engine.logger.debug "Processing source #{source}"
+        say "Source: #{source}"
         source.each_with_index do |row, index|
-          row.each do |name, value|
+          Engine.current_source_row = index + 1
+          if Engine.realtime_activity && index % 1000 == 0
+            say_without_newline "."
+          end
+          
+          begin
             # execute transforms
-            row[name] = ETL::Transform::Transform.transform(name, value, control.get_transform(name))
+            row.each do |name, value|
+              row[name] = ETL::Transform::Transform.transform(name, value, control.transform(name))
+            end
+          rescue => e
+            msg = "Error transforming from #{source} on line #{index}: #{e}"
+            source.errors << msg
+            Engine.logger.error msg
           end
-          # write the row to the destination
-          destinations.each do |destination|
-            destination.write(row)
+          
+          begin
+            # write the row to the destination
+            destinations.each do |destination|
+              Engine.current_destination = destination
+              destination.write(row)
+            end
+          rescue
+            msg = "Error writing to #{destination} on line #{index}"
+            destination.errors << msg
+            Engine.logger.error msg
           end
         end
+        say_on_own_line "Processed #{Engine.current_source_row} rows in #{distance_of_time_in_words(start_time)}"
         destinations.each do |destination|
           destination.close
         end
       end
       
+      Engine.logger.debug "Post-processing #{control.file}"
       post_process(control)
+      Engine.logger.debug "Post-processing complete"
     end
     
     private
+    # Execute all preprocessors
     def pre_process(control)
       control.pre_processors.each do |processor|
         processor.process
       end
     end
     
+    # Execute all postprocessors
     def post_process(control)
       control.post_processors.each do |processor|
         processor.process
       end
     end
+    
+    # Return the distance of time in words from the given from_time to the specified to_time. If to_time
+    # is not specified then Time.now is used. By default seconds are included...set the include_seconds
+    # argument to false to disable the seconds.
+    def distance_of_time_in_words(from_time, to_time=Time.now)
+      from_time = from_time.to_time if from_time.respond_to?(:to_time)
+      to_time = to_time.to_time if to_time.respond_to?(:to_time)
+      seconds = (to_time - from_time).round
+      distance_in_days = (seconds/(60*60*24)).round
+      seconds = seconds % (60*60*24)
+      distance_in_hours = (seconds/(60*60)).round
+      seconds = seconds % (60*60)
+      distance_in_minutes = (seconds/60).round
+      seconds = seconds % 60
+      distance_in_seconds = seconds
+      
+      s = ''
+      s << "#{distance_in_days} days," if distance_in_days > 0
+      s << "#{distance_in_hours} hours, " if distance_in_hours > 0
+      s << "#{distance_in_minutes} minutes, " if distance_in_minutes > 0
+      s << "#{distance_in_seconds} seconds"
+      s
+    end
+    
+    def approximate_distance_of_time_in_words(from_time, to_time=Time.now, include_seconds=true)
+      from_time = from_time.to_time if from_time.respond_to?(:to_time)
+      to_time = to_time.to_time if to_time.respond_to?(:to_time)
+      distance_in_minutes = (((to_time - from_time).abs)/60).round
+      distance_in_seconds = ((to_time - from_time).abs).round
+      
+      case distance_in_minutes
+        when 0..1
+          return (distance_in_minutes == 0) ? 'less than a minute' : '1 minute' unless include_seconds
+        case distance_in_seconds
+          when 0..4   then 'less than 5 seconds'
+          when 5..9   then 'less than 10 seconds'
+          when 10..19 then 'less than 20 seconds'
+          when 20..39 then 'half a minute'
+          when 40..59 then 'less than a minute'
+          else             '1 minute'
+        end
+        when 2..44           then "#{distance_in_minutes} minutes"
+        when 45..89          then 'about 1 hour'
+        when 90..1439        then "about #{(distance_in_minutes.to_f / 60.0).round} hours"
+        when 1440..2879      then '1 day'
+        when 2880..43199     then "#{(distance_in_minutes / 1440).round} days"
+        when 43200..86399    then 'about 1 month'
+        when 86400..525959   then "#{(distance_in_minutes / 43200).round} months"
+        when 525960..1051919 then 'about 1 year'
+        else                      "over #{(distance_in_minutes / 525960).round} years"
+      end
+    end
   end
 end