activewarehouse-etl 0.9.0 → 0.9.1

data/lib/etl/control/destination/database_destination.rb

@@ -47,10 +47,7 @@ module ETL #:nodoc:
       
       # Flush the currently buffered data
       def flush
-        conn = ETL::Engine.connection(target)
         conn.transaction do
-          conn.truncate(table_name) if truncate
-          
           buffer.flatten.each do |row|
             # check to see if this row's compound key constraint already exists
             # note that the compound key constraint may not utilize virtual fields
@@ -62,10 +59,10 @@ module ETL #:nodoc:
             names = []
             values = []
             order.each do |name|
-              names << name
+              names << "`#{name}`"
               values << conn.quote(row[name]) # TODO: this is probably not database agnostic
             end
-            q = "INSERT INTO #{table_name} (#{names.join(',')}) VALUES (#{values.join(',')})"
+            q = "INSERT INTO `#{table_name}` (#{names.join(',')}) VALUES (#{values.join(',')})"
             ETL::Engine.logger.debug("Executing insert: #{q}")
             conn.insert(q, "Insert row #{current_row}")
             @current_row += 1
@@ -81,6 +78,14 @@ module ETL #:nodoc:
       end
       
       private
+      def conn
+        @conn ||= begin
+          conn = ETL::Engine.connection(target)
+          conn.truncate(table_name) if truncate
+          conn
+        end
+      end
+      
       def table_name
         ETL::Engine.table(table, ETL::Engine.connection(target))
       end

data/lib/etl/control/source.rb

@@ -23,7 +23,7 @@ module ETL #:nodoc:
         # For example if name is :database then this will return a 
         # DatabaseSource class
         def class_for_name(name)
-          ETL::Control.const_get("#{name.to_s.classify}Source")
+          ETL::Control.const_get("#{name.to_s.camelize}Source")
         end
       end
       

data/lib/etl/control/source/database_source.rb

@@ -93,14 +93,8 @@ module ETL #:nodoc:
       # Get the list of columns to read. This is defined in the source
       # definition as either an Array or Hash
       def columns
-        case definition
-        when Array
-          definition.collect(&:to_sym)
-        when Hash
-          definition.keys.collect(&:to_sym)
-        else
-          raise "Definition must be either an Array or a Hash"
-        end
+        # weird default is required for writing to cache correctly
+        @columns ||= query_rows.any? ? query_rows.first.keys : ['']
       end
       
       # Returns each row from the source. If read_locally is specified then
@@ -118,7 +112,7 @@ module ETL #:nodoc:
             write_local(file)
             read_rows(file, &block)
           else
-            connection.select_all(query).each do |row|
+            query_rows.each do |row|
               row = ETL::Row.new(row.symbolize_keys)
               row.source = self
               yield row
@@ -158,7 +152,7 @@ module ETL #:nodoc:
         t = Benchmark.realtime do
           FasterCSV.open(file, 'w') do |f|
             f << columns
-            connection.select_all(query).each do |row|
+            query_rows.each do |row|
               f << columns.collect { |column| row[column.to_s] }
               lines += 1
             end
@@ -204,6 +198,10 @@ module ETL #:nodoc:
         @query = q
       end
       
+      def query_rows
+        @query_rows ||= connection.select_all(query)
+      end
+      
       # Get the database connection to use
       def connection
         ETL::Engine.connection(target)

data/lib/etl/core_ext/time/calculations.rb

@@ -1,3 +1,5 @@
+#Updated by Jack Hong on 04/05/08
+
 module ETL #:nodoc:
   module CoreExtensions #:nodoc:
     module Time #:nodoc:
@@ -18,7 +20,7 @@ module ETL #:nodoc:
         end
         def fiscal_year_month(offset_month=10)
           shifted_month = month - (offset_month - 1)
-          shifted_month += 12 if shifted_month < 0
+          shifted_month += 12 if shifted_month <= 0
           shifted_month
         end
         def fiscal_year_quarter(offset_month=10)
@@ -37,4 +39,4 @@ module ETL #:nodoc:
       end
     end
   end
-end
+end

data/lib/etl/engine.rb

@@ -19,7 +19,7 @@ module ETL #:nodoc:
       # * <tt>:rails_root</tt>: Set to the rails root to boot rails
       def init(options={})
         unless @initialized
-          puts "initializing ETL engine"
+          puts "initializing ETL engine\n\n"
           @limit = options[:limit]
           @offset = options[:offset]
           @log_write_mode = 'w' if options[:newlog]
@@ -28,8 +28,8 @@ module ETL #:nodoc:
           @rails_root = options[:rails_root]
           
           require File.join(@rails_root, 'config/environment') if @rails_root
-
           options[:config] ||= 'database.yml'
+          options[:config] = 'config/database.yml' unless File.exist?(options[:config])
           database_configuration = YAML::load(ERB.new(IO.read(options[:config])).result + "\n")
           ActiveRecord::Base.configurations.merge!(database_configuration)
           ETL::Base.configurations = database_configuration
@@ -177,9 +177,7 @@ module ETL #:nodoc:
             if temp_tables[temp_table_name].nil?
               # Create the temp table and add it to the mapping
               begin connection.drop_table(temp_table_name); rescue; end
-              connection.execute(
-                connection.add_select_into_table(temp_table_name, "SELECT * FROM #{table_name}")
-              )
+              connection.copy_table(table_name, temp_table_name)
               temp_tables[temp_table_name] = {
                 :table => table_name,
                 :connection => connection
@@ -274,6 +272,7 @@ module ETL #:nodoc:
     # Process the specified batch file
     def process_batch(batch)
       batch = ETL::Batch::Batch.resolve(batch, self)
+      say "Processing batch #{batch.file}"
     
       ETL::Engine.batch = ETL::Execution::Batch.create!(
         :batch_file => batch.file,
@@ -290,10 +289,12 @@ module ETL #:nodoc:
     # Process the specified control file
     def process_control(control)
       control = ETL::Control::Control.resolve(control)
+      say_on_own_line "Processing control #{control.file}"
       
       ETL::Engine.job = ETL::Execution::Job.create!(
         :control_file => control.file, 
-        :status => 'executing'
+        :status => 'executing',
+        :batch_id => ETL::Engine.batch ? ETL::Engine.batch.id : nil
       )
       
       execute_dependencies(control)
@@ -357,11 +358,15 @@ module ETL #:nodoc:
                   row[name] = transform.transform(name, row[name], row)
                 end
               end
+            rescue ResolverError => e
+              Engine.logger.error(e.message)
+              errors << e.message
             rescue => e
               msg = "Error transforming from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
               errors << msg
               Engine.logger.error(msg)
               e.backtrace.each { |line| Engine.logger.error(line) }
+            ensure
               begin
                 exceeded_error_threshold?(control) ? break : next
               rescue => inner_error
@@ -422,7 +427,7 @@ module ETL #:nodoc:
         destination.close
       end
       
-      say_on_own_line "Executing screens"
+      say_on_own_line "Executing before post-process screens"
       begin
         execute_screens(control)
       rescue FatalScreenError => e
@@ -443,7 +448,21 @@ module ETL #:nodoc:
       if destinations.length > 0
         say "Wrote #{Engine.rows_written} lines to destinations"
       end
-      say "Completed #{control.file} in #{distance_of_time_in_words(start_time)} with #{errors.length} errors."
+
+      say_on_own_line "Executing after post-process screens"
+      begin
+        execute_screens(control, :after_post_process)
+      rescue FatalScreenError => e
+        say "Fatal screen error during job execution: #{e.message}"
+        exit
+      rescue ScreenError => e
+        say "Screen error during job execution: #{e.message}"
+        return
+      else
+        say "Screens passed"
+      end
+
+      say_on_own_line "Completed #{control.file} in #{distance_of_time_in_words(start_time)} with #{errors.length} errors."
       say "Processing average: #{Engine.average_rows_per_second} rows/sec)"
       
       say "Avg after_reads: #{Engine.rows_read/benchmarks[:after_reads]} rows/sec" if benchmarks[:after_reads] > 0
@@ -509,9 +528,15 @@ module ETL #:nodoc:
     end
     
     # Execute all screens
-    def execute_screens(control)
+    def execute_screens(control, timing = :before_post_process)
+      screens = case timing
+        when :after_post_process
+          control.after_post_process_screens
+        else # default to before post-process screens
+          control.screens
+        end
       [:fatal,:error,:warn].each do |type|
-        control.screens[type].each do |block|
+        screens[type].each do |block|
           begin
             block.call
           rescue => e

data/lib/etl/execution/migration.rb

@@ -4,20 +4,28 @@ module ETL #:nodoc:
     # for the ETL engine
     class Migration
       class << self
+        protected
+        # Get the schema info table name
+        def schema_info_table_name
+          ActiveRecord::Migrator.schema_migrations_table_name
+        end
+        alias :schema_migrations_table_name :schema_info_table_name
+        
+        public
         # Execute the migrations
         def migrate
-          connection.initialize_schema_information
-          v = connection.select_value("SELECT version FROM #{schema_info_table_name}").to_i
-          v.upto(target - 1) do |i| 
+          connection.initialize_schema_migrations_table
+          last_migration.upto(target - 1) do |i| 
             __send__("migration_#{i+1}".to_sym)
-            update_schema_info(i+1)
+            connection.assume_migrated_upto_version(i+1)
           end
         end
+        
         protected
-        # Get the schema info table name
-        def schema_info_table_name
-          ETL::Execution::Base.table_name_prefix + "schema_info" + 
-            ETL::Execution::Base.table_name_suffix
+        def last_migration
+          connection.select_values(
+            "SELECT version FROM #{schema_migrations_table_name}"
+          ).map(&:to_i).sort.last || 0
         end
         
         # Get the connection to use during migration
@@ -27,7 +35,7 @@ module ETL #:nodoc:
         
         # Get the final target version number
         def target
-          3
+          4
         end
         
         private
@@ -62,6 +70,10 @@ module ETL #:nodoc:
           connection.add_column :jobs, :batch_id, :integer
           connection.add_index :jobs, :batch_id
         end
+        
+        def migration_4
+          connection.drop_table :records
+        end
       
         # Update the schema info table, setting the version value
         def update_schema_info(version)

data/lib/etl/generator/generator.rb

@@ -7,7 +7,7 @@ module ETL #:nodoc:
         #
         # For example, if name is :surrogate_key then a SurrogateKeyGenerator class is returned
         def class_for_name(name)
-          ETL::Generator.const_get("#{name.to_s.classify}Generator")
+          ETL::Generator.const_get("#{name.to_s.camelize}Generator")
         end
       end
       

data/lib/etl/http_tools.rb

@@ -106,20 +106,34 @@ module HttpTools
     result
   end
   
-  def parse_uri(uri_string)
-    if uri_string      
+  # Parse a URI. If options[:prefix] is set then prepend it to the keys for the hash that
+  # is returned.
+  def parse_uri(uri_string, options={})
+    prefix = options[:prefix] ||= ''
+    empty_hash = {
+      "#{prefix}scheme".to_sym => nil, 
+      "#{prefix}host".to_sym => nil, 
+      "#{prefix}port".to_sym => nil, 
+      "#{prefix}uri_path".to_sym => nil, 
+      "#{prefix}domain".to_sym => nil
+    }
+    if uri_string
       #attempt to parse uri --if it's a uri then catch the problem and set everything to nil
       begin
         uri = URI.parse(uri_string)    
-        results = {:scheme => uri.scheme, :host => uri.host, :port => uri.port, :uri_path => uri.path}
-        results[:domain] = $1 if uri.host =~ /\.?([^\.]+\.[^\.]+$)/
+        results = {
+          "#{prefix}scheme".to_sym => uri.scheme, 
+          "#{prefix}host".to_sym => uri.host, 
+          "#{prefix}port".to_sym => uri.port, 
+          "#{prefix}uri_path".to_sym => uri.path
+        }
+        results["#{prefix}domain".to_sym] = $1 if uri.host =~ /\.?([^\.]+\.[^\.]+$)/
         results
       rescue
-        {:scheme => nil, :host => nil, :port => nil, :uri_path => nil, :domain => nil}      
+        empty_hash
       end
-
     else
-      {:scheme => nil, :host => nil, :port => nil, :uri_path => nil, :domain => nil}
+      empty_hash
     end
   end
 end

data/lib/etl/parser/apache_combined_log_parser.rb

@@ -33,9 +33,11 @@ module ETL #:nodoc:
         #fields[:timestamp] =~ r%{(\d\d)/(\w\w\w)/(\d\d\d\d):(\d\d):(\d\d):(\d\d) -(\d\d\d\d)}
         d = Date._strptime(fields[:timestamp], '%d/%b/%Y:%H:%M:%S') unless fields[:timestamp].nil?
         fields[:timestamp] = Time.mktime(d[:year], d[:mon], d[:mday], d[:hour], d[:min], d[:sec], d[:sec_fraction]) unless d.nil?
+        
+        fields[:method], fields[:path] = fields[:request].split(/\s/)
 
         fields.merge!(parse_user_agent(fields[:user_agent])) unless fields[:user_agent].nil?
-        fields.merge!(parse_uri(fields[:referrer]))
+        fields.merge!(parse_uri(fields[:referrer], :prefix => 'referrer_'))
         
         fields.each do |key, value|
           fields[key] = nil if value == '-'

data/lib/etl/parser/delimited_parser.rb

@@ -44,7 +44,7 @@ module ETL #:nodoc:
         ETL::Engine.logger.debug "validating line #{line} in file #{file}"
         if row.length != fields.length
           raise_with_info( MismatchError, 
-            "The number of rows from the source (#{row.length}) does not match the number of rows in the definition (#{fields.length})", 
+            "The number of columns from the source (#{row.length}) does not match the number of columns in the definition (#{fields.length})", 
             line, file
           )
         end

data/lib/etl/parser/parser.rb

@@ -11,7 +11,7 @@ module ETL #:nodoc:
         # Example:
         #   <tt>class_for_name(:fixed_width)</tt> returns a FixedWidthParser class
         def class_for_name(name)
-          ETL::Parser.const_get("#{name.to_s.classify}Parser")
+          ETL::Parser.const_get("#{name.to_s.camelize}Parser")
         end
       end
       

data/lib/etl/processor/block_processor.rb

@@ -0,0 +1,14 @@
+module ETL
+  module Processor
+    # This processor is both a valid RowProcessor (called on each row with after_read) or a Processor (called once on pre_process or post_process)
+    class BlockProcessor < ETL::Processor::RowProcessor
+      def initialize(control, configuration)
+        super
+        @block = configuration[:block]
+      end
+      def process(row=nil)
+        @block.call(row)
+      end
+    end
+  end
+end

data/lib/etl/processor/bulk_import_processor.rb

@@ -21,6 +21,8 @@ module ETL #:nodoc:
       attr_accessor :field_enclosure
       # The line separator (defaults to a newline)
       attr_accessor :line_separator
+      # The string that indicates a NULL (defaults to an empty string)
+      attr_accessor :null_string
        
       # Initialize the processor.
       #
@@ -43,6 +45,7 @@ module ETL #:nodoc:
         @columns = configuration[:columns]
         @field_separator = (configuration[:field_separator] || ',')
         @line_separator = (configuration[:line_separator] || "\n")
+        @null_string = (configuration[:null_string] || "")
         @field_enclosure = configuration[:field_enclosure]
         
         raise ControlError, "Target must be specified" unless @target
@@ -59,8 +62,9 @@ module ETL #:nodoc:
           conn.truncate(table_name) if truncate
           options = {}
           options[:columns] = columns
-          if field_separator || field_enclosure
+          if field_separator || field_enclosure || line_separator || null_string
             options[:fields] = {}
+            options[:fields][:null_string] = null_string if null_string
             options[:fields][:delimited_by] = field_separator if field_separator
             options[:fields][:enclosed_by] = field_enclosure if field_enclosure
             options[:fields][:terminated_by] = line_separator if line_separator

data/lib/etl/processor/check_exist_processor.rb

@@ -64,6 +64,7 @@ module ETL #:nodoc:
           end
         end
         q << conditions.join(" AND ")
+        q << " LIMIT 1"
       
         #puts "query: #{q}"
         result = conn.select_one(q)

data/lib/etl/processor/encode_processor.rb

@@ -0,0 +1,55 @@
+require 'iconv'
+
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # The encode processor uses Iconv to convert a file from one encoding (eg: utf-8) to another (eg: latin1), line by line.
+    class EncodeProcessor < ETL::Processor::Processor
+      
+      # The file to load from
+      attr_reader :source_file
+      # The file to write to
+      attr_reader :target_file
+      # The source file encoding
+      attr_reader :source_encoding
+      # The target file encoding
+      attr_reader :target_encoding
+      
+      # Initialize the processor.
+      #
+      # Configuration options:
+      # * <tt>:source_file</tt>: The file to load data from
+      # * <tt>:source_encoding</tt>: The source file encoding (eg: 'latin1','utf-8'), as supported by Iconv
+      # * <tt>:target_file</tt>: The file to write data to
+      # * <tt>:target_encoding</tt>: The target file encoding
+      def initialize(control, configuration)
+        super
+        raise ControlError, "Source file must be specified" if configuration[:source_file].nil?
+        raise ControlError, "Target file must be specified" if configuration[:target_file].nil?
+        @source_file = File.join(File.dirname(control.file), configuration[:source_file])
+        @source_encoding = configuration[:source_encoding]
+        @target_file = File.join(File.dirname(control.file), configuration[:target_file])
+        @target_encoding = configuration[:target_encoding]
+        raise ControlError, "Source and target file cannot currently point to the same file" if source_file == target_file
+        begin
+          @iconv = Iconv.new(target_encoding,source_encoding)
+        rescue Iconv::InvalidEncoding
+          raise ControlError, "Either the source encoding '#{source_encoding}' or the target encoding '#{target_encoding}' is not supported"
+        end
+      end
+      
+      # Execute the processor
+      def process
+        # operate line by line to handle large files without loading them in-memory
+        # could be replaced by a system iconv call when available, for greater performance
+        File.open(source_file) do |source|
+          #puts "Opening #{target_file}"
+          File.open(target_file,'w') do |target|
+            source.each_line do |line|
+              target << @iconv.iconv(line)
+            end
+          end
+        end
+      end
+    end
+  end
+end