activewarehouse-etl 0.9.1 → 0.9.5.rc1

data/lib/etl/control/destination/update_database_destination.rb

@@ -0,0 +1,109 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # Destination which writes directly to a database. This is useful when you are dealing with
+    # a small amount of data. For larger amounts of data you should probably use the bulk
+    # loader if it is supported with your target database as it will use a much faster load
+    # method.
+    class UpdateDatabaseDestination < Destination
+      # The target connection
+      attr_reader :target
+      
+      # The table
+      attr_reader :table
+      
+      # Specify the order from the source
+      attr_reader :order
+      
+      # Specify the conditions from the source
+      attr_reader :conditions
+      
+      # Initialize the database destination
+      # 
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>mapping</tt>: The mapping
+      #
+      # Configuration options:
+      # * <tt>:database</tt>: The database name (REQUIRED)
+      # * <tt>:target</tt>: The target connection (REQUIRED)
+      # * <tt>:table</tt>: The table to write to (REQUIRED)
+      # * <tt>:unique</tt>: Set to true to only insert unique records (defaults to false)
+      # * <tt>:append_rows</tt>: Array of rows to append
+      #
+      # Mapping options:
+      # * <tt>:order</tt>: The order of fields to write (REQUIRED)
+      # * <tt>:conditions</tt>: The conditions on the fields to update (REQUIRED)
+      def initialize(control, configuration, mapping={})
+        super
+        @target = configuration[:target]
+        @table = configuration[:table]
+        @unique = configuration[:unique] ? configuration[:unique] + [scd_effective_date_field] : configuration[:unique]
+        @unique.uniq! unless @unique.nil?
+        @order = mapping[:order] ? mapping[:order] + scd_required_fields : order_from_source
+        @order.uniq! unless @order.nil?
+        @conditions = mapping[:conditions] ? mapping[:conditions] + scd_required_fields : nil
+        @conditions.uniq! unless @conditions.nil?
+        raise ControlError, "Conditions required in mapping" unless @conditions
+        raise ControlError, "Order required in mapping" unless @order
+        raise ControlError, "Table required" unless @table
+        raise ControlError, "Target required" unless @target
+      end
+      
+      # Flush the currently buffered data
+      def flush
+        conn.transaction do
+          buffer.flatten.each do |row|
+            # check to see if this row's compound key constraint already exists
+            # note that the compound key constraint may not utilize virtual fields
+            next unless row_allowed?(row)
+
+            # add any virtual fields
+            add_virtuals!(row)
+            
+            conditionsfilter = []
+            conditions.each do |cond|
+              c = " #{cond[:field]} #{cond[:comp]} #{cond[:value]} "
+              condition = c
+              begin
+                condition = eval('"' + c + '"')
+              rescue
+              end
+              conditionsfilter << condition
+            end
+
+            updatevalues = []
+            order.each do |name|
+              updatevalues << "#{conn.quote_column_name(name)} = #{conn.quote(row[name])}"
+            end
+            q = "UPDATE #{conn.quote_table_name(table_name)} SET #{updatevalues.join(',')} WHERE #{conditionsfilter.join(' AND ')}"
+            ETL::Engine.logger.debug("Executing update: #{q}")
+            conn.update(q, "Update row #{current_row}")
+            @current_row += 1
+          end
+          buffer.clear
+        end
+      end
+      
+      # Close the connection
+      def close
+        buffer << append_rows if append_rows
+        flush
+      end
+      
+      private
+      def conn
+        @conn ||= begin
+          conn = ETL::Engine.connection(target)
+          conn
+        rescue
+          raise RuntimeError, "Problem to connect to db"
+        end
+      end
+      
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+      
+    end
+  end
+end

data/lib/etl/control/source.rb

@@ -40,7 +40,8 @@ module ETL #:nodoc:
         @configuration = configuration
         @definition = definition
         
-        @store_locally = configuration[:store_locally] || true
+        @store_locally = true
+        @store_locally = configuration[:store_locally] unless configuration[:store_locally].nil?
       end
       
       # Get an array of errors that occur during reading from the source
@@ -106,4 +107,4 @@ module ETL #:nodoc:
   end
 end
 
-Dir[File.dirname(__FILE__) + "/source/*.rb"].each { |file| require(file) }
+Dir[File.dirname(__FILE__) + "/source/*.rb"].each { |file| require(file) }

data/lib/etl/control/source/database_source.rb

@@ -41,17 +41,18 @@ module ETL #:nodoc:
         super
         @target = configuration[:target]
         @table = configuration[:table]
+        @query = configuration[:query]
       end
       
       # Get a String identifier for the source
       def to_s
-        "#{host}/#{database}/#{table}"
+        "#{host}/#{database}/#{@table}"
       end
       
       # Get the local directory to use, which is a combination of the 
       # local_base, the db hostname the db database name and the db table.
       def local_directory
-        File.join(local_base, host, database, configuration[:table])
+        File.join(local_base, to_s)
       end
       
       # Get the join part of the query, defaults to nil
@@ -83,7 +84,7 @@ module ETL #:nodoc:
       # Get the number of rows in the source
       def count(use_cache=true)
         return @count if @count && use_cache
-        if store_locally || read_locally
+        if @store_locally || read_locally
           @count = count_locally
         else
           @count = connection.select_value(query.gsub(/SELECT .* FROM/, 'SELECT count(1) FROM'))
@@ -107,13 +108,16 @@ module ETL #:nodoc:
           ETL::Engine.logger.debug "Reading from local cache"
           read_rows(last_local_file, &block)
         else # Read from the original source
-          if store_locally
+          if @store_locally
             file = local_file
             write_local(file)
             read_rows(file, &block)
           else
-            query_rows.each do |row|
-              row = ETL::Row.new(row.symbolize_keys)
+            query_rows.each do |r|
+              row = ETL::Row.new()
+              r.symbolize_keys.each_pair { |key, value|
+                row[key] = value
+              }
               row.source = self
               yield row
             end
@@ -128,7 +132,7 @@ module ETL #:nodoc:
         raise "Local cache trigger file not found" unless File.exists?(local_file_trigger(file))
         
         t = Benchmark.realtime do
-          FasterCSV.open(file, :headers => true).each do |row|
+          CSV.open(file, :headers => true).each do |row|
             result_row = ETL::Row.new
             result_row.source = self
             row.each do |header, field|
@@ -150,7 +154,7 @@ module ETL #:nodoc:
       def write_local(file)
         lines = 0
         t = Benchmark.realtime do
-          FasterCSV.open(file, 'w') do |f|
+          CSV.open(file, 'w') do |f|
             f << columns
             query_rows.each do |row|
               f << columns.collect { |column| row[column.to_s] }
@@ -165,7 +169,7 @@ module ETL #:nodoc:
       # Get the query to use
       def query
         return @query if @query
-        q = "SELECT #{select} FROM #{configuration[:table]}"
+        q = "SELECT #{select} FROM #{@table}"
         q << " #{join}" if join
         
         conditions = []
@@ -217,4 +221,4 @@ module ETL #:nodoc:
       end
     end
   end
-end
+end

data/lib/etl/control/source/file_source.rb

@@ -41,7 +41,7 @@ module ETL #:nodoc:
       # Returns each row from the source
       def each
         count = 0
-        copy_sources if store_locally
+        copy_sources if @store_locally
         @parser.each do |row|
           if ETL::Engine.offset && count < ETL::Engine.offset
             count += 1
@@ -87,4 +87,4 @@ module ETL #:nodoc:
       end
     end
   end
-end
+end

data/lib/etl/engine.rb

@@ -32,7 +32,7 @@ module ETL #:nodoc:
           options[:config] = 'config/database.yml' unless File.exist?(options[:config])
           database_configuration = YAML::load(ERB.new(IO.read(options[:config])).result + "\n")
           ActiveRecord::Base.configurations.merge!(database_configuration)
-          ETL::Base.configurations = database_configuration
+          ETL::Base.configurations = HashWithIndifferentAccess.new(database_configuration)
           #puts "configurations in init: #{ActiveRecord::Base.configurations.inspect}"
           
           require 'etl/execution'
@@ -173,17 +173,19 @@ module ETL #:nodoc:
       # Modify the table name if necessary
       def table(table_name, connection)
         if use_temp_tables?
-          returning "tmp_#{table_name}" do |temp_table_name|
-            if temp_tables[temp_table_name].nil?
-              # Create the temp table and add it to the mapping
-              begin connection.drop_table(temp_table_name); rescue; end
-              connection.copy_table(table_name, temp_table_name)
-              temp_tables[temp_table_name] = {
-                :table => table_name,
-                :connection => connection
-              }
-            end
+          temp_table_name = "tmp_#{table_name}"
+
+          if temp_tables[temp_table_name].nil?
+            # Create the temp table and add it to the mapping
+            begin connection.drop_table(temp_table_name); rescue; end
+            connection.copy_table(table_name, temp_table_name)
+            temp_tables[temp_table_name] = {
+              :table => table_name,
+              :connection => connection
+            }
           end
+
+          temp_table_name
         else
           table_name
         end
@@ -308,7 +310,7 @@ module ETL #:nodoc:
       
       sources.each do |source|
         Engine.current_source = source
-        Engine.logger.debug "Processing source #{source}"
+        Engine.logger.debug "Processing source #{source.inspect}"
         say "Source: #{source}"
         say "Limiting enabled: #{Engine.limit}" if Engine.limit != nil
         say "Offset enabled: #{Engine.offset}" if Engine.offset != nil
@@ -470,8 +472,8 @@ module ETL #:nodoc:
       say "Avg transforms: #{Engine.rows_read/benchmarks[:transforms]} rows/sec" if benchmarks[:transforms] > 0
       say "Avg writes: #{Engine.rows_read/benchmarks[:writes]} rows/sec" if benchmarks[:writes] > 0
 
-      say "Avg time writing execution records: #{ETL::Execution::Record.average_time_spent}"
-      
+      # say "Avg time writing execution records: #{ETL::Execution::Record.average_time_spent}"
+      # 
       # ETL::Transform::Transform.benchmarks.each do |klass, t|
 #         say "Avg #{klass}: #{Engine.rows_read/t} rows/sec"
 #       end
@@ -553,4 +555,4 @@ module ETL #:nodoc:
       end
     end
   end
-end
+end

data/lib/etl/execution.rb

@@ -16,5 +16,4 @@ end
 require 'etl/execution/base'
 require 'etl/execution/batch'
 require 'etl/execution/job'
-require 'etl/execution/record'
 require 'etl/execution/migration'

data/lib/etl/execution/batch.rb

@@ -2,7 +2,9 @@ module ETL #:nodoc:
   module Execution #:nodoc:
     # Persistent class representing an ETL batch
     class Batch < Base
+      belongs_to :batch
+      has_many :batches
       has_many :jobs
     end
   end
-end
+end

data/lib/etl/execution/migration.rb

@@ -74,6 +74,11 @@ module ETL #:nodoc:
         def migration_4
           connection.drop_table :records
         end
+
+        def migration_5
+          connection.add_column :batches, :batch_id, :integer
+          connection.add_index :batches, :batch_id
+        end
       
         # Update the schema info table, setting the version value
         def update_schema_info(version)

data/lib/etl/parser/delimited_parser.rb

@@ -10,13 +10,32 @@ module ETL #:nodoc:
         configure
       end
       
+      def get_fields_names(file)
+        File.open(file) do |input|
+          fields = CSV.parse(input.readline).first
+          new_fields = []
+          fields.each_with_index do |field,index|
+            # compute the index of occurrence of this specific occurrence of the field (usually, will be 1)
+            occurrence_index = fields[0..index].find_all { |e| e == field }.size
+            number_of_occurrences = fields.find_all { |e| e == field }.size
+            new_field = field + (number_of_occurrences > 1 ? "_#{occurrence_index}" : "")
+            new_fields << Field.new(new_field.to_sym)
+          end
+          return new_fields
+        end
+      end
+
       # Returns each row.
       def each
         Dir.glob(file).each do |file|
           ETL::Engine.logger.debug "parsing #{file}"
+          if fields.length == 0
+            ETL::Engine.logger.debug "no columns specified so reading names from first line of #{file}"
+            @fields = get_fields_names(file)
+          end
           line = 0
           lines_skipped = 0
-          FasterCSV.foreach(file, options) do |raw_row|
+          CSV.foreach(file, options) do |raw_row|
             if lines_skipped < source.skip_lines
               ETL::Engine.logger.debug "skipping line"
               lines_skipped += 1

data/lib/etl/parser/excel_parser.rb

@@ -0,0 +1,112 @@
+require 'spreadsheet'
+
+module ETL
+  module Parser
+    class ExcelParser < ETL::Parser::Parser
+
+      attr_accessor :ignore_blank_line
+
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+
+      # Returns each row
+      def each
+        Dir.glob(file).each do |file|
+          ETL::Engine.logger.debug "parsing #{file}"
+          line = 0
+          lines_skipped = 0
+          book = Spreadsheet.open file
+          loopworksheets = []
+
+          if worksheets.empty?
+            loopworksheets = book.worksheets
+          else
+            worksheets.each do |index|
+              loopworksheets << book.worksheet(index)
+            end
+          end
+
+          loopworksheets.each do |sheet|
+            sheet.each do |raw_row|
+              if lines_skipped < source.skip_lines
+                ETL::Engine.logger.debug "skipping line"
+                lines_skipped += 1
+                next
+              end
+              line += 1
+              row = {}
+              if self.ignore_blank_line and raw_row.empty?
+                lines_skipped += 1
+                next
+              end
+              validate_row(raw_row, line, file)
+              raw_row.each_with_index do |value, index|
+                f = fields[index]
+                row[f.name] = value
+              end
+              yield row
+            end
+          end
+        end
+      end
+
+      # Get an array of defined worksheets
+      def worksheets
+        @worksheets ||= []
+      end
+
+      # Get an array of defined fields
+      def fields
+        @fields ||= []
+      end
+
+      private
+      def validate_row(row, line, file)
+        ETL::Engine.logger.debug "validating line #{line} in file #{file}"
+        if row.length != fields.length
+          raise_with_info( MismatchError, 
+            "The number of columns from the source (#{row.length}) does not match the number of columns in the definition (#{fields.length})", 
+            line, file
+          )
+        end
+      end
+      
+      private
+      def configure
+        source.definition[:worksheets].each do |worksheet|
+          if Integer(worksheet)
+            worksheets << worksheet.to_i
+          else
+            raise DefinitionError, "Each worksheet definition must be an integer"
+          end
+        end unless source.definition[:worksheets].nil?
+
+        self.ignore_blank_line = source.definition[:ignore_blank_line]
+
+        source.definition[:fields].each do |options|
+          case options
+          when Symbol
+            fields << Field.new(options)
+          when Hash
+            fields << Field.new(options[:name])
+          else
+            raise DefinitionError, "Each field definition must either be a symbol or a hash"
+          end
+        end
+      end
+      
+      class Field #:nodoc:
+        attr_reader :name
+        def initialize(name)
+          @name = name
+        end
+      end
+
+    end
+  end
+end