activewarehouse-etl 0.9.1 → 0.9.5.rc1

data/active_support_logger.patch

@@ -0,0 +1,78 @@
+Index: lib/active_support/clean_logger.rb
+===================================================================
+--- lib/active_support/clean_logger.rb	(revision 5963)
++++ lib/active_support/clean_logger.rb	(working copy)
+@@ -1,10 +1,21 @@
+ require 'logger'
+ require File.dirname(__FILE__) + '/core_ext/class/attribute_accessors'
+ 
+-class Logger #:nodoc:
++# Extensions to the built in Ruby logger.
++#
++# If you want to use the default log formatter as defined in the Ruby core, then you 
++# will need to set the formatter for the logger as in:
++#
++#   logger.formatter = Formatter.new
++#
++# You can then specify the datetime format, for example:
++#
++#   logger.datetime_format = "%Y-%m-%d"
++class Logger
++  # Set to false to disable the silencer
+   cattr_accessor :silencer
+   self.silencer = true
+-
++  
+   # Silences the logger for the duration of the block.
+   def silence(temporary_level = Logger::ERROR)
+     if silencer
+@@ -18,6 +29,35 @@
+       yield self
+     end
+   end
++  
++  alias :old_datetime_format= :datetime_format=
++  # Logging date-time format (string passed to +strftime+). Ignored if the formatter
++  # does not respond to datetime_format=.
++  def datetime_format=(datetime_format)
++    formatter.datetime_format = datetime_format if formatter.respond_to?(:datetime_format=)
++  end
++  
++  alias :old_datetime_format :datetime_format
++  # Get the logging datetime format. Returns nil if the formatter does not support
++  # datetime formatting.
++  def datetime_format
++    formatter.datetime_format if formatter.respond_to?(:datetime_format)
++  end
++  
++  alias :old_formatter :formatter
++  # Get the current formatter. The default formatter is a SimpleFormatter which only
++  # displays the log message
++  def formatter
++    @formatter ||= SimpleFormatter.new
++  end
++  
++  # Simple formatter which only displays the message.
++  class SimpleFormatter < Logger::Formatter
++    # This method is invoked when a log event occurs
++    def call(severity, timestamp, progname, msg)
++      "#{msg}\n"
++    end
++  end
+ 
+   private
+     alias old_format_message format_message
+@@ -28,11 +68,11 @@
+     # with Logger from 1.8.3 and vice versa.
+     if method_defined?(:formatter=)
+       def format_message(severity, timestamp, progname, msg)
+-        "#{msg}\n"
++        formatter.call(severity, timestamp, progname, msg)
+       end
+     else
+       def format_message(severity, timestamp, msg, progname)
+-        "#{msg}\n"
++        formatter.call(severity, timestamp, progname, msg)
+       end
+     end
+ end

data/activewarehouse-etl.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+
+require 'etl/version'
+
+Gem::Specification.new do |s|
+  s.name = %q{activewarehouse-etl}
+  s.version = ETL::VERSION
+  s.platform = Gem::Platform::RUBY
+  s.authors = ["Anthony Eden", "Thibaut Barrère"]
+  s.email = ["thibaut.barrere@gmail.com"]
+  s.homepage = "https://github.com/activewarehouse/activewarehouse-etl"
+  s.summary = %q{Pure Ruby ETL package.}
+  s.description = %q{ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application for loading data into a database.}
+
+  s.required_rubygems_version = ">= 1.3.6"
+
+  s.add_runtime_dependency('rake',                '>= 0.8.3')
+  s.add_runtime_dependency('activesupport',       '>= 2.1.0')
+  s.add_runtime_dependency('activerecord',        '>= 2.1.0')
+  s.add_runtime_dependency('fastercsv',           '>= 1.2.0')
+  s.add_runtime_dependency('adapter_extensions',  '>= 0.5.0')
+  s.add_runtime_dependency('spreadsheet')
+
+  s.files              = `git ls-files`.split("\n")
+  s.test_files         = `git ls-files -- {test}/*`.split("\n")
+  s.executables        = %w(etl)
+  s.require_path       = "lib"
+end

data/lib/etl.rb

@@ -33,13 +33,21 @@ require 'rubygems'
 
 unless defined?(REXML::VERSION)
   require 'rexml/rexml'
-  REXML::VERSION = REXML::Version
+  unless defined?(REXML::VERSION)
+    REXML::VERSION = REXML::Version
+  end
 end
 
 require 'active_support'
 require 'active_record'
 require 'adapter_extensions'
-require 'faster_csv'
+
+if RUBY_VERSION < '1.9'
+  require 'faster_csv'
+  CSV = FasterCSV unless defined?(CSV)
+else
+  require 'csv'
+end
 
 $:.unshift(File.dirname(__FILE__))
 

data/lib/etl/batch/directives.rb

@@ -37,7 +37,17 @@ module ETL #:nodoc:
       protected
       # Execute the process
       def do_execute
+        current_batch = ETL::Engine.batch
         batch.engine.process(file)
+
+        job = ETL::Engine.batch
+        if (job.kind_of? ETL::Execution::Batch and
+            current_batch[:id] != job[:id])
+          job[:batch_id] = current_batch[:id]
+          job.save!
+        end
+
+        ETL::Engine.batch = current_batch
       end
     end
     
@@ -52,4 +62,4 @@ module ETL #:nodoc:
       end
     end 
   end
-end
+end

data/lib/etl/control/control.rb

@@ -66,8 +66,8 @@ module ETL #:nodoc:
               sources << source_class.new(self, configuration, definition)
               break
             end
-            raise ControlError, "A source was specified but no matching type was found"
           end
+          raise ControlError, "A source was specified but no matching type was found" if sources.empty?
         end
       end
       
@@ -100,8 +100,8 @@ module ETL #:nodoc:
               destinations << dest_class.new(self, configuration, mapping)
               break
             end
-            raise ControlError, "A destination was specified but no matching destination type was found"
           end
+          raise ControlError, "A destination was specified but no matching destination type was found" if destinations.empty?      
         end
       end
       

data/lib/etl/control/destination.rb

@@ -129,11 +129,23 @@ module ETL #:nodoc:
       # missing, uses all of the row's fields.
       def scd_fields(row)
         @scd_fields ||= configuration[:scd_fields] || row.keys
+        ETL::Engine.logger.debug "@scd_fields is: #{@scd_fields.inspect}"
+        @scd_fields
+      end
+
+      # returns the fields that are required to identify an SCD
+      def scd_required_fields
+        if scd?
+         [scd_effective_date_field, scd_end_date_field, scd_latest_version_field]
+        else
+         []
+        end
       end
       
       def non_scd_fields(row)
-        @non_csd_fields ||= row.keys - natural_key - scd_fields(row) -
-          [primary_key, scd_effective_date_field, scd_end_date_field, scd_latest_version_field]
+        @non_scd_fields ||= row.keys - natural_key - scd_fields(row) - [primary_key] - scd_required_fields
+        ETL::Engine.logger.debug "@non_scd_fields is: #{@non_scd_fields.inspect}"
+        @non_scd_fields
       end
       
       def non_evolving_fields
@@ -280,7 +292,8 @@ module ETL #:nodoc:
           values << row[nk]
         end
         statement = statement.join(" AND ")
-        ActiveRecord::Base.send(:sanitize_sql, [statement, *values])
+        x=ActiveRecord::Base.send(:sanitize_sql_array, [statement, *values])
+        return x
       end
       
       # Do all the steps required when a SCD *has* changed.  Exact steps
@@ -353,10 +366,10 @@ module ETL #:nodoc:
         q = "SELECT * FROM #{dimension_table} WHERE #{natural_key_equality_for_row(row)}"
         q << " AND #{scd_latest_version_field}" if scd_type == 2
         
-        #puts "looking for original record"
+        ETL::Engine.logger.debug "looking for original record"
         result = connection.select_one(q)
         
-        #puts "Result: #{result.inspect}"
+        ETL::Engine.logger.debug "Result: #{result.inspect}"
         
         result ? ETL::Row[result.symbolize_keys!] : nil
       end
@@ -364,7 +377,14 @@ module ETL #:nodoc:
       # Check whether non-scd fields have changed since the last
       # load of this record.
       def has_scd_field_changes?(row)
-        scd_fields(row).any? { |csd_field| row[csd_field].to_s != @existing_row[csd_field].to_s }
+        scd_fields(row).any? { |csd_field| 
+          ETL::Engine.logger.debug "Row: #{row.inspect}"
+          ETL::Engine.logger.debug "Existing Row: #{@existing_row.inspect}"
+          ETL::Engine.logger.debug "comparing: #{row[csd_field].to_s} != #{@existing_row[csd_field].to_s}"
+          x=row[csd_field].to_s != @existing_row[csd_field].to_s 
+          ETL::Engine.logger.debug x
+          x
+        }
       end
       
       # Check whether non-scd fields have changed since the last
@@ -417,4 +437,4 @@ module ETL #:nodoc:
   end
 end
 
-Dir[File.dirname(__FILE__) + "/destination/*.rb"].each { |file| require(file) }
+Dir[File.dirname(__FILE__) + "/destination/*.rb"].each { |file| require(file) }

data/lib/etl/control/destination/database_destination.rb

@@ -38,8 +38,10 @@ module ETL #:nodoc:
         @target = configuration[:target]
         @table = configuration[:table]
         @truncate = configuration[:truncate] ||= false
-        @unique = configuration[:unique]
-        @order = mapping[:order] || order_from_source
+        @unique = configuration[:unique] ? configuration[:unique] + [scd_effective_date_field] : configuration[:unique]
+        @unique.uniq! unless @unique.nil?
+        @order = mapping[:order] ? mapping[:order] + scd_required_fields : order_from_source
+        @order.uniq! unless @order.nil?
         raise ControlError, "Order required in mapping" unless @order
         raise ControlError, "Table required" unless @table
         raise ControlError, "Target required" unless @target
@@ -59,10 +61,10 @@ module ETL #:nodoc:
             names = []
             values = []
             order.each do |name|
-              names << "`#{name}`"
-              values << conn.quote(row[name]) # TODO: this is probably not database agnostic
+              names << conn.quote_column_name(name)
+              values << conn.quote(row[name])
             end
-            q = "INSERT INTO `#{table_name}` (#{names.join(',')}) VALUES (#{values.join(',')})"
+            q = "INSERT INTO #{conn.quote_table_name(table_name)} (#{names.join(',')}) VALUES (#{values.join(',')})"
             ETL::Engine.logger.debug("Executing insert: #{q}")
             conn.insert(q, "Insert row #{current_row}")
             @current_row += 1
@@ -92,4 +94,4 @@ module ETL #:nodoc:
       
     end
   end
-end
+end

data/lib/etl/control/destination/excel_destination.rb

@@ -0,0 +1,91 @@
+require 'spreadsheet'
+
+module ETL
+  module Control
+    # Excel as the final destination.
+    class ExcelDestination < Destination
+      # The File to write to
+      attr_reader :file
+      
+      # The output order
+      attr_reader :order
+      
+      # Flag which indicates to append (default is to overwrite)
+      attr_accessor :append
+      
+      # Initialize the object.
+      # * <tt>control</tt>: The Control object
+      # * <tt>configuration</tt>: The configuration map
+      # * <tt>mapping</tt>: The output mapping
+      # 
+      # Configuration options:
+      # * <tt>:file<tt>: The file to write to (REQUIRED)
+      # * <tt>:append</tt>: Set to true to append to the file (default is to overwrite)
+      # * <tt>:unique</tt>: Set to true to only write unique records
+      # * <tt>:append_rows</tt>: Array of rows to append
+      # 
+      # Mapping options:
+      # * <tt>:order</tt>: The order array
+      def initialize(control, configuration, mapping={})
+        super
+        path = Pathname.new(configuration[:file])
+        @file = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(control.file))) + path
+        @append = configuration[:append] ||= false
+        @unique = configuration[:unique] ? configuration[:unique] + scd_required_fields : configuration[:unique]
+        @unique.uniq! unless @unique.nil?
+        @order = mapping[:order] ? mapping[:order] + scd_required_fields : order_from_source
+        @order.uniq! unless @order.nil?
+        raise ControlError, "Order required in mapping" unless @order
+      end
+      
+      # Close the destination. This will flush the buffer and close the underlying stream or connection.
+      def close
+        buffer << append_rows if append_rows
+        flush
+        book.write(file)
+      end
+
+   # Flush the destination buffer
+      def flush
+        #puts "Flushing buffer (#{file}) with #{buffer.length} rows"
+        buffer.flatten.each_with_index do |row, index|
+          #puts "row change type: #{row.change_type}"
+          # check to see if this row's compound key constraint already exists
+          # note that the compound key constraint may not utilize virtual fields
+          next unless row_allowed?(row)
+          
+          # add any virtual fields
+          add_virtuals!(row)
+          
+          # collect all of the values using the order designated in the configuration
+          values = order.collect do |name|
+            value = row[name]
+            case value
+            when Date, Time, DateTime
+              value.to_s(:db)
+            else
+              value.to_s
+            end
+          end
+          
+          # write the values
+          sheet.insert_row(index, values)
+        end
+        buffer.clear
+        #puts "After flush there are #{buffer.length} rows"
+      end
+      
+      private
+      # Get the open file excel
+      def book
+        @book ||= ( append ? Spreadsheet.open(file) : Spreadsheet::Workbook.new(file) )
+      end
+
+      private
+      # Get the open sheet
+      def sheet
+        @sheet ||= ( append ? book.worksheet(0) : book.create_worksheet() )
+      end
+    end
+  end
+end

data/lib/etl/control/destination/file_destination.rb

@@ -40,14 +40,16 @@ module ETL #:nodoc:
       # * <tt>:order</tt>: The order array
       def initialize(control, configuration, mapping={})
         super
-        @file = File.join(File.dirname(control.file), configuration[:file])
+        path = Pathname.new(configuration[:file])
+        @file = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(control.file))) + path
         @append = configuration[:append] ||= false
         @separator = configuration[:separator] ||= ','
         @eol = configuration[:eol] ||= "\n"
         @enclose = configuration[:enclose]
-        @unique = configuration[:unique]
-        
-        @order = mapping[:order] || order_from_source
+        @unique = configuration[:unique] ? configuration[:unique] + scd_required_fields : configuration[:unique]
+        @unique.uniq! unless @unique.nil?
+        @order = mapping[:order] ? mapping[:order] + scd_required_fields : order_from_source
+        @order.uniq! unless @order.nil?
         raise ControlError, "Order required in mapping" unless @order
       end
       

data/lib/etl/control/destination/insert_update_database_destination.rb

@@ -0,0 +1,133 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # Destination which writes directly to a database. This is useful when you are dealing with
+    # a small amount of data. For larger amounts of data you should probably use the bulk
+    # loader if it is supported with your target database as it will use a much faster load
+    # method.
+    class InsertUpdateDatabaseDestination < Destination
+      # The target connection
+      attr_reader :target
+      
+      # The table
+      attr_reader :table
+      
+      # Specify the order from the source
+      attr_reader :order
+      
+      # Specify the primarykey from the source
+      attr_reader :primarykey
+      
+      # Set to true to truncate the destination table first
+      attr_reader :truncate
+      
+      # Initialize the database destination
+      # 
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>mapping</tt>: The mapping
+      #
+      # Configuration options:
+      # * <tt>:database</tt>: The database name (REQUIRED)
+      # * <tt>:target</tt>: The target connection (REQUIRED)
+      # * <tt>:table</tt>: The table to write to (REQUIRED)
+      # * <tt>:truncate</tt>: Set to true to truncate before writing (defaults to false)
+      # * <tt>:unique</tt>: Set to true to only insert unique records (defaults to false)
+      # * <tt>:append_rows</tt>: Array of rows to append
+      #
+      # Mapping options:
+      # * <tt>:order</tt>: The order of fields to write (REQUIRED)
+      # * <tt>:primarykey</tt>: The primary key of fields to select insert or update (REQUIRED)
+      def initialize(control, configuration, mapping={})
+        super
+        @target = configuration[:target]
+        @table = configuration[:table]
+        @truncate = configuration[:truncate] ||= false
+        @unique = configuration[:unique] ? configuration[:unique] + [scd_effective_date_field] : configuration[:unique]
+        @unique.uniq! unless @unique.nil?
+        @order = mapping[:order] ? mapping[:order] + scd_required_fields : order_from_source
+        @order.uniq! unless @order.nil?
+        @primarykey = mapping[:primarykey] ? mapping[:primarykey] + scd_required_fields : nil
+        @primarykey.uniq! unless @primarykey.nil?
+        raise ControlError, "Primarykey required in mapping" unless @primarykey
+        raise ControlError, "Order required in mapping" unless @order
+        raise ControlError, "Table required" unless @table
+        raise ControlError, "Target required" unless @target
+      end
+      
+      # Flush the currently buffered data
+      def flush
+        conn.transaction do
+          buffer.flatten.each do |row|
+            # check to see if this row's compound key constraint already exists
+            # note that the compound key constraint may not utilize virtual fields
+            next unless row_allowed?(row)
+
+            # add any virtual fields
+            add_virtuals!(row)
+            
+            primarykeyfilter = []
+            primarykey.each do |name|
+              primarykeyfilter << "#{conn.quote_column_name(name)} = #{conn.quote(row[name])}"
+            end
+            q = "SELECT * FROM #{conn.quote_table_name(table_name)} WHERE #{primarykeyfilter.join(' AND ')}"
+            ETL::Engine.logger.debug("Executing select: #{q}")
+            res = conn.execute(q, "Select row #{current_row}")
+            none = true
+            
+            case conn
+              when ActiveRecord::ConnectionAdapters::PostgreSQLAdapter;
+                res.each { none = false }
+              when ActiveRecord::ConnectionAdapters::MysqlAdapter;
+                res.each_hash { none = false }
+              else raise "Unsupported adapter #{conn.class} for this destination"
+            end
+
+            if none
+              names = []
+              values = []
+              order.each do |name|
+                names << conn.quote_column_name(name)
+                values << conn.quote(row[name])
+              end
+              q = "INSERT INTO #{conn.quote_table_name(table_name)} (#{names.join(',')}) VALUES (#{values.join(',')})"
+              ETL::Engine.logger.debug("Executing insert: #{q}")
+              conn.insert(q, "Insert row #{current_row}")
+            else
+              updatevalues = []
+              order.each do |name|
+                updatevalues << "#{conn.quote_column_name(name)} = #{conn.quote(row[name])}"
+              end
+              q = "UPDATE #{conn.quote_table_name(table_name)} SET #{updatevalues.join(',')} WHERE #{primarykeyfilter.join(' AND ')}"
+              ETL::Engine.logger.debug("Executing update: #{q}")
+              conn.update(q, "Update row #{current_row}")
+            end
+            @current_row += 1
+          end
+          buffer.clear
+        end
+      end
+      
+      # Close the connection
+      def close
+        buffer << append_rows if append_rows
+        flush
+      end
+      
+      private
+      def conn
+        @conn ||= begin
+          conn = ETL::Engine.connection(target)
+          conn.truncate(table_name) if truncate
+          conn
+        rescue
+          raise RuntimeError, "Problem to connect to db"
+        end
+      end
+      
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+      
+    end
+  end
+end