activewarehouse-etl-sgonyea 0.9.6

data/lib/etl/control/destination/database_destination.rb

@@ -0,0 +1,97 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # Destination which writes directly to a database. This is useful when you are dealing with
+    # a small amount of data. For larger amounts of data you should probably use the bulk
+    # loader if it is supported with your target database as it will use a much faster load
+    # method.
+    class DatabaseDestination < Destination
+      # The target connection
+      attr_reader :target
+      
+      # The table
+      attr_reader :table
+      
+      # Specify the order from the source
+      attr_reader :order
+      
+      # Set to true to truncate the destination table first
+      attr_reader :truncate
+      
+      # Initialize the database destination
+      # 
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>mapping</tt>: The mapping
+      #
+      # Configuration options:
+      # * <tt>:database</tt>: The database name (REQUIRED)
+      # * <tt>:target</tt>: The target connection (REQUIRED)
+      # * <tt>:table</tt>: The table to write to (REQUIRED)
+      # * <tt>:truncate</tt>: Set to true to truncate before writing (defaults to false)
+      # * <tt>:unique</tt>: Set to true to only insert unique records (defaults to false)
+      # * <tt>:append_rows</tt>: Array of rows to append
+      #
+      # Mapping options:
+      # * <tt>:order</tt>: The order of fields to write (REQUIRED)
+      def initialize(control, configuration, mapping={})
+        super
+        @target = configuration[:target]
+        @table = configuration[:table]
+        @truncate = configuration[:truncate] ||= false
+        @unique = configuration[:unique] ? configuration[:unique] + [scd_effective_date_field] : configuration[:unique]
+        @unique.uniq! unless @unique.nil?
+        @order = mapping[:order] ? mapping[:order] + scd_required_fields : order_from_source
+        @order.uniq! unless @order.nil?
+        raise ControlError, "Order required in mapping" unless @order
+        raise ControlError, "Table required" unless @table
+        raise ControlError, "Target required" unless @target
+      end
+      
+      # Flush the currently buffered data
+      def flush
+        conn.transaction do
+          buffer.flatten.each do |row|
+            # check to see if this row's compound key constraint already exists
+            # note that the compound key constraint may not utilize virtual fields
+            next unless row_allowed?(row)
+
+            # add any virtual fields
+            add_virtuals!(row)
+            
+            names = []
+            values = []
+            order.each do |name|
+              names << conn.quote_column_name(name)
+              values << conn.quote(row[name])
+            end
+            q = "INSERT INTO #{conn.quote_table_name(table_name)} (#{names.join(',')}) VALUES (#{values.join(',')})"
+            ETL::Engine.logger.debug("Executing insert: #{q}")
+            conn.insert(q, "Insert row #{current_row}")
+            @current_row += 1
+          end
+          buffer.clear
+        end
+      end
+      
+      # Close the connection
+      def close
+        buffer << append_rows if append_rows
+        flush
+      end
+      
+      private
+      def conn
+        @conn ||= begin
+          conn = ETL::Engine.connection(target)
+          conn.truncate(table_name) if truncate
+          conn
+        end
+      end
+      
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+      
+    end
+  end
+end

data/lib/etl/control/destination/excel_destination.rb

@@ -0,0 +1,91 @@
+optional_require 'spreadsheet'
+
+module ETL
+  module Control
+    # Excel as the final destination.
+    class ExcelDestination < Destination
+      # The File to write to
+      attr_reader :file
+      
+      # The output order
+      attr_reader :order
+      
+      # Flag which indicates to append (default is to overwrite)
+      attr_accessor :append
+      
+      # Initialize the object.
+      # * <tt>control</tt>: The Control object
+      # * <tt>configuration</tt>: The configuration map
+      # * <tt>mapping</tt>: The output mapping
+      # 
+      # Configuration options:
+      # * <tt>:file<tt>: The file to write to (REQUIRED)
+      # * <tt>:append</tt>: Set to true to append to the file (default is to overwrite)
+      # * <tt>:unique</tt>: Set to true to only write unique records
+      # * <tt>:append_rows</tt>: Array of rows to append
+      # 
+      # Mapping options:
+      # * <tt>:order</tt>: The order array
+      def initialize(control, configuration, mapping={})
+        super
+        path = Pathname.new(configuration[:file])
+        @file = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(control.file))) + path
+        @append = configuration[:append] ||= false
+        @unique = configuration[:unique] ? configuration[:unique] + scd_required_fields : configuration[:unique]
+        @unique.uniq! unless @unique.nil?
+        @order = mapping[:order] ? mapping[:order] + scd_required_fields : order_from_source
+        @order.uniq! unless @order.nil?
+        raise ControlError, "Order required in mapping" unless @order
+      end
+      
+      # Close the destination. This will flush the buffer and close the underlying stream or connection.
+      def close
+        buffer << append_rows if append_rows
+        flush
+        book.write(file)
+      end
+
+   # Flush the destination buffer
+      def flush
+        #puts "Flushing buffer (#{file}) with #{buffer.length} rows"
+        buffer.flatten.each_with_index do |row, index|
+          #puts "row change type: #{row.change_type}"
+          # check to see if this row's compound key constraint already exists
+          # note that the compound key constraint may not utilize virtual fields
+          next unless row_allowed?(row)
+          
+          # add any virtual fields
+          add_virtuals!(row)
+          
+          # collect all of the values using the order designated in the configuration
+          values = order.collect do |name|
+            value = row[name]
+            case value
+            when Date, Time, DateTime
+              value.to_s(:db)
+            else
+              value.to_s
+            end
+          end
+          
+          # write the values
+          sheet.insert_row(index, values)
+        end
+        buffer.clear
+        #puts "After flush there are #{buffer.length} rows"
+      end
+      
+      private
+      # Get the open file excel
+      def book
+        @book ||= ( append ? Spreadsheet.open(file) : Spreadsheet::Workbook.new(file) )
+      end
+
+      private
+      # Get the open sheet
+      def sheet
+        @sheet ||= ( append ? book.worksheet(0) : book.create_worksheet() )
+      end
+    end
+  end
+end

data/lib/etl/control/destination/file_destination.rb

@@ -0,0 +1,126 @@
+# This source file contains the ETL::Control::FileDestination
+
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # File as the final destination.
+    class FileDestination < Destination
+      # The File to write to
+      attr_reader :file
+      
+      # The output order
+      attr_reader :order
+      
+      # Flag which indicates to append (default is to overwrite)
+      attr_accessor :append
+      
+      # The separator
+      attr_accessor :separator
+      
+      # The end of line marker
+      attr_accessor :eol
+      
+      # The enclosure character
+      attr_accessor :enclose
+      
+      # Initialize the object.
+      # * <tt>control</tt>: The Control object
+      # * <tt>configuration</tt>: The configuration map
+      # * <tt>mapping</tt>: The output mapping
+      # 
+      # Configuration options:
+      # * <tt>:file<tt>: The file to write to (REQUIRED)
+      # * <tt>:append</tt>: Set to true to append to the file (default is to overwrite)
+      # * <tt>:separator</tt>: Record separator (default is a comma)
+      # * <tt>:eol</tt>: End of line marker (default is \n)
+      # * <tt>:enclose</tt>: Enclosure character (default is none)
+      # * <tt>:unique</tt>: Set to true to only write unique records
+      # * <tt>:append_rows</tt>: Array of rows to append
+      # 
+      # Mapping options:
+      # * <tt>:order</tt>: The order array
+      def initialize(control, configuration, mapping={})
+        super
+        path = Pathname.new(configuration[:file])
+        @file = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(control.file))) + path
+        @append = configuration[:append] ||= false
+        @separator = configuration[:separator] ||= ','
+        @eol = configuration[:eol] ||= "\n"
+        @enclose = configuration[:enclose]
+        @unique = configuration[:unique] ? configuration[:unique] + scd_required_fields : configuration[:unique]
+        @unique.uniq! unless @unique.nil?
+        @order = mapping[:order] ? mapping[:order] + scd_required_fields : order_from_source
+        @order.uniq! unless @order.nil?
+        raise ControlError, "Order required in mapping" unless @order
+      end
+      
+      # Close the destination. This will flush the buffer and close the underlying stream or connection.
+      def close
+        buffer << append_rows if append_rows
+        flush
+        f.close
+      end
+      
+      # Flush the destination buffer
+      def flush
+        #puts "Flushing buffer (#{file}) with #{buffer.length} rows"
+        buffer.flatten.each do |row|
+          #puts "row change type: #{row.change_type}"
+          # check to see if this row's compound key constraint already exists
+          # note that the compound key constraint may not utilize virtual fields
+          next unless row_allowed?(row)
+          
+          # add any virtual fields
+          add_virtuals!(row)
+          
+          # collect all of the values using the order designated in the configuration
+          values = order.collect do |name|
+            value = row[name]
+            case value
+            when Date, Time, DateTime
+              value.to_s(:db)
+            else
+              value.to_s
+            end
+          end
+          
+          values.collect! { |v| v.gsub(/\\/, '\\\\\\\\')}
+          values.collect! { |v| v.gsub(separator, "\\#{separator}")}
+          values.collect! { |v| v.gsub(/\n|\r/, '')}
+          
+          # enclose the value if required
+          if !enclose.nil?
+            values.collect! { |v| enclose + v.gsub(/(#{enclose})/, '\\\\\1') + enclose }
+          end
+          
+          # write the values joined by the separator defined in the configuration
+          f.write(values.join(separator))
+          
+          # write the end-of-line
+          f.write(eol)
+        end
+        f.flush
+        buffer.clear
+        #puts "After flush there are #{buffer.length} rows"
+      end
+      
+      private
+      # Get the open file stream
+      def f
+        @f ||= open(file, mode)
+      end
+      
+      def options
+        @options ||= {
+          :col_sep => separator,
+          :row_sep => eol,
+          :force_quotes => !enclose.nil?
+        }
+      end
+      
+      # Get the appropriate mode to open the file stream
+      def mode
+        append ? 'a' : 'w'
+      end
+    end
+  end
+end

data/lib/etl/control/destination/insert_update_database_destination.rb

@@ -0,0 +1,136 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # Destination which writes directly to a database. This is useful when you are dealing with
+    # a small amount of data. For larger amounts of data you should probably use the bulk
+    # loader if it is supported with your target database as it will use a much faster load
+    # method.
+    class InsertUpdateDatabaseDestination < Destination
+      # The target connection
+      attr_reader :target
+      
+      # The table
+      attr_reader :table
+      
+      # Specify the order from the source
+      attr_reader :order
+      
+      # Specify the primarykey from the source
+      attr_reader :primarykey
+      
+      # Set to true to truncate the destination table first
+      attr_reader :truncate
+      
+      # Initialize the database destination
+      # 
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>mapping</tt>: The mapping
+      #
+      # Configuration options:
+      # * <tt>:database</tt>: The database name (REQUIRED)
+      # * <tt>:target</tt>: The target connection (REQUIRED)
+      # * <tt>:table</tt>: The table to write to (REQUIRED)
+      # * <tt>:truncate</tt>: Set to true to truncate before writing (defaults to false)
+      # * <tt>:unique</tt>: Set to true to only insert unique records (defaults to false)
+      # * <tt>:append_rows</tt>: Array of rows to append
+      #
+      # Mapping options:
+      # * <tt>:order</tt>: The order of fields to write (REQUIRED)
+      # * <tt>:primarykey</tt>: The primary key of fields to select insert or update (REQUIRED)
+      def initialize(control, configuration, mapping={})
+        super
+        @target = configuration[:target]
+        @table = configuration[:table]
+        @truncate = configuration[:truncate] ||= false
+        @unique = configuration[:unique] ? configuration[:unique] + [scd_effective_date_field] : configuration[:unique]
+        @unique.uniq! unless @unique.nil?
+        @order = mapping[:order] ? mapping[:order] + scd_required_fields : order_from_source
+        @order.uniq! unless @order.nil?
+        @primarykey = mapping[:primarykey] ? mapping[:primarykey] + scd_required_fields : nil
+        @primarykey.uniq! unless @primarykey.nil?
+        raise ControlError, "Primarykey required in mapping" unless @primarykey
+        raise ControlError, "Order required in mapping" unless @order
+        raise ControlError, "Table required" unless @table
+        raise ControlError, "Target required" unless @target
+      end
+      
+      # Flush the currently buffered data
+      def flush
+        conn.transaction do
+          buffer.flatten.each do |row|
+            # check to see if this row's compound key constraint already exists
+            # note that the compound key constraint may not utilize virtual fields
+            next unless row_allowed?(row)
+
+            # add any virtual fields
+            add_virtuals!(row)
+            
+            primarykeyfilter = []
+            primarykey.each do |name|
+              primarykeyfilter << "#{conn.quote_column_name(name)} = #{conn.quote(row[name])}"
+            end
+            q = "SELECT * FROM #{conn.quote_table_name(table_name)} WHERE #{primarykeyfilter.join(' AND ')}"
+            ETL::Engine.logger.debug("Executing select: #{q}")
+            res = conn.execute(q, "Select row #{current_row}")
+            none = true
+            
+            case conn
+              when ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
+                res.each { none = false }
+              when ActiveRecord::ConnectionAdapters::MysqlAdapter
+                res.each_hash { none = false }
+                res.free
+              when ActiveRecord::ConnectionAdapters::Mysql2Adapter
+                res.each { none = false }
+              else raise "Unsupported adapter #{conn.class} for this destination"
+            end
+
+            if none
+              names = []
+              values = []
+              order.each do |name|
+                names << conn.quote_column_name(name)
+                values << conn.quote(row[name])
+              end
+              q = "INSERT INTO #{conn.quote_table_name(table_name)} (#{names.join(',')}) VALUES (#{values.join(',')})"
+              ETL::Engine.logger.debug("Executing insert: #{q}")
+              conn.insert(q, "Insert row #{current_row}")
+            else
+              updatevalues = []
+              order.each do |name|
+                updatevalues << "#{conn.quote_column_name(name)} = #{conn.quote(row[name])}"
+              end
+              q = "UPDATE #{conn.quote_table_name(table_name)} SET #{updatevalues.join(',')} WHERE #{primarykeyfilter.join(' AND ')}"
+              ETL::Engine.logger.debug("Executing update: #{q}")
+              conn.update(q, "Update row #{current_row}")
+            end
+            @current_row += 1
+          end
+          buffer.clear
+        end
+      end
+      
+      # Close the connection
+      def close
+        buffer << append_rows if append_rows
+        flush
+      end
+      
+      private
+      def conn
+        @conn ||= begin
+          conn = ETL::Engine.connection(target)
+          conn.truncate(table_name) if truncate
+          conn
+        rescue
+          raise RuntimeError, "Problem to connect to db"
+        end
+      end
+      
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+      
+    end
+  end
+end