colincasey-activewarehouse-etl 0.9.4

data/lib/etl/control/destination/database_destination.rb

@@ -0,0 +1,95 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # Destination which writes directly to a database. This is useful when you are dealing with
+    # a small amount of data. For larger amounts of data you should probably use the bulk
+    # loader if it is supported with your target database as it will use a much faster load
+    # method.
+    class DatabaseDestination < Destination
+      # The target connection
+      attr_reader :target
+      
+      # The table
+      attr_reader :table
+      
+      # Specify the order from the source
+      attr_reader :order
+      
+      # Set to true to truncate the destination table first
+      attr_reader :truncate
+      
+      # Initialize the database destination
+      # 
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>mapping</tt>: The mapping
+      #
+      # Configuration options:
+      # * <tt>:database</tt>: The database name (REQUIRED)
+      # * <tt>:target</tt>: The target connection (REQUIRED)
+      # * <tt>:table</tt>: The table to write to (REQUIRED)
+      # * <tt>:truncate</tt>: Set to true to truncate before writing (defaults to false)
+      # * <tt>:unique</tt>: Set to true to only insert unique records (defaults to false)
+      # * <tt>:append_rows</tt>: Array of rows to append
+      #
+      # Mapping options:
+      # * <tt>:order</tt>: The order of fields to write (REQUIRED)
+      def initialize(control, configuration, mapping={})
+        super
+        @target = configuration[:target]
+        @table = configuration[:table]
+        @truncate = configuration[:truncate] ||= false
+        @unique = configuration[:unique]
+        @order = mapping[:order] || order_from_source
+        raise ControlError, "Order required in mapping" unless @order
+        raise ControlError, "Table required" unless @table
+        raise ControlError, "Target required" unless @target
+      end
+      
+      # Flush the currently buffered data
+      def flush
+        conn.transaction do
+          buffer.flatten.each do |row|
+            # check to see if this row's compound key constraint already exists
+            # note that the compound key constraint may not utilize virtual fields
+            next unless row_allowed?(row)
+
+            # add any virtual fields
+            add_virtuals!(row)
+            
+            names = []
+            values = []
+            order.each do |name|
+              names << "`#{name}`"
+              values << conn.quote(row[name]) # TODO: this is probably not database agnostic
+            end
+            q = "INSERT INTO `#{table_name}` (#{names.join(',')}) VALUES (#{values.join(',')})"
+            ETL::Engine.logger.debug("Executing insert: #{q}")
+            conn.insert(q, "Insert row #{current_row}")
+            @current_row += 1
+          end
+          buffer.clear
+        end
+      end
+      
+      # Close the connection
+      def close
+        buffer << append_rows if append_rows
+        flush
+      end
+      
+      private
+      def conn
+        @conn ||= begin
+          conn = ETL::Engine.connection(target)
+          conn.truncate(table_name) if truncate
+          conn
+        end
+      end
+      
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+      
+    end
+  end
+end

data/lib/etl/control/destination/file_destination.rb

@@ -0,0 +1,124 @@
+# This source file contains the ETL::Control::FileDestination
+
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # File as the final destination.
+    class FileDestination < Destination
+      # The File to write to
+      attr_reader :file
+      
+      # The output order
+      attr_reader :order
+      
+      # Flag which indicates to append (default is to overwrite)
+      attr_accessor :append
+      
+      # The separator
+      attr_accessor :separator
+      
+      # The end of line marker
+      attr_accessor :eol
+      
+      # The enclosure character
+      attr_accessor :enclose
+      
+      # Initialize the object.
+      # * <tt>control</tt>: The Control object
+      # * <tt>configuration</tt>: The configuration map
+      # * <tt>mapping</tt>: The output mapping
+      # 
+      # Configuration options:
+      # * <tt>:file<tt>: The file to write to (REQUIRED)
+      # * <tt>:append</tt>: Set to true to append to the file (default is to overwrite)
+      # * <tt>:separator</tt>: Record separator (default is a comma)
+      # * <tt>:eol</tt>: End of line marker (default is \n)
+      # * <tt>:enclose</tt>: Enclosure character (default is none)
+      # * <tt>:unique</tt>: Set to true to only write unique records
+      # * <tt>:append_rows</tt>: Array of rows to append
+      # 
+      # Mapping options:
+      # * <tt>:order</tt>: The order array
+      def initialize(control, configuration, mapping={})
+        super
+        @file = File.join(File.dirname(control.file), configuration[:file])
+        @append = configuration[:append] ||= false
+        @separator = configuration[:separator] ||= ','
+        @eol = configuration[:eol] ||= "\n"
+        @enclose = configuration[:enclose]
+        @unique = configuration[:unique]
+        
+        @order = mapping[:order] || order_from_source
+        raise ControlError, "Order required in mapping" unless @order
+      end
+      
+      # Close the destination. This will flush the buffer and close the underlying stream or connection.
+      def close
+        buffer << append_rows if append_rows
+        flush
+        f.close
+      end
+      
+      # Flush the destination buffer
+      def flush
+        #puts "Flushing buffer (#{file}) with #{buffer.length} rows"
+        buffer.flatten.each do |row|
+          #puts "row change type: #{row.change_type}"
+          # check to see if this row's compound key constraint already exists
+          # note that the compound key constraint may not utilize virtual fields
+          next unless row_allowed?(row)
+          
+          # add any virtual fields
+          add_virtuals!(row)
+          
+          # collect all of the values using the order designated in the configuration
+          values = order.collect do |name|
+            value = row[name]
+            case value
+            when Date, Time, DateTime
+              value.to_s(:db)
+            else
+              value.to_s
+            end
+          end
+          
+          values.collect! { |v| v.gsub(/\\/, '\\\\\\\\')}
+          values.collect! { |v| v.gsub(separator, "\\#{separator}")}
+          values.collect! { |v| v.gsub(/\n|\r/, '')}
+          
+          # enclose the value if required
+          if !enclose.nil?
+            values.collect! { |v| enclose + v.gsub(/(#{enclose})/, '\\\\\1') + enclose }
+          end
+          
+          # write the values joined by the separator defined in the configuration
+          f.write(values.join(separator))
+          
+          # write the end-of-line
+          f.write(eol)
+        end
+        f.flush
+        buffer.clear
+        #puts "After flush there are #{buffer.length} rows"
+      end
+      
+      private
+      # Get the open file stream
+      def f
+        @f ||= open(file, mode)
+      end
+      
+      def options
+        @options ||= {
+          :col_sep => separator,
+          :row_sep => eol,
+          :force_quotes => !enclose.nil?
+        }
+      end
+      
+      # Get the appropriate mode to open the file stream
+      def mode
+        append ? 'a' : 'w'
+      end
+    end
+  end
+end

data/lib/etl/control/destination/yaml_destination.rb

@@ -0,0 +1,74 @@
+require 'yaml'
+
+module ETL #:nodoc:
+  module Control #:nodoc:
+    class YamlDestination < Destination
+      attr_reader :file, :append, :only, :except
+      # Initialize the object.
+      # * <tt>control</tt>: The Control object
+      # * <tt>configuration</tt>: The configuration map
+      # * <tt>mapping</tt>: The output mapping
+      #
+      # Configuration options:
+      # * <tt>:file<tt>: The file to write to (REQUIRED)
+      # * <tt>:append</tt>: Set to true to append to the file (default is to overwrite)
+      # * <tt>:only</tt>
+      # * <tt>:except</tt>
+      def initialize(control, configuration, mapping={})
+        super
+        @file = File.join(File.dirname(control.file), configuration[:file])
+        @append = configuration[:append] ||= false
+        @only = configuration[:only]
+        @except = configuration[:except]
+        raise ControlError, "the :only and :except options must be used seperately, do not specify both" if @only && @except
+      end
+
+      # Close the destination. This will flush the buffer and close the underlying stream or connection.
+      def close
+        flush
+        f.close
+      end
+
+      # Flush the destination buffer
+      def flush
+        #puts "Flushing buffer (#{file}) with #{buffer.length} rows"
+        buffer.flatten.each do |row|
+          # check to see if this row's compound key constraint already exists
+          # note that the compound key constraint may not utilize virtual fields
+          next unless row_allowed?(row)
+          # add any virtual fields
+          add_virtuals!(row)
+
+          yaml = {}
+          row.each do |key, value|
+            next if only && !only.include?(key)
+            next if except && except.include?(key)
+
+            case value
+            when Date, Time, DateTime
+              value = value.to_s(:db)
+            end
+
+            yaml[key] = value
+          end
+         
+          # write the values
+          YAML.dump(yaml, f)
+        end
+        f.flush
+        buffer.clear
+      end
+
+      private
+      # Get the open file stream
+      def f
+        @f ||= File.open(file, mode)
+      end
+
+      # Get the appropriate mode to open the file stream
+      def mode
+        append ? 'a' : 'w'
+      end
+    end
+  end
+end

data/lib/etl/control/source.rb

@@ -0,0 +1,109 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # ETL source. Subclasses must implement the <tt>each</tt> method.
+    class Source
+      include Enumerable
+      
+      # The control object
+      attr_accessor :control
+      
+      # The configuration Hash
+      attr_accessor :configuration
+      
+      # The definition Hash
+      attr_accessor :definition
+      
+      # Returns true if the source data should be stored locally for archival
+      # Default behavior will return true.
+      attr_accessor :store_locally
+      
+      class << self
+        # Convert the name to a Source class.
+        # 
+        # For example if name is :database then this will return a 
+        # DatabaseSource class
+        def class_for_name(name)
+          ETL::Control.const_get("#{name.to_s.camelize}Source")
+        end
+      end
+      
+      # Initialize the Source instance
+      # * <tt>control</tt>: The control object
+      # * <tt>configuration</tt>: The configuration hash
+      # * <tt>definition</tt>: The source layout definition
+      #
+      # Configuration options:
+      # * <tt>:store_locally</tt>: Set to false to not store source data
+      #   locally (defaults to true)
+      def initialize(control, configuration, definition)
+        @control = control
+        @configuration = configuration
+        @definition = definition
+        
+        @store_locally = configuration[:store_locally] || true
+      end
+      
+      # Get an array of errors that occur during reading from the source
+      def errors
+        @errors ||= []
+      end
+      
+      # Get a timestamp value as a string
+      def timestamp
+        Engine.timestamp
+      end
+      
+      # The base directory where local files are stored.
+      attr_accessor :local_base
+      
+      # Get the local base, defaults to 'source_data'
+      def local_base
+        @local_base ||= 'source_data'
+      end
+      
+      # The local directory for storing. This method must be overriden by 
+      # subclasses
+      def local_directory
+        raise "local_directory method is abstract"
+      end
+      
+      # Return the local file for storing the raw source data. Each call to 
+      # this method will result in a timestamped file, so you cannot expect 
+      # to call it multiple times and reference the same file
+      # 
+      # Optional sequence can be specified if there are multiple source files
+      def local_file(sequence=nil)
+        filename = timestamp.to_s
+        filename += sequence.to_s if sequence
+        
+        local_dir = local_directory
+        FileUtils.mkdir_p(local_dir)
+        File.join(local_dir, "#{filename}.csv")
+      end
+      
+      # Get the last fully written local file
+      def last_local_file
+        File.join(local_directory, File.basename(last_local_file_trigger, '.trig'))
+      end
+      
+      # Get the last local file trigger
+      def last_local_file_trigger
+        Dir.glob(File.join(local_directory, '*.trig')).last
+      end
+      
+      # Get the local trigger file that is used to indicate that the file has
+      # been completely written
+      def local_file_trigger(file)
+        Pathname.new(file.to_s + '.trig')
+      end
+      
+      # Return true if the source should read locally.
+      def read_locally
+        Engine.read_locally
+      end
+      
+    end
+  end
+end
+
+Dir[File.dirname(__FILE__) + "/source/*.rb"].each { |file| require(file) }

data/lib/etl/control/source/database_source.rb

@@ -0,0 +1,220 @@
+require 'fileutils'
+
+module ETL #:nodoc:
+  class Source < ::ActiveRecord::Base #:nodoc:
+    # Connection for database sources
+  end
+  
+  module Control #:nodoc:
+    # Source object which extracts data from a database using ActiveRecord.
+    class DatabaseSource < Source
+      attr_accessor :target
+      attr_accessor :table
+      
+      # Initialize the source.
+      #
+      # Arguments:
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>definition</tt>: The source definition
+      #
+      # Required configuration options:
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:table</tt>: The source table name
+      # * <tt>:database</tt>: The database name
+      # 
+      # Other options:
+      # * <tt>:join</tt>: Optional join part for the query (ignored unless 
+      #   specified)
+      # * <tt>:select</tt>: Optional select part for the query (defaults to 
+      #   '*')
+      # * <tt>:group</tt>: Optional group by part for the query (ignored 
+      #   unless specified)
+      # * <tt>:order</tt>: Optional order part for the query (ignored unless 
+      #   specified)
+      # * <tt>:new_records_only</tt>: Specify the column to use when comparing
+      #   timestamps against the last successful ETL job execution for the
+      #   current control file.
+      # * <tt>:store_locally</tt>: Set to false to not store a copy of the 
+      #   source data locally in a flat file (defaults to true)
+      def initialize(control, configuration, definition)
+        super
+        @target = configuration[:target]
+        @table = configuration[:table]
+      end
+      
+      # Get a String identifier for the source
+      def to_s
+        "#{host}/#{database}/#{table}"
+      end
+      
+      # Get the local directory to use, which is a combination of the 
+      # local_base, the db hostname the db database name and the db table.
+      def local_directory
+        File.join(local_base, host, database, configuration[:table])
+      end
+      
+      # Get the join part of the query, defaults to nil
+      def join
+        configuration[:join]
+      end
+      
+      # Get the select part of the query, defaults to '*'
+      def select
+        configuration[:select] || '*'
+      end
+      
+      # Get the group by part of the query, defaults to nil
+      def group
+        configuration[:group]
+      end
+      
+      # Get the order for the query, defaults to nil
+      def order
+        configuration[:order]
+      end
+      
+      # Return the column which is used for in the where clause to identify
+      # new rows
+      def new_records_only
+        configuration[:new_records_only]
+      end
+      
+      # Get the number of rows in the source
+      def count(use_cache=true)
+        return @count if @count && use_cache
+        if store_locally || read_locally
+          @count = count_locally
+        else
+          @count = connection.select_value(query.gsub(/SELECT .* FROM/, 'SELECT count(1) FROM'))
+        end
+      end
+      
+      # Get the list of columns to read. This is defined in the source
+      # definition as either an Array or Hash
+      def columns
+        # weird default is required for writing to cache correctly
+        @columns ||= query_rows.any? ? query_rows.first.keys : ['']
+      end
+      
+      # Returns each row from the source. If read_locally is specified then
+      # this method will attempt to read from the last stored local file. 
+      # If no locally stored file exists or if the trigger file for the last
+      # locally stored file does not exist then this method will raise an
+      # error.
+      def each(&block)
+        if read_locally # Read from the last stored source
+          ETL::Engine.logger.debug "Reading from local cache"
+          read_rows(last_local_file, &block)
+        else # Read from the original source
+          if store_locally
+            file = local_file
+            write_local(file)
+            read_rows(file, &block)
+          else
+            query_rows.each do |row|
+              row = ETL::Row.new(row.symbolize_keys)
+              row.source = self
+              yield row
+            end
+          end
+        end
+      end
+      
+      private
+      # Read rows from the local cache
+      def read_rows(file)
+        raise "Local cache file not found" unless File.exists?(file)
+        raise "Local cache trigger file not found" unless File.exists?(local_file_trigger(file))
+        
+        t = Benchmark.realtime do
+          FasterCSV.open(file, :headers => true).each do |row|
+            result_row = ETL::Row.new
+            result_row.source = self
+            row.each do |header, field|
+              result_row[header.to_sym] = field
+            end
+            yield result_row
+          end
+        end
+        ETL::Engine.average_rows_per_second = ETL::Engine.rows_read / t
+      end
+      
+      def count_locally
+        counter = 0
+        File.open(last_local_file, 'r').each { |line| counter += 1 }
+        counter
+      end
+      
+      # Write rows to the local cache
+      def write_local(file)
+        lines = 0
+        t = Benchmark.realtime do
+          FasterCSV.open(file, 'w') do |f|
+            f << columns
+            query_rows.each do |row|
+              f << columns.collect { |column| row[column.to_s] }
+              lines += 1
+            end
+          end
+          File.open(local_file_trigger(file), 'w') {|f| }
+        end
+        ETL::Engine.logger.info "Stored locally in #{t}s (avg: #{lines/t} lines/sec)"
+      end
+      
+      # Get the query to use
+      def query
+        return @query if @query
+        q = "SELECT #{select} FROM #{configuration[:table]}"
+        q << " #{join}" if join
+        
+        conditions = []
+        if new_records_only
+          last_completed = ETL::Execution::Job.maximum('created_at', 
+            :conditions => ['control_file = ? and completed_at is not null', control.file]
+          )
+          if last_completed
+            conditions << "#{new_records_only} > #{connection.quote(last_completed.to_s(:db))}"
+          end
+        end
+        
+        conditions << configuration[:conditions] if configuration[:conditions]
+        if conditions.length > 0
+          q << " WHERE #{conditions.join(' AND ')}"
+        end
+        
+        q << " GROUP BY #{group}" if group
+        q << " ORDER BY #{order}" if order
+
+        if ETL::Engine.limit || ETL::Engine.offset
+          options = {}
+          options[:limit] = ETL::Engine.limit if ETL::Engine.limit
+          options[:offset] = ETL::Engine.offset if ETL::Engine.offset
+          connection.add_limit_offset!(q, options)
+        end
+        
+        q = q.gsub(/\n/,' ')
+        ETL::Engine.logger.info "Query: #{q}"
+        @query = q
+      end
+      
+      def query_rows
+        @query_rows ||= connection.select_all(query)
+      end
+      
+      # Get the database connection to use
+      def connection
+        ETL::Engine.connection(target)
+      end
+      
+      # Get the host, defaults to 'localhost'
+      def host
+        ETL::Base.configurations[target.to_s]['host'] || 'localhost'
+      end
+      
+      def database
+        ETL::Base.configurations[target.to_s]['database']
+      end
+    end
+  end
+end