activewarehouse-etl-sgonyea 0.9.6

data/lib/etl/control/destination/update_database_destination.rb

@@ -0,0 +1,109 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # Destination which writes directly to a database. This is useful when you are dealing with
+    # a small amount of data. For larger amounts of data you should probably use the bulk
+    # loader if it is supported with your target database as it will use a much faster load
+    # method.
+    class UpdateDatabaseDestination < Destination
+      # The target connection
+      attr_reader :target
+      
+      # The table
+      attr_reader :table
+      
+      # Specify the order from the source
+      attr_reader :order
+      
+      # Specify the conditions from the source
+      attr_reader :conditions
+      
+      # Initialize the database destination
+      # 
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>mapping</tt>: The mapping
+      #
+      # Configuration options:
+      # * <tt>:database</tt>: The database name (REQUIRED)
+      # * <tt>:target</tt>: The target connection (REQUIRED)
+      # * <tt>:table</tt>: The table to write to (REQUIRED)
+      # * <tt>:unique</tt>: Set to true to only insert unique records (defaults to false)
+      # * <tt>:append_rows</tt>: Array of rows to append
+      #
+      # Mapping options:
+      # * <tt>:order</tt>: The order of fields to write (REQUIRED)
+      # * <tt>:conditions</tt>: The conditions on the fields to update (REQUIRED)
+      def initialize(control, configuration, mapping={})
+        super
+        @target = configuration[:target]
+        @table = configuration[:table]
+        @unique = configuration[:unique] ? configuration[:unique] + [scd_effective_date_field] : configuration[:unique]
+        @unique.uniq! unless @unique.nil?
+        @order = mapping[:order] ? mapping[:order] + scd_required_fields : order_from_source
+        @order.uniq! unless @order.nil?
+        @conditions = mapping[:conditions] ? mapping[:conditions] + scd_required_fields : nil
+        @conditions.uniq! unless @conditions.nil?
+        raise ControlError, "Conditions required in mapping" unless @conditions
+        raise ControlError, "Order required in mapping" unless @order
+        raise ControlError, "Table required" unless @table
+        raise ControlError, "Target required" unless @target
+      end
+      
+      # Flush the currently buffered data
+      def flush
+        conn.transaction do
+          buffer.flatten.each do |row|
+            # check to see if this row's compound key constraint already exists
+            # note that the compound key constraint may not utilize virtual fields
+            next unless row_allowed?(row)
+
+            # add any virtual fields
+            add_virtuals!(row)
+            
+            conditionsfilter = []
+            conditions.each do |cond|
+              c = " #{cond[:field]} #{cond[:comp]} #{cond[:value]} "
+              condition = c
+              begin
+                condition = eval('"' + c + '"')
+              rescue
+              end
+              conditionsfilter << condition
+            end
+
+            updatevalues = []
+            order.each do |name|
+              updatevalues << "#{conn.quote_column_name(name)} = #{conn.quote(row[name])}"
+            end
+            q = "UPDATE #{conn.quote_table_name(table_name)} SET #{updatevalues.join(',')} WHERE #{conditionsfilter.join(' AND ')}"
+            ETL::Engine.logger.debug("Executing update: #{q}")
+            conn.update(q, "Update row #{current_row}")
+            @current_row += 1
+          end
+          buffer.clear
+        end
+      end
+      
+      # Close the connection
+      def close
+        buffer << append_rows if append_rows
+        flush
+      end
+      
+      private
+      def conn
+        @conn ||= begin
+          conn = ETL::Engine.connection(target)
+          conn
+        rescue
+          raise RuntimeError, "Problem to connect to db"
+        end
+      end
+      
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+      
+    end
+  end
+end

data/lib/etl/control/destination/yaml_destination.rb

@@ -0,0 +1,74 @@
+require 'yaml'
+
+module ETL #:nodoc:
+  module Control #:nodoc:
+    class YamlDestination < Destination
+      attr_reader :file, :append, :only, :except
+      # Initialize the object.
+      # * <tt>control</tt>: The Control object
+      # * <tt>configuration</tt>: The configuration map
+      # * <tt>mapping</tt>: The output mapping
+      #
+      # Configuration options:
+      # * <tt>:file<tt>: The file to write to (REQUIRED)
+      # * <tt>:append</tt>: Set to true to append to the file (default is to overwrite)
+      # * <tt>:only</tt>
+      # * <tt>:except</tt>
+      def initialize(control, configuration, mapping={})
+        super
+        @file = File.join(File.dirname(control.file), configuration[:file])
+        @append = configuration[:append] ||= false
+        @only = configuration[:only]
+        @except = configuration[:except]
+        raise ControlError, "the :only and :except options must be used seperately, do not specify both" if @only && @except
+      end
+
+      # Close the destination. This will flush the buffer and close the underlying stream or connection.
+      def close
+        flush
+        f.close
+      end
+
+      # Flush the destination buffer
+      def flush
+        #puts "Flushing buffer (#{file}) with #{buffer.length} rows"
+        buffer.flatten.each do |row|
+          # check to see if this row's compound key constraint already exists
+          # note that the compound key constraint may not utilize virtual fields
+          next unless row_allowed?(row)
+          # add any virtual fields
+          add_virtuals!(row)
+
+          yaml = {}
+          row.each do |key, value|
+            next if only && !only.include?(key)
+            next if except && except.include?(key)
+
+            case value
+            when Date, Time, DateTime
+              value = value.to_s(:db)
+            end
+
+            yaml[key] = value
+          end
+         
+          # write the values
+          YAML.dump(yaml, f)
+        end
+        f.flush
+        buffer.clear
+      end
+
+      private
+      # Get the open file stream
+      def f
+        @f ||= File.open(file, mode)
+      end
+
+      # Get the appropriate mode to open the file stream
+      def mode
+        append ? 'a' : 'w'
+      end
+    end
+  end
+end

data/lib/etl/control/source.rb

@@ -0,0 +1,132 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # ETL source. Subclasses must implement the <tt>each</tt> method.
+    class Source
+      include Enumerable
+      
+      # The control object
+      attr_accessor :control
+      
+      # The configuration Hash
+      attr_accessor :configuration
+      
+      # The definition Hash
+      attr_accessor :definition
+      
+      # Returns true if the source data should be stored locally for archival
+      # Default behavior will return true.
+      attr_accessor :store_locally
+      
+      class << self
+        # Convert the name to a Source class.
+        # 
+        # For example if name is :database then this will return a 
+        # DatabaseSource class
+        def class_for_name(name)
+          ETL::Control.const_get("#{name.to_s.camelize}Source")
+        end
+      end
+      
+      # Initialize the Source instance
+      # * <tt>control</tt>: The control object
+      # * <tt>configuration</tt>: The configuration hash
+      # * <tt>definition</tt>: The source layout definition
+      #
+      # Configuration options:
+      # * <tt>:store_locally</tt>: Set to false to not store source data
+      #   locally (defaults to true)
+      def initialize(control, configuration, definition)
+        @control = control
+        @configuration = configuration
+        @definition = definition
+        
+        @store_locally = configuration[:store_locally].nil? ? true : configuration[:store_locally]
+      end
+      
+      # Get an array of errors that occur during reading from the source
+      def errors
+        @errors ||= []
+      end
+      
+      # Get a timestamp value as a string
+      def timestamp
+        Engine.timestamp
+      end
+      
+      # The base directory where local files are stored.
+      attr_accessor :local_base
+      
+      # Get the local base, defaults to 'source_data'
+      def local_base
+        @local_base ||= 'source_data'
+      end
+      
+      # The local directory for storing. This method must be overriden by 
+      # subclasses
+      def local_directory
+        raise "local_directory method is abstract"
+      end
+      
+      # Return the local file for storing the raw source data. Each call to 
+      # this method will result in a timestamped file, so you cannot expect 
+      # to call it multiple times and reference the same file
+      # 
+      # Optional sequence can be specified if there are multiple source files
+      def local_file(sequence=nil)
+        filename = timestamp.to_s
+        filename += sequence.to_s if sequence
+        
+        local_dir = local_directory
+        FileUtils.mkdir_p(local_dir)
+        File.join(local_dir, "#{filename}.csv")
+      end
+      
+      # Get the last fully written local file
+      def last_local_file
+        File.join(local_directory, File.basename(last_local_file_trigger, '.trig'))
+      end
+      
+      # Get the last local file trigger filename using timestamp in filenames.
+      # Filename is in the format YYYYMMDDHHMMSS.csv.trig, but in the case of a
+      # file source there is an unpadded sequence number before the file
+      # extension.  This code may not return the correct "last" file in that
+      # case (in particular when there are 10 or more source files).  However,
+      # at this point only the database source calls the method, and it wouldn't
+      # make sense for a file source to use it if multiple files are expected
+      def last_local_file_trigger
+        trig_files = []
+        trig_ext = '.csv.trig'
+
+        # Store the basename (without extension) of all files that end in the
+        # desired extension
+        Dir.glob(File.join(local_directory, "*" + trig_ext)) do |f|
+            # Extract the basename of each file with the extension snipped off
+            trig_files << File.basename(f, trig_ext) if File.file?(f)
+        end
+
+        # Throw an exception if no trigger files are available
+        raise "Local cache trigger file not found" if trig_files.empty?
+
+        # Sort trigger file strings and get the last one
+        last_trig = trig_files.sort {|a,b| a <=> b}.last
+
+        # Return the file path including extension
+        File.join(local_directory, last_trig + trig_ext)
+      end
+      
+      # Get the local trigger file that is used to indicate that the file has
+      # been completely written
+      def local_file_trigger(file)
+        Pathname.new(file.to_s + '.trig')
+      end
+      
+      # Return true if the source should read locally.
+      def read_locally
+        Engine.read_locally
+      end
+      
+    end
+  end
+end
+
+Dir[File.dirname(__FILE__) + "/source/*.rb"].each { |file| require(file) }

data/lib/etl/control/source/database_source.rb

@@ -0,0 +1,224 @@
+require 'fileutils'
+
+module ETL #:nodoc:
+  class Source < ::ActiveRecord::Base #:nodoc:
+    # Connection for database sources
+  end
+  
+  module Control #:nodoc:
+    # Source object which extracts data from a database using ActiveRecord.
+    class DatabaseSource < Source
+      attr_accessor :target
+      attr_accessor :table
+      
+      # Initialize the source.
+      #
+      # Arguments:
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>definition</tt>: The source definition
+      #
+      # Required configuration options:
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:table</tt>: The source table name
+      # * <tt>:database</tt>: The database name
+      # 
+      # Other options:
+      # * <tt>:join</tt>: Optional join part for the query (ignored unless 
+      #   specified)
+      # * <tt>:select</tt>: Optional select part for the query (defaults to 
+      #   '*')
+      # * <tt>:group</tt>: Optional group by part for the query (ignored 
+      #   unless specified)
+      # * <tt>:order</tt>: Optional order part for the query (ignored unless 
+      #   specified)
+      # * <tt>:new_records_only</tt>: Specify the column to use when comparing
+      #   timestamps against the last successful ETL job execution for the
+      #   current control file.
+      # * <tt>:store_locally</tt>: Set to false to not store a copy of the 
+      #   source data locally in a flat file (defaults to true)
+      def initialize(control, configuration, definition)
+        super
+        @target = configuration[:target]
+        @table = configuration[:table]
+        @query = configuration[:query]
+      end
+      
+      # Get a String identifier for the source
+      def to_s
+        "#{host}/#{database}/#{@table}"
+      end
+      
+      # Get the local directory to use, which is a combination of the 
+      # local_base, the db hostname the db database name and the db table.
+      def local_directory
+        File.join(local_base, to_s)
+      end
+      
+      # Get the join part of the query, defaults to nil
+      def join
+        configuration[:join]
+      end
+      
+      # Get the select part of the query, defaults to '*'
+      def select
+        configuration[:select] || '*'
+      end
+      
+      # Get the group by part of the query, defaults to nil
+      def group
+        configuration[:group]
+      end
+      
+      # Get the order for the query, defaults to nil
+      def order
+        configuration[:order]
+      end
+      
+      # Return the column which is used for in the where clause to identify
+      # new rows
+      def new_records_only
+        configuration[:new_records_only]
+      end
+      
+      # Get the number of rows in the source
+      def count(use_cache=true)
+        return @count if @count && use_cache
+        if @store_locally || read_locally
+          @count = count_locally
+        else
+          @count = connection.select_value(query.gsub(/SELECT .* FROM/, 'SELECT count(1) FROM'))
+        end
+      end
+      
+      # Get the list of columns to read. This is defined in the source
+      # definition as either an Array or Hash
+      def columns
+        # weird default is required for writing to cache correctly
+        @columns ||= query_rows.any? ? query_rows.first.keys : ['']
+      end
+      
+      # Returns each row from the source. If read_locally is specified then
+      # this method will attempt to read from the last stored local file. 
+      # If no locally stored file exists or if the trigger file for the last
+      # locally stored file does not exist then this method will raise an
+      # error.
+      def each(&block)
+        if read_locally # Read from the last stored source
+          ETL::Engine.logger.debug "Reading from local cache"
+          read_rows(last_local_file, &block)
+        else # Read from the original source
+          if @store_locally
+            file = local_file
+            write_local(file)
+            read_rows(file, &block)
+          else
+            query_rows.each do |r|
+              row = ETL::Row.new()
+              r.symbolize_keys.each_pair { |key, value|
+                row[key] = value
+              }
+              row.source = self
+              yield row
+            end
+          end
+        end
+      end
+      
+      private
+      # Read rows from the local cache
+      def read_rows(file)
+        raise "Local cache file not found" unless File.exists?(file)
+        raise "Local cache trigger file not found" unless File.exists?(local_file_trigger(file))
+        
+        t = Benchmark.realtime do
+          CSV.open(file, :headers => true).each do |row|
+            result_row = ETL::Row.new
+            result_row.source = self
+            row.each do |header, field|
+              result_row[header.to_sym] = field
+            end
+            yield result_row
+          end
+        end
+        ETL::Engine.average_rows_per_second = ETL::Engine.rows_read / t
+      end
+      
+      def count_locally
+        counter = 0
+        File.open(last_local_file, 'r').each { |line| counter += 1 }
+        counter
+      end
+      
+      # Write rows to the local cache
+      def write_local(file)
+        lines = 0
+        t = Benchmark.realtime do
+          CSV.open(file, 'w') do |f|
+            f << columns
+            query_rows.each do |row|
+              f << columns.collect { |column| row[column.to_s] }
+              lines += 1
+            end
+          end
+          File.open(local_file_trigger(file), 'w') {|f| }
+        end
+        ETL::Engine.logger.info "Stored locally in #{t}s (avg: #{lines/t} lines/sec)"
+      end
+      
+      # Get the query to use
+      def query
+        return @query if @query
+        q = "SELECT #{select} FROM #{@table}"
+        q << " #{join}" if join
+        
+        conditions = []
+        if new_records_only
+          last_completed = ETL::Execution::Job.maximum('created_at', 
+            :conditions => ['control_file = ? and completed_at is not null', control.file]
+          )
+          if last_completed
+            conditions << "#{new_records_only} > #{connection.quote(last_completed.to_s(:db))}"
+          end
+        end
+        
+        conditions << configuration[:conditions] if configuration[:conditions]
+        if conditions.length > 0
+          q << " WHERE #{conditions.join(' AND ')}"
+        end
+        
+        q << " GROUP BY #{group}" if group
+        q << " ORDER BY #{order}" if order
+
+        if ETL::Engine.limit || ETL::Engine.offset
+          options = {}
+          options[:limit] = ETL::Engine.limit if ETL::Engine.limit
+          options[:offset] = ETL::Engine.offset if ETL::Engine.offset
+          connection.add_limit_offset!(q, options)
+        end
+        
+        q = q.gsub(/\n/,' ')
+        ETL::Engine.logger.info "Query: #{q}"
+        @query = q
+      end
+      
+      def query_rows
+        @query_rows ||= connection.select_all(query)
+      end
+      
+      # Get the database connection to use
+      def connection
+        ETL::Engine.connection(target)
+      end
+      
+      # Get the host, defaults to 'localhost'
+      def host
+        ETL::Base.configurations[target.to_s]['host'] || 'localhost'
+      end
+      
+      def database
+        ETL::Base.configurations[target.to_s]['database']
+      end
+    end
+  end
+end