darrell-activewarehouse-etl 0.9.1.4

data/lib/etl/control/source/database_source.rb

@@ -0,0 +1,220 @@
+require 'fileutils'
+
+module ETL #:nodoc:
+  class Source < ::ActiveRecord::Base #:nodoc:
+    # Connection for database sources
+  end
+  
+  module Control #:nodoc:
+    # Source object which extracts data from a database using ActiveRecord.
+    class DatabaseSource < Source
+      attr_accessor :target
+      attr_accessor :table
+      
+      # Initialize the source.
+      #
+      # Arguments:
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>definition</tt>: The source definition
+      #
+      # Required configuration options:
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:table</tt>: The source table name
+      # * <tt>:database</tt>: The database name
+      # 
+      # Other options:
+      # * <tt>:join</tt>: Optional join part for the query (ignored unless 
+      #   specified)
+      # * <tt>:select</tt>: Optional select part for the query (defaults to 
+      #   '*')
+      # * <tt>:group</tt>: Optional group by part for the query (ignored 
+      #   unless specified)
+      # * <tt>:order</tt>: Optional order part for the query (ignored unless 
+      #   specified)
+      # * <tt>:new_records_only</tt>: Specify the column to use when comparing
+      #   timestamps against the last successful ETL job execution for the
+      #   current control file.
+      # * <tt>:store_locally</tt>: Set to false to not store a copy of the 
+      #   source data locally in a flat file (defaults to true)
+      def initialize(control, configuration, definition)
+        super
+        @target = configuration[:target]
+        @table = configuration[:table]
+      end
+      
+      # Get a String identifier for the source
+      def to_s
+        "#{host}/#{database}/#{table}"
+      end
+      
+      # Get the local directory to use, which is a combination of the 
+      # local_base, the db hostname the db database name and the db table.
+      def local_directory
+        File.join(local_base, host, database, configuration[:table])
+      end
+      
+      # Get the join part of the query, defaults to nil
+      def join
+        configuration[:join]
+      end
+      
+      # Get the select part of the query, defaults to '*'
+      def select
+        configuration[:select] || '*'
+      end
+      
+      # Get the group by part of the query, defaults to nil
+      def group
+        configuration[:group]
+      end
+      
+      # Get the order for the query, defaults to nil
+      def order
+        configuration[:order]
+      end
+      
+      # Return the column which is used for in the where clause to identify
+      # new rows
+      def new_records_only
+        configuration[:new_records_only]
+      end
+      
+      # Get the number of rows in the source
+      def count(use_cache=true)
+        return @count if @count && use_cache
+        if store_locally || read_locally
+          @count = count_locally
+        else
+          @count = connection.select_value(query.gsub(/SELECT .* FROM/, 'SELECT count(1) FROM'))
+        end
+      end
+      
+      # Get the list of columns to read. This is defined in the source
+      # definition as either an Array or Hash
+      def columns
+        # weird default is required for writing to cache correctly
+        @columns ||= query_rows.any? ? query_rows.first.keys : ['']
+      end
+      
+      # Returns each row from the source. If read_locally is specified then
+      # this method will attempt to read from the last stored local file. 
+      # If no locally stored file exists or if the trigger file for the last
+      # locally stored file does not exist then this method will raise an
+      # error.
+      def each(&block)
+        if read_locally # Read from the last stored source
+          ETL::Engine.logger.debug "Reading from local cache"
+          read_rows(last_local_file, &block)
+        else # Read from the original source
+          if store_locally
+            file = local_file
+            write_local(file)
+            read_rows(file, &block)
+          else
+            query_rows.each do |row|
+              row = ETL::Row.new(row.symbolize_keys)
+              row.source = self
+              yield row
+            end
+          end
+        end
+      end
+      
+      private
+      # Read rows from the local cache
+      def read_rows(file)
+        raise "Local cache file not found" unless File.exists?(file)
+        raise "Local cache trigger file not found" unless File.exists?(local_file_trigger(file))
+        
+        t = Benchmark.realtime do
+          FasterCSV.open(file, :headers => true).each do |row|
+            result_row = ETL::Row.new
+            result_row.source = self
+            row.each do |header, field|
+              result_row[header.to_sym] = field
+            end
+            yield result_row
+          end
+        end
+        ETL::Engine.average_rows_per_second = ETL::Engine.rows_read / t
+      end
+      
+      def count_locally
+        counter = 0
+        File.open(last_local_file, 'r').each { |line| counter += 1 }
+        counter
+      end
+      
+      # Write rows to the local cache
+      def write_local(file)
+        lines = 0
+        t = Benchmark.realtime do
+          FasterCSV.open(file, 'w') do |f|
+            f << columns
+            query_rows.each do |row|
+              f << columns.collect { |column| row[column.to_s] }
+              lines += 1
+            end
+          end
+          File.open(local_file_trigger(file), 'w') {|f| }
+        end
+        ETL::Engine.logger.info "Stored locally in #{t}s (avg: #{lines/t} lines/sec)"
+      end
+      
+      # Get the query to use
+      def query
+        return @query if @query
+        q = "SELECT #{select} FROM #{configuration[:table]}"
+        q << " #{join}" if join
+        
+        conditions = []
+        if new_records_only
+          last_completed = ETL::Execution::Job.maximum('created_at', 
+            :conditions => ['control_file = ? and completed_at is not null', control.file]
+          )
+          if last_completed
+            conditions << "#{new_records_only} > #{connection.quote(last_completed.to_s(:db))}"
+          end
+        end
+        
+        conditions << configuration[:conditions] if configuration[:conditions]
+        if conditions.length > 0
+          q << " WHERE #{conditions.join(' AND ')}"
+        end
+        
+        q << " GROUP BY #{group}" if group
+        q << " ORDER BY #{order}" if order
+
+        if ETL::Engine.limit || ETL::Engine.offset
+          options = {}
+          options[:limit] = ETL::Engine.limit if ETL::Engine.limit
+          options[:offset] = ETL::Engine.offset if ETL::Engine.offset
+          connection.add_limit_offset!(q, options)
+        end
+        
+        q = q.gsub(/\n/,' ')
+        ETL::Engine.logger.info "Query: #{q}"
+        @query = q
+      end
+      
+      def query_rows
+        @query_rows ||= connection.select_all(query)
+      end
+      
+      # Get the database connection to use
+      def connection
+        ETL::Engine.connection(target)
+      end
+      
+      # Get the host, defaults to 'localhost'
+      def host
+        ETL::Base.configurations[target.to_s]['host'] || 'localhost'
+      end
+      
+      def database
+        ETL::Base.configurations[target.to_s]['database']
+      end
+    end
+  end
+end

data/lib/etl/control/source/enumerable_source.rb

@@ -0,0 +1,11 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # Use an Enumerable as a source
+    class EnumerableSource < ETL::Control::Source
+      # Iterate through the enumerable
+      def each(&block)
+        configuration[:enumerable].each(&block)
+      end
+    end
+  end
+end

data/lib/etl/control/source/file_source.rb

@@ -0,0 +1,90 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # A File source.
+    class FileSource < Source
+      # The number of lines to skip, default is 0
+      attr_accessor :skip_lines
+      
+      # Accessor for the underlying parser
+      attr_accessor :parser
+      
+      # The source file
+      attr_accessor :file
+      
+      # Initialize the source
+      #
+      # Configuration options:
+      # * <tt>:file</tt>: The source file
+      # * <tt>:parser</tt>: One of the following: a parser name as a String or
+      #   symbol, a class which extends from Parser, a Hash with :name and 
+      #   optionally an :options key. Whether or not the parser uses the 
+      #   options is dependent on which parser is used. See the documentation 
+      #   for each parser for information on what options it accepts.
+      # * <tt>:skip_lines</tt>: The number of lines to skip (defaults to 0)
+      # * <tt>:store_locally</tt>: Set to false to not store a copy of the 
+      #   source data locally for archival
+      def initialize(control, configuration, definition)
+        super
+        configure
+      end
+      
+      # Get a String identifier for the source
+      def to_s
+        file
+      end
+      
+      # Get the local storage directory
+      def local_directory
+        File.join(local_base, File.basename(file, File.extname(file)))
+      end
+      
+      # Returns each row from the source
+      def each
+        count = 0
+        copy_sources if store_locally
+        @parser.each do |row|
+          if ETL::Engine.offset && count < ETL::Engine.offset
+            count += 1
+          else
+            row = ETL::Row[row]
+            row.source = self
+            yield row
+          end
+        end
+      end
+      
+      private
+      # Copy source data to a local directory structure
+      def copy_sources
+        sequence = 0
+        path = Pathname.new(file)
+        path = path.absolute? ? path : Pathname.new(File.dirname(control.file)) + path
+        Pathname.glob(path).each do |f|
+          next if f.directory?
+          lf = local_file(sequence)
+          FileUtils.cp(f, lf)
+          File.open(local_file_trigger(lf), 'w') {|f| }
+          sequence += 1
+        end
+      end
+      
+      # Configure the source
+      def configure
+        @file = configuration[:file]
+        case configuration[:parser]
+        when Class
+          @parser = configuration[:parser].new(self)
+        when String, Symbol
+          @parser = ETL::Parser::Parser.class_for_name(configuration[:parser]).new(self)
+        when Hash
+          name = configuration[:parser][:name]
+          options = configuration[:parser][:options]
+          @parser = ETL::Parser::Parser.class_for_name(name).new(self, options)
+        else
+          raise ControlError, "Configuration option :parser must be a Class, String or Symbol"
+        end
+        @skip_lines = configuration[:skip_lines] ||= 0
+      end
+    end
+  end
+end

data/lib/etl/control/source/model_source.rb

@@ -0,0 +1,39 @@
+#RAILS_ENV = 'development'
+#require '../config/environment'
+
+module ETL #:nodoc:
+  module Control #:nodoc:
+    class ModelSource < Source   
+      
+      def columns
+        case definition
+        when Array
+          definition.collect(&:to_sym)
+        when Hash
+          definition.keys.collect(&:to_sym)
+        else
+          raise "Definition must be either an Array or a Hash"
+        end
+      end
+      
+      def railsmodel
+        configuration[:model]
+      end
+      
+      def order
+        configuration[:order] || "id"
+      end
+         
+      def each(&block)
+          railsmodel.to_s.camelize.constantize.find(:all,:order=>order).each do |row|
+            result_row = ETL::Row.new
+            result_row.source = self
+            columns.each do |column|
+              result_row[column.to_sym] = row.send(column)
+            end
+            yield result_row
+        end
+      end
+    end
+  end
+end

data/lib/etl/control/source.rb

@@ -0,0 +1,109 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # ETL source. Subclasses must implement the <tt>each</tt> method.
+    class Source
+      include Enumerable
+      
+      # The control object
+      attr_accessor :control
+      
+      # The configuration Hash
+      attr_accessor :configuration
+      
+      # The definition Hash
+      attr_accessor :definition
+      
+      # Returns true if the source data should be stored locally for archival
+      # Default behavior will return true.
+      attr_accessor :store_locally
+      
+      class << self
+        # Convert the name to a Source class.
+        # 
+        # For example if name is :database then this will return a 
+        # DatabaseSource class
+        def class_for_name(name)
+          ETL::Control.const_get("#{name.to_s.camelize}Source")
+        end
+      end
+      
+      # Initialize the Source instance
+      # * <tt>control</tt>: The control object
+      # * <tt>configuration</tt>: The configuration hash
+      # * <tt>definition</tt>: The source layout definition
+      #
+      # Configuration options:
+      # * <tt>:store_locally</tt>: Set to false to not store source data
+      #   locally (defaults to true)
+      def initialize(control, configuration, definition)
+        @control = control
+        @configuration = configuration
+        @definition = definition
+        
+        @store_locally = configuration[:store_locally] || true
+      end
+      
+      # Get an array of errors that occur during reading from the source
+      def errors
+        @errors ||= []
+      end
+      
+      # Get a timestamp value as a string
+      def timestamp
+        Engine.timestamp
+      end
+      
+      # The base directory where local files are stored.
+      attr_accessor :local_base
+      
+      # Get the local base, defaults to 'source_data'
+      def local_base
+        @local_base ||= 'source_data'
+      end
+      
+      # The local directory for storing. This method must be overriden by 
+      # subclasses
+      def local_directory
+        raise "local_directory method is abstract"
+      end
+      
+      # Return the local file for storing the raw source data. Each call to 
+      # this method will result in a timestamped file, so you cannot expect 
+      # to call it multiple times and reference the same file
+      # 
+      # Optional sequence can be specified if there are multiple source files
+      def local_file(sequence=nil)
+        filename = timestamp.to_s
+        filename += sequence.to_s if sequence
+        
+        local_dir = local_directory
+        FileUtils.mkdir_p(local_dir)
+        File.join(local_dir, "#{filename}.csv")
+      end
+      
+      # Get the last fully written local file
+      def last_local_file
+        File.join(local_directory, File.basename(last_local_file_trigger, '.trig'))
+      end
+      
+      # Get the last local file trigger
+      def last_local_file_trigger
+        Dir.glob(File.join(local_directory, '*.trig')).last
+      end
+      
+      # Get the local trigger file that is used to indicate that the file has
+      # been completely written
+      def local_file_trigger(file)
+        Pathname.new(file.to_s + '.trig')
+      end
+      
+      # Return true if the source should read locally.
+      def read_locally
+        Engine.read_locally
+      end
+      
+    end
+  end
+end
+
+Dir[File.dirname(__FILE__) + "/source/*.rb"].each { |file| require(file) }

data/lib/etl/control.rb

@@ -0,0 +1,3 @@
+require 'etl/control/control'
+require 'etl/control/source'
+require 'etl/control/destination'

data/lib/etl/core_ext/time/calculations.rb

@@ -0,0 +1,42 @@
+#Updated by Jack Hong on 04/05/08
+
+module ETL #:nodoc:
+  module CoreExtensions #:nodoc:
+    module Time #:nodoc:
+      # Enables the use of time calculations within Time itself
+      module Calculations
+        def week
+          cyw = ((yday - 1) / 7) + 1
+          cyw = 52 if cyw == 53
+          cyw
+        end
+        def quarter
+          ((month - 1) / 3) + 1
+        end
+        def fiscal_year_week(offset_month=10)
+          fyw = ((fiscal_year_yday(offset_month) - 1) / 7) + 1
+          fyw = 52 if fyw == 53
+          fyw
+        end
+        def fiscal_year_month(offset_month=10)
+          shifted_month = month - (offset_month - 1)
+          shifted_month += 12 if shifted_month <= 0
+          shifted_month
+        end
+        def fiscal_year_quarter(offset_month=10)
+          ((fiscal_year_month(offset_month) - 1) / 3) + 1
+        end
+        def fiscal_year(offset_month=10)
+          month >= offset_month ? year + 1 : year
+        end
+        def fiscal_year_yday(offset_month=10)
+          offset_days = 0
+          1.upto(offset_month - 1) { |m| offset_days += ::Time.days_in_month(m, year) }
+          shifted_year_day = yday - offset_days
+          shifted_year_day += 365 if shifted_year_day <= 0
+          shifted_year_day
+        end
+      end
+    end
+  end
+end

data/lib/etl/core_ext/time.rb

@@ -0,0 +1,5 @@
+require File.dirname(__FILE__) + '/time/calculations'
+
+class Time#:nodoc:
+  include ETL::CoreExtensions::Time::Calculations
+end

data/lib/etl/core_ext.rb

@@ -0,0 +1 @@
+require 'etl/core_ext/time'