darrell-activewarehouse-etl 0.9.1.4

data/lib/etl/batch/directives.rb

@@ -0,0 +1,55 @@
+module ETL #:nodoc:
+  module Batch #:nodoc:
+    # Abstract base class for directives
+    class Directive
+      # Method to access the batch object
+      attr_reader :batch
+      
+      # Initialize the directive with the given batch object
+      def initialize(batch)
+        @batch = batch
+      end
+      
+      # Execute the directive
+      def execute
+        do_execute
+      end
+      
+      protected
+      # Implemented by subclasses
+      def do_execute
+        raise RuntimeError, "Directive must implement do_execute method"
+      end
+    end
+    
+    # Directive indicating that the specified ETL control file should be 
+    # run
+    class Run < Directive
+      # The file to execute
+      attr_reader :file
+      
+      # Initialize the directive with the given batch object and file
+      def initialize(batch, file)
+        super(batch)
+        @file = file
+      end
+      
+      protected
+      # Execute the process
+      def do_execute
+        batch.engine.process(file)
+      end
+    end
+    
+    # Directive indicating temp tables should be used.
+    class UseTempTables < Directive
+      def initialize(batch)
+        super(batch)
+      end
+      protected
+      def do_execute
+        ETL::Engine.use_temp_tables = true
+      end
+    end 
+  end
+end

data/lib/etl/batch.rb

@@ -0,0 +1,2 @@
+require 'etl/batch/batch'
+require 'etl/batch/directives'

data/lib/etl/builder/date_dimension_builder.rb

@@ -0,0 +1,96 @@
+module ETL #:nodoc:
+  module Builder #:nodoc:
+    # A builder which will build a data structure which can be used to populate a date dimension using 
+    # commonly used date dimension columns.
+    class DateDimensionBuilder
+      # Specify the start date for the first record
+      attr_accessor :start_date
+      
+      # Specify the end date for the last record
+      attr_accessor :end_date
+      
+      # Define any holiday indicators
+      attr_accessor :holiday_indicators
+
+      # Add offset month for fiscal year
+      attr_accessor :fiscal_year_offset_month
+      
+      # Define the weekday indicators. The default array begins on Sunday and goes to Saturday.
+      cattr_accessor :weekday_indicators
+      @@weekday_indicators = ['Weekend','Weekday','Weekday','Weekday','Weekday','Weekday','Weekend']
+      
+      # Initialize the builder.
+      # 
+      # * <tt>start_date</tt>: The start date. Defaults to 5 years ago from today.
+      # * <tt>end_date</tt>: The end date. Defaults to now.
+      def initialize(start_date=Time.now.years_ago(5), end_date=Time.now, fiscal_year_offset_month=10)
+        @start_date = start_date.to_date
+        @end_date = end_date.to_date
+        @fiscal_year_offset_month = fiscal_year_offset_month.to_i
+        @holiday_indicators = []
+      end
+      
+      # Returns an array of hashes representing records in the dimension.
+      def build(options={})
+        (start_date..end_date).map { |date| record_from_date(date) }
+      end
+
+      private
+
+      # Returns a hash representing a record in the dimension. The values for each record are 
+      # accessed by name.
+      def record_from_date(date)
+        time = date.to_time # need methods only available in Time
+        record = {}
+        record[:date] = time.strftime("%m/%d/%Y")
+        record[:full_date_description] = time.strftime("%B %d,%Y")
+        record[:day_of_week] = time.strftime("%A")
+        record[:day_in_week] = record[:day_of_week] # alias
+        #record[:day_number_in_epoch] = time.to_i / 24
+        #record[:week_number_in_epoch] = time.to_i / (24 * 7)
+        #record[:month_number_in_epoch] = time.to_i / (24 * 7 * 30)
+        record[:day_number_in_calendar_month] = time.day
+        record[:day_number_in_calendar_year] = time.yday
+        record[:day_number_in_fiscal_month] = time.day # should this be different from CY?
+        record[:day_number_in_fiscal_year] = time.fiscal_year_yday(fiscal_year_offset_month)
+        #record[:last_day_in_week_indicator] = 
+        #record[:last_day_in_month_indicator] =
+        #record[:calendar_week_ending_date] = 
+        record[:calendar_week] = "Week #{time.week}"
+        record[:calendar_week_number] = time.week
+        record[:calendar_week_number_in_year] = time.week # DEPRECATED
+        record[:calendar_month_name] = time.strftime("%B")
+        record[:calendar_month_number_in_year] = time.month # DEPRECATED
+        record[:calendar_month_number] = time.month
+        record[:calendar_year_month] = time.strftime("%Y-%m")
+        record[:calendar_quarter] = "Q#{time.quarter}"
+        record[:calendar_quarter_number] = time.quarter
+        record[:calendar_quarter_number_in_year] = time.quarter # DEPRECATED
+        record[:calendar_year_quarter] = "#{time.strftime('%Y')}-#{record[:calendar_quarter]}"
+        #record[:calendar_half_year] = 
+        record[:calendar_year] = "#{time.year}"
+        record[:fiscal_week] = "FY Week #{time.fiscal_year_week(fiscal_year_offset_month)}"
+        record[:fiscal_week_number_in_year] = time.fiscal_year_week(fiscal_year_offset_month) # DEPRECATED
+        record[:fiscal_week_number] = time.fiscal_year_week(fiscal_year_offset_month)
+        record[:fiscal_month] = time.fiscal_year_month(fiscal_year_offset_month)
+        record[:fiscal_month_number] = time.fiscal_year_month(fiscal_year_offset_month)
+        record[:fiscal_month_number_in_year] = time.fiscal_year_month(fiscal_year_offset_month) # DEPRECATED
+        record[:fiscal_year_month] = "FY#{time.fiscal_year(fiscal_year_offset_month)}-" + time.fiscal_year_month(fiscal_year_offset_month).to_s.rjust(2, '0')
+        record[:fiscal_quarter] = "FY Q#{time.fiscal_year_quarter(fiscal_year_offset_month)}"
+        record[:fiscal_year_quarter] = "FY#{time.fiscal_year(fiscal_year_offset_month)}-Q#{time.fiscal_year_quarter(fiscal_year_offset_month)}"
+        record[:fiscal_quarter_number] = time.fiscal_year_quarter(fiscal_year_offset_month) # DEPRECATED
+        record[:fiscal_year_quarter_number] = time.fiscal_year_quarter(fiscal_year_offset_month)
+        #record[:fiscal_half_year] = 
+        record[:fiscal_year] = "FY#{time.fiscal_year(fiscal_year_offset_month)}"
+        record[:fiscal_year_number] = time.fiscal_year(fiscal_year_offset_month)
+        record[:holiday_indicator] = holiday_indicators.include?(date) ? 'Holiday' : 'Nonholiday'
+        record[:weekday_indicator] = weekday_indicators[time.wday]
+        record[:selling_season] = 'None'
+        record[:major_event] = 'None'
+        record[:sql_date_stamp] = date
+
+        record
+      end
+    end
+  end
+end

data/lib/etl/builder/time_dimension_builder.rb

@@ -0,0 +1,31 @@
+module ETL #:nodoc:
+  module Builder #:nodoc:
+    # Builder that creates a simple time dimension.
+    class TimeDimensionBuilder
+      def initialize
+        # Returns an array of hashes representing records in the dimension. The values for each record are 
+        # accessed by name.
+        def build(options={})
+          records = []
+          0.upto(23) do |t_hour|
+            0.upto(59) do |t_minute|
+              0.upto(59) do |t_second|
+                t_hour_string = t_hour.to_s.rjust(2, '0')
+                t_minute_string = t_minute.to_s.rjust(2, '0')
+                t_second_string = t_second.to_s.rjust(2, '0')
+                record = {}
+                record[:hour] = t_hour
+                record[:minute] = t_minute
+                record[:second] = t_second
+                record[:minute_description] = "#{t_hour_string}:#{t_minute_string}"
+                record[:full_description] = "#{t_hour_string}:#{t_minute_string}:#{t_second_string}"
+                records << record
+              end
+            end
+          end
+          records
+        end
+      end
+    end
+  end
+end

data/lib/etl/builder.rb

@@ -0,0 +1,2 @@
+require 'etl/builder/date_dimension_builder'
+require 'etl/builder/time_dimension_builder'

data/lib/etl/commands/etl.rb

@@ -0,0 +1,89 @@
+#--
+# Copyright (c) 2006 Anthony Eden
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'benchmark'
+require 'getoptlong'
+
+# Print a usage statement
+def usage #:nodoc:
+  puts "Usage: etl file [file file ...]" # TODO: add the command line options
+end
+
+def execute
+  opts = GetoptLong.new(
+    [ '--version', '-v', GetoptLong::NO_ARGUMENT],
+    [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+    [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT ],
+    [ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ],
+    [ '--offset', '-o', GetoptLong::REQUIRED_ARGUMENT],
+    [ '--newlog', '-n', GetoptLong::NO_ARGUMENT ],
+    [ '--skip-bulk-import', '-s', GetoptLong::NO_ARGUMENT ],
+    [ '--read-locally', GetoptLong::NO_ARGUMENT],
+    [ '--rails-root', GetoptLong::REQUIRED_ARGUMENT]
+  )
+  
+  options = {}
+  opts.each do |opt, arg|
+    case opt
+    when '--version'
+      puts "ActiveWarehouse ETL version #{ETL::VERSION::STRING}"
+      return
+    when '--help'
+      usage
+      return
+    when '--config'
+      options[:config] = arg
+    when '--limit'
+      options[:limit] = arg.to_i
+    when '--offset'
+      options[:offset] = arg.to_i
+    when '--newlog'
+      options[:newlog] = true
+    when '--skip-bulk-import'
+      puts "skip bulk import enabled"
+      options[:skip_bulk_import] = true
+    when '--read-locally'
+      puts "read locally enabled"
+      options[:read_locally] = true
+    when '--rails-root'
+      options[:rails_root] = arg
+      puts "rails root set to #{options[:rails_root]}"
+    end
+  end
+
+  if ARGV.length < 1
+    usage
+  else
+    puts "Starting ETL process"
+
+    ETL::Engine.init(options)
+    ARGV.each do |f|
+      ETL::Engine.realtime_activity = true
+      ETL::Engine.process(f)
+    end
+  
+    puts "ETL process complete\n\n"
+  end
+end
+
+execute

data/lib/etl/control/control.rb

@@ -0,0 +1,405 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # The Context is passed to eval. 
+    class Context
+      require 'test/unit/assertions'
+      include Test::Unit::Assertions
+      attr_reader :control
+      
+      class << self
+        # Create a Context instance
+        def create(control)
+          Context.new(control).get_binding
+        end
+      end
+      
+      # Initialize the context
+      def initialize(control)
+        @control = control
+      end
+      
+      # Get the control file
+      def file
+        control.file
+      end
+      
+      # Set the allowed error threshold
+      def set_error_threshold(error_threshold)
+        control.error_threshold = error_threshold
+      end
+      
+      # Define a list of control files that this file depends on. Those control
+      # files will be executed prior to this control file. The list may 
+      # contain symbols that will be converted to file names by calling 
+      # to_s + '.ctl', or they may be strings in which case they will be used 
+      # as is
+      def depends_on(*args)
+        (dependencies << args).flatten!
+      end
+      
+      # Get the defined dependencies
+      def dependencies
+        control.dependencies
+      end
+      
+      # Define a source.
+      def source(name, configuration={}, definition={})
+        if configuration[:type]
+          case configuration[:type]
+          when Class
+            source_class = configuration[:type]
+            sources << source_class.new(self, configuration, definition)
+          when String, Symbol
+            source_class = ETL::Control::Source.class_for_name(configuration[:type])
+            sources << source_class.new(self, configuration, definition)
+          else
+            if configuration[:type].is_a?(ETL::Control::Source)
+              sources << configuration[:type]
+            else
+              raise ControlError, "Type must be a Class, String, Symbol or object extending ETL::Control::Source"
+            end
+          end
+        else
+          source_types.each do |source_type|
+            if configuration[source_type]
+              source_class = ETL::Control::Source.class_for_name(source_type)
+              sources << source_class.new(self, configuration, definition)
+              break
+            end
+          end
+          raise ControlError, "A source was specified but no matching type was found" if sources.empty?
+        end
+      end
+      
+      # Get the defined source
+      def sources
+        control.sources
+      end
+      
+      # Define a destination
+      def destination(name, configuration={}, mapping={})
+        if configuration[:type]
+          case configuration[:type]
+          when Class
+            dest_class = configuration[:type]
+            destinations << dest_class.new(self, configuration, mapping)
+          when String, Symbol
+            dest_class = ETL::Control::Destination.class_for_name(configuration[:type])
+            destinations << dest_class.new(self, configuration, mapping)
+          else
+            if configuration[:type].is_a?(ETL::Control::Destination)
+              destinations << configuration[:type]
+            else
+              raise ControlError, "Type must be a Class, String, Symbol or object extending ETL::Control::Destination"
+            end
+          end
+        else
+          destination_types.each do |dest_type|
+            if configuration[dest_type]
+              dest_class = ETL::Control::Destination.class_for_name(dest_type)
+              destinations << dest_class.new(self, configuration, mapping)
+              break
+            end
+          end
+          raise ControlError, "A destination was specified but no matching destination type was found" if destinations.empty?      
+        end
+      end
+      
+      # Get the defined destinations
+      def destinations
+        control.destinations
+      end
+      
+      # Define a transform
+      def transform(name, transformer=nil, configuration={}, &block)
+        if transformer
+          case transformer
+          when String, Symbol
+            class_name = "#{transformer.to_s.camelize}Transform"
+            begin
+              transform_class = ETL::Transform.const_get(class_name)
+              transforms << transform_class.new(self, name, configuration)
+            rescue NameError => e
+              raise ControlError, "Unable to find transformer #{class_name}: #{e}"
+            end
+          when Class
+            transforms << transformer.new(self, transformer.name, configuration)
+          else
+            #transformer.class.inspect
+            if transformer.is_a?(ETL::Transform::Transform)
+              Engine.logger.debug "Adding transformer #{transformer.inspect} for field #{name}"
+              t = transformer.dup
+              t.name = name
+              transforms << t 
+            else
+              raise ControlError, "Transformer must be a String, Symbol, Class or Transform instance"
+            end
+          end
+        elsif block_given?
+          transforms << ETL::Transform::BlockTransform.new(self, name, :block => block)
+        else
+          raise ControlError, "Either a transformer or a block must be specified"
+        end
+      end
+      
+      # Get the defined transforms
+      def transforms
+        control.transforms
+      end
+      
+      # Define a before post-process screen block. The type argument must be
+      # one of :fatal, :error or :warn
+      def screen(type, &block)
+        screens[type] << block
+      end
+      
+      # Get the before post-process screen blocks
+      def screens
+        control.screens
+      end
+      
+      # Define an after post-proces screen block. The type argument must be
+      # one of :fatal, :error or :warn
+      def after_post_process_screen(type, &block)
+        after_post_process_screens[type] << block
+      end
+      
+      # Get the after post-process screen blocks
+      def after_post_process_screens
+        control.after_post_process_screens
+      end
+      
+      # Rename the source field to the destination field
+      def rename(source, destination)
+        after_read :rename, :source => source, :dest => destination
+      end
+      
+      # Copy the source field to the destination field
+      def copy(source, destination)
+        after_read :copy_field, :source => source, :dest => destination
+      end
+      
+      protected
+      # This method is used to define a processor and insert into the specified processor
+      # collection.
+      def define_processor(name, processor_collection, configuration, proc)
+        case name
+        when String, Symbol, nil
+          name ||= 'block'
+          class_name = "#{name.to_s.camelize}Processor"
+          begin
+            processor_class = ETL::Processor.const_get(class_name)
+            if name == 'block'
+              raise ControlError, "A block must be passed for block processor" if proc.nil?
+              configuration[:block] = proc
+            end
+            processor_collection << processor_class.new(self, configuration)
+          rescue NameError => e
+            raise ControlError, "Unable to find processor #{class_name}: #{e}"
+          end
+        when Class
+          processor_collection << name.new(self, configuration)
+        else
+          raise ControlError, "The process declaration requires a String, Symbol, Class, or a Block to be passed"
+        end
+      end
+      
+      public
+      # Define an "after read" processor. This must be a row-level processor.
+      def after_read(name='block', configuration={}, &block)
+        define_processor(name, after_read_processors, configuration, block)
+      end
+      
+      # Get the defined "after read" processors
+      def after_read_processors
+        control.after_read_processors
+      end
+      
+      # Define a "before write" processor. This must be a row-level processor.
+      def before_write(name='block', configuration={}, &block)
+        define_processor(name, before_write_processors, configuration, block)
+      end
+      
+      # Get the defined "before write" processors
+      def before_write_processors
+        control.before_write_processors
+      end
+      
+      # Define a pre-processor
+      def pre_process(name='block', configuration={}, &block)
+        define_processor(name, pre_processors, configuration, block)
+      end
+      
+      # Get the defined pre-processors
+      def pre_processors
+        control.pre_processors
+      end
+      
+      # Define a post-processor
+      def post_process(name='block', configuration={}, &block)
+        define_processor(name, post_processors, configuration, block)
+      end
+      
+      # Get the defined post-processors
+      def post_processors
+        control.post_processors
+      end
+      
+      # Get the binding object
+      def get_binding
+        binding
+      end
+      
+      protected
+      # Get an array of supported source types
+      def source_types
+        control.source_types
+      end
+      
+      # Get an array of supported destination types
+      def destination_types
+        control.destination_types
+      end
+      
+    end
+    
+    # Object representation of a control file
+    class Control
+      # The File object
+      attr_reader :file
+      
+      # The error threshold
+      attr_accessor :error_threshold
+      
+      class << self
+        # Parse a control file and return a Control instance
+        def parse(control_file)
+          control_file = control_file.path if control_file.instance_of?(File)
+          control = ETL::Control::Control.new(control_file)
+          # TODO: better handling of parser errors. Return the line in the control file where the error occurs.
+          eval(IO.readlines(control_file).join("\n"), Context.create(control), control_file)
+          control.validate
+          control
+        end
+        
+        def parse_text(text)
+          control = ETL::Control::Control.new(nil)
+          eval(text, Context.create(control), 'inline')
+          control.validate
+          control
+        end
+        
+        # Resolve the given object to an ETL::Control::Control instance. Acceptable arguments
+        # are:
+        # * The path to a control file as a String
+        # * A File object referencing the control file
+        # * The ETL::Control::Control object (which will just be returned)
+        #
+        # Raises a ControlError if any other type is given
+        def resolve(control)
+          case control
+          when String
+            ETL::Control::Control.parse(File.new(control))
+          when File
+            ETL::Control::Control.parse(control)
+          when ETL::Control::Control
+            control
+          else
+            raise ControlError, "Control must be a String, File or Control object"
+          end
+        end
+      end
+      
+      # Initialize the instance with the given File object
+      def initialize(file)
+        @file = file
+      end
+      
+      # Get a list of dependencies
+      def dependencies
+        @dependencies ||= []
+      end
+      
+      # Get the defined source
+      def sources
+        @sources ||= []
+      end
+      
+      # Get the defined destinations
+      def destinations
+        @destinations ||= []
+      end
+      
+      # Get the transforms with the specified name
+      # def transform(name)
+#         transforms[name] ||= []
+#       end
+      
+      def after_read_processors
+        @after_read_processors ||= []
+      end
+      
+      # Get all of the "before write" processors
+      def before_write_processors
+        @before_write_processors ||= []
+      end
+      
+      # Get an Array of preprocessors
+      def pre_processors
+        @pre_processors ||= []
+      end
+      
+      # Get an Array of post processors
+      def post_processors
+        @post_processors ||= []
+      end
+      
+      # Get an Array of all transforms for this control
+      def transforms
+        @transforms ||= []
+      end
+      
+      # A hash of the screens executed before post-process
+      def screens
+        @screens ||= {
+          :fatal => [],
+          :error => [],
+          :warn => []
+        }
+      end
+      
+      # A hash of the screens executed after post-process
+      def after_post_process_screens
+        @after_post_process_screens ||= {
+          :fatal => [],
+          :error => [],
+          :warn => []
+        }
+      end
+      
+      # Get the error threshold. Defaults to 100.
+      def error_threshold
+        @error_threshold ||= 100
+      end
+      
+      # Validate the control file
+      def validate
+        #unless sources.length > 0
+        #  raise ControlError, "Configuration must include one of the following for the source: #{source_types.join(',')}"
+        #end
+        #unless destinations.length > 0
+        #  raise ControlError, "Configuration must include one of the following for the destination: #{destination_types.join(',')}"
+        #end
+      end
+      
+      def source_types
+        [:file, :database]
+      end
+      
+      def destination_types
+        [:file, :database]
+      end
+      
+    end
+  end
+end