activewarehouse-etl 0.8.4 → 0.9.0

data/lib/etl/builder.rb

@@ -0,0 +1 @@
+require 'etl/builder/date_dimension_builder'

data/lib/etl/builder/date_dimension_builder.rb

@@ -0,0 +1,83 @@
+module ETL #:nodoc:
+  module Builder #:nodoc:
+    # A builder which will build a data structure which can be used to populate a date dimension using 
+    # commonly used date dimension columns.
+    class DateDimensionBuilder
+      # Specify the start date for the first record
+      attr_accessor :start_date
+      
+      # Specify the end date for the last record
+      attr_accessor :end_date
+      
+      # Define any holiday indicators
+      attr_accessor :holiday_indicators
+      
+      # Define the weekday indicators. The default array begins on Sunday and goes to Saturday.
+      cattr_accessor :weekday_indicators
+      @@weekday_indicators = ['Weekend','Weekday','Weekday','Weekday','Weekday','Weekday','Weekend']
+      
+      # Initialize the builder.
+      # 
+      # * <tt>start_date</tt>: The start date. Defaults to 5 years ago from today.
+      # * <tt>end_date</tt>: The end date. Defaults to now.
+      def initialize(start_date=Time.now.years_ago(5), end_date=Time.now)
+        @start_date = start_date
+        @end_date = end_date
+        @holiday_indicators = []
+      end
+      
+      # Returns an array of hashes representing records in the dimension. The values for each record are 
+      # accessed by name.
+      def build(options={})
+        records = []
+        date = start_date.to_time
+        while date <= end_date.to_time
+          record = {}
+          record[:date] = date.strftime("%m/%d/%Y")
+          record[:full_date_description] = date.strftime("%B %d,%Y")
+          record[:day_of_week] = date.strftime("%A")
+          #record[:day_number_in_epoch] = date.to_i / 24
+          #record[:week_number_in_epoch] = date.to_i / (24 * 7)
+          #record[:month_number_in_epoch] = date.to_i / (24 * 7 * 30)
+          record[:day_number_in_calendar_month] = date.day
+          record[:day_number_in_calendar_year] = date.yday
+          record[:day_number_in_fiscal_month] = date.day # should this be different from CY?
+          record[:day_number_in_fiscal_year] = date.fiscal_year_yday
+          #record[:last_day_in_week_indicator] = 
+          #record[:last_day_in_month_indicator] =
+          #record[:calendar_week_ending_date] = 
+          record[:calendar_week] = "Week #{date.week}"
+          record[:calendar_week_number_in_year] = date.week
+          record[:calendar_month_name] = date.strftime("%B")
+          record[:calendar_month_number_in_year] = date.month
+          record[:calendar_year_month] = date.strftime("%Y-%m")
+          record[:calendar_quarter] = "Q#{date.quarter}"
+          record[:calendar_quarter_number_in_year] = date.quarter
+          record[:calendar_year_quarter] = "#{date.strftime('%Y')}-#{record[:calendar_quarter]}"
+          #record[:calendar_half_year] = 
+          record[:calendar_year] = "#{date.year}"
+          record[:fiscal_week] = "FY Week #{date.fiscal_year_week}"
+          record[:fiscal_week_number_in_year] = date.fiscal_year_week
+          record[:fiscal_month] = date.fiscal_year_month
+          record[:fiscal_month_number_in_year] = date.fiscal_year_month
+          record[:fiscal_year_month] = "FY#{date.fiscal_year}-" + date.fiscal_year_month.to_s.rjust(2, '0')
+          record[:fiscal_quarter] = "FY Q#{date.fiscal_year_quarter}"
+          record[:fiscal_year_quarter] = "FY#{date.fiscal_year}-Q#{date.fiscal_year_quarter}"
+          record[:fiscal_year_quarter_number] = date.fiscal_year_quarter
+          #record[:fiscal_half_year] = 
+          record[:fiscal_year] = "FY#{date.fiscal_year}"
+          record[:fiscal_year_number] = date.fiscal_year
+          record[:holiday_indicator] = holiday_indicators.include?(date) ? 'Holiday' : 'Nonholiday'
+          record[:weekday_indicator] = weekday_indicators[date.wday]
+          record[:selling_season] = 'None'
+          record[:major_event] = 'None'
+          record[:sql_date_stamp] = date
+          
+          records << record
+          date = date.tomorrow
+        end
+        records
+      end
+    end
+  end
+end

data/lib/etl/commands/etl.rb

@@ -24,54 +24,67 @@
 require 'benchmark'
 require 'getoptlong'
 
-opts = GetoptLong.new(
-  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
-  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT ],
-  [ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ],
-  [ '--offset', '-o', GetoptLong::REQUIRED_ARGUMENT],
-  [ '--newlog', '-n', GetoptLong::NO_ARGUMENT ],
-  [ '--skip-bulk-import', '-s', GetoptLong::NO_ARGUMENT ],
-  [ '--read-locally', GetoptLong::NO_ARGUMENT]
-)
-
 # Print a usage statement
 def usage #:nodoc:
-  puts "Usage: etl ctl_file [ctl_file2 ctl_file3 ...]" # TODO: add the command line options
+  puts "Usage: etl file [file file ...]" # TODO: add the command line options
 end
 
-options = {}
-opts.each do |opt, arg|
-  case opt
-  when '--help'
-    usage
-  when '--config'
-    options[:config] = arg
-  when '--limit'
-    options[:limit] = arg.to_i
-  when '--offset'
-    options[:offset] = arg.to_i
-  when '--newlog'
-    options[:newlog] = true
-  when '--skip-bulk-import'
-    puts "skip bulk import enabled"
-    options[:skip_bulk_import] = true
-  when '--read-locally'
-    puts "read locally enabled"
-    options[:read_locally] = true
+def execute
+  opts = GetoptLong.new(
+    [ '--version', '-v', GetoptLong::NO_ARGUMENT],
+    [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+    [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT ],
+    [ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ],
+    [ '--offset', '-o', GetoptLong::REQUIRED_ARGUMENT],
+    [ '--newlog', '-n', GetoptLong::NO_ARGUMENT ],
+    [ '--skip-bulk-import', '-s', GetoptLong::NO_ARGUMENT ],
+    [ '--read-locally', GetoptLong::NO_ARGUMENT],
+    [ '--rails-root', GetoptLong::REQUIRED_ARGUMENT]
+  )
+  
+  options = {}
+  opts.each do |opt, arg|
+    case opt
+    when '--version'
+      puts "ActiveWarehouse ETL version #{ETL::VERSION::STRING}"
+      return
+    when '--help'
+      usage
+      return
+    when '--config'
+      options[:config] = arg
+    when '--limit'
+      options[:limit] = arg.to_i
+    when '--offset'
+      options[:offset] = arg.to_i
+    when '--newlog'
+      options[:newlog] = true
+    when '--skip-bulk-import'
+      puts "skip bulk import enabled"
+      options[:skip_bulk_import] = true
+    when '--read-locally'
+      puts "read locally enabled"
+      options[:read_locally] = true
+    when '--rails-root'
+      options[:rails_root] = arg
+      puts "rails root set to #{options[:rails_root]}"
+    end
   end
-end
 
-if ARGV.length < 1
-  usage
-else
-  puts "Starting ETL process"
+  if ARGV.length < 1
+    usage
+  else
+    puts "Starting ETL process"
 
-  ETL::Engine.init(options)
-  ARGV.each do |f|
-    puts "Processing #{f}"
-    ETL::Engine.realtime_activity = true
-    ETL::Engine.process(f)
-  end
+    ETL::Engine.init(options)
+    ARGV.each do |f|
+      puts "Processing #{f}"
+      ETL::Engine.realtime_activity = true
+      ETL::Engine.process(f)
+    end
   
-  puts "ETL process complete"
-end
+    puts "ETL process complete"
+  end
+end
+
+execute

data/lib/etl/control/control.rb

@@ -2,6 +2,8 @@ module ETL #:nodoc:
   module Control #:nodoc:
     # The Context is passed to eval. 
     class Context
+      require 'test/unit/assertions'
+      include Test::Unit::Assertions
       attr_reader :control
       
       class << self
@@ -26,10 +28,11 @@ module ETL #:nodoc:
         control.error_threshold = error_threshold
       end
       
-      # Define a list of control files that this file depends on. Those control files
-      # will be executed prior to this control file. The list may contain symbols that will
-      # be converted to file names by calling to_s + '.ctl', or they may be strings in which
-      # case they will be used as is
+      # Define a list of control files that this file depends on. Those control
+      # files will be executed prior to this control file. The list may 
+      # contain symbols that will be converted to file names by calling 
+      # to_s + '.ctl', or they may be strings in which case they will be used 
+      # as is
       def depends_on(*args)
         dependencies << args
       end
@@ -53,7 +56,7 @@ module ETL #:nodoc:
             if configuration[:type].is_a?(ETL::Control::Source)
               sources << configuration[:type]
             else
-              raise "Configuration must extend ETL::Control::Source"
+              raise ControlError, "Type must be a Class, String, Symbol or object extending ETL::Control::Source"
             end
           end
         else
@@ -61,7 +64,9 @@ module ETL #:nodoc:
             if configuration[source_type]
               source_class = ETL::Control::Source.class_for_name(source_type)
               sources << source_class.new(self, configuration, definition)
+              break
             end
+            raise ControlError, "A source was specified but no matching type was found"
           end
         end
       end
@@ -73,10 +78,29 @@ module ETL #:nodoc:
       
       # Define a destination
       def destination(name, configuration={}, mapping={})
-        destination_types.each do |dest_type|
-          if configuration[dest_type]
-            dest_class = ETL::Control::Destination.class_for_name(dest_type)
+        if configuration[:type]
+          case configuration[:type]
+          when Class
+            dest_class = configuration[:type]
             destinations << dest_class.new(self, configuration, mapping)
+          when String, Symbol
+            dest_class = ETL::Control::Destination.class_for_name(configuration[:type])
+            destinations << dest_class.new(self, configuration, mapping)
+          else
+            if configuration[:type].is_a?(ETL::Control::Destination)
+              destinations << configuration[:type]
+            else
+              raise ControlError, "Type must be a Class, String, Symbol or object extending ETL::Control::Destination"
+            end
+          end
+        else
+          destination_types.each do |dest_type|
+            if configuration[dest_type]
+              dest_class = ETL::Control::Destination.class_for_name(dest_type)
+              destinations << dest_class.new(self, configuration, mapping)
+              break
+            end
+            raise ControlError, "A destination was specified but no matching destination type was found"
           end
         end
       end
@@ -121,6 +145,17 @@ module ETL #:nodoc:
         control.transforms
       end
       
+      # Define a screen block. The type argument must be one of :fatal, :error
+      # or :warn
+      def screen(type, &block)
+        screens[type] << block
+      end
+      
+      # Get the screen blocks
+      def screens
+        control.screens
+      end
+      
       # Rename the source field to the destination field
       def rename(source, destination)
         after_read :rename, :source => source, :dest => destination
@@ -222,7 +257,6 @@ module ETL #:nodoc:
         # Parse a control file and return a Control instance
         def parse(control_file)
           control_file = control_file.path if control_file.instance_of?(File)
-          # logger.debug "Parsing control file #{control_file.path}"
           control = ETL::Control::Control.new(control_file)
           # TODO: better handling of parser errors. Return the line in the control file where the error occurs.
           eval(IO.readlines(control_file).join("\n"), Context.create(control), control_file)
@@ -230,6 +264,13 @@ module ETL #:nodoc:
           control
         end
         
+        def parse_text(text)
+          control = ETL::Control::Control.new(nil)
+          eval(text, Context.create(control), 'inline')
+          control.validate
+          control
+        end
+        
         # Resolve the given object to an ETL::Control::Control instance. Acceptable arguments
         # are:
         # * The path to a control file as a String
@@ -300,6 +341,14 @@ module ETL #:nodoc:
         @transforms ||= []
       end
       
+      def screens
+        @screens ||= {
+          :fatal => [],
+          :error => [],
+          :warn => []
+        }
+      end
+      
       # Get the error threshold. Defaults to 100.
       def error_threshold
         @error_threshold ||= 100

data/lib/etl/control/destination.rb

@@ -157,7 +157,12 @@ module ETL #:nodoc:
       
       # Get the dimension table if specified
       def dimension_table
-        configuration[:scd][:dimension_table] if scd?
+        ETL::Engine.table(configuration[:scd][:dimension_table], dimension_target) if scd?
+      end
+      
+      # Get the dimension target if specified
+      def dimension_target
+        configuration[:scd][:dimension_target] if scd?
       end
       
       # Process a row to determine the change type
@@ -209,10 +214,16 @@ module ETL #:nodoc:
             
             if scd_type == 2
               ETL::Engine.logger.debug "type 2 SCD"
+              
+              raise ConfigurationError, "dimension_table setting required" unless dimension_table
+              raise ConfigurationError, "dimension_target setting required" unless dimension_target
+              
+              conn = ETL::Engine.connection(dimension_target)
+              
               q = "SELECT * FROM #{dimension_table} WHERE "
               q << natural_key.collect { |nk| "#{nk} = '#{row[nk]}'" }.join(" AND ")
               #puts "looking for original record"
-              result = ETL::ActiveRecord::Base.connection.select_one(q)
+              result = conn.select_one(q)
               if result
                 #puts "Result: #{result.inspect}"
                 original_record = ETL::Row[result.symbolize_keys!]
@@ -223,6 +234,15 @@ module ETL #:nodoc:
                 # need to figure out how to delete that old record before inserting the
                 # updated version of the record
                 
+                q = "DELETE FROM #{dimension_table} WHERE "
+                q << natural_key.collect { |nk| "#{nk} = '#{row[nk]}'" }.join(" AND ")
+                
+                num_rows_affected = conn.delete(q)
+                ETL::Engine.logger.debug "deleted old row"
+                
+                # do this?
+                #raise "Should have deleted a single record" if num_rows_affected != 1
+                
                 buffer << original_record
               end
               
@@ -239,9 +259,14 @@ module ETL #:nodoc:
           else
             ETL::Engine.logger.debug "CRC matches, skipping"
             
+            raise ConfigurationError, "dimension_table setting required" unless dimension_table
+            raise ConfigurationError, "dimension_target setting required" unless dimension_target
+            
+            conn = ETL::Engine.connection(dimension_target)
+            
             q = "SELECT * FROM #{dimension_table} WHERE "
             q << natural_key.collect { |nk| "#{nk} = '#{row[nk]}'" }.join(" AND ")
-            result = ETL::ActiveRecord::Base.connection.select_one(q)
+            result = conn.select_one(q)
             if result
               # This was necessary when truncating and then loading, however I
               # am getting reluctant to having the ETL process do the truncation
@@ -297,7 +322,7 @@ module ETL #:nodoc:
               generator = generators[key] ||= value.new
               row[key] = generator.next
             when Symbol
-              generator = generators[key] ||= ETL::Generator::Generator.class_for_name(value).new
+              generator = generators[key] ||= ETL::Generator::Generator.class_for_name(value).new(options)
               row[key] = generator.next
             when Proc
               row[key] = value.call(row)

data/lib/etl/control/destination/database_destination.rb

@@ -5,6 +5,12 @@ module ETL #:nodoc:
     # loader if it is supported with your target database as it will use a much faster load
     # method.
     class DatabaseDestination < Destination
+      # The target connection
+      attr_reader :target
+      
+      # The table
+      attr_reader :table
+      
       # Specify the order from the source
       attr_reader :order
       
@@ -19,31 +25,31 @@ module ETL #:nodoc:
       #
       # Configuration options:
       # * <tt>:database</tt>: The database name (REQUIRED)
+      # * <tt>:target</tt>: The target connection (REQUIRED)
       # * <tt>:table</tt>: The table to write to (REQUIRED)
       # * <tt>:truncate</tt>: Set to true to truncate before writing (defaults to false)
       # * <tt>:unique</tt>: Set to true to only insert unique records (defaults to false)
-      # * <tt>:adapter</tt>: The adapter to use (defaults to :mysql)
-      # * <tt>:username</tt>: The database username (defaults to 'root')
-      # * <tt>:password</tt>: The password to the database (defaults to nothing)
-      # * <tt>:host</tt>: The host for the database (defaults to 'localhost')
       # * <tt>:append_rows</tt>: Array of rows to append
       #
       # Mapping options:
       # * <tt>:order</tt>: The order of fields to write (REQUIRED)
       def initialize(control, configuration, mapping={})
         super
+        @target = configuration[:target]
+        @table = configuration[:table]
         @truncate = configuration[:truncate] ||= false
         @unique = configuration[:unique]
         @order = mapping[:order] || order_from_source
         raise ControlError, "Order required in mapping" unless @order
-        connect
+        raise ControlError, "Table required" unless @table
+        raise ControlError, "Target required" unless @target
       end
       
       # Flush the currently buffered data
       def flush
-        conn = ETL::ActiveRecord::Base.connection
+        conn = ETL::Engine.connection(target)
         conn.transaction do
-          conn.truncate(configuration[:table]) if truncate
+          conn.truncate(table_name) if truncate
           
           buffer.flatten.each do |row|
             # check to see if this row's compound key constraint already exists
@@ -59,7 +65,7 @@ module ETL #:nodoc:
               names << name
               values << conn.quote(row[name]) # TODO: this is probably not database agnostic
             end
-            q = "INSERT INTO #{configuration[:table]} (#{names.join(',')}) VALUES (#{values.join(',')})"
+            q = "INSERT INTO #{table_name} (#{names.join(',')}) VALUES (#{values.join(',')})"
             ETL::Engine.logger.debug("Executing insert: #{q}")
             conn.insert(q, "Insert row #{current_row}")
             @current_row += 1
@@ -72,29 +78,13 @@ module ETL #:nodoc:
       def close
         buffer << append_rows if append_rows
         flush
-        ETL::ActiveRecord::Base.connection.disconnect!
       end
       
       private
-      # Connect to the database. 
-      #
-      # Required options:
-      # * <tt>:database</tt>: The database name
-      # 
-      # Options:
-      # * <tt>:adapter</tt>: The adapter to use (defaults to :mysql)
-      # * <tt>:username</tt>: The database username (defaults to 'root')
-      # * <tt>:password</tt>: The password to the database (defaults to nothing)
-      # * <tt>:host<tt>: The host for the database (defaults to 'localhost')
-      def connect
-        ETL::ActiveRecord::Base.establish_connection(
-            :adapter  => (configuration[:adapter] || :mysql),
-            :username => (configuration[:username] || 'root'),
-            :host     => (configuration[:host] || 'localhost'),
-            :password => configuration[:password],
-            :database => configuration[:database] 
-        )
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
       end
+      
     end
   end
 end