aeden-activewarehouse-etl 0.9.1

data/etl/bin/etl

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+#--
+# Copyright (c) 2006 Anthony Eden
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+$:.unshift(File.dirname(__FILE__) + '/../lib/')
+require 'etl'
+require 'etl/commands/etl'

data/etl/bin/etl.cmd

@@ -0,0 +1,8 @@
+@echo off
+
+rem The purpose of this Windows script is to let you use the etl command line with a non-gem version of AW-ETL (eg: unpacked gem, pistoned trunk).
+rem Just add the current folder on top of your PATH variable to use it instead of the etl command provided with the gem release.
+
+rem %~dp0 returns the absolute path where the current script is. We just append 'etl' to it, and forward all the arguments with %*
+
+ruby "%~dp0etl" %*

data/etl/examples/database.example.yml

@@ -0,0 +1,16 @@
+etl_execution:
+  adapter: mysql
+  username: root
+  host: localhost
+  database: etl_execution
+  encoding: utf8
+datawarehouse:
+  adapter: mysql
+  username: root
+  host: localhost
+  database: datawarehouse_development
+operational:
+  adapter: mysql
+  username: root
+  host: localhost
+  database: operational_production

data/etl/lib/etl.rb

@@ -0,0 +1,97 @@
+# This source file requires all of the necessary gems and source files for ActiveWarehouse ETL. If you
+# load this source file all of the other required files and gems will also be brought into the 
+# runtime.
+
+#--
+# Copyright (c) 2006-2007 Anthony Eden
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'logger'
+require 'yaml'
+require 'erb'
+
+require 'rubygems'
+
+unless Kernel.respond_to?(:gem)
+  Kernel.send :alias_method, :gem, :require_gem
+end
+
+unless defined?(REXML::VERSION)
+  require 'rexml/rexml'
+  REXML::VERSION = REXML::Version
+end
+
+unless defined?(ActiveSupport)
+  gem 'activesupport'
+  require 'active_support'
+end
+
+unless defined?(ActiveRecord)
+  gem 'activerecord'
+  require 'active_record'
+end
+
+unless defined?(AdapterExtensions)
+  gem 'adapter_extensions'
+  require 'adapter_extensions'
+end
+
+unless defined?(FasterCSV)
+  gem 'fastercsv'
+  require 'faster_csv'
+end
+
+$:.unshift(File.dirname(__FILE__))
+
+require 'etl/core_ext'
+require 'etl/util'
+require 'etl/http_tools'
+require 'etl/builder'
+require 'etl/version'
+require 'etl/engine'
+require 'etl/control'
+require 'etl/batch'
+require 'etl/row'
+require 'etl/parser'
+require 'etl/transform'
+require 'etl/processor'
+require 'etl/generator'
+require 'etl/screen'
+
+module ETL #:nodoc:
+  class ETLError < StandardError #:nodoc:
+  end
+  class ControlError < ETLError #:nodoc:
+  end
+  class DefinitionError < ControlError #:nodoc:
+  end
+  class ConfigurationError < ControlError #:nodoc:
+  end
+  class MismatchError < ETLError #:nodoc:
+  end
+  class ResolverError < ETLError #:nodoc:
+  end
+  class ScreenError < ETLError #:nodoc:
+  end
+  class FatalScreenError < ScreenError #:nodoc:
+  end
+end

data/etl/lib/etl/batch.rb

@@ -0,0 +1,2 @@
+require 'etl/batch/batch'
+require 'etl/batch/directives'

data/etl/lib/etl/batch/batch.rb

@@ -0,0 +1,111 @@
+module ETL #:nodoc:
+  module Batch
+    class Context
+      attr_reader :batch
+      
+      class << self
+        # Create a context that is used when evaluating the batch file
+        def create(batch)
+          Context.new(batch).get_binding
+        end
+      end
+      
+      def initialize(batch)
+        @batch = batch
+      end
+      
+      def file
+        batch.file
+      end
+      
+      def get_binding
+        binding
+      end
+      
+      def run(file)
+        batch.run(File.dirname(self.file) + "/" + file)
+      end
+      
+      def use_temp_tables(value=true)
+        batch.use_temp_tables(value)
+      end
+      
+    end
+    class Batch
+      attr_accessor :file
+      attr_accessor :engine
+      
+      class << self
+        # Resolve the given object to an ETL::Control::Control instance. Acceptable arguments
+        # are:
+        # * The path to a control file as a String
+        # * A File object referencing the control file
+        # * The ETL::Control::Control object (which will just be returned)
+        #
+        # Raises a ControlError if any other type is given
+        def resolve(batch, engine)
+          batch = do_resolve(batch)
+          batch.engine = engine
+          batch
+        end
+        
+        protected
+        def parse(batch_file)
+          batch_file = batch_file.path if batch_file.instance_of?(File)
+          batch = ETL::Batch::Batch.new(batch_file)
+          eval(IO.readlines(batch_file).join("\n"), Context.create(batch), batch_file)
+          batch
+        end
+        
+        def do_resolve(batch)
+          case batch
+          when String
+            ETL::Batch::Batch.parse(File.new(batch))
+          when File
+            ETL::Batch::Batch.parse(batch)
+          when ETL::Batch::Batch
+            batch
+          else
+            raise RuntimeError, "Batch must be a String, File or Batch object"
+          end
+        end
+      end
+      
+      def initialize(file)
+        @file = file
+      end
+      
+      def run(file)
+        directives << Run.new(self, file)
+      end
+      
+      def use_temp_tables(value = true)
+        directives << UseTempTables.new(self)
+      end
+      
+      def execute
+        engine.say "Executing batch"
+        before_execute
+        directives.each do |directive|
+          directive.execute
+        end
+        engine.say "Finishing batch"
+        after_execute
+        engine.say "Batch complete"
+      end
+      
+      def directives
+        @directives ||= []
+      end
+      
+      def before_execute
+        
+      end
+      
+      def after_execute
+        ETL::Engine.finish # TODO: should be moved to the directive?
+        ETL::Engine.use_temp_tables = false # reset the temp tables
+      end
+    end
+  end
+end

data/etl/lib/etl/batch/directives.rb

@@ -0,0 +1,55 @@
+module ETL #:nodoc:
+  module Batch #:nodoc:
+    # Abstract base class for directives
+    class Directive
+      # Method to access the batch object
+      attr_reader :batch
+      
+      # Initialize the directive with the given batch object
+      def initialize(batch)
+        @batch = batch
+      end
+      
+      # Execute the directive
+      def execute
+        do_execute
+      end
+      
+      protected
+      # Implemented by subclasses
+      def do_execute
+        raise RuntimeError, "Directive must implement do_execute method"
+      end
+    end
+    
+    # Directive indicating that the specified ETL control file should be 
+    # run
+    class Run < Directive
+      # The file to execute
+      attr_reader :file
+      
+      # Initialize the directive with the given batch object and file
+      def initialize(batch, file)
+        super(batch)
+        @file = file
+      end
+      
+      protected
+      # Execute the process
+      def do_execute
+        batch.engine.process(file)
+      end
+    end
+    
+    # Directive indicating temp tables should be used.
+    class UseTempTables < Directive
+      def initialize(batch)
+        super(batch)
+      end
+      protected
+      def do_execute
+        ETL::Engine.use_temp_tables = true
+      end
+    end 
+  end
+end

data/etl/lib/etl/builder.rb

@@ -0,0 +1,2 @@
+require 'etl/builder/date_dimension_builder'
+require 'etl/builder/time_dimension_builder'

data/etl/lib/etl/builder/date_dimension_builder.rb

@@ -0,0 +1,96 @@
+module ETL #:nodoc:
+  module Builder #:nodoc:
+    # A builder which will build a data structure which can be used to populate a date dimension using 
+    # commonly used date dimension columns.
+    class DateDimensionBuilder
+      # Specify the start date for the first record
+      attr_accessor :start_date
+      
+      # Specify the end date for the last record
+      attr_accessor :end_date
+      
+      # Define any holiday indicators
+      attr_accessor :holiday_indicators
+
+      # Add offset month for fiscal year
+      attr_accessor :fiscal_year_offset_month
+      
+      # Define the weekday indicators. The default array begins on Sunday and goes to Saturday.
+      cattr_accessor :weekday_indicators
+      @@weekday_indicators = ['Weekend','Weekday','Weekday','Weekday','Weekday','Weekday','Weekend']
+      
+      # Initialize the builder.
+      # 
+      # * <tt>start_date</tt>: The start date. Defaults to 5 years ago from today.
+      # * <tt>end_date</tt>: The end date. Defaults to now.
+      def initialize(start_date=Time.now.years_ago(5), end_date=Time.now, fiscal_year_offset_month=10)
+        @start_date = start_date.to_date
+        @end_date = end_date.to_date
+        @fiscal_year_offset_month = fiscal_year_offset_month.to_i
+        @holiday_indicators = []
+      end
+      
+      # Returns an array of hashes representing records in the dimension.
+      def build(options={})
+        (start_date..end_date).map { |date| record_from_date(date) }
+      end
+
+      private
+
+      # Returns a hash representing a record in the dimension. The values for each record are 
+      # accessed by name.
+      def record_from_date(date)
+        time = date.to_time # need methods only available in Time
+        record = {}
+        record[:date] = time.strftime("%m/%d/%Y")
+        record[:full_date_description] = time.strftime("%B %d,%Y")
+        record[:day_of_week] = time.strftime("%A")
+        record[:day_in_week] = record[:day_of_week] # alias
+        #record[:day_number_in_epoch] = time.to_i / 24
+        #record[:week_number_in_epoch] = time.to_i / (24 * 7)
+        #record[:month_number_in_epoch] = time.to_i / (24 * 7 * 30)
+        record[:day_number_in_calendar_month] = time.day
+        record[:day_number_in_calendar_year] = time.yday
+        record[:day_number_in_fiscal_month] = time.day # should this be different from CY?
+        record[:day_number_in_fiscal_year] = time.fiscal_year_yday(fiscal_year_offset_month)
+        #record[:last_day_in_week_indicator] = 
+        #record[:last_day_in_month_indicator] =
+        #record[:calendar_week_ending_date] = 
+        record[:calendar_week] = "Week #{time.week}"
+        record[:calendar_week_number] = time.week
+        record[:calendar_week_number_in_year] = time.week # DEPRECATED
+        record[:calendar_month_name] = time.strftime("%B")
+        record[:calendar_month_number_in_year] = time.month # DEPRECATED
+        record[:calendar_month_number] = time.month
+        record[:calendar_year_month] = time.strftime("%Y-%m")
+        record[:calendar_quarter] = "Q#{time.quarter}"
+        record[:calendar_quarter_number] = time.quarter
+        record[:calendar_quarter_number_in_year] = time.quarter # DEPRECATED
+        record[:calendar_year_quarter] = "#{time.strftime('%Y')}-#{record[:calendar_quarter]}"
+        #record[:calendar_half_year] = 
+        record[:calendar_year] = "#{time.year}"
+        record[:fiscal_week] = "FY Week #{time.fiscal_year_week(fiscal_year_offset_month)}"
+        record[:fiscal_week_number_in_year] = time.fiscal_year_week(fiscal_year_offset_month) # DEPRECATED
+        record[:fiscal_week_number] = time.fiscal_year_week(fiscal_year_offset_month)
+        record[:fiscal_month] = time.fiscal_year_month(fiscal_year_offset_month)
+        record[:fiscal_month_number] = time.fiscal_year_month(fiscal_year_offset_month)
+        record[:fiscal_month_number_in_year] = time.fiscal_year_month(fiscal_year_offset_month) # DEPRECATED
+        record[:fiscal_year_month] = "FY#{time.fiscal_year(fiscal_year_offset_month)}-" + time.fiscal_year_month(fiscal_year_offset_month).to_s.rjust(2, '0')
+        record[:fiscal_quarter] = "FY Q#{time.fiscal_year_quarter(fiscal_year_offset_month)}"
+        record[:fiscal_year_quarter] = "FY#{time.fiscal_year(fiscal_year_offset_month)}-Q#{time.fiscal_year_quarter(fiscal_year_offset_month)}"
+        record[:fiscal_quarter_number] = time.fiscal_year_quarter(fiscal_year_offset_month) # DEPRECATED
+        record[:fiscal_year_quarter_number] = time.fiscal_year_quarter(fiscal_year_offset_month)
+        #record[:fiscal_half_year] = 
+        record[:fiscal_year] = "FY#{time.fiscal_year(fiscal_year_offset_month)}"
+        record[:fiscal_year_number] = time.fiscal_year(fiscal_year_offset_month)
+        record[:holiday_indicator] = holiday_indicators.include?(date) ? 'Holiday' : 'Nonholiday'
+        record[:weekday_indicator] = weekday_indicators[time.wday]
+        record[:selling_season] = 'None'
+        record[:major_event] = 'None'
+        record[:sql_date_stamp] = date
+
+        record
+      end
+    end
+  end
+end

data/etl/lib/etl/commands/etl.rb

@@ -0,0 +1,89 @@
+#--
+# Copyright (c) 2006 Anthony Eden
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'benchmark'
+require 'getoptlong'
+
+# Print a usage statement
+def usage #:nodoc:
+  puts "Usage: etl file [file file ...]" # TODO: add the command line options
+end
+
+def execute
+  opts = GetoptLong.new(
+    [ '--version', '-v', GetoptLong::NO_ARGUMENT],
+    [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+    [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT ],
+    [ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ],
+    [ '--offset', '-o', GetoptLong::REQUIRED_ARGUMENT],
+    [ '--newlog', '-n', GetoptLong::NO_ARGUMENT ],
+    [ '--skip-bulk-import', '-s', GetoptLong::NO_ARGUMENT ],
+    [ '--read-locally', GetoptLong::NO_ARGUMENT],
+    [ '--rails-root', GetoptLong::REQUIRED_ARGUMENT]
+  )
+  
+  options = {}
+  opts.each do |opt, arg|
+    case opt
+    when '--version'
+      puts "ActiveWarehouse ETL version #{ETL::VERSION::STRING}"
+      return
+    when '--help'
+      usage
+      return
+    when '--config'
+      options[:config] = arg
+    when '--limit'
+      options[:limit] = arg.to_i
+    when '--offset'
+      options[:offset] = arg.to_i
+    when '--newlog'
+      options[:newlog] = true
+    when '--skip-bulk-import'
+      puts "skip bulk import enabled"
+      options[:skip_bulk_import] = true
+    when '--read-locally'
+      puts "read locally enabled"
+      options[:read_locally] = true
+    when '--rails-root'
+      options[:rails_root] = arg
+      puts "rails root set to #{options[:rails_root]}"
+    end
+  end
+
+  if ARGV.length < 1
+    usage
+  else
+    puts "Starting ETL process"
+
+    ETL::Engine.init(options)
+    ARGV.each do |f|
+      ETL::Engine.realtime_activity = true
+      ETL::Engine.process(f)
+    end
+  
+    puts "ETL process complete\n\n"
+  end
+end
+
+execute