activewarehouse-etl 0.9.0 → 0.9.1

data/CHANGELOG

@@ -165,7 +165,7 @@
 0.8.4 - May 24, 2007
 * Added fix for backslash in file writer
 
-0.9.0 - 
+0.9.0 - August 9, 2007
 * Added support for batch processing through .ebf files. These files are
   essentially control files that apply settings to an entire ETL process.
 * Implemented support for screen blocks. These blocks can be used to test
@@ -175,4 +175,24 @@
   connection information in the control files.
 * Implemented temp table support throughout.
 * DateDimensionBuilder now included in ActiveWarehouse ETL directly.
-* Time calculations for fiscal year now included in ActiveWarehouse ETL.
+* Time calculations for fiscal year now included in ActiveWarehouse ETL.
+
+0.9.1 -
+* SQLResolver now uses ETL::Engine.table so it may utilize temp tables. (aeden)
+* Added Thibaut Barrère's encode processor.
+* Added MockSource and MockDestination test helpers (thbar)
+* Added the block processor. Can call a block once (pre/post processor)
+  or once for each row (after_read/before_write row processor) (thbar)
+* Changed temp table to use new AdapterExtension copy_table method (aeden)
+* Added bin/etl.cmd windows batch - just add the bin folder to your PATH
+  and it will let you call etl on an unpacked/pistoned version of AW-ETL (thbar)
+* Upgraded to support Rails 2.1. No longer compatible with older versions of Rails.
+* Added ETL::Builder::TimeDimensionBuilder
+* Added :default option to ForeignKeyLookupTransform that will be used if no
+  foreign key is found.
+* Added :cache option to ForeignKeyLookupTransform that will preload the FK
+  mappings if the underlying resolver supports it. Currently supported by 
+  SQLResolver.
+* A Class extending ETL::Transform::Transform may now be passed as a transformer.
+  For example, in the control file you would define the transform as:
+  transform :a_field, MyTransform, {:option1 => 'option1'}.

data/README

@@ -1,5 +1,14 @@
 Ruby Extract-Transform-Load (ETL) tool.
 
+== Requirements
+
+* Ruby 1.8.5 or higher
+* Rubygems
+
+== Online Documentation
+
+Available at http://activewarehouse.rubyforge.org/docs/activewarehouse-etl.html
+
 == Features
 
 Current supported features:
@@ -67,6 +76,9 @@ Command line options:
 == Control File Examples
 Control file examples can be found in the examples directory.
 
+== Running Tests
+The tests require Shoulda 1.x.
+
 == Feedback
 This is a work in progress. Comments should be made on the 
 activewarehouse-discuss mailing list at the moment. Contributions are always

data/Rakefile

@@ -7,16 +7,13 @@ require 'rake/contrib/rubyforgepublisher'
 
 require File.join(File.dirname(__FILE__), 'lib/etl', 'version')
 
-PKG_BUILD       = ENV['PKG_BUILD'] ? '.' + ENV['PKG_BUILD'] : ''
-PKG_NAME        = 'activewarehouse-etl'
-PKG_VERSION     = ETL::VERSION::STRING + PKG_BUILD
-PKG_FILE_NAME   = "#{PKG_NAME}-#{PKG_VERSION}"
-PKG_DESTINATION = ENV["PKG_DESTINATION"] || "../#{PKG_NAME}"
-
-RELEASE_NAME  = "REL #{PKG_VERSION}"
-
-RUBY_FORGE_PROJECT = "activewarehouse"
-RUBY_FORGE_USER    = "aeden"
+module AWETL
+  PKG_BUILD       = ENV['PKG_BUILD'] ? '.' + ENV['PKG_BUILD'] : ''
+  PKG_NAME        = 'activewarehouse-etl'
+  PKG_VERSION     = ETL::VERSION::STRING + PKG_BUILD
+  PKG_FILE_NAME   = "#{PKG_NAME}-#{PKG_VERSION}"
+  PKG_DESTINATION = ENV["PKG_DESTINATION"] || "../#{PKG_NAME}"
+end
 
 desc 'Default: run unit tests.'
 task :default => :test
@@ -45,54 +42,62 @@ namespace :rcov do
     mkdir 'coverage' unless File.exist?('coverage')
     rcov = "rcov --aggregate coverage.data --text-summary -Ilib"
     system("#{rcov} test/*_test.rb")
-    system("open coverage/index.html") if PLATFORM['darwin']
+    # system("open coverage/index.html") if PLATFORM['darwin']
   end
 end
 
-PKG_FILES = FileList[
-  'CHANGELOG',
-  'LICENSE',
-  'README',
-  'TODO',
-  'Rakefile',
-  'bin/**/*',
-  'doc/**/*',
-  'lib/**/*',
-  'examples/**/*',
-] - [ 'test' ]
-
-spec = Gem::Specification.new do |s|
-  s.name = 'activewarehouse-etl'
-  s.version = PKG_VERSION
-  s.summary = "Pure Ruby ETL package."
-  s.description = <<-EOF
-    ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application for loading data into a database.
-  EOF
-
-  s.add_dependency('rake',                '>= 0.7.1')
-  s.add_dependency('activesupport',       '>= 1.3.1')
-  s.add_dependency('activerecord',        '>= 1.14.4')
-  s.add_dependency('fastercsv',           '>= 1.2.0')
-  s.add_dependency('adapter_extensions',  '>= 0.1.0')
-
-  s.rdoc_options << '--exclude' << '.'
-  s.has_rdoc = false
-
-  s.files = PKG_FILES.to_a.delete_if {|f| f.include?('.svn')}
-  s.require_path = 'lib'
-
-  s.bindir = "bin" # Use these for applications.
-  s.executables = ['etl']
-  s.default_executable = "etl"
-
-  s.author = "Anthony Eden"
-  s.email = "anthonyeden@gmail.com"
-  s.homepage = "http://activewarehouse.rubyforge.org/etl"
-  s.rubyforge_project = "activewarehouse"
+# Gem Spec
+
+module AWETL
+  def self.package_files(package_prefix)
+    FileList[
+      "#{package_prefix}CHANGELOG",
+      "#{package_prefix}LICENSE",
+      "#{package_prefix}README",
+      "#{package_prefix}TODO",
+      "#{package_prefix}Rakefile",
+      "#{package_prefix}bin/**/*",
+      "#{package_prefix}doc/**/*",
+      "#{package_prefix}lib/**/*",
+      "#{package_prefix}examples/**/*",
+    ] - [ "#{package_prefix}test" ]
+  end
+
+  def self.spec(package_prefix = '')
+    Gem::Specification.new do |s|
+      s.name = 'activewarehouse-etl'
+      s.version = AWETL::PKG_VERSION
+      s.summary = "Pure Ruby ETL package."
+      s.description = <<-EOF
+        ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application for loading data into a database.
+      EOF
+
+      s.add_dependency('rake',                '>= 0.7.1')
+      s.add_dependency('activesupport',       '>= 1.3.1')
+      s.add_dependency('activerecord',        '>= 1.14.4')
+      s.add_dependency('fastercsv',           '>= 1.2.0')
+      s.add_dependency('adapter_extensions',  '>= 0.1.0')
+
+      s.rdoc_options << '--exclude' << '.'
+      s.has_rdoc = false
+
+      s.files = package_files(package_prefix).to_a.delete_if {|f| f.include?('.svn')}
+      s.require_path = 'lib'
+
+      s.bindir = "#{package_prefix}bin" # Use these for applications.
+      s.executables = ['etl']
+      s.default_executable = "etl"
+
+      s.author = "Anthony Eden"
+      s.email = "anthonyeden@gmail.com"
+      s.homepage = "http://activewarehouse.rubyforge.org/etl"
+      s.rubyforge_project = "activewarehouse"
+    end
+  end
 end
 
-Rake::GemPackageTask.new(spec) do |pkg|
-  pkg.gem_spec = spec
+Rake::GemPackageTask.new(AWETL.spec) do |pkg|
+  pkg.gem_spec = AWETL.spec
   pkg.need_tar = true
   pkg.need_zip = true
 end
@@ -112,10 +117,10 @@ task :lines do
       codelines += 1
     end
     puts "L: #{sprintf("%4d", lines)}, LOC #{sprintf("%4d", codelines)} | #{file_name}"
-    
+
     total_lines     += lines
     total_codelines += codelines
-    
+
     lines, codelines = 0, 0
   end
 
@@ -127,7 +132,7 @@ task :release => [ :package ] do
   `rubyforge login`
 
   for ext in %w( gem tgz zip )
-    release_command = "rubyforge add_release activewarehouse #{PKG_NAME} 'REL #{PKG_VERSION}' pkg/#{PKG_NAME}-#{PKG_VERSION}.#{ext}"
+    release_command = "rubyforge add_release activewarehouse #{AWETL::PKG_NAME} 'REL #{AWETL::PKG_VERSION}' pkg/#{AWETL::PKG_NAME}-#{AWETL::PKG_VERSION}.#{ext}"
     puts release_command
     system(release_command)
   end
@@ -143,6 +148,6 @@ task :reinstall => [:package] do
   windows = RUBY_PLATFORM =~ /mswin/
   sudo = windows ? '' : 'sudo'
   gem = windows ? 'gem.bat' : 'gem'
-  `#{sudo} #{gem} uninstall -x -i #{PKG_NAME}`
-  `#{sudo} #{gem} install pkg/#{PKG_NAME}-#{PKG_VERSION}`
-end
+  `#{sudo} #{gem} uninstall #{AWETL::PKG_NAME} -x`
+  `#{sudo} #{gem} install pkg/#{AWETL::PKG_NAME}-#{AWETL::PKG_VERSION}`
+end

data/bin/etl.cmd

@@ -0,0 +1,8 @@
+@echo off
+
+rem The purpose of this Windows script is to let you use the etl command line with a non-gem version of AW-ETL (eg: unpacked gem, pistoned trunk).
+rem Just add the current folder on top of your PATH variable to use it instead of the etl command provided with the gem release.
+
+rem %~dp0 returns the absolute path where the current script is. We just append 'etl' to it, and forward all the arguments with %*
+
+ruby "%~dp0etl" %*

data/examples/database.example.yml

@@ -3,4 +3,14 @@ etl_execution:
   username: root
   host: localhost
   database: etl_execution
-  encoding: utf8
+  encoding: utf8
+datawarehouse:
+  adapter: mysql
+  username: root
+  host: localhost
+  database: datawarehouse_development
+operational:
+  adapter: mysql
+  username: root
+  host: localhost
+  database: operational_production

data/lib/etl.rb

@@ -31,29 +31,15 @@ require 'erb'
 
 require 'rubygems'
 
-unless Kernel.respond_to?(:gem)
-  Kernel.send :alias_method, :gem, :require_gem
+unless defined?(REXML::VERSION)
+  require 'rexml/rexml'
+  REXML::VERSION = REXML::Version
 end
 
-unless defined?(ActiveSupport)
-  gem 'activesupport'
-  require 'active_support'
-end
-
-unless defined?(ActiveRecord)
-  gem 'activerecord'
-  require 'active_record'
-end
-
-unless defined?(AdapterExtensions)
-  gem 'adapter_extensions'
-  require 'adapter_extensions'
-end
-
-unless defined?(FasterCSV)
-  gem 'fastercsv'
-  require 'faster_csv'
-end
+require 'active_support'
+require 'active_record'
+require 'adapter_extensions'
+require 'faster_csv'
 
 $:.unshift(File.dirname(__FILE__))
 
@@ -79,6 +65,8 @@ module ETL #:nodoc:
   end
   class DefinitionError < ControlError #:nodoc:
   end
+  class ConfigurationError < ControlError #:nodoc:
+  end
   class MismatchError < ETLError #:nodoc:
   end
   class ResolverError < ETLError #:nodoc:

data/lib/etl/builder.rb

@@ -1 +1,2 @@
-require 'etl/builder/date_dimension_builder'
+require 'etl/builder/date_dimension_builder'
+require 'etl/builder/time_dimension_builder'

data/lib/etl/builder/date_dimension_builder.rb

@@ -11,6 +11,9 @@ module ETL #:nodoc:
       
       # Define any holiday indicators
       attr_accessor :holiday_indicators
+
+      # Add offset month for fiscal year
+      attr_accessor :fiscal_year_offset_month
       
       # Define the weekday indicators. The default array begins on Sunday and goes to Saturday.
       cattr_accessor :weekday_indicators
@@ -20,64 +23,74 @@ module ETL #:nodoc:
       # 
       # * <tt>start_date</tt>: The start date. Defaults to 5 years ago from today.
       # * <tt>end_date</tt>: The end date. Defaults to now.
-      def initialize(start_date=Time.now.years_ago(5), end_date=Time.now)
-        @start_date = start_date
-        @end_date = end_date
+      def initialize(start_date=Time.now.years_ago(5), end_date=Time.now, fiscal_year_offset_month=10)
+        @start_date = start_date.to_date
+        @end_date = end_date.to_date
+        @fiscal_year_offset_month = fiscal_year_offset_month.to_i
         @holiday_indicators = []
       end
       
-      # Returns an array of hashes representing records in the dimension. The values for each record are 
-      # accessed by name.
+      # Returns an array of hashes representing records in the dimension.
       def build(options={})
-        records = []
-        date = start_date.to_time
-        while date <= end_date.to_time
-          record = {}
-          record[:date] = date.strftime("%m/%d/%Y")
-          record[:full_date_description] = date.strftime("%B %d,%Y")
-          record[:day_of_week] = date.strftime("%A")
-          #record[:day_number_in_epoch] = date.to_i / 24
-          #record[:week_number_in_epoch] = date.to_i / (24 * 7)
-          #record[:month_number_in_epoch] = date.to_i / (24 * 7 * 30)
-          record[:day_number_in_calendar_month] = date.day
-          record[:day_number_in_calendar_year] = date.yday
-          record[:day_number_in_fiscal_month] = date.day # should this be different from CY?
-          record[:day_number_in_fiscal_year] = date.fiscal_year_yday
-          #record[:last_day_in_week_indicator] = 
-          #record[:last_day_in_month_indicator] =
-          #record[:calendar_week_ending_date] = 
-          record[:calendar_week] = "Week #{date.week}"
-          record[:calendar_week_number_in_year] = date.week
-          record[:calendar_month_name] = date.strftime("%B")
-          record[:calendar_month_number_in_year] = date.month
-          record[:calendar_year_month] = date.strftime("%Y-%m")
-          record[:calendar_quarter] = "Q#{date.quarter}"
-          record[:calendar_quarter_number_in_year] = date.quarter
-          record[:calendar_year_quarter] = "#{date.strftime('%Y')}-#{record[:calendar_quarter]}"
-          #record[:calendar_half_year] = 
-          record[:calendar_year] = "#{date.year}"
-          record[:fiscal_week] = "FY Week #{date.fiscal_year_week}"
-          record[:fiscal_week_number_in_year] = date.fiscal_year_week
-          record[:fiscal_month] = date.fiscal_year_month
-          record[:fiscal_month_number_in_year] = date.fiscal_year_month
-          record[:fiscal_year_month] = "FY#{date.fiscal_year}-" + date.fiscal_year_month.to_s.rjust(2, '0')
-          record[:fiscal_quarter] = "FY Q#{date.fiscal_year_quarter}"
-          record[:fiscal_year_quarter] = "FY#{date.fiscal_year}-Q#{date.fiscal_year_quarter}"
-          record[:fiscal_year_quarter_number] = date.fiscal_year_quarter
-          #record[:fiscal_half_year] = 
-          record[:fiscal_year] = "FY#{date.fiscal_year}"
-          record[:fiscal_year_number] = date.fiscal_year
-          record[:holiday_indicator] = holiday_indicators.include?(date) ? 'Holiday' : 'Nonholiday'
-          record[:weekday_indicator] = weekday_indicators[date.wday]
-          record[:selling_season] = 'None'
-          record[:major_event] = 'None'
-          record[:sql_date_stamp] = date
-          
-          records << record
-          date = date.tomorrow
-        end
-        records
+        (start_date..end_date).map { |date| record_from_date(date) }
+      end
+
+      private
+
+      # Returns a hash representing a record in the dimension. The values for each record are 
+      # accessed by name.
+      def record_from_date(date)
+        time = date.to_time # need methods only available in Time
+        record = {}
+        record[:date] = time.strftime("%m/%d/%Y")
+        record[:full_date_description] = time.strftime("%B %d,%Y")
+        record[:day_of_week] = time.strftime("%A")
+        record[:day_in_week] = record[:day_of_week] # alias
+        #record[:day_number_in_epoch] = time.to_i / 24
+        #record[:week_number_in_epoch] = time.to_i / (24 * 7)
+        #record[:month_number_in_epoch] = time.to_i / (24 * 7 * 30)
+        record[:day_number_in_calendar_month] = time.day
+        record[:day_number_in_calendar_year] = time.yday
+        record[:day_number_in_fiscal_month] = time.day # should this be different from CY?
+        record[:day_number_in_fiscal_year] = time.fiscal_year_yday(fiscal_year_offset_month)
+        #record[:last_day_in_week_indicator] = 
+        #record[:last_day_in_month_indicator] =
+        #record[:calendar_week_ending_date] = 
+        record[:calendar_week] = "Week #{time.week}"
+        record[:calendar_week_number] = time.week
+        record[:calendar_week_number_in_year] = time.week # DEPRECATED
+        record[:calendar_month_name] = time.strftime("%B")
+        record[:calendar_month_number_in_year] = time.month # DEPRECATED
+        record[:calendar_month_number] = time.month
+        record[:calendar_year_month] = time.strftime("%Y-%m")
+        record[:calendar_quarter] = "Q#{time.quarter}"
+        record[:calendar_quarter_number] = time.quarter
+        record[:calendar_quarter_number_in_year] = time.quarter # DEPRECATED
+        record[:calendar_year_quarter] = "#{time.strftime('%Y')}-#{record[:calendar_quarter]}"
+        #record[:calendar_half_year] = 
+        record[:calendar_year] = "#{time.year}"
+        record[:fiscal_week] = "FY Week #{time.fiscal_year_week(fiscal_year_offset_month)}"
+        record[:fiscal_week_number_in_year] = time.fiscal_year_week(fiscal_year_offset_month) # DEPRECATED
+        record[:fiscal_week_number] = time.fiscal_year_week(fiscal_year_offset_month)
+        record[:fiscal_month] = time.fiscal_year_month(fiscal_year_offset_month)
+        record[:fiscal_month_number] = time.fiscal_year_month(fiscal_year_offset_month)
+        record[:fiscal_month_number_in_year] = time.fiscal_year_month(fiscal_year_offset_month) # DEPRECATED
+        record[:fiscal_year_month] = "FY#{time.fiscal_year(fiscal_year_offset_month)}-" + time.fiscal_year_month(fiscal_year_offset_month).to_s.rjust(2, '0')
+        record[:fiscal_quarter] = "FY Q#{time.fiscal_year_quarter(fiscal_year_offset_month)}"
+        record[:fiscal_year_quarter] = "FY#{time.fiscal_year(fiscal_year_offset_month)}-Q#{time.fiscal_year_quarter(fiscal_year_offset_month)}"
+        record[:fiscal_quarter_number] = time.fiscal_year_quarter(fiscal_year_offset_month) # DEPRECATED
+        record[:fiscal_year_quarter_number] = time.fiscal_year_quarter(fiscal_year_offset_month)
+        #record[:fiscal_half_year] = 
+        record[:fiscal_year] = "FY#{time.fiscal_year(fiscal_year_offset_month)}"
+        record[:fiscal_year_number] = time.fiscal_year(fiscal_year_offset_month)
+        record[:holiday_indicator] = holiday_indicators.include?(date) ? 'Holiday' : 'Nonholiday'
+        record[:weekday_indicator] = weekday_indicators[time.wday]
+        record[:selling_season] = 'None'
+        record[:major_event] = 'None'
+        record[:sql_date_stamp] = date
+
+        record
       end
     end
   end
-end
+end

data/lib/etl/builder/time_dimension_builder.rb

@@ -0,0 +1,31 @@
+module ETL #:nodoc:
+  module Builder #:nodoc:
+    # Builder that creates a simple time dimension.
+    class TimeDimensionBuilder
+      def initialize
+        # Returns an array of hashes representing records in the dimension. The values for each record are 
+        # accessed by name.
+        def build(options={})
+          records = []
+          0.upto(23) do |t_hour|
+            0.upto(59) do |t_minute|
+              0.upto(59) do |t_second|
+                t_hour_string = t_hour.to_s.rjust(2, '0')
+                t_minute_string = t_minute.to_s.rjust(2, '0')
+                t_second_string = t_second.to_s.rjust(2, '0')
+                record = {}
+                record[:hour] = t_hour
+                record[:minute] = t_minute
+                record[:second] = t_second
+                record[:minute_description] = "#{t_hour_string}:#{t_minute_string}"
+                record[:full_description] = "#{t_hour_string}:#{t_minute_string}:#{t_second_string}"
+                records << record
+              end
+            end
+          end
+          records
+        end
+      end
+    end
+  end
+end