etl 0.9.5.rc1

data/TODO

@@ -0,0 +1,28 @@
+TODO
+
+* Add build-in support for audit_dimension
+* Do not rerun the processing if it isn't needed, i.e. the source and control files have not been modified (allow forced override)
+* Provide greater control in error handling
+** Allow a error threshold
+** Don't die completely if a parse error, just stop processing that specific file if error threshold is reached
+** Allow mismatch row length error in delimited parser to be ignored
+* Improve error messages throughout, but especially in problems with the control files
+* Add support for paritioned views during the insert process. Use specifiable columns as the trigger columns for determining the data output destination.
+* Check if a temp table exists and the last job run was successful, in which case skip during the current run
+* Create models for each of the tables in each of the databases defined in ETL::Engine.connections
+
+Audit Record
+  
+Process-Level
+  * Start Time
+  * End Time
+  * (Duration)
+  * Rows Read
+  * Rows Written
+  * Rows Rejected
+  * Errors
+  * Destination
+Record-Level
+  * Source
+  * Timestamp
+  * Transformation Log

data/activewarehouse-etl.gemspec

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+
+require 'etl/version'
+
+Gem::Specification.new do |s|
+  s.name = %q{etl}
+  s.version = ETL::VERSION
+  s.platform = Gem::Platform::RUBY
+  s.authors = ["Anthony Eden", "Thibaut Barrère"]
+  s.email = ["thibaut.barrere@gmail.com"]
+  s.homepage = "https://github.com/activewarehouse/activewarehouse-etl"
+  s.summary = %q{Pure Ruby ETL package.}
+  s.description = %q{ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application for loading data into a database.}
+
+  s.required_rubygems_version = ">= 1.3.6"
+
+  s.add_runtime_dependency('activesupport',       '>= 2.1.0')
+  s.add_runtime_dependency('activerecord',        '>= 2.1.0')
+  s.add_runtime_dependency('fastercsv',           '>= 1.2.0')
+  s.add_runtime_dependency('adapter_extensions',  '>= 0.9.5.rc1')
+
+  s.add_development_dependency('rake')
+  s.add_development_dependency('rspec', '~>2.6.0')
+  s.add_development_dependency('shoulda', '~>2.11.3')
+  s.add_development_dependency('flexmock', '~>0.9.0')
+  s.add_development_dependency('mysql', '~>2.8.1')
+  s.add_development_dependency('mysql2', '~>0.3.7')
+  s.add_development_dependency('spreadsheet', '~>0.6.5.4')
+  s.add_development_dependency('yard')
+  s.add_development_dependency('RedCloth')
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/bin/etl

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+#--
+# Copyright (c) 2006 Anthony Eden
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+$:.unshift(File.dirname(__FILE__) + '/../lib/')
+require 'etl'
+require 'etl/commands/etl'

data/bin/etl.cmd

@@ -0,0 +1,8 @@
+@echo off
+
+rem The purpose of this Windows script is to let you use the etl command line with a non-gem version of AW-ETL (eg: unpacked gem, pistoned trunk).
+rem Just add the current folder on top of your PATH variable to use it instead of the etl command provided with the gem release.
+
+rem %~dp0 returns the absolute path where the current script is. We just append 'etl' to it, and forward all the arguments with %*
+
+ruby "%~dp0etl" %*

data/examples/database.example.yml

@@ -0,0 +1,16 @@
+etl_execution:
+  adapter: mysql
+  username: root
+  host: localhost
+  database: etl_execution
+  encoding: utf8
+datawarehouse:
+  adapter: mysql
+  username: root
+  host: localhost
+  database: datawarehouse_development
+operational:
+  adapter: mysql
+  username: root
+  host: localhost
+  database: operational_production

data/lib/etl.rb

@@ -0,0 +1,97 @@
+# This source file requires all of the necessary gems and source files for ActiveWarehouse ETL. If you
+# load this source file all of the other required files and gems will also be brought into the 
+# runtime.
+
+#--
+# Copyright (c) 2006-2007 Anthony Eden
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'logger'
+require 'yaml'
+require 'erb'
+
+require 'rubygems'
+
+unless defined?(REXML::VERSION)
+  require 'rexml/rexml'
+  unless defined?(REXML::VERSION)
+    REXML::VERSION = REXML::Version
+  end
+end
+
+require 'active_support'
+require 'active_record'
+require 'adapter_extensions'
+
+if RUBY_VERSION < '1.9'
+  require 'faster_csv'
+  CSV = FasterCSV unless defined?(CSV)
+else
+  require 'csv'
+end
+
+# patch for https://github.com/activewarehouse/activewarehouse-etl/issues/24
+# allow components to require optional gems
+class Object
+  def optional_require(feature)
+    begin
+      require feature
+    rescue LoadError
+    end
+  end
+end
+
+$:.unshift(File.dirname(__FILE__))
+
+require 'etl/core_ext'
+require 'etl/util'
+require 'etl/http_tools'
+require 'etl/builder'
+require 'etl/version'
+require 'etl/engine'
+require 'etl/control'
+require 'etl/batch'
+require 'etl/row'
+require 'etl/parser'
+require 'etl/transform'
+require 'etl/processor'
+require 'etl/generator'
+require 'etl/screen'
+
+module ETL #:nodoc:
+  class ETLError < StandardError #:nodoc:
+  end
+  class ControlError < ETLError #:nodoc:
+  end
+  class DefinitionError < ControlError #:nodoc:
+  end
+  class ConfigurationError < ControlError #:nodoc:
+  end
+  class MismatchError < ETLError #:nodoc:
+  end
+  class ResolverError < ETLError #:nodoc:
+  end
+  class ScreenError < ETLError #:nodoc:
+  end
+  class FatalScreenError < ScreenError #:nodoc:
+  end
+end

data/lib/etl/batch.rb

@@ -0,0 +1,2 @@
+require 'etl/batch/batch'
+require 'etl/batch/directives'

data/lib/etl/batch/batch.rb

@@ -0,0 +1,111 @@
+module ETL #:nodoc:
+  module Batch
+    class Context
+      attr_reader :batch
+      
+      class << self
+        # Create a context that is used when evaluating the batch file
+        def create(batch)
+          Context.new(batch).get_binding
+        end
+      end
+      
+      def initialize(batch)
+        @batch = batch
+      end
+      
+      def file
+        batch.file
+      end
+      
+      def get_binding
+        binding
+      end
+      
+      def run(file)
+        batch.run(File.dirname(self.file) + "/" + file)
+      end
+      
+      def use_temp_tables(value=true)
+        batch.use_temp_tables(value)
+      end
+      
+    end
+    class Batch
+      attr_accessor :file
+      attr_accessor :engine
+      
+      class << self
+        # Resolve the given object to an ETL::Control::Control instance. Acceptable arguments
+        # are:
+        # * The path to a control file as a String
+        # * A File object referencing the control file
+        # * The ETL::Control::Control object (which will just be returned)
+        #
+        # Raises a ControlError if any other type is given
+        def resolve(batch, engine)
+          batch = do_resolve(batch)
+          batch.engine = engine
+          batch
+        end
+        
+        protected
+        def parse(batch_file)
+          batch_file = batch_file.path if batch_file.instance_of?(File)
+          batch = ETL::Batch::Batch.new(batch_file)
+          eval(IO.readlines(batch_file).join("\n"), Context.create(batch), batch_file)
+          batch
+        end
+        
+        def do_resolve(batch)
+          case batch
+          when String
+            ETL::Batch::Batch.parse(File.new(batch))
+          when File
+            ETL::Batch::Batch.parse(batch)
+          when ETL::Batch::Batch
+            batch
+          else
+            raise RuntimeError, "Batch must be a String, File or Batch object"
+          end
+        end
+      end
+      
+      def initialize(file)
+        @file = file
+      end
+      
+      def run(file)
+        directives << Run.new(self, file)
+      end
+      
+      def use_temp_tables(value = true)
+        directives << UseTempTables.new(self)
+      end
+      
+      def execute
+        engine.say "Executing batch"
+        before_execute
+        directives.each do |directive|
+          directive.execute
+        end
+        engine.say "Finishing batch"
+        after_execute
+        engine.say "Batch complete"
+      end
+      
+      def directives
+        @directives ||= []
+      end
+      
+      def before_execute
+        
+      end
+      
+      def after_execute
+        ETL::Engine.finish # TODO: should be moved to the directive?
+        ETL::Engine.use_temp_tables = false # reset the temp tables
+      end
+    end
+  end
+end

data/lib/etl/batch/directives.rb

@@ -0,0 +1,65 @@
+module ETL #:nodoc:
+  module Batch #:nodoc:
+    # Abstract base class for directives
+    class Directive
+      # Method to access the batch object
+      attr_reader :batch
+      
+      # Initialize the directive with the given batch object
+      def initialize(batch)
+        @batch = batch
+      end
+      
+      # Execute the directive
+      def execute
+        do_execute
+      end
+      
+      protected
+      # Implemented by subclasses
+      def do_execute
+        raise RuntimeError, "Directive must implement do_execute method"
+      end
+    end
+    
+    # Directive indicating that the specified ETL control file should be 
+    # run
+    class Run < Directive
+      # The file to execute
+      attr_reader :file
+      
+      # Initialize the directive with the given batch object and file
+      def initialize(batch, file)
+        super(batch)
+        @file = file
+      end
+      
+      protected
+      # Execute the process
+      def do_execute
+        current_batch = ETL::Engine.batch
+        batch.engine.process(file)
+
+        job = ETL::Engine.batch
+        if (job.kind_of? ETL::Execution::Batch and
+            current_batch[:id] != job[:id])
+          job[:batch_id] = current_batch[:id]
+          job.save!
+        end
+
+        ETL::Engine.batch = current_batch
+      end
+    end
+    
+    # Directive indicating temp tables should be used.
+    class UseTempTables < Directive
+      def initialize(batch)
+        super(batch)
+      end
+      protected
+      def do_execute
+        ETL::Engine.use_temp_tables = true
+      end
+    end 
+  end
+end

data/lib/etl/builder.rb

@@ -0,0 +1,2 @@
+require 'etl/builder/date_dimension_builder'
+require 'etl/builder/time_dimension_builder'

data/lib/etl/builder/date_dimension_builder.rb

@@ -0,0 +1,96 @@
+module ETL #:nodoc:
+  module Builder #:nodoc:
+    # A builder which will build a data structure which can be used to populate a date dimension using 
+    # commonly used date dimension columns.
+    class DateDimensionBuilder
+      # Specify the start date for the first record
+      attr_accessor :start_date
+      
+      # Specify the end date for the last record
+      attr_accessor :end_date
+      
+      # Define any holiday indicators
+      attr_accessor :holiday_indicators
+
+      # Add offset month for fiscal year
+      attr_accessor :fiscal_year_offset_month
+      
+      # Define the weekday indicators. The default array begins on Sunday and goes to Saturday.
+      cattr_accessor :weekday_indicators
+      @@weekday_indicators = ['Weekend','Weekday','Weekday','Weekday','Weekday','Weekday','Weekend']
+      
+      # Initialize the builder.
+      # 
+      # * <tt>start_date</tt>: The start date. Defaults to 5 years ago from today.
+      # * <tt>end_date</tt>: The end date. Defaults to now.
+      def initialize(start_date=Time.now.years_ago(5), end_date=Time.now, fiscal_year_offset_month=10)
+        @start_date = start_date.to_date
+        @end_date = end_date.to_date
+        @fiscal_year_offset_month = fiscal_year_offset_month.to_i
+        @holiday_indicators = []
+      end
+      
+      # Returns an array of hashes representing records in the dimension.
+      def build(options={})
+        (start_date..end_date).map { |date| record_from_date(date) }
+      end
+
+      private
+
+      # Returns a hash representing a record in the dimension. The values for each record are 
+      # accessed by name.
+      def record_from_date(date)
+        time = date.to_time # need methods only available in Time
+        record = {}
+        record[:date] = time.strftime("%m/%d/%Y")
+        record[:full_date_description] = time.strftime("%B %d,%Y")
+        record[:day_of_week] = time.strftime("%A")
+        record[:day_in_week] = record[:day_of_week] # alias
+        #record[:day_number_in_epoch] = time.to_i / 24
+        #record[:week_number_in_epoch] = time.to_i / (24 * 7)
+        #record[:month_number_in_epoch] = time.to_i / (24 * 7 * 30)
+        record[:day_number_in_calendar_month] = time.day
+        record[:day_number_in_calendar_year] = time.yday
+        record[:day_number_in_fiscal_month] = time.day # should this be different from CY?
+        record[:day_number_in_fiscal_year] = time.fiscal_year_yday(fiscal_year_offset_month)
+        #record[:last_day_in_week_indicator] = 
+        #record[:last_day_in_month_indicator] =
+        #record[:calendar_week_ending_date] = 
+        record[:calendar_week] = "Week #{time.week}"
+        record[:calendar_week_number] = time.week
+        record[:calendar_week_number_in_year] = time.week # DEPRECATED
+        record[:calendar_month_name] = time.strftime("%B")
+        record[:calendar_month_number_in_year] = time.month # DEPRECATED
+        record[:calendar_month_number] = time.month
+        record[:calendar_year_month] = time.strftime("%Y-%m")
+        record[:calendar_quarter] = "Q#{time.quarter}"
+        record[:calendar_quarter_number] = time.quarter
+        record[:calendar_quarter_number_in_year] = time.quarter # DEPRECATED
+        record[:calendar_year_quarter] = "#{time.strftime('%Y')}-#{record[:calendar_quarter]}"
+        #record[:calendar_half_year] = 
+        record[:calendar_year] = "#{time.year}"
+        record[:fiscal_week] = "FY Week #{time.fiscal_year_week(fiscal_year_offset_month)}"
+        record[:fiscal_week_number_in_year] = time.fiscal_year_week(fiscal_year_offset_month) # DEPRECATED
+        record[:fiscal_week_number] = time.fiscal_year_week(fiscal_year_offset_month)
+        record[:fiscal_month] = time.fiscal_year_month(fiscal_year_offset_month)
+        record[:fiscal_month_number] = time.fiscal_year_month(fiscal_year_offset_month)
+        record[:fiscal_month_number_in_year] = time.fiscal_year_month(fiscal_year_offset_month) # DEPRECATED
+        record[:fiscal_year_month] = "FY#{time.fiscal_year(fiscal_year_offset_month)}-" + time.fiscal_year_month(fiscal_year_offset_month).to_s.rjust(2, '0')
+        record[:fiscal_quarter] = "FY Q#{time.fiscal_year_quarter(fiscal_year_offset_month)}"
+        record[:fiscal_year_quarter] = "FY#{time.fiscal_year(fiscal_year_offset_month)}-Q#{time.fiscal_year_quarter(fiscal_year_offset_month)}"
+        record[:fiscal_quarter_number] = time.fiscal_year_quarter(fiscal_year_offset_month) # DEPRECATED
+        record[:fiscal_year_quarter_number] = time.fiscal_year_quarter(fiscal_year_offset_month)
+        #record[:fiscal_half_year] = 
+        record[:fiscal_year] = "FY#{time.fiscal_year(fiscal_year_offset_month)}"
+        record[:fiscal_year_number] = time.fiscal_year(fiscal_year_offset_month)
+        record[:holiday_indicator] = holiday_indicators.include?(date) ? 'Holiday' : 'Nonholiday'
+        record[:weekday_indicator] = weekday_indicators[time.wday]
+        record[:selling_season] = 'None'
+        record[:major_event] = 'None'
+        record[:sql_date_stamp] = date
+
+        record
+      end
+    end
+  end
+end