activewarehouse-etl-sgonyea 0.9.6

data/TODO

@@ -0,0 +1,28 @@
+TODO
+
+* Add build-in support for audit_dimension
+* Do not rerun the processing if it isn't needed, i.e. the source and control files have not been modified (allow forced override)
+* Provide greater control in error handling
+** Allow a error threshold
+** Don't die completely if a parse error, just stop processing that specific file if error threshold is reached
+** Allow mismatch row length error in delimited parser to be ignored
+* Improve error messages throughout, but especially in problems with the control files
+* Add support for paritioned views during the insert process. Use specifiable columns as the trigger columns for determining the data output destination.
+* Check if a temp table exists and the last job run was successful, in which case skip during the current run
+* Create models for each of the tables in each of the databases defined in ETL::Engine.connections
+
+Audit Record
+  
+Process-Level
+  * Start Time
+  * End Time
+  * (Duration)
+  * Rows Read
+  * Rows Written
+  * Rows Rejected
+  * Errors
+  * Destination
+Record-Level
+  * Source
+  * Timestamp
+  * Transformation Log

data/active_support_logger.patch

@@ -0,0 +1,78 @@
+Index: lib/active_support/clean_logger.rb
+===================================================================
+--- lib/active_support/clean_logger.rb	(revision 5963)
++++ lib/active_support/clean_logger.rb	(working copy)
+@@ -1,10 +1,21 @@
+ require 'logger'
+ require File.dirname(__FILE__) + '/core_ext/class/attribute_accessors'
+ 
+-class Logger #:nodoc:
++# Extensions to the built in Ruby logger.
++#
++# If you want to use the default log formatter as defined in the Ruby core, then you 
++# will need to set the formatter for the logger as in:
++#
++#   logger.formatter = Formatter.new
++#
++# You can then specify the datetime format, for example:
++#
++#   logger.datetime_format = "%Y-%m-%d"
++class Logger
++  # Set to false to disable the silencer
+   cattr_accessor :silencer
+   self.silencer = true
+-
++  
+   # Silences the logger for the duration of the block.
+   def silence(temporary_level = Logger::ERROR)
+     if silencer
+@@ -18,6 +29,35 @@
+       yield self
+     end
+   end
++  
++  alias :old_datetime_format= :datetime_format=
++  # Logging date-time format (string passed to +strftime+). Ignored if the formatter
++  # does not respond to datetime_format=.
++  def datetime_format=(datetime_format)
++    formatter.datetime_format = datetime_format if formatter.respond_to?(:datetime_format=)
++  end
++  
++  alias :old_datetime_format :datetime_format
++  # Get the logging datetime format. Returns nil if the formatter does not support
++  # datetime formatting.
++  def datetime_format
++    formatter.datetime_format if formatter.respond_to?(:datetime_format)
++  end
++  
++  alias :old_formatter :formatter
++  # Get the current formatter. The default formatter is a SimpleFormatter which only
++  # displays the log message
++  def formatter
++    @formatter ||= SimpleFormatter.new
++  end
++  
++  # Simple formatter which only displays the message.
++  class SimpleFormatter < Logger::Formatter
++    # This method is invoked when a log event occurs
++    def call(severity, timestamp, progname, msg)
++      "#{msg}\n"
++    end
++  end
+ 
+   private
+     alias old_format_message format_message
+@@ -28,11 +68,11 @@
+     # with Logger from 1.8.3 and vice versa.
+     if method_defined?(:formatter=)
+       def format_message(severity, timestamp, progname, msg)
+-        "#{msg}\n"
++        formatter.call(severity, timestamp, progname, msg)
+       end
+     else
+       def format_message(severity, timestamp, msg, progname)
+-        "#{msg}\n"
++        formatter.call(severity, timestamp, progname, msg)
+       end
+     end
+ end

data/activewarehouse-etl.gemspec

@@ -0,0 +1,36 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+
+require 'etl/version'
+
+Gem::Specification.new do |s|
+  s.name = %q{activewarehouse-etl-sgonyea}
+  s.version = '0.9.6'
+  s.platform = Gem::Platform::RUBY
+  s.authors = ["Anthony Eden", "Thibaut Barrère"]
+  s.email = ["thibaut.barrere@gmail.com"]
+  s.homepage = "https://github.com/activewarehouse/activewarehouse-etl"
+  s.summary = %q{Pure Ruby ETL package.}
+  s.description = %q{ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application for loading data into a database.}
+
+  s.required_rubygems_version = ">= 1.3.6"
+
+  s.add_runtime_dependency('rake',                '>= 0.8.3')
+  s.add_runtime_dependency('activesupport',       '>= 2.1.0')
+  s.add_runtime_dependency('activerecord',        '>= 2.1.0')
+  s.add_runtime_dependency('fastercsv',           '>= 1.2.0')
+  s.add_runtime_dependency('adapter_extensions',  '>= 0.9.5.rc1')
+
+  s.add_development_dependency('shoulda', '~>2.11.3')
+  s.add_development_dependency('flexmock', '~>0.9.0')
+  s.add_development_dependency('mysql', '~>2.8.1')
+  s.add_development_dependency('mysql2', '~>0.3.7')
+  s.add_development_dependency('rdoc')
+  s.add_development_dependency('spreadsheet', '~>0.6.5.4')
+
+  s.files              = `git ls-files`.split("\n")
+  s.test_files         = `git ls-files -- {test}/*`.split("\n")
+  s.executables        = %w(etl)
+  s.require_path       = "lib"
+end

data/bin/etl

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+#--
+# Copyright (c) 2006 Anthony Eden
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+$:.unshift(File.dirname(__FILE__) + '/../lib/')
+require 'etl'
+require 'etl/commands/etl'

data/bin/etl.cmd

@@ -0,0 +1,8 @@
+@echo off
+
+rem The purpose of this Windows script is to let you use the etl command line with a non-gem version of AW-ETL (eg: unpacked gem, pistoned trunk).
+rem Just add the current folder on top of your PATH variable to use it instead of the etl command provided with the gem release.
+
+rem %~dp0 returns the absolute path where the current script is. We just append 'etl' to it, and forward all the arguments with %*
+
+ruby "%~dp0etl" %*

data/examples/database.example.yml

@@ -0,0 +1,16 @@
+etl_execution:
+  adapter: mysql
+  username: root
+  host: localhost
+  database: etl_execution
+  encoding: utf8
+datawarehouse:
+  adapter: mysql
+  username: root
+  host: localhost
+  database: datawarehouse_development
+operational:
+  adapter: mysql
+  username: root
+  host: localhost
+  database: operational_production

data/lib/etl.rb

@@ -0,0 +1,97 @@
+# This source file requires all of the necessary gems and source files for ActiveWarehouse ETL. If you
+# load this source file all of the other required files and gems will also be brought into the 
+# runtime.
+
+#--
+# Copyright (c) 2006-2007 Anthony Eden
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'logger'
+require 'yaml'
+require 'erb'
+
+require 'rubygems'
+
+unless defined?(REXML::VERSION)
+  require 'rexml/rexml'
+  unless defined?(REXML::VERSION)
+    REXML::VERSION = REXML::Version
+  end
+end
+
+require 'active_support'
+require 'active_record'
+require 'adapter_extensions'
+
+if RUBY_VERSION < '1.9'
+  require 'faster_csv'
+  CSV = FasterCSV unless defined?(CSV)
+else
+  require 'csv'
+end
+
+# patch for https://github.com/activewarehouse/activewarehouse-etl/issues/24
+# allow components to require optional gems
+class Object
+  def optional_require(feature)
+    begin
+      require feature
+    rescue LoadError
+    end
+  end
+end
+
+$:.unshift(File.dirname(__FILE__))
+
+require 'etl/core_ext'
+require 'etl/util'
+require 'etl/http_tools'
+require 'etl/builder'
+require 'etl/version'
+require 'etl/engine'
+require 'etl/control'
+require 'etl/batch'
+require 'etl/row'
+require 'etl/parser'
+require 'etl/transform'
+require 'etl/processor'
+require 'etl/generator'
+require 'etl/screen'
+
+module ETL #:nodoc:
+  class ETLError < StandardError #:nodoc:
+  end
+  class ControlError < ETLError #:nodoc:
+  end
+  class DefinitionError < ControlError #:nodoc:
+  end
+  class ConfigurationError < ControlError #:nodoc:
+  end
+  class MismatchError < ETLError #:nodoc:
+  end
+  class ResolverError < ETLError #:nodoc:
+  end
+  class ScreenError < ETLError #:nodoc:
+  end
+  class FatalScreenError < ScreenError #:nodoc:
+  end
+end

data/lib/etl/batch.rb

@@ -0,0 +1,2 @@
+require 'etl/batch/batch'
+require 'etl/batch/directives'

data/lib/etl/batch/batch.rb

@@ -0,0 +1,111 @@
+module ETL #:nodoc:
+  module Batch
+    class Context
+      attr_reader :batch
+      
+      class << self
+        # Create a context that is used when evaluating the batch file
+        def create(batch)
+          Context.new(batch).get_binding
+        end
+      end
+      
+      def initialize(batch)
+        @batch = batch
+      end
+      
+      def file
+        batch.file
+      end
+      
+      def get_binding
+        binding
+      end
+      
+      def run(file)
+        batch.run(File.dirname(self.file) + "/" + file)
+      end
+      
+      def use_temp_tables(value=true)
+        batch.use_temp_tables(value)
+      end
+      
+    end
+    class Batch
+      attr_accessor :file
+      attr_accessor :engine
+      
+      class << self
+        # Resolve the given object to an ETL::Control::Control instance. Acceptable arguments
+        # are:
+        # * The path to a control file as a String
+        # * A File object referencing the control file
+        # * The ETL::Control::Control object (which will just be returned)
+        #
+        # Raises a ControlError if any other type is given
+        def resolve(batch, engine)
+          batch = do_resolve(batch)
+          batch.engine = engine
+          batch
+        end
+        
+        protected
+        def parse(batch_file)
+          batch_file = batch_file.path if batch_file.instance_of?(File)
+          batch = ETL::Batch::Batch.new(batch_file)
+          eval(IO.readlines(batch_file).join("\n"), Context.create(batch), batch_file)
+          batch
+        end
+        
+        def do_resolve(batch)
+          case batch
+          when String
+            ETL::Batch::Batch.parse(File.new(batch))
+          when File
+            ETL::Batch::Batch.parse(batch)
+          when ETL::Batch::Batch
+            batch
+          else
+            raise RuntimeError, "Batch must be a String, File or Batch object"
+          end
+        end
+      end
+      
+      def initialize(file)
+        @file = file
+      end
+      
+      def run(file)
+        directives << Run.new(self, file)
+      end
+      
+      def use_temp_tables(value = true)
+        directives << UseTempTables.new(self)
+      end
+      
+      def execute
+        engine.say "Executing batch"
+        before_execute
+        directives.each do |directive|
+          directive.execute
+        end
+        engine.say "Finishing batch"
+        after_execute
+        engine.say "Batch complete"
+      end
+      
+      def directives
+        @directives ||= []
+      end
+      
+      def before_execute
+        
+      end
+      
+      def after_execute
+        ETL::Engine.finish # TODO: should be moved to the directive?
+        ETL::Engine.use_temp_tables = false # reset the temp tables
+      end
+    end
+  end
+end

data/lib/etl/batch/directives.rb

@@ -0,0 +1,65 @@
+module ETL #:nodoc:
+  module Batch #:nodoc:
+    # Abstract base class for directives
+    class Directive
+      # Method to access the batch object
+      attr_reader :batch
+      
+      # Initialize the directive with the given batch object
+      def initialize(batch)
+        @batch = batch
+      end
+      
+      # Execute the directive
+      def execute
+        do_execute
+      end
+      
+      protected
+      # Implemented by subclasses
+      def do_execute
+        raise RuntimeError, "Directive must implement do_execute method"
+      end
+    end
+    
+    # Directive indicating that the specified ETL control file should be 
+    # run
+    class Run < Directive
+      # The file to execute
+      attr_reader :file
+      
+      # Initialize the directive with the given batch object and file
+      def initialize(batch, file)
+        super(batch)
+        @file = file
+      end
+      
+      protected
+      # Execute the process
+      def do_execute
+        current_batch = ETL::Engine.batch
+        batch.engine.process(file)
+
+        job = ETL::Engine.batch
+        if (job.kind_of? ETL::Execution::Batch and
+            current_batch[:id] != job[:id])
+          job[:batch_id] = current_batch[:id]
+          job.save!
+        end
+
+        ETL::Engine.batch = current_batch
+      end
+    end
+    
+    # Directive indicating temp tables should be used.
+    class UseTempTables < Directive
+      def initialize(batch)
+        super(batch)
+      end
+      protected
+      def do_execute
+        ETL::Engine.use_temp_tables = true
+      end
+    end 
+  end
+end