darrell-activewarehouse-etl 0.9.1.4 → 0.9.1.6

data/README

@@ -86,6 +86,7 @@ The tests require:
 - gem install shoulda
 - gem install flexmock
 - gem install pg (if you want to run the tests on pg)
+- gem install spreadsheet
 
 The tests subfolder contains examples database.yml for mysql and postgres.
 
@@ -96,4 +97,4 @@ To run the tests:
 == Feedback
 This is a work in progress. Comments should be made on the 
 activewarehouse-discuss mailing list at the moment. Contributions are always
-welcome.
+welcome.

data/Rakefile

@@ -9,7 +9,7 @@ require File.join(File.dirname(__FILE__), 'lib/etl', 'version')
 
 module AWETL
   PKG_BUILD       = ENV['PKG_BUILD'] ? '.' + ENV['PKG_BUILD'] : ''
-  PKG_NAME        = 'activewarehouse-etl'
+  PKG_NAME        = 'darrell-activewarehouse-etl'
   PKG_VERSION     = ETL::VERSION::STRING + PKG_BUILD
   PKG_FILE_NAME   = "#{PKG_NAME}-#{PKG_VERSION}"
   PKG_DESTINATION = ENV["PKG_DESTINATION"] || "../#{PKG_NAME}"
@@ -87,7 +87,7 @@ module AWETL
 
   def self.spec(package_prefix = '')
     Gem::Specification.new do |s|
-      s.name = 'activewarehouse-etl'
+      s.name = 'darrell-activewarehouse-etl'
       s.version = AWETL::PKG_VERSION
       s.summary = "Pure Ruby ETL package."
       s.description = <<-EOF
@@ -172,4 +172,4 @@ task :reinstall => [:package] do
   gem = windows ? 'gem.bat' : 'gem'
   `#{sudo} #{gem} uninstall #{AWETL::PKG_NAME} -x`
   `#{sudo} #{gem} install pkg/#{AWETL::PKG_NAME}-#{AWETL::PKG_VERSION}`
-end
+end

data/lib/etl/control/destination.rb

@@ -209,7 +209,12 @@ module ETL #:nodoc:
           return
         end
         
-        @timestamp = Time.now
+        @timestamp = case configuration[:scd][:timestamp]
+                     when Time, Date then configuration[:scd][:timestamp]
+                     when Symbol then row[configuration[:scd][:timestamp]]
+                     when nil then Time.now
+                     else raise "Unknown timestamp: #{configuration[:scd][:timestamp].inspect}. Use Time or Date for a specific time, a symbol for a value from each row, or nil for the current time"
+                     end
 
         # See if the scd_fields of the current record have changed
         # from the last time this record was loaded into the data
@@ -318,6 +323,12 @@ module ETL #:nodoc:
           ETL::Engine.logger.debug "expiring original record"
           @existing_row[scd_end_date_field] = @timestamp
           @existing_row[scd_latest_version_field] = false
+
+          if configuration[:scd][:merge_nils]
+            scd_fields(row).each do |f|
+              row[f] ||= @existing_row[f]
+            end
+          end
           
           buffer << @existing_row
 
@@ -381,17 +392,16 @@ module ETL #:nodoc:
       # Check whether non-scd fields have changed since the last
       # load of this record.
       def has_scd_field_changes?(row)
-        scd_fields(row).any? { |csd_field| 
-          ETL::Engine.logger.debug "Row: #{row.inspect}"
-          ETL::Engine.logger.debug "Existing Row: #{@existing_row.inspect}"
-          ETL::Engine.logger.debug "comparing: #{row[csd_field].to_s} != #{@existing_row[csd_field].to_s}"
-          if row[csd_field].to_s != @existing_row[csd_field].to_s
-            x=true
-          else
-            x=false
-          end
-          ETL::Engine.logger.debug "Fields differ?: #{x}"
-          x
+        fields = scd_fields(row)
+        ETL::Engine.logger.debug "         Row: %s" % row.slice(*fields).inspect
+        ETL::Engine.logger.debug "Existing Row: %s" % @existing_row.slice(*fields).inspect
+
+        fields.any? { |csd_field| 
+          mismatch = configuration[:scd][:merge_nils] ? !row[csd_field].nil? : true
+          mismatch = mismatch && (row[csd_field].to_s != @existing_row[csd_field].to_s)
+
+          ETL::Engine.logger.debug "#{csd_field}: " + (mismatch ? row[csd_field].to_s + " != " + @existing_row[csd_field].to_s : @existing_row[csd_field].to_s)
+          mismatch
         }
       end
       

data/lib/etl/control/source.rb

@@ -40,7 +40,8 @@ module ETL #:nodoc:
         @configuration = configuration
         @definition = definition
         
-        @store_locally = configuration[:store_locally] || true
+        @store_locally = true
+        @store_locally = configuration[:store_locally] unless configuration[:store_locally].nil?
       end
       
       # Get an array of errors that occur during reading from the source
@@ -106,4 +107,4 @@ module ETL #:nodoc:
   end
 end
 
-Dir[File.dirname(__FILE__) + "/source/*.rb"].each { |file| require(file) }
+Dir[File.dirname(__FILE__) + "/source/*.rb"].each { |file| require(file) }

data/lib/etl/control/source/database_source.rb

@@ -41,17 +41,18 @@ module ETL #:nodoc:
         super
         @target = configuration[:target]
         @table = configuration[:table]
+        @query = configuration[:query]
       end
       
       # Get a String identifier for the source
       def to_s
-        "#{host}/#{database}/#{table}"
+        "#{host}/#{database}/#{@table}"
       end
       
       # Get the local directory to use, which is a combination of the 
       # local_base, the db hostname the db database name and the db table.
       def local_directory
-        File.join(local_base, host, database, configuration[:table])
+        File.join(local_base, to_s)
       end
       
       # Get the join part of the query, defaults to nil
@@ -83,7 +84,7 @@ module ETL #:nodoc:
       # Get the number of rows in the source
       def count(use_cache=true)
         return @count if @count && use_cache
-        if store_locally || read_locally
+        if @store_locally || read_locally
           @count = count_locally
         else
           @count = connection.select_value(query.gsub(/SELECT .* FROM/, 'SELECT count(1) FROM'))
@@ -107,13 +108,16 @@ module ETL #:nodoc:
           ETL::Engine.logger.debug "Reading from local cache"
           read_rows(last_local_file, &block)
         else # Read from the original source
-          if store_locally
+          if @store_locally
             file = local_file
             write_local(file)
             read_rows(file, &block)
           else
-            query_rows.each do |row|
-              row = ETL::Row.new(row.symbolize_keys)
+            query_rows.each do |r|
+              row = ETL::Row.new()
+              r.symbolize_keys.each_pair { |key, value|
+                row[key] = value
+              }
               row.source = self
               yield row
             end
@@ -165,7 +169,7 @@ module ETL #:nodoc:
       # Get the query to use
       def query
         return @query if @query
-        q = "SELECT #{select} FROM #{configuration[:table]}"
+        q = "SELECT #{select} FROM #{@table}"
         q << " #{join}" if join
         
         conditions = []
@@ -217,4 +221,4 @@ module ETL #:nodoc:
       end
     end
   end
-end
+end

data/lib/etl/control/source/file_source.rb

@@ -41,7 +41,7 @@ module ETL #:nodoc:
       # Returns each row from the source
       def each
         count = 0
-        copy_sources if store_locally
+        copy_sources if @store_locally
         @parser.each do |row|
           if ETL::Engine.offset && count < ETL::Engine.offset
             count += 1
@@ -87,4 +87,4 @@ module ETL #:nodoc:
       end
     end
   end
-end
+end

data/lib/etl/engine.rb

@@ -1,3 +1,4 @@
+require 'active_support/all'
 module ETL #:nodoc:
   
   class Base < ActiveRecord::Base
@@ -32,7 +33,7 @@ module ETL #:nodoc:
           options[:config] = 'config/database.yml' unless File.exist?(options[:config])
           database_configuration = YAML::load(ERB.new(IO.read(options[:config])).result + "\n")
           ActiveRecord::Base.configurations.merge!(database_configuration)
-          ETL::Base.configurations = database_configuration
+          ETL::Base.configurations = HashWithIndifferentAccess.new(database_configuration)
           #puts "configurations in init: #{ActiveRecord::Base.configurations.inspect}"
           
           require 'etl/execution'

data/lib/etl/parser/delimited_parser.rb

@@ -10,10 +10,29 @@ module ETL #:nodoc:
         configure
       end
       
+      def get_fields_names(file)
+        File.open(file) do |input|
+          fields = FasterCSV.parse(input.readline).first
+          new_fields = []
+          fields.each_with_index do |field,index|
+            # compute the index of occurrence of this specific occurrence of the field (usually, will be 1)
+            occurrence_index = fields[0..index].find_all { |e| e == field }.size
+            number_of_occurrences = fields.find_all { |e| e == field }.size
+            new_field = field + (number_of_occurrences > 1 ? "_#{occurrence_index}" : "")
+            new_fields << Field.new(new_field.to_sym)
+          end
+          return new_fields
+        end
+      end
+
       # Returns each row.
       def each
         Dir.glob(file).each do |file|
           ETL::Engine.logger.debug "parsing #{file}"
+          if fields.length == 0
+            ETL::Engine.logger.debug "no columns specified so reading names from first line of #{file}"
+            @fields = get_fields_names(file)
+          end
           line = 0
           lines_skipped = 0
           FasterCSV.foreach(file, options) do |raw_row|

data/lib/etl/parser/excel_parser.rb

@@ -0,0 +1,112 @@
+require 'spreadsheet'
+
+module ETL
+  module Parser
+    class ExcelParser < ETL::Parser::Parser
+
+      attr_accessor :ignore_blank_line
+
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+
+      # Returns each row
+      def each
+        Dir.glob(file).each do |file|
+          ETL::Engine.logger.debug "parsing #{file}"
+          line = 0
+          lines_skipped = 0
+          book = Spreadsheet.open file
+          loopworksheets = []
+
+          if worksheets.empty?
+            loopworksheets = book.worksheets
+          else
+            worksheets.each do |index|
+              loopworksheets << book.worksheet(index)
+            end
+          end
+
+          loopworksheets.each do |sheet|
+            sheet.each do |raw_row|
+              if lines_skipped < source.skip_lines
+                ETL::Engine.logger.debug "skipping line"
+                lines_skipped += 1
+                next
+              end
+              line += 1
+              row = {}
+              if self.ignore_blank_line and raw_row.empty?
+                lines_skipped += 1
+                next
+              end
+              validate_row(raw_row, line, file)
+              raw_row.each_with_index do |value, index|
+                f = fields[index]
+                row[f.name] = value
+              end
+              yield row
+            end
+          end
+        end
+      end
+
+      # Get an array of defined worksheets
+      def worksheets
+        @worksheets ||= []
+      end
+
+      # Get an array of defined fields
+      def fields
+        @fields ||= []
+      end
+
+      private
+      def validate_row(row, line, file)
+        ETL::Engine.logger.debug "validating line #{line} in file #{file}"
+        if row.length != fields.length
+          raise_with_info( MismatchError, 
+            "The number of columns from the source (#{row.length}) does not match the number of columns in the definition (#{fields.length})", 
+            line, file
+          )
+        end
+      end
+      
+      private
+      def configure
+        source.definition[:worksheets].each do |worksheet|
+          if Integer(worksheet)
+            worksheets << worksheet.to_i
+          else
+            raise DefinitionError, "Each worksheet definition must be an integer"
+          end
+        end unless source.definition[:worksheets].nil?
+
+        self.ignore_blank_line = source.definition[:ignore_blank_line]
+
+        source.definition[:fields].each do |options|
+          case options
+          when Symbol
+            fields << Field.new(options)
+          when Hash
+            fields << Field.new(options[:name])
+          else
+            raise DefinitionError, "Each field definition must either be a symbol or a hash"
+          end
+        end
+      end
+      
+      class Field #:nodoc:
+        attr_reader :name
+        def initialize(name)
+          @name = name
+        end
+      end
+
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification 
 name: darrell-activewarehouse-etl
 version: !ruby/object:Gem::Version 
-  hash: 11
+  hash: 15
   prerelease: false
   segments: 
   - 0
   - 9
   - 1
-  - 4
-  version: 0.9.1.4
+  - 6
+  version: 0.9.1.6
 platform: ruby
 authors: 
 - Anthony Eden
@@ -171,6 +171,7 @@ files:
 - lib/etl/generator/surrogate_key_generator.rb
 - lib/etl/parser/apache_combined_log_parser.rb
 - lib/etl/parser/delimited_parser.rb
+- lib/etl/parser/excel_parser.rb
 - lib/etl/parser/fixed_width_parser.rb
 - lib/etl/parser/parser.rb
 - lib/etl/parser/sax_parser.rb