activewarehouse-etl 0.7.2 → 0.8.0

data/CHANGELOG

@@ -101,4 +101,21 @@
 * Fixed source caching
 
 0.7.2 - Apr 8, 2007
-* Fixed quoting bug in CheckExistProcessor
+* Fixed quoting bug in CheckExistProcessor
+
+0.8.0 - Apr 12, 2007
+* Source now available through the current row source accessor.
+* Added new_rows_only configuration option to DatabaseSource. A date field must be specified and
+  only records that are greater than the date value in that field, relative to the last successful
+  execution, will be returned from the source.
+* Added an (untested) count feature which returns the number of rows for processing.
+* If no natural key is defined then an empty array will now be used, resulting in the row being
+  written to the output without going through change checks.
+* Mapping argument in destination is now optional. An empty hash will be used if the mapping
+  hash is not specified. If the mapping hash is not specified then the order will be determined
+  using the originating source's order.
+* ActiveRecord configurations loaded from database.yml by the etl tool will be merged with
+  ActiveRecord::Base.configurations.
+* Fixed several bugs in how record change detection was implemented.
+* Fixed how the read_locally functionality was implemented so that it will find that last
+  completed local source copy using the source's trigger file (untested).

data/lib/etl/control/destination.rb

@@ -162,10 +162,16 @@ module ETL #:nodoc:
       
       # Process a row to determine the change type
       def process_change(row)
+        ETL::Engine.logger.debug "Processing row: #{row.inspect}"
         return unless row
         
         # Change processing can only occur if the natural key exists in the row 
-        supports_change = true
+        ETL::Engine.logger.debug "Checking for natural key existence"
+        if natural_key.length == 0
+          buffer << row
+          return
+        end
+        
         natural_key.each do |key| 
           unless row.has_key?(key)
             buffer << row
@@ -173,7 +179,7 @@ module ETL #:nodoc:
           end
         end
         
-        ETL::Engine.logger.debug "checking scd fields"
+        ETL::Engine.logger.debug "Checking for SCD fields"
         s = String.new
         if scd_fields
           scd_fields.each { |f| s << row[f].to_s }
@@ -193,7 +199,7 @@ module ETL #:nodoc:
         
         timestamp = Time.now
         
-        ETL::Engine.logger.debug "checking record change type"
+        ETL::Engine.logger.debug "Checking record change type"
         if record
           if record.crc != crc.to_s
             # SCD Type 1: only the new row should be added
@@ -206,7 +212,7 @@ module ETL #:nodoc:
               q = "SELECT * FROM #{dimension_table} WHERE "
               q << natural_key.collect { |nk| "#{nk} = '#{row[nk]}'" }.join(" AND ")
               #puts "looking for original record"
-              result = ActiveRecord::Base.connection.select_one(q)
+              result = ETL::ActiveRecord::Base.connection.select_one(q)
               if result
                 #puts "Result: #{result.inspect}"
                 original_record = ETL::Row[result.symbolize_keys!]
@@ -230,9 +236,12 @@ module ETL #:nodoc:
             
             q = "SELECT * FROM #{dimension_table} WHERE "
             q << natural_key.collect { |nk| "#{nk} = '#{row[nk]}'" }.join(" AND ")
-            result = ActiveRecord::Base.connection.select_one(q)
+            result = ETL::ActiveRecord::Base.connection.select_one(q)
             if result
-              buffer << ETL::Row[result.symbolize_keys!]
+              # This was necessary when truncating and then loading, however I
+              # am getting reluctant to having the ETL process do the truncation
+              # as part of the bulk load, favoring using a preprocessor instead.
+              # buffer << ETL::Row[result.symbolize_keys!]
             else
               # The record never made it into the database, so add the effective and end date
               # and add it into the bulk load file
@@ -262,8 +271,6 @@ module ETL #:nodoc:
             )
           end
         end
-      rescue => e
-        puts e
       end
       
       # Add any virtual fields to the row. Virtual rows will get their value 
@@ -288,7 +295,11 @@ module ETL #:nodoc:
             when Proc
               row[key] = value.call(row)
             else
-              row[key] = value
+              if value.is_a?(ETL::Generator::Generator)
+                row[key] = value.next
+              else
+                row[key] = value
+              end
             end
           end
         end
@@ -304,7 +315,7 @@ module ETL #:nodoc:
         when String, Symbol
           [configuration[:natural_key].to_sym]
         else
-          [:id]
+          [] # no natural key defined
         end
       end
     end

data/lib/etl/control/destination/database_destination.rb

@@ -30,7 +30,7 @@ module ETL #:nodoc:
       #
       # Mapping options:
       # * <tt>:order</tt>: The order of fields to write (REQUIRED)
-      def initialize(control, configuration, mapping)
+      def initialize(control, configuration, mapping={})
         super
         @truncate = configuration[:truncate] ||= false
         @unique = configuration[:unique]
@@ -60,7 +60,7 @@ module ETL #:nodoc:
               values << "'#{row[name]}'" # TODO: this is probably not database agnostic
             end
             q = "INSERT INTO #{configuration[:table]} (#{names.join(',')}) VALUES (#{values.join(',')})"
-            # ETL::Engine.logger.debug("Query: #{q}")
+            ETL::Engine.logger.debug("Executing insert: #{q}")
             conn.insert(q, "Insert row #{current_row}")
             @current_row += 1
           end

data/lib/etl/control/destination/file_destination.rb

@@ -38,7 +38,7 @@ module ETL #:nodoc:
       # 
       # Mapping options:
       # * <tt>:order</tt>: The order array
-      def initialize(control, configuration, mapping)
+      def initialize(control, configuration, mapping={})
         super
         @file = File.join(File.dirname(control.file), configuration[:file])
         @append = configuration[:append] ||= false

data/lib/etl/control/source.rb

@@ -81,6 +81,16 @@ module ETL #:nodoc:
         File.join(local_dir, "#{filename}.csv")
       end
       
+      # Get the last fully written local file
+      def last_local_file
+        File.join(local_directory, File.basename(last_local_file_trigger, '.trig'))
+      end
+      
+      # Get the last local file trigger
+      def last_local_file_trigger
+        Dir.glob(File.join(local_directory, '*.trig')).last
+      end
+      
       # Get the local trigger file that is used to indicate that the file has
       # been completely written
       def local_file_trigger(file)

data/lib/etl/control/source/database_source.rb

@@ -65,6 +65,22 @@ module ETL #:nodoc:
         configuration[:order]
       end
       
+      # Return the column which is used for in the where clause to identify
+      # new rows
+      def new_records_only
+        configuration[:new_records_only]
+      end
+      
+      # Get the number of rows in the source
+      def count(use_cache=true)
+        return @count if @count && use_cache
+        if store_locally || read_locally
+          @count = count_locally
+        else
+          @count = connection.select_value(query.gsub(/SELECT .* FROM/, 'SELECT count(1) FROM'))
+        end
+      end
+      
       # Get the list of columns to read. This is defined in the source
       # definition as either an Array or Hash
       def columns
@@ -86,7 +102,7 @@ module ETL #:nodoc:
       def each(&block)
         if read_locally # Read from the last stored source
           ETL::Engine.logger.debug "Reading from local cache"
-          read_rows(local_file, &block)
+          read_rows(last_local_file, &block)
         else # Read from the original source
           if store_locally
             file = local_file
@@ -94,7 +110,8 @@ module ETL #:nodoc:
             read_rows(file, &block)
           else
             connection.select_all(query).each do |row|
-              row = Row.new(row.symbolize_keys)
+              row = ETL::Row.new(row.symbolize_keys)
+              row.source = self
               yield row
             end
           end
@@ -110,6 +127,7 @@ module ETL #:nodoc:
         t = Benchmark.realtime do
           FasterCSV.open(file, :headers => true).each do |row|
             result_row = ETL::Row.new
+            result_row.source = self
             row.each do |header, field|
               result_row[header.to_sym] = field
             end
@@ -119,6 +137,12 @@ module ETL #:nodoc:
         ETL::Engine.average_rows_per_second = ETL::Engine.rows_read / t
       end
       
+      def count_locally
+        counter = 0
+        File.open(last_local_file, 'r').each { |line| counter += 1 }
+        counter
+      end
+      
       # Write rows to the local cache
       def write_local(file)
         lines = 0
@@ -140,6 +164,16 @@ module ETL #:nodoc:
         return @query if @query
         q = "SELECT #{select} FROM #{configuration[:table]}"
         q << " #{join}" if join
+        
+        if new_records_only
+          last_completed = ETL::Execution::Job.maximum('created_at', 
+            :conditions => ['control_file = ? and completed_at is not null', control.file]
+          )
+          if last_completed
+            q << " WHERE #{new_records_only} > #{connection.quote(last_completed.to_s(:db))}"
+          end
+        end
+        
         q << " ORDER BY #{order}" if order
         if ETL::Engine.limit || ETL::Engine.offset
           options = {}
@@ -147,7 +181,7 @@ module ETL #:nodoc:
           options[:offset] = ETL::Engine.offset if ETL::Engine.offset
           connection.add_limit_offset!(q, options)
         end
-        #q << " LIMIT #{ETL::Engine.limit}" unless ETL::Engine.limit.nil?
+        
         q = q.gsub(/\n/,' ')
         ETL::Engine.logger.info "Query: #{q}"
         @query = q

data/lib/etl/control/source/file_source.rb

@@ -45,6 +45,7 @@ module ETL #:nodoc:
           # TODO skip rows if offset is defined
           # TODO stop processing if limit is reached
           row = ETL::Row[row]
+          row.source = self
           yield row
         end
       end

data/lib/etl/engine.rb

@@ -20,7 +20,8 @@ module ETL #:nodoc:
           @read_locally = options[:read_locally]
           options[:config] ||= 'database.yml'
           database_configuration = YAML::load(ERB.new(IO.read(options[:config])).result + "\n")
-          ActiveRecord::Base.configurations = database_configuration
+          ETL::ActiveRecord::Base.configurations = database_configuration
+          ActiveRecord::Base.configurations.merge!(ETL::ActiveRecord::Base.configurations)
           require 'etl/execution'
           ETL::Execution::Base.establish_connection :etl_execution
           ETL::Execution::Execution.migrate

data/lib/etl/execution/base.rb

@@ -1,7 +1,7 @@
 module ETL #:nodoc:
   module Execution #:nodoc:
     # Base class for ETL execution information
-    class Base < ActiveRecord::Base
+    class Base < ETL::ActiveRecord::Base
       self.abstract_class = true
       establish_connection :etl_execution
     end

data/lib/etl/generator/surrogate_key_generator.rb

@@ -4,6 +4,15 @@ module ETL #:nodoc:
   module Generator #:nodoc:
     # Surrogate key generator.
     class SurrogateKeyGenerator < Generator
+      # Initialize the generator
+      def initialize(options={})
+        if options[:query]
+          @surrogate_key = ETL::ActiveRecord::Base.connection.select_value(options[:query])
+          @surrogate_key = 0 if @surrogate_key.blank?
+          @surrogate_key = @surrogate_key.to_i
+        end
+      end
+
       # Get the next surrogate key
       def next
         @surrogate_key ||= 0

data/lib/etl/parser/delimited_parser.rb

@@ -41,7 +41,7 @@ module ETL #:nodoc:
       
       private
       def validate_row(row, line, file)
-        ETL::Engine.logger.debug "validing line #{line} in file #{file}"
+        ETL::Engine.logger.debug "validating line #{line} in file #{file}"
         if row.length != fields.length
           raise_with_info( MismatchError, 
             "The number of rows from the source (#{row.length}) does not match the number of rows in the definition (#{fields.length})", 

data/lib/etl/processor/check_exist_processor.rb

@@ -30,7 +30,7 @@ module ETL #:nodoc:
         @columns = configuration[:columns]
         
         q = "SELECT COUNT(*) FROM #{table}"
-        @should_check = ActiveRecord::Base.connection.select_value(q).to_i > 0 
+        @should_check = ETL::ActiveRecord::Base.connection.select_value(q).to_i > 0 
       end
       
       # Return true if the given key should be skipped
@@ -51,12 +51,12 @@ module ETL #:nodoc:
       # Process the row
       def process(row)
         return row unless should_check?
-        connection = ActiveRecord::Base.connection
+        connection = ETL::ActiveRecord::Base.connection
         q = "SELECT * FROM #{table} WHERE "
         conditions = []
         row.each do |k,v| 
           if columns.nil? || columns.include?(k.to_sym)
-            conditions << "#{k} = #{connection.quote(v)}" unless skip?(k)
+            conditions << "#{k} = #{connection.quote(v)}" unless skip?(k.to_sym)
           end
         end
         q << conditions.join(" AND ")

data/lib/etl/processor/surrogate_key_processor.rb

@@ -8,7 +8,9 @@ module ETL #:nodoc:
       # Initialize the surrogate key generator
       def initialize(control, configuration)
         super
-        @surrogate_key = ActiveRecord::Base.connection.select_value(configuration[:query])
+        if configuration[:query]
+          @surrogate_key = ETL::ActiveRecord::Base.connection.select_value(configuration[:query])
+        end
         #puts "initial surrogate key: #{@surrogate_key}"
         @surrogate_key = 0 if @surrogate_key.blank?
         @surrogate_key = @surrogate_key.to_i

data/lib/etl/row.rb

@@ -3,6 +3,9 @@
 module ETL #:nodoc:
   # This class represents a single row currently passing through the ETL pipeline
   class Row < Hash
+    # Accessor for the originating source
+    attr_accessor :source
+    
     # All change types
     CHANGE_TYPES = [:insert, :update, :delete]
     

data/lib/etl/version.rb

@@ -1,8 +1,8 @@
 module ETL#:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 7
-    TINY  = 2
+    MINOR = 8
+    TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0.10
 specification_version: 1
 name: activewarehouse-etl
 version: !ruby/object:Gem::Version 
-  version: 0.7.2
-date: 2007-04-08 00:00:00 -04:00
+  version: 0.8.0
+date: 2007-04-12 00:00:00 -04:00
 summary: Pure Ruby ETL package.
 require_paths: 
 - lib