activewarehouse-etl 0.8.1 → 0.8.2

data/CHANGELOG

@@ -125,4 +125,12 @@
 * Added :type configuration option to the source directive, allowing the source type to be 
   explicitly specified. The source type can be a string or symbol (in which case the class will
   be constructed by appending Source to the type name), a class (which will be instantiated
-  and passed the control, configuration and mapping) and finally an actual Source instance.
+  and passed the control, configuration and mapping) and finally an actual Source instance.
+  
+0.8.2 -
+* Fixed bug with premature destination closing.
+* Added indexes to execution records table.
+* Added a PrintRowProcessor.
+* Added support for conditions and "group by" in the database source.
+* Added after_initialize hook in Processor base class.
+* Added examples directory

data/README

@@ -65,7 +65,7 @@ Command line options:
 * <tt>--read-locally</tt>: Read from the local cache (skip source extraction)
 
 == Control File Examples
-Control file examples can be found in the test directory.
+Control file examples can be found in the examples directory.
 
 == Feedback
 This is a work in progress. Comments should be made on the 

data/Rakefile

@@ -47,6 +47,7 @@ PKG_FILES = FileList[
   'bin/**/*',
   'doc/**/*',
   'lib/**/*',
+  'examples/**/*',
 ] - [ 'test' ]
 
 spec = Gem::Specification.new do |s|

data/examples/database.example.yml

@@ -0,0 +1,6 @@
+etl_execution:
+  adapter: mysql
+  username: root
+  host: localhost
+  database: etl_execution
+  encoding: utf8

data/lib/etl/control/destination.rb

@@ -263,12 +263,14 @@ module ETL #:nodoc:
           
           # Record the record
           if ETL::Engine.job # only record the execution if there is a job
-            ETL::Execution::Record.create!(
-              :control_file => control.file,
-              :natural_key => nk,
-              :crc => crc,
-              :job_id => ETL::Engine.job.id
-            )
+            ETL::Execution::Record.time_spent += Benchmark.realtime do
+              ETL::Execution::Record.create!(
+                :control_file => control.file,
+                :natural_key => nk,
+                :crc => crc,
+                :job_id => ETL::Engine.job.id
+              )
+            end
           end
         end
       end

data/lib/etl/control/source/database_source.rb

@@ -60,6 +60,11 @@ module ETL #:nodoc:
         configuration[:select] || '*'
       end
       
+      # Get the group by part of the query, defaults to nil
+      def group
+        configuration[:group]
+      end
+      
       # Get the order for the query, defaults to nil
       def order
         configuration[:order]
@@ -165,16 +170,24 @@ module ETL #:nodoc:
         q = "SELECT #{select} FROM #{configuration[:table]}"
         q << " #{join}" if join
         
+        conditions = []
         if new_records_only
           last_completed = ETL::Execution::Job.maximum('created_at', 
             :conditions => ['control_file = ? and completed_at is not null', control.file]
           )
           if last_completed
-            q << " WHERE #{new_records_only} > #{connection.quote(last_completed.to_s(:db))}"
+            conditions << "#{new_records_only} > #{connection.quote(last_completed.to_s(:db))}"
           end
         end
         
+        conditions << configuration[:conditions] if configuration[:conditions]
+        if conditions.length > 0
+          q << " WHERE #{conditions.join(' AND ')}"
+        end
+        
+        q << " GROUP BY #{group}" if group
         q << " ORDER BY #{order}" if order
+
         if ETL::Engine.limit || ETL::Engine.offset
           options = {}
           options[:limit] = ETL::Engine.limit if ETL::Engine.limit

data/lib/etl/control/source/enumerable_source.rb

@@ -1,9 +1,8 @@
-module ETL
-  module Control
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # Use an Enumerable as a source
     class EnumerableSource < ETL::Control::Source
-      def initialize(control, configuration, definition)
-        super
-      end
+      # Iterate through the enumerable
       def each(&block)
         configuration[:enumerable].each(&block)
       end

data/lib/etl/engine.rb

@@ -237,7 +237,11 @@ module ETL #:nodoc:
               errors << msg
               Engine.logger.error(msg)
               e.backtrace.each { |line| Engine.logger.error(line) }
-              exceeded_error_threshold?(control) ? break : next
+              begin
+                exceeded_error_threshold?(control) ? break : next
+              rescue => inner_error
+                puts inner_error
+              end
             end
           end
           
@@ -286,9 +290,10 @@ module ETL #:nodoc:
           return
         end
         
-        destinations.each do |destination|
-          destination.close
-        end
+      end
+      
+      destinations.each do |destination|
+        destination.close
       end
       
       say_on_own_line "Executing post processes"
@@ -310,6 +315,8 @@ module ETL #:nodoc:
       say "Avg before_writes: #{Engine.rows_read/benchmarks[:before_writes]} rows/sec" if benchmarks[:before_writes] > 0
       say "Avg transforms: #{Engine.rows_read/benchmarks[:transforms]} rows/sec" if benchmarks[:transforms] > 0
       say "Avg writes: #{Engine.rows_read/benchmarks[:writes]} rows/sec" if benchmarks[:writes] > 0
+
+      say "Avg time writing execution records: #{ETL::Execution::Record.average_time_spent}"
       
       # ETL::Transform::Transform.benchmarks.each do |klass, t|
 #         say "Avg #{klass}: #{Engine.rows_read/t} rows/sec"

data/lib/etl/execution/migration.rb

@@ -24,7 +24,7 @@ module ETL #:nodoc:
         
         # Get the final target version number
         def target
-          1
+          2
         end
         
         private
@@ -43,6 +43,13 @@ module ETL #:nodoc:
           end
           update_schema_info(1)
         end
+        
+        def migration_2 #:nodoc:
+          connection.add_index :records, :control_file
+          connection.add_index :records, :natural_key
+          connection.add_index :records, :job_id
+          update_schema_info(2)
+        end
       
         # Update the schema info table, setting the version value
         def update_schema_info(version)

data/lib/etl/execution/record.rb

@@ -3,6 +3,16 @@ module ETL #:nodoc:
     # Represents a single record
     class Record < ETL::Execution::Base
       belongs_to :table
+      class << self
+        attr_accessor :time_spent
+        def time_spent
+          @time_spent ||= 0
+        end
+        def average_time_spent
+          return 0 if time_spent == 0
+          ETL::Engine.rows_read / time_spent
+        end
+      end
     end
   end
 end

data/lib/etl/processor/check_exist_processor.rb

@@ -25,7 +25,7 @@ module ETL #:nodoc:
       #   key
       def initialize(control, configuration)
         super
-        @skip = configuration[:skip]
+        @skip = configuration[:skip] || []
         @table = configuration[:table]
         @columns = configuration[:columns]
         

data/lib/etl/processor/print_row_processor.rb

@@ -0,0 +1,12 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Debugging processor for printing the current row
+    class PrintRowProcessor < ETL::Processor::RowProcessor
+      # Process the row
+      def process(row)
+        puts row.inspect
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/processor.rb

@@ -5,6 +5,7 @@ module ETL #:nodoc:
       def initialize(control, configuration)
         @control = control
         @configuration = configuration
+        after_initialize if respond_to?(:after_initialize)
       end
       protected
       # Get the control object

data/lib/etl/transform/foreign_key_lookup_transform.rb

@@ -27,7 +27,7 @@ module ETL #:nodoc:
           raise ResolverError, "Foreign key for #{value} not found and no resolver specified" unless resolver
           raise ResolverError, "Resolver does not appear to respond to resolve method" unless resolver.respond_to?(:resolve)
           fk = resolver.resolve(value)
-          raise ResolverError, "Unable to resolve #{value} to foreign key" unless fk
+          raise ResolverError, "Unable to resolve #{value} to foreign key for #{name} in row #{ETL::Engine.rows_read}" unless fk
           @collection[value] = fk
         end
         fk
@@ -61,4 +61,16 @@ class ActiveRecordResolver
     rec = ar_class.__send__(find_method, value)
     rec.nil? ? nil : rec.id
   end
+end
+
+class SQLResolver
+  def initialize(table, field, connection=nil)
+    @table = table
+    @field = field
+    @connection = connection
+  end
+  def resolve(value)
+    conn = @connection ||= ActiveRecord::Base.connection
+    conn.select_value("SELECT id FROM #{table} WHERE #{field} = #{conn.quote(value)}")
+  end
 end

data/lib/etl/version.rb

@@ -2,7 +2,7 @@ module ETL#:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 8
-    TINY  = 1
+    TINY  = 2
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0.10
 specification_version: 1
 name: activewarehouse-etl
 version: !ruby/object:Gem::Version 
-  version: 0.8.1
-date: 2007-04-12 00:00:00 -04:00
+  version: 0.8.2
+date: 2007-04-15 00:00:00 -04:00
 summary: Pure Ruby ETL package.
 require_paths: 
 - lib
@@ -83,6 +83,7 @@ files:
 - lib/etl/processor/check_unique_processor.rb
 - lib/etl/processor/copy_field_processor.rb
 - lib/etl/processor/hierarchy_exploder_processor.rb
+- lib/etl/processor/print_row_processor.rb
 - lib/etl/processor/processor.rb
 - lib/etl/processor/rename_processor.rb
 - lib/etl/processor/row_processor.rb
@@ -104,6 +105,7 @@ files:
 - lib/etl/transform/transform.rb
 - lib/etl/transform/trim_transform.rb
 - lib/etl/transform/type_transform.rb
+- examples/database.example.yml
 test_files: []
 
 rdoc_options: