activewarehouse-etl 0.8.2 → 0.8.3

data/CHANGELOG

@@ -127,10 +127,13 @@
   be constructed by appending Source to the type name), a class (which will be instantiated
   and passed the control, configuration and mapping) and finally an actual Source instance.
   
-0.8.2 -
+0.8.2 - April 15, 2007
 * Fixed bug with premature destination closing.
 * Added indexes to execution records table.
 * Added a PrintRowProcessor.
 * Added support for conditions and "group by" in the database source.
 * Added after_initialize hook in Processor base class.
-* Added examples directory
+* Added examples directory
+
+0.8.3 - May 13, 2007
+* Added patches from Andy Triboletti

data/lib/etl/control/destination.rb

@@ -218,6 +218,11 @@ module ETL #:nodoc:
                 original_record = ETL::Row[result.symbolize_keys!]
                 original_record[scd_end_date_field] = timestamp
                 ETL::Engine.logger.debug "writing original record"
+                
+                # if there is no truncate then the row will exist twice in the database
+                # need to figure out how to delete that old record before inserting the
+                # updated version of the record
+                
                 buffer << original_record
               end
               

data/lib/etl/control/destination/database_destination.rb

@@ -57,7 +57,7 @@ module ETL #:nodoc:
             values = []
             order.each do |name|
               names << name
-              values << "'#{row[name]}'" # TODO: this is probably not database agnostic
+              values << conn.quote(row[name]) # TODO: this is probably not database agnostic
             end
             q = "INSERT INTO #{configuration[:table]} (#{names.join(',')}) VALUES (#{values.join(',')})"
             ETL::Engine.logger.debug("Executing insert: #{q}")

data/lib/etl/engine.rb

@@ -204,7 +204,7 @@ module ETL #:nodoc:
           # rows rather than the single row
           rows = [row]
           
-          benchmarks[:after_reads] += Benchmark.realtime do
+          t = Benchmark.realtime do
             begin
               Engine.logger.debug "Processing after read"
               control.after_read_processors.each do |processor|
@@ -221,8 +221,9 @@ module ETL #:nodoc:
               exceeded_error_threshold?(control) ? break : next
             end
           end
+          benchmarks[:after_reads] += t unless t.nil?
           
-          benchmarks[:transforms] += Benchmark.realtime do
+          t = Benchmark.realtime do
             begin
               # execute transforms
               Engine.logger.debug "Executing transforms"
@@ -244,8 +245,9 @@ module ETL #:nodoc:
               end
             end
           end
+          benchmarks[:transforms] += t unless t.nil?
           
-          benchmarks[:before_writes] += Benchmark.realtime do
+          t = Benchmark.realtime do
             begin
               # execute row-level "before write" processing
               Engine.logger.debug "Processing before write"
@@ -264,8 +266,9 @@ module ETL #:nodoc:
               exceeded_error_threshold?(control) ? break : next
             end
           end
+          benchmarks[:before_writes] += t unless t.nil?
           
-          benchmarks[:writes] += Benchmark.realtime do
+          t = Benchmark.realtime do
             begin
               # write the row to the destination
               destinations.each_with_index do |destination, index|
@@ -283,6 +286,7 @@ module ETL #:nodoc:
               exceeded_error_threshold?(control) ? break : next
             end
           end
+          benchmarks[:writes] += t unless t.nil?
         end
         
         if exceeded_error_threshold?(control)

data/lib/etl/http_tools.rb

@@ -7,7 +7,8 @@ module HttpTools
   # Code taken from http://gemtacular.com/gems/ParseUserAgent
   def parse_user_agent(user_agent)
     if '-' == user_agent
-      raise 'Invalid User Agent'
+      #raise 'Invalid User Agent'
+      puts 'Invalid User Agent'
     end
     
     browser, browser_version_major, browser_version_minor, ostype, os, os_version = nil
@@ -106,12 +107,17 @@ module HttpTools
   end
   
   def parse_uri(uri_string)
-    if uri_string
-      uri = URI.parse(uri_string)
-      
-      results = {:scheme => uri.scheme, :host => uri.host, :port => uri.port, :uri_path => uri.path}
-      results[:domain] = $1 if uri.host =~ /\.?([^\.]+\.[^\.]+$)/
-      results
+    if uri_string      
+      #attempt to parse uri --if it's a uri then catch the problem and set everything to nil
+      begin
+        uri = URI.parse(uri_string)    
+        results = {:scheme => uri.scheme, :host => uri.host, :port => uri.port, :uri_path => uri.path}
+        results[:domain] = $1 if uri.host =~ /\.?([^\.]+\.[^\.]+$)/
+        results
+      rescue
+        {:scheme => nil, :host => nil, :port => nil, :uri_path => nil, :domain => nil}      
+      end
+
     else
       {:scheme => nil, :host => nil, :port => nil, :uri_path => nil, :domain => nil}
     end

data/lib/etl/parser/apache_combined_log_parser.rb

@@ -31,10 +31,10 @@ module ETL #:nodoc:
           :user_agent => $9,
         }
         #fields[:timestamp] =~ r%{(\d\d)/(\w\w\w)/(\d\d\d\d):(\d\d):(\d\d):(\d\d) -(\d\d\d\d)}
-        d = Date._strptime(fields[:timestamp], '%d/%b/%Y:%H:%M:%S')
-        fields[:timestamp] = Time.mktime(d[:year], d[:mon], d[:mday], d[:hour], d[:min], d[:sec], d[:sec_fraction])
-        
-        fields.merge!(parse_user_agent(fields[:user_agent]))
+        d = Date._strptime(fields[:timestamp], '%d/%b/%Y:%H:%M:%S') unless fields[:timestamp].nil?
+        fields[:timestamp] = Time.mktime(d[:year], d[:mon], d[:mday], d[:hour], d[:min], d[:sec], d[:sec_fraction]) unless d.nil?
+
+        fields.merge!(parse_user_agent(fields[:user_agent])) unless fields[:user_agent].nil?
         fields.merge!(parse_uri(fields[:referrer]))
         
         fields.each do |key, value|

data/lib/etl/version.rb

@@ -2,7 +2,7 @@ module ETL#:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 8
-    TINY  = 2
+    TINY  = 3
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

metadata

@@ -1,10 +1,10 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.0.10
+rubygems_version: 0.9.2
 specification_version: 1
 name: activewarehouse-etl
 version: !ruby/object:Gem::Version 
-  version: 0.8.2
-date: 2007-04-15 00:00:00 -04:00
+  version: 0.8.3
+date: 2007-05-13 00:00:00 -04:00
 summary: Pure Ruby ETL package.
 require_paths: 
 - lib
@@ -36,48 +36,41 @@ files:
 - Rakefile
 - bin/etl
 - lib/etl
-- lib/etl.rb
 - lib/etl/commands
-- lib/etl/control
-- lib/etl/control.rb
-- lib/etl/engine.rb
-- lib/etl/execution
-- lib/etl/execution.rb
-- lib/etl/generator
-- lib/etl/generator.rb
-- lib/etl/http_tools.rb
-- lib/etl/parser
-- lib/etl/parser.rb
-- lib/etl/processor
-- lib/etl/processor.rb
-- lib/etl/row.rb
-- lib/etl/screen
-- lib/etl/transform
-- lib/etl/transform.rb
-- lib/etl/version.rb
 - lib/etl/commands/etl.rb
+- lib/etl/control
 - lib/etl/control/control.rb
 - lib/etl/control/destination
-- lib/etl/control/destination.rb
-- lib/etl/control/source
-- lib/etl/control/source.rb
 - lib/etl/control/destination/database_destination.rb
 - lib/etl/control/destination/file_destination.rb
+- lib/etl/control/destination.rb
+- lib/etl/control/source
 - lib/etl/control/source/database_source.rb
 - lib/etl/control/source/enumerable_source.rb
 - lib/etl/control/source/file_source.rb
+- lib/etl/control/source.rb
+- lib/etl/control.rb
+- lib/etl/engine.rb
+- lib/etl/execution
 - lib/etl/execution/base.rb
 - lib/etl/execution/job.rb
 - lib/etl/execution/migration.rb
 - lib/etl/execution/record.rb
+- lib/etl/execution.rb
+- lib/etl/generator
 - lib/etl/generator/generator.rb
 - lib/etl/generator/surrogate_key_generator.rb
+- lib/etl/generator.rb
+- lib/etl/http_tools.rb
+- lib/etl/parser
 - lib/etl/parser/apache_combined_log_parser.rb
 - lib/etl/parser/delimited_parser.rb
 - lib/etl/parser/fixed_width_parser.rb
 - lib/etl/parser/parser.rb
 - lib/etl/parser/sax_parser.rb
 - lib/etl/parser/xml_parser.rb
+- lib/etl/parser.rb
+- lib/etl/processor
 - lib/etl/processor/bulk_import_processor.rb
 - lib/etl/processor/check_exist_processor.rb
 - lib/etl/processor/check_unique_processor.rb
@@ -90,7 +83,11 @@ files:
 - lib/etl/processor/sequence_processor.rb
 - lib/etl/processor/surrogate_key_processor.rb
 - lib/etl/processor/truncate_processor.rb
+- lib/etl/processor.rb
+- lib/etl/row.rb
+- lib/etl/screen
 - lib/etl/screen/row_count_screen.rb
+- lib/etl/transform
 - lib/etl/transform/block_transform.rb
 - lib/etl/transform/date_to_string_transform.rb
 - lib/etl/transform/decode_transform.rb
@@ -105,6 +102,9 @@ files:
 - lib/etl/transform/transform.rb
 - lib/etl/transform/trim_transform.rb
 - lib/etl/transform/type_transform.rb
+- lib/etl/transform.rb
+- lib/etl/version.rb
+- lib/etl.rb
 - examples/database.example.yml
 test_files: []