activewarehouse-etl 0.5.2 → 0.6.0

data/lib/etl/transform/sha1_transform.rb

@@ -8,7 +8,7 @@ module ETL #:nodoc:
         super
       end
       # Transform the value with a SHA1 digest algorithm.
-      def transform(value)
+      def transform(name, value, row)
         Digest::SHA1.hexdigest(value)
       end
     end

data/lib/etl/transform/string_to_date_transform.rb

@@ -5,9 +5,9 @@ module ETL #:nodoc:
       def initialize(control, configuration={})
         super
       end
-      # Transform the value using Time.parse
-      def transform(value)
-        t = Date.parse(value)
+      # Transform the value using Date.parse
+      def transform(name, value, row)
+        Date.parse(value)
       end
     end
   end

data/lib/etl/transform/string_to_datetime_transform.rb

@@ -0,0 +1,17 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a DateTime instance
+    class StringToDateTimeTransform < ETL::Transform::Transform
+      def initialize(control, configuration={})
+        super
+      end
+      # Transform the value using DateTime.parse.
+      #
+      # WARNING: This transform is slow (due to the Ruby implementation), but if you need to 
+      # parse timestamps before or after the values supported by the Time.parse.
+      def transform(name, value, row)
+        DateTime.parse(value)
+      end
+    end
+  end
+end

data/lib/etl/transform/string_to_time_transform.rb

@@ -0,0 +1,14 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a Time instance
+    class StringToTimeTransform < ETL::Transform::Transform
+      def initialize(control, configuration={})
+        super
+      end
+      # Transform the value using Time.parse
+      def transform(name, value, row)
+        Time.parse(value)
+      end
+    end
+  end
+end

data/lib/etl/transform/transform.rb

@@ -20,14 +20,14 @@ module ETL#:nodoc:
         # Transform the specified value using the given transforms. The transforms can either be
         # Proc objects or objects which extend from Transform and implement the method <tt>transform(value)</tt>.
         # Any other object will result in a ControlError being raised.
-        def transform(name, value, transforms)
-          # logger.debug "Transforming field #{name}" if transforms.length > 0
+        def transform(name, value, row, transforms)
           transforms.each do |transform|
+            Engine.logger.debug "Transforming field #{name} with #{transform.inspect}"
             case transform
             when Proc
-              value = transform.call(value)
+              value = transform.call([name, value, row])
             when Transform
-              value = transform.transform(value)
+              value = transform.transform(name, value, row)
             else
               raise ControlError, "Unsupported transform configuration type: #{transform}"
             end
@@ -43,6 +43,10 @@ module ETL#:nodoc:
         @control = control
         @configuration = configuration
       end
+      
+      def transform(name, value, row)
+        raise "transform is an abstract method"
+      end
     end
   end
 end

data/lib/etl/transform/type_transform.rb

@@ -6,8 +6,8 @@ module ETL #:nodoc:
         super
         @type = configuration[:type]
       end
-      # Transform the value using Time.parse
-      def transform(value)
+      # Transform the value
+      def transform(name, value, row)
         case @type
         when :string
           value.to_s

data/lib/etl/version.rb

@@ -1,8 +1,8 @@
 module ETL#:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 5
-    TINY  = 2
+    MINOR = 6
+    TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0.10
 specification_version: 1
 name: activewarehouse-etl
 version: !ruby/object:Gem::Version 
-  version: 0.5.2
-date: 2007-02-19 00:00:00 -05:00
+  version: 0.6.0
+date: 2007-03-08 00:00:00 -05:00
 summary: Pure Ruby ETL package.
 require_paths: 
 - lib
@@ -36,14 +36,13 @@ files:
 - bin/etl
 - lib/etl
 - lib/etl.rb
-- lib/etl/active_record_ext
-- lib/etl/active_record_ext.rb
 - lib/etl/commands
 - lib/etl/control
 - lib/etl/control.rb
 - lib/etl/engine.rb
 - lib/etl/generator
 - lib/etl/generator.rb
+- lib/etl/http_tools.rb
 - lib/etl/parser
 - lib/etl/parser.rb
 - lib/etl/processor
@@ -51,8 +50,6 @@ files:
 - lib/etl/transform
 - lib/etl/transform.rb
 - lib/etl/version.rb
-- lib/etl/active_record_ext/connection_adapters
-- lib/etl/active_record_ext/connection_adapters/mysql_adapter.rb
 - lib/etl/commands/etl.rb
 - lib/etl/control/control.rb
 - lib/etl/control/destination
@@ -65,19 +62,26 @@ files:
 - lib/etl/control/source/file_source.rb
 - lib/etl/generator/generator.rb
 - lib/etl/generator/surrogate_key_generator.rb
+- lib/etl/parser/apache_combined_log_parser.rb
 - lib/etl/parser/delimited_parser.rb
 - lib/etl/parser/fixed_width_parser.rb
 - lib/etl/parser/parser.rb
 - lib/etl/parser/sax_parser.rb
 - lib/etl/parser/xml_parser.rb
 - lib/etl/processor/bulk_import_processor.rb
+- lib/etl/processor/hierarchy_exploder_processor.rb
 - lib/etl/processor/processor.rb
+- lib/etl/processor/row_processor.rb
 - lib/etl/processor/truncate_processor.rb
 - lib/etl/transform/date_to_string_transform.rb
 - lib/etl/transform/decode_transform.rb
+- lib/etl/transform/default_transform.rb
 - lib/etl/transform/foreign_key_lookup_transform.rb
+- lib/etl/transform/hierarchy_lookup_transform.rb
 - lib/etl/transform/sha1_transform.rb
 - lib/etl/transform/string_to_date_transform.rb
+- lib/etl/transform/string_to_datetime_transform.rb
+- lib/etl/transform/string_to_time_transform.rb
 - lib/etl/transform/transform.rb
 - lib/etl/transform/type_transform.rb
 test_files: []
@@ -110,7 +114,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.3.1.5618
+        version: 1.3.1
     version: 
 - !ruby/object:Gem::Dependency 
   name: activerecord
@@ -119,7 +123,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.14.4.5618
+        version: 1.14.4
     version: 
 - !ruby/object:Gem::Dependency 
   name: fastercsv
@@ -130,3 +134,12 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 1.0.0
     version: 
+- !ruby/object:Gem::Dependency 
+  name: adapter_extensions
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.1.0
+    version: 

data/lib/etl/active_record_ext.rb

@@ -1 +0,0 @@
-require 'etl/active_record_ext/connection_adapters/mysql_adapter'

data/lib/etl/active_record_ext/connection_adapters/mysql_adapter.rb

@@ -1,34 +0,0 @@
-require 'active_record/connection_adapters/abstract_adapter'
-
-module ActiveRecord #:nodoc:
-  module ConnectionAdapters #:nodoc:
-    class MysqlAdapter < AbstractAdapter
-      # Execute a truncate statement on the table. Note that in MySQL a truncate will *NOT* reset
-      # the auto_increment
-      def truncate(table_name)
-        execute("TRUNCATE #{table_name}")
-      end
-      
-      # Bulk load the data in the specified file. This implementation always uses the LOCAL keyword
-      # so the file must be found locally, not on the remote server, to be loaded.
-      #
-      # Options:
-      # * <tt>:ignore</tt> -- Ignore the specified number of lines from the source file
-      # * <tt>:columns</tt> -- Array of column names defining the source file column order
-      # * <tt>:fields</tt> -- Hash of options for fields:
-      # ** <tt>:delimited_by</tt> -- The field delimiter
-      # ** <tt>:enclosed_by</tt> -- The field enclosure
-      def bulk_load(file, table_name, options={})
-        q = "LOAD DATA LOCAL INFILE '#{file}' INTO TABLE #{table_name}"
-        if options[:fields]
-          q << " FIELDS"
-          q << " TERMINATED BY '#{options[:fields][:delimited_by]}'" if options[:fields][:delimited_by]
-          q << " ENCLOSED BY '#{options[:fields][:enclosed_by]}'" if options[:fields][:enclosed_by]
-        end
-        q << " IGNORE #{options[:ignore]} LINES" if options[:ignore]
-        q << " (#{options[:columns].join(',')})" if options[:columns]
-        execute(q)
-      end
-    end
-  end
-end