activewarehouse-etl 0.9.0 → 0.9.1

data/lib/etl/transform/date_to_string_transform.rb

@@ -12,6 +12,7 @@ module ETL #:nodoc:
       end
       # Transform the value using strftime
       def transform(name, value, row)
+        return value unless value.respond_to?(:strftime)
         value.strftime(@format)
       end
     end

data/lib/etl/transform/foreign_key_lookup_transform.rb

@@ -5,6 +5,9 @@ module ETL #:nodoc:
       # The resolver to use if the foreign key is not found in the collection
       attr_accessor :resolver
       
+      # The default foreign key to use if none is found.
+      attr_accessor :default
+      
       # Initialize the foreign key lookup transform.
       #
       # Configuration options:
@@ -12,12 +15,21 @@ module ETL #:nodoc:
       #  an empty Hash will be used. This Hash will be used to cache values that have been resolved already
       #  for future use.
       # *<tt>:resolver</tt>: Object or Class which implements the method resolve(value)
+      # *<tt>:default</tt>: A default foreign key to use if no foreign key is found
       def initialize(control, name, configuration={})
         super
         
         @collection = (configuration[:collection] || {})
         @resolver = configuration[:resolver]
         @resolver = @resolver.new if @resolver.is_a?(Class)
+        @default = configuration[:default]
+        if configuration[:cache] ||= true
+          if resolver.respond_to?(:load_cache)
+            resolver.load_cache
+          else
+            ETL::Engine.logger.info "#{resolver.class.name} does not support caching"
+          end
+        end
       end
       
       # Transform the value by resolving it to a foriegn key
@@ -27,7 +39,8 @@ module ETL #:nodoc:
           raise ResolverError, "Foreign key for #{value} not found and no resolver specified" unless resolver
           raise ResolverError, "Resolver does not appear to respond to resolve method" unless resolver.respond_to?(:resolve)
           fk = resolver.resolve(value)
-          raise ResolverError, "Unable to resolve #{value} to foreign key for #{name} in row #{ETL::Engine.rows_read}" unless fk
+          fk ||= @default
+          raise ResolverError, "Unable to resolve #{value} to foreign key for #{name} in row #{ETL::Engine.rows_read}. You may want to specify a :default value." unless fk
           @collection[value] = fk
         end
         fk
@@ -81,6 +94,58 @@ class SQLResolver
     @connection ||= ActiveRecord::Base.connection
   end
   def resolve(value)
-    @connection.select_value("SELECT id FROM #{@table} WHERE #{@field} = #{@connection.quote(value)}")
+    if @use_cache
+      cache[value]
+    else
+      q = "SELECT id FROM #{table_name} WHERE #{@field} = #{@connection.quote(value)}"
+      ETL::Engine.logger.debug("Executing query: #{q}")
+      @connection.select_value(q)
+    end
+  end
+  def table_name
+    ETL::Engine.table(@table, @connection)
+  end
+  def cache
+    @cache ||= {}
+  end
+  def load_cache
+    @use_cache = true
+    q = "SELECT id, #{@field} FROM #{table_name}"
+    @connection.select_all(q).each do |record|
+      cache[record[@field]] = record['id']
+    end
+  end
+end
+
+class FlatFileResolver
+  # Initialize the flat file resolver. Expects to open a comma-delimited file. 
+  # Returns the column with the given result_field_index.
+  #
+  # The matches argument is a Hash with the key as the column index to search and
+  # the value of the Hash as a String to match exactly. It will only match the first
+  # result.
+  def initialize(file, match_index, result_field_index)
+    @file = file
+    @match_index = match_index
+    @result_field_index = result_field_index
+  end
+  
+  # Get the rows from the file specified in the initializer.
+  def rows
+    @rows ||= FasterCSV.read(@file)
+  end
+  protected :rows
+  
+  # Match the row field from the column indicated by the match_index with the given 
+  # value and return the field value from the column identified by the result_field_index.
+  def resolve(value)
+    rows.each do |row|
+      #puts "checking #{row.inspect} for #{value}"
+      if row[@match_index] == value
+        #puts "match found!, returning #{row[@result_field_index]}"
+        return row[@result_field_index]
+      end
+    end
+    nil
   end
 end

data/lib/etl/transform/string_to_date_transform.rb

@@ -4,7 +4,12 @@ module ETL #:nodoc:
     class StringToDateTransform < ETL::Transform::Transform
       # Transform the value using Date.parse
       def transform(name, value, row)
-        Date.parse(value)
+        return value if value.nil?
+        begin
+          Date.parse(value)
+        rescue => e
+          return value
+        end
       end
     end
   end

data/lib/etl/transform/string_to_datetime_transform.rb

@@ -7,7 +7,7 @@ module ETL #:nodoc:
       # WARNING: This transform is slow (due to the Ruby implementation), but if you need to 
       # parse timestamps before or after the values supported by the Time.parse.
       def transform(name, value, row)
-        DateTime.parse(value)
+        DateTime.parse(value) unless value.nil?
       end
     end
   end

data/lib/etl/transform/string_to_time_transform.rb

@@ -4,7 +4,7 @@ module ETL #:nodoc:
     class StringToTimeTransform < ETL::Transform::Transform
       # Transform the value using Time.parse
       def transform(name, value, row)
-        Time.parse(value)
+        Time.parse(value) unless value.nil?
       end
     end
   end

data/lib/etl/version.rb

@@ -2,7 +2,7 @@ module ETL#:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 9
-    TINY  = 0
+    TINY  = 1
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

metadata

@@ -1,33 +1,75 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.2
-specification_version: 1
 name: activewarehouse-etl
 version: !ruby/object:Gem::Version 
-  version: 0.9.0
-date: 2007-08-09 00:00:00 -04:00
-summary: Pure Ruby ETL package.
-require_paths: 
-- lib
-email: anthonyeden@gmail.com
-homepage: http://activewarehouse.rubyforge.org/etl
-rubyforge_project: activewarehouse
-description: ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application for loading data into a database.
-autorequire: 
-default_executable: etl
-bindir: bin
-has_rdoc: false
-required_ruby_version: !ruby/object:Gem::Version::Requirement 
-  requirements: 
-  - - ">"
-    - !ruby/object:Gem::Version 
-      version: 0.0.0
-  version: 
+  version: 0.9.1
 platform: ruby
-signing_key: 
-cert_chain: 
-post_install_message: 
 authors: 
 - Anthony Eden
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-01-14 00:00:00 -05:00
+default_executable: etl
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.7.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.3.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.14.4
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: adapter_extensions
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.1.0
+    version: 
+description: ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application for loading data into a database.
+email: anthonyeden@gmail.com
+executables: 
+- etl
+extensions: []
+
+extra_rdoc_files: []
+
 files: 
 - CHANGELOG
 - LICENSE
@@ -35,6 +77,7 @@ files:
 - TODO
 - Rakefile
 - bin/etl
+- bin/etl.cmd
 - lib/etl
 - lib/etl/batch
 - lib/etl/batch/batch.rb
@@ -42,6 +85,7 @@ files:
 - lib/etl/batch.rb
 - lib/etl/builder
 - lib/etl/builder/date_dimension_builder.rb
+- lib/etl/builder/time_dimension_builder.rb
 - lib/etl/builder.rb
 - lib/etl/commands
 - lib/etl/commands/etl.rb
@@ -85,10 +129,12 @@ files:
 - lib/etl/parser/xml_parser.rb
 - lib/etl/parser.rb
 - lib/etl/processor
+- lib/etl/processor/block_processor.rb
 - lib/etl/processor/bulk_import_processor.rb
 - lib/etl/processor/check_exist_processor.rb
 - lib/etl/processor/check_unique_processor.rb
 - lib/etl/processor/copy_field_processor.rb
+- lib/etl/processor/encode_processor.rb
 - lib/etl/processor/hierarchy_exploder_processor.rb
 - lib/etl/processor/print_row_processor.rb
 - lib/etl/processor/processor.rb
@@ -123,62 +169,32 @@ files:
 - lib/etl/version.rb
 - lib/etl.rb
 - examples/database.example.yml
-test_files: []
-
+has_rdoc: false
+homepage: http://activewarehouse.rubyforge.org/etl
+post_install_message: 
 rdoc_options: 
 - --exclude
 - .
-extra_rdoc_files: []
-
-executables: 
-- etl
-extensions: []
-
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
 requirements: []
 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: rake
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 0.7.1
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: activesupport
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.3.1
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: activerecord
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.14.4
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: fastercsv
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.2.0
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: adapter_extensions
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 0.1.0
-    version: 
+rubyforge_project: activewarehouse
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: Pure Ruby ETL package.
+test_files: []
+