activewarehouse-etl 0.9.5.rc1 → 1.0.0.rc1

data/lib/etl/parser/{delimited_parser.rb → csv_parser.rb}

@@ -1,7 +1,7 @@
 module ETL #:nodoc:
   module Parser #:nodoc:
-    # Parses delimited files
-    class DelimitedParser < ETL::Parser::Parser
+    # Parses CSV files
+    class CsvParser < ETL::Parser::Parser
       # Initialize the parser
       # * <tt>source</tt>: The Source object
       # * <tt>options</tt>: Hash of options for the parser, defaults to an empty hash
@@ -12,7 +12,7 @@ module ETL #:nodoc:
       
       def get_fields_names(file)
         File.open(file) do |input|
-          fields = CSV.parse(input.readline).first
+          fields = CSV.parse(input.readline, options).first
           new_fields = []
           fields.each_with_index do |field,index|
             # compute the index of occurrence of this specific occurrence of the field (usually, will be 1)

data/lib/etl/parser/excel_parser.rb

@@ -1,4 +1,4 @@
-require 'spreadsheet'
+optional_require 'spreadsheet'
 
 module ETL
   module Parser

data/lib/etl/parser/nokogiri_xml_parser.rb

@@ -0,0 +1,83 @@
+optional_require 'nokogiri'
+require 'open-uri'
+optional_require 'zlib'
+
+module ETL
+  module Parser
+    class NokogiriXmlParser < ETL::Parser::Parser
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      # Returns each row
+      def each
+        Dir.glob(file).each do |source|
+
+          doc = nil
+
+          gzip = false
+          magic = "1F8B".to_i(base=16)  # Check for gzip archives
+          if File.exist?(source)
+            gzip = true if magic == (
+              File.open(source).read(2).unpack("H2H2").to_s.to_i(base=16))
+          end
+
+          if gzip
+            doc = Nokogiri::XML(Zlib::GzipReader.open(source))
+          else
+            doc = Nokogiri::XML(open(source))
+          end
+          
+          doc.xpath(@collection_xpath).each do |nodeset|
+            row = {}
+
+            fields.each do |f|
+              value = nodeset.xpath(f.xpath).text
+              row[f.name] = value
+            end
+            yield row
+          end
+
+        end
+      end
+      
+      # Get an array of defined fields
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def configure
+        @collection_xpath = source.definition[:collection]
+        if @collection_xpath.nil?
+          raise ":collection => 'XPath' argument required"
+        end
+        source.definition[:fields].each do |options|
+          case options
+          when Symbol
+            fields << Field.new(options, options.to_s)
+          when Hash
+            options[:xpath] ||= options[:name]
+            fields << Field.new(options[:name], options[:xpath].to_s)
+          else
+            raise DefinitionError, 
+              "Each field definition must either be an symbol " +
+              "or a hash of options for the field"
+          end
+        end
+      end
+      
+      class Field
+        attr_reader :name, :xpath
+        def initialize(name, xpath)
+          @name = name
+          @xpath = xpath
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/bulk_import_processor.rb

@@ -23,6 +23,10 @@ module ETL #:nodoc:
       attr_accessor :line_separator
       # The string that indicates a NULL (defaults to an empty string)
       attr_accessor :null_string
+      # boolean that indicates disable keys before, then enable after load (MySql only optimization)
+      attr_accessor :disable_keys
+      # replace existing records, not just insert
+      attr_accessor :replace
        
       # Initialize the processor.
       #
@@ -36,6 +40,7 @@ module ETL #:nodoc:
       # * <tt>:field_separator</tt>: The field separator. Defaults to a comma
       # * <tt>:line_separator</tt>: The line separator. Defaults to a newline
       # * <tt>:field_enclosure</tt>: The field enclosure charcaters
+      # * <tt>:disable_keys</tt>: Set to true to disable keys before, then enable after load (MySql only optimization)
       def initialize(control, configuration)
         super
         @target = configuration[:target]
@@ -49,6 +54,8 @@ module ETL #:nodoc:
         @line_separator = (configuration[:line_separator] || "\n")
         @null_string = (configuration[:null_string] || "")
         @field_enclosure = configuration[:field_enclosure]
+        @disable_keys = configuration[:disable_keys] || false
+        @replace = configuration[:replace] || false
         
         raise ControlError, "Target must be specified" unless @target
         raise ControlError, "Table must be specified" unless @table
@@ -64,6 +71,10 @@ module ETL #:nodoc:
           conn.truncate(table_name) if truncate
           options = {}
           options[:columns] = columns
+          
+          options[:disable_keys] = true if disable_keys
+          options[:replace] = true if replace
+          
           if field_separator || field_enclosure || line_separator || null_string
             options[:fields] = {}
             options[:fields][:null_string] = null_string if null_string

data/lib/etl/processor/check_exist_processor.rb

@@ -21,11 +21,10 @@ module ETL #:nodoc:
       
       # Initialize the processor
       # Configuration options:
-      # * <tt>:skip</tt>: A symbol or array of column names that should not 
-      #   be checked
+      # * <tt>:columns</tt>: An array of symbols for columns that should be included in the query conditions. If this option is not specified then all of the columns in the row will be included in the conditions (unless :skip is specified).
+      # * <tt>:skip</tt>: A symbol or array of symbols that should not be included in the existence check. If this option is not specified then all of the columns will be included in the existence check (unless :columns is specified).
+      # * <tt>:target</tt>: The target connection
       # * <tt>:table</tt>: The table name
-      # * <tt>:columns</tt>: An array of columns which represent the natural
-      #   key
       def initialize(control, configuration)
         super
         @skip = configuration[:skip] || []
@@ -58,6 +57,7 @@ module ETL #:nodoc:
         conn = ETL::Engine.connection(target)
         q = "SELECT * FROM #{table_name} WHERE "
         conditions = []
+        ensure_columns_available_in_row!(row, columns, 'for existence check')
         row.each do |k,v| 
           if columns.nil? || columns.include?(k.to_sym)
             conditions << "#{k} = #{conn.quote(v)}" unless skip?(k.to_sym)
@@ -66,12 +66,12 @@ module ETL #:nodoc:
         q << conditions.join(" AND ")
         q << " LIMIT 1"
       
-        #puts "query: #{q}"
         result = conn.select_one(q)
         return row if result.nil?
       end
       
-      private
+private
+
       def table_name
         ETL::Engine.table(table, ETL::Engine.connection(target))
       end

data/lib/etl/processor/check_unique_processor.rb

@@ -23,7 +23,11 @@ module ETL #:nodoc:
       
       # Process the row. This implementation will only return a row if it
       # it's key combination has not already been seen.
+      #
+      # An error will be raised if the row doesn't include the keys.
       def process(row)
+        ensure_columns_available_in_row!(row, keys, 'for unicity check')
+        
         key = (keys.collect { |k| row[k] }).join('|')
         unless compound_key_constraints[key]
           compound_key_constraints[key] = 1

data/lib/etl/processor/database_join_processor.rb

@@ -21,6 +21,9 @@ module ETL
         @target = configuration[:target]
         @query = configuration[:query]
         @fields = configuration[:fields]
+        raise ControlError, ":target must be specified" unless @target
+        raise ControlError, ":query must be specified" unless @query
+        raise ControlError, ":fields must be specified" unless @fields
       end
       
       # Get a String identifier for the source
@@ -40,10 +43,28 @@ module ETL
         ETL::Engine.logger.debug("Executing select: #{q}")
         res = connection.execute(q)
 
-        res.each_hash do |r|
-          @fields.each do |field|
-            row[field.to_sym] = r[field]
-          end
+        # TODO - refactor this and move it (and similar code around) to adapter_extensions
+        case connection.class.name
+          when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter";
+            res.each do |r|
+              @fields.each do |field|
+                row[field.to_sym] = r[field.to_s]
+              end
+            end
+          when "ActiveRecord::ConnectionAdapters::Mysql2Adapter";
+            res.each(:as => :hash) do |r|
+              @fields.each do |field|
+                row[field.to_sym] = r[field.to_s]
+              end
+            end
+          when "ActiveRecord::ConnectionAdapters::MysqlAdapter";
+            res.each_hash do |r|
+              @fields.each do |field|
+                row[field.to_sym] = r[field.to_s]
+              end
+            end
+            res.free
+          else raise "Unsupported adapter #{connection.class} for this destination"
         end
 
         return row

data/lib/etl/processor/encode_processor.rb

@@ -1,5 +1,3 @@
-require 'iconv'
-
 module ETL #:nodoc:
   module Processor #:nodoc:
     # The encode processor uses Iconv to convert a file from one encoding (eg: utf-8) to another (eg: latin1), line by line.

data/lib/etl/processor/ensure_fields_presence_processor.rb

@@ -0,0 +1,24 @@
+module ETL
+  module Processor
+    # Ensure that each specified field is available
+    class EnsureFieldsPresenceProcessor < ETL::Processor::RowProcessor
+      
+      # Initialize the processor.
+      #
+      # Configuration options:
+      # * <tt>:fields</tt>: An array of keys whose presence should be verified in each row
+      def initialize(control, configuration)
+        super
+        @fields = configuration[:fields]
+        raise ControlError, ":fields must be specified" unless @fields
+      end
+      
+      def process(row)
+        missing_fields = configuration[:fields].map(&:to_s) - row.keys.map(&:to_s)
+        raise(ETL::ControlError, 
+          "Row missing required field(s) #{missing_fields.join(',')} in row. Available fields are : #{row.keys.join(',')}") unless missing_fields.empty?
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/imapattachment_downloader_processor.rb

@@ -1,5 +1,5 @@
-require 'net/imap'
-require 'tmail'
+optional_require 'net/imap'
+optional_require 'tmail'
 
 module ETL
   module Processor

data/lib/etl/processor/pop3attachment_downloader_processor.rb

@@ -1,5 +1,5 @@
-require 'net/pop'
-require 'tmail'
+optional_require 'net/pop'
+optional_require 'tmail'
 
 module ETL
   module Processor

data/lib/etl/processor/row_processor.rb

@@ -12,6 +12,16 @@ module ETL #:nodoc:
       def process(row)
         raise "process_row is an abstract method"
       end
+
+      # Ensure a given row keys include all the provided columns
+      # and raise an error using the provided message if it doesn't
+      def ensure_columns_available_in_row!(row, columns, message)
+        unless columns.nil?
+          columns.each do |k|
+            raise(ETL::ControlError, "Row missing required field #{k.inspect} #{message}") unless row.keys.include?(k)
+          end
+        end
+      end
     end
   end
 end

data/lib/etl/processor/sftp_downloader_processor.rb

@@ -1,4 +1,4 @@
-require 'net/sftp'
+optional_require 'net/sftp'
 
 module ETL
   module Processor

data/lib/etl/processor/sftp_uploader_processor.rb

@@ -1,4 +1,4 @@
-require 'net/sftp'
+optional_require 'net/sftp'
 
 module ETL
   module Processor

data/lib/etl/processor/truncate_processor.rb

@@ -14,16 +14,19 @@ module ETL #:nodoc:
       # Options:
       # * <tt>:target</tt>: The target connection
       # * <tt>:table</tt>: The table name
+      # * <tt>:options</tt>: Optional truncate options
       def initialize(control, configuration)
         super
         #@file = File.join(File.dirname(control.file), configuration[:file])
         @target = configuration[:target] || {}
         @table = configuration[:table]
+        @options = configuration[:options] 
       end
       
       def process
         conn = ETL::Engine.connection(target)
-        conn.truncate(table_name)
+        @options ||= 'RESTART IDENTITY' if conn.class.name =~ /postgres/i
+        conn.truncate(table_name, @options)
       end
       
       private

data/lib/etl/processor/zip_file_processor.rb

@@ -1,4 +1,4 @@
-require 'zip/zip'
+optional_require 'zip/zip'
 
 module ETL
   module Processor

data/lib/etl/transform/foreign_key_lookup_transform.rb

@@ -16,6 +16,7 @@ module ETL #:nodoc:
       #  for future use.
       # *<tt>:resolver</tt>: Object or Class which implements the method resolve(value)
       # *<tt>:default</tt>: A default foreign key to use if no foreign key is found
+      # *<tt>:cache</tt>: If true and the resolver responds to load_cache, load_cache will be called
       def initialize(control, name, configuration={})
         super
         
@@ -23,7 +24,10 @@ module ETL #:nodoc:
         @resolver = configuration[:resolver]
         @resolver = @resolver.new if @resolver.is_a?(Class)
         @default = configuration[:default]
-        if configuration[:cache] ||= true
+        
+        configuration[:cache] = true if configuration[:cache].nil?
+        
+        if configuration[:cache]
           if resolver.respond_to?(:load_cache)
             resolver.load_cache
           else
@@ -87,44 +91,57 @@ class SQLResolver
   # referencing a connection defined in the ETL database.yml file or an actual
   # ActiveRecord connection instance. If the connection is not specified then
   # the ActiveRecord::Base.connection will be used.
-  def initialize(table, field, connection=nil)
-    @table = table
-    @field = field
+  def initialize(atable, afield, connection=nil)
+    # puts "table: #{atable.inspect} field:#{afield.inspect}"
+    @table = atable
+    @field = afield
     @connection = (connection.respond_to?(:quote) ? connection : ETL::Engine.connection(connection)) if connection
     @connection ||= ActiveRecord::Base.connection
   end
+  
   def resolve(value)
+    return nil if value.nil?
+    r = nil
     if @use_cache
-      cache[cache_key(value)]
+      r = cache[value]
+      # puts "resolve failed: #{value.class.name}:#{value.inspect} from: #{@table}.#{@field}" unless r
     else
       q = "SELECT id FROM #{table_name} WHERE #{wheres(value)}"
-      @connection.select_value(q)
+      # puts q
+      r = @connection.select_value(q)
     end
+    r
   end
+  
   def table_name
     ETL::Engine.table(@table, @connection)
   end
+  
   def cache
     @cache ||= {}
   end
+  
   def load_cache
-    @use_cache = true
     q = "SELECT id, #{field.join(', ')} FROM #{table_name}"
+    # puts q
     @connection.select_all(q).each do |record|
-      cache[cache_key(record.values_at(*field))] = record['id']
+      ck = @field.kind_of?(Array) ? record.values_at(*@field) : record[@field]
+      # puts "load_cache key: #{ck.class.name}:#{ck.inspect}"
+      # puts "  #{@field.class.name}:#{@field.inspect}"
+      # puts "  #{record[@field].class.name}:#{record[@field].inspect}"
+      cache[ck] = record['id']
     end
+    @use_cache = true
   end
 
   private
+
   def field
-    unless @field.kind_of?(Array)
-      @field = [ @field ]
+    if @field.kind_of?(Array)
+      @field
+    else
+      [ @field ]
     end
-    @field
-  end
-
-  def cache_key(value)
-    value.hash
   end
 
   def wheres(value)
@@ -135,6 +152,31 @@ class SQLResolver
   end
 end
 
+class IncrementalCacheSQLResolver < SQLResolver
+
+  def initialize(atable, afield, connection=nil)
+    super
+  end
+  
+  def resolve(value)
+    return nil if value.nil?
+    r = cache[value]
+    unless r
+      q = "SELECT id FROM #{table_name} WHERE #{wheres(value)}"
+      r = @connection.select_value(q)
+      if r
+        cache[value] = r
+      end
+    end
+    r
+  end
+
+  def load_cache
+    @cache = {}
+  end
+
+end
+
 class FlatFileResolver
   # Initialize the flat file resolver. Expects to open a comma-delimited file. 
   # Returns the column with the given result_field_index.