activewarehouse-etl-sgonyea 0.9.6

data/lib/etl/processor/ensure_fields_presence_processor.rb

@@ -0,0 +1,24 @@
+module ETL
+  module Processor
+    # Ensure that each specified field is available
+    class EnsureFieldsPresenceProcessor < ETL::Processor::RowProcessor
+      
+      # Initialize the processor.
+      #
+      # Configuration options:
+      # * <tt>:fields</tt>: An array of keys whose presence should be verified in each row
+      def initialize(control, configuration)
+        super
+        @fields = configuration[:fields]
+        raise ControlError, ":fields must be specified" unless @fields
+      end
+      
+      def process(row)
+        missing_fields = configuration[:fields] - row.keys
+        raise(ETL::ControlError, 
+          "Row missing required field(s) #{missing_fields.join(',')} in row. Available fields are : #{row.keys.join(',')}") unless missing_fields.empty?
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/escape_csv_processor.rb

@@ -0,0 +1,77 @@
+require 'fileutils'
+
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    class EscapeCsvProcessor < ETL::Processor::Processor
+      
+      # The file to load from
+      attr_reader :source_file
+      # The file to write to
+      attr_reader :target_file
+      # whether to use a temporary file or not
+      attr_reader :use_temp_file
+
+      attr_reader :filters
+      attr_reader :charcount
+
+      # Initialize the processor.
+      #
+      # Configuration options:
+      # * <tt>:source_file</tt>: The file to load data from
+      # * <tt>:target_file</tt>: The file to write data to
+      # * <tt>:file</tt>: short-cut which will set the same value to both source_file and target_file
+      def initialize(control, configuration)
+        super
+        if configuration[:file]
+          @use_temp_file = true
+          configuration[:source_file] = configuration[:file]
+          configuration[:target_file] = configuration[:file] + '.tmp'
+        end
+        path = Pathname.new(configuration[:source_file])
+        @source_file = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(configuration[:source_file]))) + path
+        path = Pathname.new(configuration[:target_file])
+        @target_file = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(configuration[:target_file]))) + path
+        @filters = configuration[:filters] || [{:replace => '\"', :result => '""'}]
+        @charcount = configuration[:charcount]
+        raise ControlError, "Source file must be specified" if @source_file.nil?
+        raise ControlError, "Target file must be specified" if @target_file.nil?
+        raise ControlError, "Source and target file cannot currently point to the same file" if @source_file == @target_file
+      end
+      
+      # Execute the processor
+      def process
+        reader = File.open(@source_file, 'r')
+        writer = File.open(@target_file, 'w')
+
+        reader.each_line do |line|
+          reading = line
+          @filters.each do |filter|
+            if (!filter[:replace].nil? &&
+                !filter[:result].nil?)
+              result = reading.gsub(Regexp.new(filter[:replace]), filter[:result])
+              reading = result
+            end
+          end unless @filters.nil?
+          @charcount.each do |count|
+            if (!count[:char].nil? &&
+                !count[:count].nil?)
+              c = reading.count count[:char]
+              if c != count[:count]
+                reading = nil
+              end
+            end
+          end unless @charcount.nil?
+          writer.write(reading) unless reading.nil?
+        end
+
+        reader.close
+        writer.close
+
+        if use_temp_file
+          FileUtils.rm(source_file)
+          FileUtils.mv(target_file,source_file)
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/filter_row_processor.rb

@@ -0,0 +1,51 @@
+module ETL
+  module Processor
+    class FilterRowProcessor < ETL::Processor::RowProcessor
+      attr_reader :condition
+      attr_reader :outtrue
+      attr_reader :outfalse
+
+      def initialize(control, configuration)
+        @condition = configuration[:condition]
+        @outtrue = configuration[:outtrue]
+        @outfalse = configuration[:outfalse]
+        super
+      end
+      
+      def process(row)
+        return nil if row.nil?
+
+        if eval_condition(row, @condition)
+          return [] if @outtrue.nil?
+
+          eval(@outtrue)
+        else
+          eval(@outfalse) unless @outfalse.nil?
+        end
+
+        return row
+      end
+
+      private
+      def eval_condition(row, cond)
+
+        first = cond[1]
+        if (cond[1].class == Array)
+          first = eval_condition(row, cond[1])
+        end
+
+        second = cond[2]
+        if (cond[2].class == Array)
+          second = eval_condition(row, cond[2])
+        end
+
+        return eval("#{cond[0]}#{first}#{second}") if cond[0] == "!"
+
+        eval("#{first}#{cond[0]}#{second}")
+      rescue => e
+        return false
+      end
+
+    end
+  end
+end

data/lib/etl/processor/ftp_downloader_processor.rb

@@ -0,0 +1,68 @@
+# Written by Susan Potter  under open source MIT license.
+# August 12, 2007.
+
+require 'net/ftp'
+
+module ETL
+  module Processor
+    # Custom processor to download files via FTP
+    class FtpDownloaderProcessor < ETL::Processor::Processor
+      attr_reader :host
+      attr_reader :port
+      attr_reader :remote_dir
+      attr_reader :files
+      attr_reader :username
+      attr_reader :local_dir
+      
+      # configuration options include:
+      # * host - hostname or IP address of FTP server (required)
+      # * port - port number for FTP server (default: 21)
+      # * remote_dir - remote path on FTP server (default: /)
+      # * files - list of files to download from FTP server (default: [])
+      # * username - username for FTP server authentication (default: anonymous)
+      # * password - password for FTP server authentication (default: nil)
+      # * local_dir - local output directory to save downloaded files (default: '')
+      # 
+      # As an example you might write something like the following in your control process file:
+      #  pre_process :ftp_downloader, {
+      #    :host => 'ftp.sec.gov',
+      #    :path => 'edgar/Feed/2007/QTR2',
+      #    :files => ['20070402.nc.tar.gz', '20070403.nc.tar.gz', '20070404.nc.tar.gz', 
+      #               '20070405.nc.tar.gz', '20070406.nc.tar.gz'],
+      #    :local_dir => '/data/sec/2007/04',
+      #  }
+      # The above example will anonymously download via FTP the first week's worth of SEC filing feed data
+      # from the second quarter of 2007 and download the files to the local directory +/data/sec/2007/04+.
+      def initialize(control, configuration)
+        @host = configuration[:host]
+        @port = configuration[:port] || 21
+        @remote_dir = configuration[:remote_dir] || '/'
+        @files = configuration[:files] || []
+        @username = configuration[:username] || 'anonymous'
+        @password = configuration[:password]
+        @local_dir = configuration[:local_dir] || ''
+      end
+      
+      def process
+        Net::FTP.open(@host) do |conn|
+          conn.connect(@host, @port)
+          conn.login(@username, @password)
+          @files.each do |f|
+            conn.getbinaryfile(remote_file(f), local_file(f))
+          end
+        end
+      end
+      
+      private
+      attr_accessor :password
+      
+      def local_file(name)
+        File.join(@local_dir, name)
+      end
+      
+      def remote_file(name)
+        File.join(@remote_dir, name)
+      end
+    end
+  end
+end

data/lib/etl/processor/ftp_uploader_processor.rb

@@ -0,0 +1,65 @@
+require 'net/ftp'
+
+module ETL
+  module Processor
+    # Custom processor to download files via FTP
+    class FtpUploaderProcessor < ETL::Processor::Processor
+      attr_reader :host
+      attr_reader :port
+      attr_reader :remote_dir
+      attr_reader :files
+      attr_reader :username
+      attr_reader :local_dir
+      
+      # configuration options include:
+      # * host - hostname or IP address of FTP server (required)
+      # * port - port number for FTP server (default: 21)
+      # * remote_dir - remote path on FTP server (default: /)
+      # * files - list of files to download from FTP server (default: [])
+      # * username - username for FTP server authentication (default: anonymous)
+      # * password - password for FTP server authentication (default: nil)
+      # * local_dir - local output directory to save downloaded files (default: '')
+      # 
+      # As an example you might write something like the following in your control process file:
+      #  pre_process :ftp_uploader, {
+      #    :host => 'ftp.sec.gov',
+      #    :path => 'edgar/Feed/2007/QTR2',
+      #    :files => ['20070402.nc.tar.gz', '20070403.nc.tar.gz', '20070404.nc.tar.gz', 
+      #               '20070405.nc.tar.gz', '20070406.nc.tar.gz'],
+      #    :local_dir => '/data/sec/2007/04',
+      #  }
+      # The above example will anonymously download via FTP the first week's worth of SEC filing feed data
+      # from the second quarter of 2007 and download the files to the local directory +/data/sec/2007/04+.
+      def initialize(control, configuration)
+        @host = configuration[:host]
+        @port = configuration[:port] || 21
+        @remote_dir = configuration[:remote_dir] || '/'
+        @files = configuration[:files] || []
+        @username = configuration[:username] || 'anonymous'
+        @password = configuration[:password]
+        @local_dir = configuration[:local_dir] || ''
+      end
+      
+      def process
+        Net::FTP.open(@host) do |conn|
+          conn.connect(@host, @port)
+          conn.login(@username, @password)
+          @files.each do |f|
+            conn.putbinaryfile(local_file(f), remote_file(f))
+          end
+        end
+      end
+      
+      private
+      attr_accessor :password
+      
+      def local_file(name)
+        File.join(@local_dir, name)
+      end
+      
+      def remote_file(name)
+        File.join(@remote_dir, name)
+      end
+    end
+  end
+end

data/lib/etl/processor/hierarchy_exploder_processor.rb

@@ -0,0 +1,55 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row-level processor that will convert a single row into multiple rows designed to be inserted
+    # into a hierarchy bridge table.
+    class HierarchyExploderProcessor < ETL::Processor::RowProcessor
+      attr_accessor :id_field
+      attr_accessor :parent_id_field
+      
+      # Initialize the processor
+      #
+      # Configuration options:
+      # * <tt>:connection</tt>: The ActiveRecord adapter connection
+      # * <tt>:id_field</tt>: The name of the id field (defaults to 'id')
+      # * <tt>:parent_id_field</tt>: The name of the parent id field (defaults to 'parent_id')
+      #
+      # TODO: Allow resolver to be implemented in a customizable fashion, i.e. don't rely
+      # on AR as the only resolution method.
+      def initialize(control, configuration={})
+        @id_field = configuration[:id_field] || 'id'
+        @parent_id_field = configuration[:parent_id_field] || 'parent_id'
+        super
+      end
+  
+      # Process the row expanding it into hierarchy values
+      def process(row)
+        rows = []
+        target = configuration[:target]
+        table = configuration[:table]
+        conn = ETL::Engine.connection(target)
+        build_rows([row[:id]], row[:id], row[:id], row[:parent_id].nil?, 0, rows, table, conn)
+        rows
+      end
+  
+      protected
+      # Recursive function that will add a row for the current level and then call build_rows
+      # for all of the children of the current level
+      def build_rows(ids, parent_id, row_id, root, level, rows, table, conn)
+        ids.each do |id|
+          child_ids = conn.select_values("SELECT #{id_field} FROM #{table} WHERE #{parent_id_field} = #{id}")
+      
+          row = {
+            :parent_id => row_id, 
+            :child_id => id, 
+            :num_levels_from_parent => level, 
+            :is_bottom => (child_ids.empty? ? 1 : 0),
+            :is_top => (root ? 1 : 0),
+          }
+          rows << row
+      
+          build_rows(child_ids, id, row_id, false, level + 1, rows, table, conn)
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/imapattachment_downloader_processor.rb

@@ -0,0 +1,91 @@
+optional_require 'net/imap'
+optional_require 'tmail'
+
+module ETL
+  module Processor
+    # Custom processor to download files via Imap Attachment
+    class ImapattachmentDownloaderProcessor < ETL::Processor::Processor
+      attr_reader :host
+      attr_reader :ssl
+      attr_reader :port
+      attr_reader :delete
+      attr_reader :filters
+      attr_reader :folder
+      attr_reader :username
+      attr_reader :local_dir
+      
+      # configuration options include:
+      # * host - hostname or IP address of IMAP server (required)
+      # * ssl  - activate encryption (default false)
+      # * port - port number for IMAP server (default: 220 or 993)
+      # * delete - delete message after reading (default false)
+      # * filters  - filter mails (default [])
+      # * folder  - folder to select mails from (default INBOX)
+      # * username - username for IMAP server authentication (default: anonymous)
+      # * password - password for IMAP server authentication (default: nil)
+      # * local_dir - local output directory to save downloaded files (default: '')
+      #
+      def initialize(control, configuration)
+        @host = configuration[:host]
+        @ssl = configuration[:ssl] || false
+        @port = configuration[:port] || (@ssl ? 993 : 220 )
+        @delete = configuration[:delete] || false
+        @filters = configuration[:filters] || []
+        @folder = configuration[:folder] || 'INBOX'
+        @username = configuration[:username] || 'anonymous'
+        @password = configuration[:password]
+        @local_dir = configuration[:local_dir] || ''
+      end
+      
+      def process
+        conn = Net::IMAP.new(@host, @port, @ssl)
+        conn.login(@username, @password)
+
+        conn.select(@folder)
+        conn.uid_search(["NOT", "DELETED"]).each do |msguuid|
+          mail = TMail::Mail.parse( conn.uid_fetch(msguuid, 'RFC822').first.attr['RFC822'] )
+          next if mail.attachments.blank?
+          if applyfilter(mail, @filters)
+            mail.attachments.each do |attachment|
+              filename = attachment.original_filename
+              File.open(local_file(filename), "w") {|f|
+                f << attachment.gets(nil)
+              }
+            end
+
+            conn.store(msguuid, "+FLAGS", [:Deleted]) if @delete
+          end
+        end
+        conn.expunge
+        conn.close
+      end
+      
+      private
+      attr_accessor :password
+      
+      def local_file(name)
+        File.join(@local_dir, name)
+      end
+
+      def applyfilter(mail, cond)
+        return true if (cond.nil? or cond.size < 3)
+
+        first = cond[1]
+        if (cond[1].class == Array)
+          first = eval_condition(row, cond[1])
+        end
+
+        second = cond[2]
+        if (cond[2].class == Array)
+          second = eval_condition(row, cond[2])
+        end
+
+        return eval("#{cond[0]}#{first}#{second}") if cond[0] == "!"
+
+        eval("#{first}#{cond[0]}#{second}")
+      rescue => e
+        return false
+      end
+    end
+  end
+end

data/lib/etl/processor/pop3attachment_downloader_processor.rb

@@ -0,0 +1,90 @@
+optional_require 'net/pop'
+optional_require 'tmail'
+
+module ETL
+  module Processor
+    # Custom processor to download files via Pop3 Attachment
+    class Pop3attachmentDownloaderProcessor < ETL::Processor::Processor
+      attr_reader :host
+      attr_reader :ssl
+      attr_reader :port
+      attr_reader :delete
+      attr_reader :filters
+      attr_reader :username
+      attr_reader :local_dir
+      
+      # configuration options include:
+      # * host - hostname or IP address of POP3 server (required)
+      # * ssl  - activate encryption (default false)
+      # * port - port number for POP3 server (default: Net::POP3.default_port or Net::POP3.default_pop3s_port)
+      # * delete - delete message after reading (default false)
+      # * filters  - filter mails (default [])
+      # * username - username for POP3 server authentication (default: anonymous)
+      # * password - password for POP3 server authentication (default: nil)
+      # * local_dir - local output directory to save downloaded files (default: '')
+      #
+      def initialize(control, configuration)
+        @host = configuration[:host]
+        @ssl = configuration[:ssl] || false
+        @port = configuration[:port] || (@ssl ? Net::POP3.default_pop3s_port : Net::POP3.default_port )
+        @delete = configuration[:delete] || false
+        @filters = configuration[:filters] || []
+        @username = configuration[:username] || 'anonymous'
+        @password = configuration[:password]
+        @local_dir = configuration[:local_dir] || ''
+      end
+      
+      def process
+        Net::POP3.enable_ssl(OpenSSL::SSL::VERIFY_NONE) if @ssl
+        conn = Net::POP3.new(@host, @port)
+        conn.start(@username, @password)
+        if !conn.mails.empty?
+          conn.each_mail do |message|
+            stringmail = message.pop
+            mail = TMail::Mail.parse(stringmail)
+            next if mail.attachments.blank?
+            if applyfilter(mail, @filters)
+              mail.attachments.each do |attachment|
+                filename = attachment.original_filename
+                File.open(local_file(filename), "w") {|f|
+                  f << attachment.gets(nil)
+                }
+              end
+
+              message.delete if @delete
+            end
+          end
+        end
+
+        conn.finish
+      end
+      
+      private
+      attr_accessor :password
+      
+      def local_file(name)
+        File.join(@local_dir, name)
+      end
+
+      def applyfilter(mail, cond)
+        return true if (cond.nil? or cond.size < 3)
+
+        first = cond[1]
+        if (cond[1].class == Array)
+          first = eval_condition(row, cond[1])
+        end
+
+        second = cond[2]
+        if (cond[2].class == Array)
+          second = eval_condition(row, cond[2])
+        end
+
+        return eval("#{cond[0]}#{first}#{second}") if cond[0] == "!"
+
+        eval("#{first}#{cond[0]}#{second}")
+      rescue => e
+        return false
+      end
+    end
+  end
+end