activewarehouse-etl 0.9.1 → 0.9.5.rc1

data/lib/etl/processor/bulk_import_processor.rb

@@ -38,8 +38,10 @@ module ETL #:nodoc:
       # * <tt>:field_enclosure</tt>: The field enclosure charcaters
       def initialize(control, configuration)
         super
-        @file = File.join(File.dirname(control.file), configuration[:file])
         @target = configuration[:target]
+        path = Pathname.new(configuration[:file])
+        @file = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(control.file))) + path
+
         @table = configuration[:table]
         @truncate = configuration[:truncate] ||= false
         @columns = configuration[:columns]
@@ -78,4 +80,4 @@ module ETL #:nodoc:
       end
     end
   end
-end
+end

data/lib/etl/processor/database_join_processor.rb

@@ -0,0 +1,68 @@
+module ETL
+  module Processor
+    class DatabaseJoinProcessor < ETL::Processor::RowProcessor
+      attr_reader :target
+      attr_reader :query
+      attr_reader :fields
+
+      # Initialize the procesor.
+      #
+      # Arguments:
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>definition</tt>: The source definition
+      #
+      # Required configuration options:
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:query</tt>: The join query
+      # * <tt>:fields</tt>: The fields to add to the row
+      def initialize(control, configuration)
+        super
+        @target = configuration[:target]
+        @query = configuration[:query]
+        @fields = configuration[:fields]
+      end
+      
+      # Get a String identifier for the source
+      def to_s
+        "#{host}/#{database}"
+      end
+      
+      def process(row)
+        return nil if row.nil?
+
+        q = @query
+        begin
+          q = eval('"' + @query + '"')
+        rescue
+        end
+
+        ETL::Engine.logger.debug("Executing select: #{q}")
+        res = connection.execute(q)
+
+        res.each_hash do |r|
+          @fields.each do |field|
+            row[field.to_sym] = r[field]
+          end
+        end
+
+        return row
+      end
+
+      private
+      # Get the database connection to use
+      def connection
+        ETL::Engine.connection(target)
+      end
+      
+      # Get the host, defaults to 'localhost'
+      def host
+        ETL::Base.configurations[target.to_s]['host'] || 'localhost'
+      end
+      
+      def database
+        ETL::Base.configurations[target.to_s]['database']
+      end
+    end
+  end
+end

data/lib/etl/processor/escape_csv_processor.rb

@@ -0,0 +1,77 @@
+require 'fileutils'
+
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    class EscapeCsvProcessor < ETL::Processor::Processor
+      
+      # The file to load from
+      attr_reader :source_file
+      # The file to write to
+      attr_reader :target_file
+      # whether to use a temporary file or not
+      attr_reader :use_temp_file
+
+      attr_reader :filters
+      attr_reader :charcount
+
+      # Initialize the processor.
+      #
+      # Configuration options:
+      # * <tt>:source_file</tt>: The file to load data from
+      # * <tt>:target_file</tt>: The file to write data to
+      # * <tt>:file</tt>: short-cut which will set the same value to both source_file and target_file
+      def initialize(control, configuration)
+        super
+        if configuration[:file]
+          @use_temp_file = true
+          configuration[:source_file] = configuration[:file]
+          configuration[:target_file] = configuration[:file] + '.tmp'
+        end
+        path = Pathname.new(configuration[:source_file])
+        @source_file = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(configuration[:source_file]))) + path
+        path = Pathname.new(configuration[:target_file])
+        @target_file = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(configuration[:target_file]))) + path
+        @filters = configuration[:filters] || [{:replace => '\"', :result => '""'}]
+        @charcount = configuration[:charcount]
+        raise ControlError, "Source file must be specified" if @source_file.nil?
+        raise ControlError, "Target file must be specified" if @target_file.nil?
+        raise ControlError, "Source and target file cannot currently point to the same file" if @source_file == @target_file
+      end
+      
+      # Execute the processor
+      def process
+        reader = File.open(@source_file, 'r')
+        writer = File.open(@target_file, 'w')
+
+        reader.each_line do |line|
+          reading = line
+          @filters.each do |filter|
+            if (!filter[:replace].nil? &&
+                !filter[:result].nil?)
+              result = reading.gsub(Regexp.new(filter[:replace]), filter[:result])
+              reading = result
+            end
+          end unless @filters.nil?
+          @charcount.each do |count|
+            if (!count[:char].nil? &&
+                !count[:count].nil?)
+              c = reading.count count[:char]
+              if c != count[:count]
+                reading = nil
+              end
+            end
+          end unless @charcount.nil?
+          writer.write(reading) unless reading.nil?
+        end
+
+        reader.close
+        writer.close
+
+        if use_temp_file
+          FileUtils.rm(source_file)
+          FileUtils.mv(target_file,source_file)
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/filter_row_processor.rb

@@ -0,0 +1,51 @@
+module ETL
+  module Processor
+    class FilterRowProcessor < ETL::Processor::RowProcessor
+      attr_reader :condition
+      attr_reader :outtrue
+      attr_reader :outfalse
+
+      def initialize(control, configuration)
+        @condition = configuration[:condition]
+        @outtrue = configuration[:outtrue]
+        @outfalse = configuration[:outfalse]
+        super
+      end
+      
+      def process(row)
+        return nil if row.nil?
+
+        if eval_condition(row, @condition)
+          return [] if @outtrue.nil?
+
+          eval(@outtrue)
+        else
+          eval(@outfalse) unless @outfalse.nil?
+        end
+
+        return row
+      end
+
+      private
+      def eval_condition(row, cond)
+
+        first = cond[1]
+        if (cond[1].class == Array)
+          first = eval_condition(row, cond[1])
+        end
+
+        second = cond[2]
+        if (cond[2].class == Array)
+          second = eval_condition(row, cond[2])
+        end
+
+        return eval("#{cond[0]}#{first}#{second}") if cond[0] == "!"
+
+        eval("#{first}#{cond[0]}#{second}")
+      rescue => e
+        return false
+      end
+
+    end
+  end
+end

data/lib/etl/processor/ftp_downloader_processor.rb

@@ -0,0 +1,68 @@
+# Written by Susan Potter  under open source MIT license.
+# August 12, 2007.
+
+require 'net/ftp'
+
+module ETL
+  module Processor
+    # Custom processor to download files via FTP
+    class FtpDownloaderProcessor < ETL::Processor::Processor
+      attr_reader :host
+      attr_reader :port
+      attr_reader :remote_dir
+      attr_reader :files
+      attr_reader :username
+      attr_reader :local_dir
+      
+      # configuration options include:
+      # * host - hostname or IP address of FTP server (required)
+      # * port - port number for FTP server (default: 21)
+      # * remote_dir - remote path on FTP server (default: /)
+      # * files - list of files to download from FTP server (default: [])
+      # * username - username for FTP server authentication (default: anonymous)
+      # * password - password for FTP server authentication (default: nil)
+      # * local_dir - local output directory to save downloaded files (default: '')
+      # 
+      # As an example you might write something like the following in your control process file:
+      #  pre_process :ftp_downloader, {
+      #    :host => 'ftp.sec.gov',
+      #    :path => 'edgar/Feed/2007/QTR2',
+      #    :files => ['20070402.nc.tar.gz', '20070403.nc.tar.gz', '20070404.nc.tar.gz', 
+      #               '20070405.nc.tar.gz', '20070406.nc.tar.gz'],
+      #    :local_dir => '/data/sec/2007/04',
+      #  }
+      # The above example will anonymously download via FTP the first week's worth of SEC filing feed data
+      # from the second quarter of 2007 and download the files to the local directory +/data/sec/2007/04+.
+      def initialize(control, configuration)
+        @host = configuration[:host]
+        @port = configuration[:port] || 21
+        @remote_dir = configuration[:remote_dir] || '/'
+        @files = configuration[:files] || []
+        @username = configuration[:username] || 'anonymous'
+        @password = configuration[:password]
+        @local_dir = configuration[:local_dir] || ''
+      end
+      
+      def process
+        Net::FTP.open(@host) do |conn|
+          conn.connect(@host, @port)
+          conn.login(@username, @password)
+          @files.each do |f|
+            conn.getbinaryfile(remote_file(f), local_file(f))
+          end
+        end
+      end
+      
+      private
+      attr_accessor :password
+      
+      def local_file(name)
+        File.join(@local_dir, name)
+      end
+      
+      def remote_file(name)
+        File.join(@remote_dir, name)
+      end
+    end
+  end
+end

data/lib/etl/processor/ftp_uploader_processor.rb

@@ -0,0 +1,65 @@
+require 'net/ftp'
+
+module ETL
+  module Processor
+    # Custom processor to download files via FTP
+    class FtpUploaderProcessor < ETL::Processor::Processor
+      attr_reader :host
+      attr_reader :port
+      attr_reader :remote_dir
+      attr_reader :files
+      attr_reader :username
+      attr_reader :local_dir
+      
+      # configuration options include:
+      # * host - hostname or IP address of FTP server (required)
+      # * port - port number for FTP server (default: 21)
+      # * remote_dir - remote path on FTP server (default: /)
+      # * files - list of files to download from FTP server (default: [])
+      # * username - username for FTP server authentication (default: anonymous)
+      # * password - password for FTP server authentication (default: nil)
+      # * local_dir - local output directory to save downloaded files (default: '')
+      # 
+      # As an example you might write something like the following in your control process file:
+      #  pre_process :ftp_uploader, {
+      #    :host => 'ftp.sec.gov',
+      #    :path => 'edgar/Feed/2007/QTR2',
+      #    :files => ['20070402.nc.tar.gz', '20070403.nc.tar.gz', '20070404.nc.tar.gz', 
+      #               '20070405.nc.tar.gz', '20070406.nc.tar.gz'],
+      #    :local_dir => '/data/sec/2007/04',
+      #  }
+      # The above example will anonymously download via FTP the first week's worth of SEC filing feed data
+      # from the second quarter of 2007 and download the files to the local directory +/data/sec/2007/04+.
+      def initialize(control, configuration)
+        @host = configuration[:host]
+        @port = configuration[:port] || 21
+        @remote_dir = configuration[:remote_dir] || '/'
+        @files = configuration[:files] || []
+        @username = configuration[:username] || 'anonymous'
+        @password = configuration[:password]
+        @local_dir = configuration[:local_dir] || ''
+      end
+      
+      def process
+        Net::FTP.open(@host) do |conn|
+          conn.connect(@host, @port)
+          conn.login(@username, @password)
+          @files.each do |f|
+            conn.putbinaryfile(local_file(f), remote_file(f))
+          end
+        end
+      end
+      
+      private
+      attr_accessor :password
+      
+      def local_file(name)
+        File.join(@local_dir, name)
+      end
+      
+      def remote_file(name)
+        File.join(@remote_dir, name)
+      end
+    end
+  end
+end

data/lib/etl/processor/imapattachment_downloader_processor.rb

@@ -0,0 +1,91 @@
+require 'net/imap'
+require 'tmail'
+
+module ETL
+  module Processor
+    # Custom processor to download files via Imap Attachment
+    class ImapattachmentDownloaderProcessor < ETL::Processor::Processor
+      attr_reader :host
+      attr_reader :ssl
+      attr_reader :port
+      attr_reader :delete
+      attr_reader :filters
+      attr_reader :folder
+      attr_reader :username
+      attr_reader :local_dir
+      
+      # configuration options include:
+      # * host - hostname or IP address of IMAP server (required)
+      # * ssl  - activate encryption (default false)
+      # * port - port number for IMAP server (default: 220 or 993)
+      # * delete - delete message after reading (default false)
+      # * filters  - filter mails (default [])
+      # * folder  - folder to select mails from (default INBOX)
+      # * username - username for IMAP server authentication (default: anonymous)
+      # * password - password for IMAP server authentication (default: nil)
+      # * local_dir - local output directory to save downloaded files (default: '')
+      #
+      def initialize(control, configuration)
+        @host = configuration[:host]
+        @ssl = configuration[:ssl] || false
+        @port = configuration[:port] || (@ssl ? 993 : 220 )
+        @delete = configuration[:delete] || false
+        @filters = configuration[:filters] || []
+        @folder = configuration[:folder] || 'INBOX'
+        @username = configuration[:username] || 'anonymous'
+        @password = configuration[:password]
+        @local_dir = configuration[:local_dir] || ''
+      end
+      
+      def process
+        conn = Net::IMAP.new(@host, @port, @ssl)
+        conn.login(@username, @password)
+
+        conn.select(@folder)
+        conn.uid_search(["NOT", "DELETED"]).each do |msguuid|
+          mail = TMail::Mail.parse( conn.uid_fetch(msguuid, 'RFC822').first.attr['RFC822'] )
+          next if mail.attachments.blank?
+          if applyfilter(mail, @filters)
+            mail.attachments.each do |attachment|
+              filename = attachment.original_filename
+              File.open(local_file(filename), "w") {|f|
+                f << attachment.gets(nil)
+              }
+            end
+
+            conn.store(msguuid, "+FLAGS", [:Deleted]) if @delete
+          end
+        end
+        conn.expunge
+        conn.close
+      end
+      
+      private
+      attr_accessor :password
+      
+      def local_file(name)
+        File.join(@local_dir, name)
+      end
+
+      def applyfilter(mail, cond)
+        return true if (cond.nil? or cond.size < 3)
+
+        first = cond[1]
+        if (cond[1].class == Array)
+          first = eval_condition(row, cond[1])
+        end
+
+        second = cond[2]
+        if (cond[2].class == Array)
+          second = eval_condition(row, cond[2])
+        end
+
+        return eval("#{cond[0]}#{first}#{second}") if cond[0] == "!"
+
+        eval("#{first}#{cond[0]}#{second}")
+      rescue => e
+        return false
+      end
+    end
+  end
+end

data/lib/etl/processor/pop3attachment_downloader_processor.rb

@@ -0,0 +1,90 @@
+require 'net/pop'
+require 'tmail'
+
+module ETL
+  module Processor
+    # Custom processor to download files via Pop3 Attachment
+    class Pop3attachmentDownloaderProcessor < ETL::Processor::Processor
+      attr_reader :host
+      attr_reader :ssl
+      attr_reader :port
+      attr_reader :delete
+      attr_reader :filters
+      attr_reader :username
+      attr_reader :local_dir
+      
+      # configuration options include:
+      # * host - hostname or IP address of POP3 server (required)
+      # * ssl  - activate encryption (default false)
+      # * port - port number for POP3 server (default: Net::POP3.default_port or Net::POP3.default_pop3s_port)
+      # * delete - delete message after reading (default false)
+      # * filters  - filter mails (default [])
+      # * username - username for POP3 server authentication (default: anonymous)
+      # * password - password for POP3 server authentication (default: nil)
+      # * local_dir - local output directory to save downloaded files (default: '')
+      #
+      def initialize(control, configuration)
+        @host = configuration[:host]
+        @ssl = configuration[:ssl] || false
+        @port = configuration[:port] || (@ssl ? Net::POP3.default_pop3s_port : Net::POP3.default_port )
+        @delete = configuration[:delete] || false
+        @filters = configuration[:filters] || []
+        @username = configuration[:username] || 'anonymous'
+        @password = configuration[:password]
+        @local_dir = configuration[:local_dir] || ''
+      end
+      
+      def process
+        Net::POP3.enable_ssl(OpenSSL::SSL::VERIFY_NONE) if @ssl
+        conn = Net::POP3.new(@host, @port)
+        conn.start(@username, @password)
+        if !conn.mails.empty?
+          conn.each_mail do |message|
+            stringmail = message.pop
+            mail = TMail::Mail.parse(stringmail)
+            next if mail.attachments.blank?
+            if applyfilter(mail, @filters)
+              mail.attachments.each do |attachment|
+                filename = attachment.original_filename
+                File.open(local_file(filename), "w") {|f|
+                  f << attachment.gets(nil)
+                }
+              end
+
+              message.delete if @delete
+            end
+          end
+        end
+
+        conn.finish
+      end
+      
+      private
+      attr_accessor :password
+      
+      def local_file(name)
+        File.join(@local_dir, name)
+      end
+
+      def applyfilter(mail, cond)
+        return true if (cond.nil? or cond.size < 3)
+
+        first = cond[1]
+        if (cond[1].class == Array)
+          first = eval_condition(row, cond[1])
+        end
+
+        second = cond[2]
+        if (cond[2].class == Array)
+          second = eval_condition(row, cond[2])
+        end
+
+        return eval("#{cond[0]}#{first}#{second}") if cond[0] == "!"
+
+        eval("#{first}#{cond[0]}#{second}")
+      rescue => e
+        return false
+      end
+    end
+  end
+end