activewarehouse-etl 0.9.1 → 0.9.5.rc1

data/lib/etl/processor/sftp_downloader_processor.rb

@@ -0,0 +1,63 @@
+require 'net/sftp'
+
+module ETL
+  module Processor
+    # Custom processor to download files via SFTP
+    class SftpDownloaderProcessor < ETL::Processor::Processor
+      attr_reader :host
+      attr_reader :port
+      attr_reader :remote_dir
+      attr_reader :files
+      attr_reader :username
+      attr_reader :local_dir
+      
+      # configuration options include:
+      # * host - hostname or IP address of FTP server (required)
+      # * port - port number for FTP server (default: 22)
+      # * remote_dir - remote path on FTP server (default: /)
+      # * files - list of files to download from FTP server (default: [])
+      # * username - username for FTP server authentication (default: anonymous)
+      # * password - password for FTP server authentication (default: nil)
+      # * local_dir - local output directory to save downloaded files (default: '')
+      # 
+      # As an example you might write something like the following in your control process file:
+      #  pre_process :sftp_downloader, {
+      #    :host => 'sftp.sec.gov',
+      #    :path => 'edgar/Feed/2007/QTR2',
+      #    :files => ['20070402.nc.tar.gz', '20070403.nc.tar.gz', '20070404.nc.tar.gz', 
+      #               '20070405.nc.tar.gz', '20070406.nc.tar.gz'],
+      #    :local_dir => '/data/sec/2007/04',
+      #  }
+      # The above example will anonymously download via SFTP the first week's worth of SEC filing feed data
+      # from the second quarter of 2007 and download the files to the local directory +/data/sec/2007/04+.
+      def initialize(control, configuration)
+        @host = configuration[:host]
+        @port = configuration[:port] || 22
+        @remote_dir = configuration[:remote_dir] || '/'
+        @files = configuration[:files] || []
+        @username = configuration[:username] || 'anonymous'
+        @password = configuration[:password]
+        @local_dir = configuration[:local_dir] || ''
+      end
+      
+      def process
+        Net::SFTP.start(@host, @username, {:port => @port, :password => @password}) do |conn|
+          @files.each do |f|
+            conn.download!(remote_file(f), local_file(f))
+          end
+        end
+      end
+      
+      private
+      attr_accessor :password
+      
+      def local_file(name)
+        File.join(@local_dir, name)
+      end
+      
+      def remote_file(name)
+        File.join(@remote_dir, name)
+      end
+    end
+  end
+end

data/lib/etl/processor/sftp_uploader_processor.rb

@@ -0,0 +1,63 @@
+require 'net/sftp'
+
+module ETL
+  module Processor
+    # Custom processor to download files via SFTP
+    class SftpUploaderProcessor < ETL::Processor::Processor
+      attr_reader :host
+      attr_reader :port
+      attr_reader :remote_dir
+      attr_reader :files
+      attr_reader :username
+      attr_reader :local_dir
+      
+      # configuration options include:
+      # * host - hostname or IP address of FTP server (required)
+      # * port - port number for FTP server (default: 22)
+      # * remote_dir - remote path on FTP server (default: /)
+      # * files - list of files to download from FTP server (default: [])
+      # * username - username for FTP server authentication (default: anonymous)
+      # * password - password for FTP server authentication (default: nil)
+      # * local_dir - local output directory to save downloaded files (default: '')
+      # 
+      # As an example you might write something like the following in your control process file:
+      #  pre_process :sftp_uploader, {
+      #    :host => 'sftp.sec.gov',
+      #    :path => 'edgar/Feed/2007/QTR2',
+      #    :files => ['20070402.nc.tar.gz', '20070403.nc.tar.gz', '20070404.nc.tar.gz', 
+      #               '20070405.nc.tar.gz', '20070406.nc.tar.gz'],
+      #    :local_dir => '/data/sec/2007/04',
+      #  }
+      # The above example will anonymously download via SFTP the first week's worth of SEC filing feed data
+      # from the second quarter of 2007 and download the files to the local directory +/data/sec/2007/04+.
+      def initialize(control, configuration)
+        @host = configuration[:host]
+        @port = configuration[:port] || 22
+        @remote_dir = configuration[:remote_dir] || '/'
+        @files = configuration[:files] || []
+        @username = configuration[:username] || 'anonymous'
+        @password = configuration[:password]
+        @local_dir = configuration[:local_dir] || ''
+      end
+      
+      def process
+        Net::SFTP.start(@host, @username, {:port => @port, :password => @password}) do |conn|
+          @files.each do |f|
+            conn.upload!(local_file(f), remote_file(f))
+          end
+        end
+      end
+      
+      private
+      attr_accessor :password
+      
+      def local_file(name)
+        File.join(@local_dir, name)
+      end
+      
+      def remote_file(name)
+        File.join(@remote_dir, name)
+      end
+    end
+  end
+end

data/lib/etl/processor/zip_file_processor.rb

@@ -0,0 +1,27 @@
+require 'zip/zip'
+
+module ETL
+  module Processor
+    # Custom processor to zip files
+    class ZipFileProcessor < ETL::Processor::Processor
+      attr_reader :infile
+      attr_reader :destination
+      
+      # configuration options include:
+      # * infile - File to zip (required)
+      # * destination - Zip file name (default: #{infile}.zip)
+      def initialize(control, configuration)
+        path = Pathname.new(configuration[:infile])
+        @infile = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(configuration[:infile]))) + path
+        @destination = configuration[:destination] || "#{infile}.zip"
+      end
+      
+      def process
+        Zip::ZipFile.open(@destination, Zip::ZipFile::CREATE) do |zipfile|
+          zipfile.add(@infile.basename, @infile)
+        end
+      end
+      
+    end
+  end
+end

data/lib/etl/transform/calculation_transform.rb

@@ -0,0 +1,71 @@
+module ETL
+  module Transform
+    class CalculationTransform < ETL::Transform::Transform
+      attr_reader :function
+      attr_reader :fields
+
+      def initialize(control, name, configuration)
+        @function = configuration[:function]
+        @fields = configuration[:fields]
+        super
+      end
+      
+      def transform(name, value, row)
+        return nil if row.nil?
+        return nil if row[@fields[0]].nil?
+
+        if (@function.eql? "A + B")
+          result = ""
+          @fields.each do |field|
+            next if field.nil?
+
+            string = ""
+            if field.to_s.eql? field
+              string = field
+              begin
+                string = eval('"' + field + '"')
+              rescue
+              end
+            else
+              string = row[field]
+            end
+            next if string.nil?
+
+            result = result + string
+          end
+
+          row[name] = result
+        end
+
+        if (@function.eql? "date A")
+          first = row[@fields[0]]
+          row[name] = Time.parse(first)
+        end
+
+        if (@function.eql? "trim A")
+          first = row[@fields[0]]
+          row[name] = first.strip
+        end
+
+        if (@function.eql? "lower A")
+          first = row[@fields[0]]
+          row[name] = first.downcase
+        end
+
+        if (@function.eql? "upper A")
+          first = row[@fields[0]]
+          row[name] = first.upcase
+        end
+
+        if (@function.eql? "encoding A")
+          # Bug from ruby 1.8 http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/
+          first = row[@fields[0]]
+          row[name] = Iconv.conv(@fields[1], @fields[2], first + ' ')[0..-2]
+        end
+
+        row[name]
+      end
+
+    end
+  end
+end

data/lib/etl/transform/foreign_key_lookup_transform.rb

@@ -95,10 +95,9 @@ class SQLResolver
   end
   def resolve(value)
     if @use_cache
-      cache[value]
+      cache[cache_key(value)]
     else
-      q = "SELECT id FROM #{table_name} WHERE #{@field} = #{@connection.quote(value)}"
-      ETL::Engine.logger.debug("Executing query: #{q}")
+      q = "SELECT id FROM #{table_name} WHERE #{wheres(value)}"
       @connection.select_value(q)
     end
   end
@@ -110,11 +109,30 @@ class SQLResolver
   end
   def load_cache
     @use_cache = true
-    q = "SELECT id, #{@field} FROM #{table_name}"
+    q = "SELECT id, #{field.join(', ')} FROM #{table_name}"
     @connection.select_all(q).each do |record|
-      cache[record[@field]] = record['id']
+      cache[cache_key(record.values_at(*field))] = record['id']
     end
   end
+
+  private
+  def field
+    unless @field.kind_of?(Array)
+      @field = [ @field ]
+    end
+    @field
+  end
+
+  def cache_key(value)
+    value.hash
+  end
+
+  def wheres(value)
+    value  = [ value ]  unless value.kind_of?(Array)
+    field.zip(value).collect { |a|
+      "#{a[0]} = #{@connection.quote(a[1])}"
+    }.join(' AND ')
+  end
 end
 
 class FlatFileResolver
@@ -132,7 +150,7 @@ class FlatFileResolver
   
   # Get the rows from the file specified in the initializer.
   def rows
-    @rows ||= FasterCSV.read(@file)
+    @rows ||= CSV.read(@file)
   end
   protected :rows
   
@@ -148,4 +166,4 @@ class FlatFileResolver
     end
     nil
   end
-end
+end

data/lib/etl/transform/ordinalize_transform.rb

@@ -1,3 +1,5 @@
+require 'active_support/core_ext/integer/inflections.rb'
+
 module ETL #:nodoc:
   module Transform #:nodoc:
     # Transform a number to an ordinalized version using the ActiveSupport ordinalize
@@ -9,4 +11,4 @@ module ETL #:nodoc:
       end
     end
   end
-end
+end

data/lib/etl/transform/split_fields_transform.rb

@@ -0,0 +1,27 @@
+module ETL
+  module Transform
+    class SplitFieldsTransform < ETL::Transform::Transform
+      attr_reader :delimiter
+      attr_reader :new_fields
+
+      def initialize(control, name, configuration)
+        @delimiter = configuration[:delimiter] || ','
+        @new_fields = configuration[:new_fields]
+        super
+      end
+      
+      def transform(name, value, row)
+        return nil if row.nil?
+        return nil if row[name].nil?
+
+        fields = row[name].split(@delimiter)
+        @new_fields.each_with_index do |new, index|
+          row[new] = fields[index]
+        end
+
+        row[name]
+      end
+
+    end
+  end
+end

data/lib/etl/version.rb

@@ -1,9 +1,3 @@
 module ETL#:nodoc:
-  module VERSION #:nodoc:
-    MAJOR = 0
-    MINOR = 9
-    TINY  = 1
-
-    STRING = [MAJOR, MINOR, TINY].join('.')
-  end
+  VERSION = "0.9.5.rc1"
 end

data/test-matrix.yml

@@ -0,0 +1,10 @@
+rvm:
+  - 1.8.7
+  - 1.9.2
+#  - jruby-1.6.2
+rails:
+  - 2.3.x
+  - 3.0.x
+database:
+  - mysql
+  - postgresql

data/test/.gitignore

@@ -0,0 +1 @@
+database*.yml

data/test/.ignore

@@ -0,0 +1,2 @@
+database.yml
+*.txt

data/test/all.ebf

@@ -0,0 +1,6 @@
+# This is an ETL Batch File and defines a means for executing
+# a collection of ETL scripts as a single process.
+
+use_temp_tables
+run 'batched1.ctl'
+run 'batched2.ctl'

data/test/apache_combined_log.ctl

@@ -0,0 +1,11 @@
+source :in, {
+  :file => 'data/apache_combined_log.txt',
+  :parser => :apache_combined_log
+}
+
+destination :out, {
+  :file => 'output/apache_combined_log.txt'
+}, 
+{
+  :order => []
+}

data/test/batch_test.rb

@@ -0,0 +1,41 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class BatchTest < Test::Unit::TestCase
+  attr_reader :file, :db_yaml, :engine
+  def setup
+    @file =  File.dirname(__FILE__) + '/all.ebf'
+    @db_yaml = File.dirname(__FILE__) + '/database.yml'
+    @engine = ETL::Engine.new
+  end
+  def teardown
+    
+  end
+  def test_etl_batch_file
+    #`etl #{file} -c #{db_yaml}`
+  end
+  def test_batch
+    assert_nothing_raised do
+      batch = ETL::Batch::Batch.resolve(file, engine)
+      batch.execute
+    end
+  end
+  def test_batch_with_file
+    assert_nothing_raised do
+      batch = ETL::Batch::Batch.resolve(File.new(file), engine)
+      batch.execute
+    end
+  end
+  def test_batch_with_batch_object
+    assert_nothing_raised do
+      batch_instance = ETL::Batch::Batch.new(File.new(file))
+      batch_instance.engine = engine
+      batch = ETL::Batch::Batch.resolve(batch_instance, engine)
+      batch.execute
+    end
+  end
+  def test_batch_with_object_should_fail
+    assert_raise(RuntimeError) do
+      batch = ETL::Batch::Batch.resolve(0, engine)
+    end
+  end
+end

data/test/batch_with_error.ebf

@@ -0,0 +1,6 @@
+# This is an ETL Batch File and defines a means for executing
+# a collection of ETL scripts as a single process.
+
+use_temp_tables
+run 'delimited_with_bulk_load.ctl'
+run 'screen_test_fatal.ctl'

data/test/block_processor.ctl

@@ -0,0 +1,6 @@
+source :in, { :type => :mock, :name => :block_processed_input } 
+
+after_read { |row| row[:added_by_after_read] = "after-" +row[:first_name]; row }
+before_write { |row| row[:added_by_before_write] = "Row #{Engine.current_source_row}"; [row,{:new_row => 'added by post_processor'}] }
+
+destination :out, { :type => :mock, :name => :block_processed_output  }

data/test/block_processor_error.ctl

@@ -0,0 +1 @@
+pre_process { raise ControlError.new( "Cough!") }

data/test/block_processor_pre_post_process.ctl

@@ -0,0 +1,4 @@
+source :in, { :type => :mock, :name => :another_input } 
+pre_process { TestWitness.call("I'm called from pre_process") }
+post_process { TestWitness.call("I'm called from post_process") }
+destination :out, { :type => :mock, :name => :another_output  }