activewarehouse-etl-sgonyea 0.9.6

data/lib/etl/processor/print_row_processor.rb

@@ -0,0 +1,12 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Debugging processor for printing the current row
+    class PrintRowProcessor < ETL::Processor::RowProcessor
+      # Process the row
+      def process(row)
+        puts row.inspect
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/processor.rb

@@ -0,0 +1,25 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Base class for pre and post processors. Subclasses must implement the +process+ method.
+    class Processor
+      def initialize(control, configuration)
+        @control = control
+        @configuration = configuration
+        after_initialize if respond_to?(:after_initialize)
+      end
+      protected
+      # Get the control object
+      def control
+        @control
+      end
+      # Get the configuration Hash
+      def configuration
+        @configuration
+      end
+      # Get the engine logger
+      def log
+        Engine.logger
+      end
+    end
+  end
+end

data/lib/etl/processor/rename_processor.rb

@@ -0,0 +1,24 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row level processor to rename a field in the row.
+    #
+    # Configuration options:
+    # * <tt>:source</tt>: the source field name
+    # * <tt>:dest</tt>: The destination field name
+    class RenameProcessor < ETL::Processor::RowProcessor
+      def process(row)
+        source_value = row[configuration[:source]]
+        case source_value
+        when Numeric
+          row[configuration[:dest]] = source_value
+        when nil
+          row[configuration[:dest]] = nil
+        else
+          row[configuration[:dest]] = source_value.dup
+        end
+        row.delete(configuration[:source])
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/require_non_blank_processor.rb

@@ -0,0 +1,26 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A processor which requires that the particular fields are non-blank in
+    # order for the row to be retained.
+    class RequireNonBlankProcessor < ETL::Processor::RowProcessor
+      # An array of fields to check
+      attr_reader :fields
+      
+      # Initialize the processor
+      #
+      # Options:
+      # * <tt>:fields</tt>: An array of fields to check, for example:
+      #   [:first_name,:last_name]
+      def initialize(control, configuration)
+        super
+        @fields = configuration[:fields] || []
+      end
+      
+      # Process the row.
+      def process(row)
+        fields.each { |field| return if row[field].blank? }
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/row_processor.rb

@@ -0,0 +1,27 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Processor which processes a specific row. Unlike a transformer, which deals with a specific
+    # value in the row, row processors can process an entire row at once, which can be used to 
+    # explode a single row into multiple rows (for example)
+    class RowProcessor < Processor
+      # Initialize the processor
+      def initialize(control, configuration)
+        super
+      end
+      # Process the specified row. This method must return the row.
+      def process(row)
+        raise "process_row is an abstract method"
+      end
+
+      # Ensure a given row keys include all the provided columns
+      # and raise an error using the provided message if it doesn't
+      def ensure_columns_available_in_row!(row, columns, message)
+        unless columns.nil?
+          columns.each do |k|
+            raise(ETL::ControlError, "Row missing required field #{k.inspect} #{message}") unless row.keys.include?(k)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/sequence_processor.rb

@@ -0,0 +1,23 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row level processor to generate a sequence.
+    #
+    # Configuration options:
+    # * <tt>:context</tt>: A context name, if none is specified then the context will be
+    #   the current ETL run
+    # * <tt>:dest</tt>: The destination field name
+    class SequenceProcessor < ETL::Processor::RowProcessor
+      def process(row)
+        sequences[configuration[:context]] ||= 0
+        row[configuration[:dest]] = sequences[configuration[:context]] += 1
+        row
+      end
+      
+      protected
+      # Get a Hash of sequences
+      def sequences
+        @sequences ||= {}
+      end
+    end
+  end
+end

data/lib/etl/processor/sftp_downloader_processor.rb

@@ -0,0 +1,63 @@
+optional_require 'net/sftp'
+
+module ETL
+  module Processor
+    # Custom processor to download files via SFTP
+    class SftpDownloaderProcessor < ETL::Processor::Processor
+      attr_reader :host
+      attr_reader :port
+      attr_reader :remote_dir
+      attr_reader :files
+      attr_reader :username
+      attr_reader :local_dir
+      
+      # configuration options include:
+      # * host - hostname or IP address of FTP server (required)
+      # * port - port number for FTP server (default: 22)
+      # * remote_dir - remote path on FTP server (default: /)
+      # * files - list of files to download from FTP server (default: [])
+      # * username - username for FTP server authentication (default: anonymous)
+      # * password - password for FTP server authentication (default: nil)
+      # * local_dir - local output directory to save downloaded files (default: '')
+      # 
+      # As an example you might write something like the following in your control process file:
+      #  pre_process :sftp_downloader, {
+      #    :host => 'sftp.sec.gov',
+      #    :path => 'edgar/Feed/2007/QTR2',
+      #    :files => ['20070402.nc.tar.gz', '20070403.nc.tar.gz', '20070404.nc.tar.gz', 
+      #               '20070405.nc.tar.gz', '20070406.nc.tar.gz'],
+      #    :local_dir => '/data/sec/2007/04',
+      #  }
+      # The above example will anonymously download via SFTP the first week's worth of SEC filing feed data
+      # from the second quarter of 2007 and download the files to the local directory +/data/sec/2007/04+.
+      def initialize(control, configuration)
+        @host = configuration[:host]
+        @port = configuration[:port] || 22
+        @remote_dir = configuration[:remote_dir] || '/'
+        @files = configuration[:files] || []
+        @username = configuration[:username] || 'anonymous'
+        @password = configuration[:password]
+        @local_dir = configuration[:local_dir] || ''
+      end
+      
+      def process
+        Net::SFTP.start(@host, @username, {:port => @port, :password => @password}) do |conn|
+          @files.each do |f|
+            conn.download!(remote_file(f), local_file(f))
+          end
+        end
+      end
+      
+      private
+      attr_accessor :password
+      
+      def local_file(name)
+        File.join(@local_dir, name)
+      end
+      
+      def remote_file(name)
+        File.join(@remote_dir, name)
+      end
+    end
+  end
+end

data/lib/etl/processor/sftp_uploader_processor.rb

@@ -0,0 +1,63 @@
+optional_require 'net/sftp'
+
+module ETL
+  module Processor
+    # Custom processor to download files via SFTP
+    class SftpUploaderProcessor < ETL::Processor::Processor
+      attr_reader :host
+      attr_reader :port
+      attr_reader :remote_dir
+      attr_reader :files
+      attr_reader :username
+      attr_reader :local_dir
+      
+      # configuration options include:
+      # * host - hostname or IP address of FTP server (required)
+      # * port - port number for FTP server (default: 22)
+      # * remote_dir - remote path on FTP server (default: /)
+      # * files - list of files to download from FTP server (default: [])
+      # * username - username for FTP server authentication (default: anonymous)
+      # * password - password for FTP server authentication (default: nil)
+      # * local_dir - local output directory to save downloaded files (default: '')
+      # 
+      # As an example you might write something like the following in your control process file:
+      #  pre_process :sftp_uploader, {
+      #    :host => 'sftp.sec.gov',
+      #    :path => 'edgar/Feed/2007/QTR2',
+      #    :files => ['20070402.nc.tar.gz', '20070403.nc.tar.gz', '20070404.nc.tar.gz', 
+      #               '20070405.nc.tar.gz', '20070406.nc.tar.gz'],
+      #    :local_dir => '/data/sec/2007/04',
+      #  }
+      # The above example will anonymously download via SFTP the first week's worth of SEC filing feed data
+      # from the second quarter of 2007 and download the files to the local directory +/data/sec/2007/04+.
+      def initialize(control, configuration)
+        @host = configuration[:host]
+        @port = configuration[:port] || 22
+        @remote_dir = configuration[:remote_dir] || '/'
+        @files = configuration[:files] || []
+        @username = configuration[:username] || 'anonymous'
+        @password = configuration[:password]
+        @local_dir = configuration[:local_dir] || ''
+      end
+      
+      def process
+        Net::SFTP.start(@host, @username, {:port => @port, :password => @password}) do |conn|
+          @files.each do |f|
+            conn.upload!(local_file(f), remote_file(f))
+          end
+        end
+      end
+      
+      private
+      attr_accessor :password
+      
+      def local_file(name)
+        File.join(@local_dir, name)
+      end
+      
+      def remote_file(name)
+        File.join(@remote_dir, name)
+      end
+    end
+  end
+end

data/lib/etl/processor/surrogate_key_processor.rb

@@ -0,0 +1,53 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A row level processor that provides surrogate keys
+    class SurrogateKeyProcessor < ETL::Processor::RowProcessor
+      attr_accessor :destination
+      attr_accessor :table
+      attr_accessor :column
+      attr_accessor :target
+      
+      # Initialize the surrogate key generator
+      #
+      # Configuration options
+      # * <tt>:query</tt>: If specified it contains a query to be used to 
+      #   locate the last surrogate key. If this is specified then :target
+      #   must also be specified.
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:destination</tt>: The destination column name (defaults to :id)
+      def initialize(control, configuration)
+        super
+        @table = configuration[:table]
+        @column = configuration[:column] || 'id'
+        @target = configuration[:target]
+        if configuration[:query]
+          raise ControlError, "Query option is no longer value, use :column and :table instead"
+        end
+        if table
+          @surrogate_key = ETL::Engine.connection(target).select_value("SELECT max(#{column}) FROM #{table_name}")
+        end
+        #puts "initial surrogate key: #{@surrogate_key}"
+        @surrogate_key = 0 if @surrogate_key.blank?
+        @surrogate_key = @surrogate_key.to_i
+        #puts "surrogate key: #{@surrogate_key}"
+        @destination = configuration[:destination] || :id
+      end
+      
+      # Add a surrogate key to the row
+      def process(row)
+        if row
+          #puts "processing row #{row.inspect}"
+          @surrogate_key += 1
+          #puts "adding surrogate key to row: #{@surrogate_key}"
+          row[destination] = @surrogate_key
+          row
+        end
+      end
+      
+      private
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end

data/lib/etl/processor/truncate_processor.rb

@@ -0,0 +1,40 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A processor which will truncate a table. Use as a pre-processor for cleaning out a table
+    # prior to loading
+    class TruncateProcessor < ETL::Processor::Processor
+      # Defines the table to truncate
+      attr_reader :table
+      
+      # Defines the database connection to use
+      attr_reader :target
+      
+      # Initialize the processor
+      #
+      # Options:
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:table</tt>: The table name
+      # * <tt>:options</tt>: Optional truncate options
+      def initialize(control, configuration)
+        super
+        #@file = File.join(File.dirname(control.file), configuration[:file])
+        @target = configuration[:target] || {}
+        @table = configuration[:table]
+        @options = configuration[:options] 
+      end
+      
+      def process
+        conn = ETL::Engine.connection(target)
+        if conn.is_a?(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter)
+          @options ||= 'RESTART IDENTITY'
+        end
+        conn.truncate(table_name, @options)
+      end
+      
+      private
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end

data/lib/etl/processor/zip_file_processor.rb

@@ -0,0 +1,27 @@
+optional_require 'zip/zip'
+
+module ETL
+  module Processor
+    # Custom processor to zip files
+    class ZipFileProcessor < ETL::Processor::Processor
+      attr_reader :infile
+      attr_reader :destination
+      
+      # configuration options include:
+      # * infile - File to zip (required)
+      # * destination - Zip file name (default: #{infile}.zip)
+      def initialize(control, configuration)
+        path = Pathname.new(configuration[:infile])
+        @infile = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(configuration[:infile]))) + path
+        @destination = configuration[:destination] || "#{infile}.zip"
+      end
+      
+      def process
+        Zip::ZipFile.open(@destination, Zip::ZipFile::CREATE) do |zipfile|
+          zipfile.add(@infile.basename, @infile)
+        end
+      end
+      
+    end
+  end
+end

data/lib/etl/row.rb

@@ -0,0 +1,20 @@
+# This source file contains the ETL::Row class.
+
+module ETL #:nodoc:
+  # This class represents a single row currently passing through the ETL pipeline
+  class Row < Hash
+    # Accessor for the originating source
+    attr_accessor :source
+    
+    # All change types
+    CHANGE_TYPES = [:insert, :update, :delete]
+    
+    # Accessor for the row's change type
+    attr_accessor :change_type
+    
+    # Get the change type, defaults to :insert
+    def change_type
+      @change_type ||= :insert
+    end
+  end
+end

data/lib/etl/screen.rb

@@ -0,0 +1,14 @@
+# This source file contains the ETL::Screen module and requires all of the 
+# screens
+
+module ETL #:nodoc:
+  # The ETL::Screen module contains pre-built screens useful for checking the 
+  # ETL state during execution. Screens may be fatal, which will result in 
+  # termination of the ETL process, errors, which will result in the
+  # termination of just the current ETL control file, or warnings, which will
+  # result in a warning message.
+  module Screen
+  end
+end
+
+Dir[File.dirname(__FILE__) + "/screen/*.rb"].each { |file| require(file) }

data/lib/etl/screen/row_count_screen.rb

@@ -0,0 +1,20 @@
+module ETL
+  module Screen
+    # This screen validates the number of rows which will be bulk loaded
+    # against the results from some sort of a row count query. If there 
+    # is a difference then the screen will not pass
+    class RowCountScreen
+      attr_accessor :control, :configuration
+      def initialize(control, configuration={})
+        @control = control
+        @configuration = configuration
+        execute
+      end
+      def execute
+        unless Engine.rows_written == configuration[:rows]
+          raise "Rows written (#{Engine.rows_written}) does not match expected rows (#{configuration[:rows]})"
+        end
+      end
+    end
+  end
+end