activewarehouse-etl 0.8.4 → 0.9.0

data/lib/etl/execution.rb

@@ -14,6 +14,7 @@ module ETL #:nodoc
 end
 
 require 'etl/execution/base'
+require 'etl/execution/batch'
 require 'etl/execution/job'
 require 'etl/execution/record'
 require 'etl/execution/migration'

data/lib/etl/execution/base.rb

@@ -1,7 +1,7 @@
 module ETL #:nodoc:
   module Execution #:nodoc:
     # Base class for ETL execution information
-    class Base < ETL::ActiveRecord::Base
+    class Base < ActiveRecord::Base
       self.abstract_class = true
       establish_connection :etl_execution
     end

data/lib/etl/execution/batch.rb

@@ -0,0 +1,8 @@
+module ETL #:nodoc:
+  module Execution #:nodoc:
+    # Persistent class representing an ETL batch
+    class Batch < Base
+      has_many :jobs
+    end
+  end
+end

data/lib/etl/execution/job.rb

@@ -2,6 +2,7 @@ module ETL #:nodoc:
   module Execution #:nodoc:
     # Persistent class representing an ETL job
     class Job < Base
+      belongs_to :batch
     end
   end
 end

data/lib/etl/execution/migration.rb

@@ -8,7 +8,10 @@ module ETL #:nodoc:
         def migrate
           connection.initialize_schema_information
           v = connection.select_value("SELECT version FROM #{schema_info_table_name}").to_i
-          v.upto(target - 1) { |i| __send__("migration_#{i+1}".to_sym) }
+          v.upto(target - 1) do |i| 
+            __send__("migration_#{i+1}".to_sym)
+            update_schema_info(i+1)
+          end
         end
         protected
         # Get the schema info table name
@@ -24,7 +27,7 @@ module ETL #:nodoc:
         
         # Get the final target version number
         def target
-          2
+          3
         end
         
         private
@@ -41,14 +44,23 @@ module ETL #:nodoc:
             t.column :crc, :string, :null => false
             t.column :job_id, :integer, :null => false
           end
-          update_schema_info(1)
         end
         
         def migration_2 #:nodoc:
           connection.add_index :records, :control_file
           connection.add_index :records, :natural_key
           connection.add_index :records, :job_id
-          update_schema_info(2)
+        end
+        
+        def migration_3 #:nodoc:
+          connection.create_table :batches do |t|
+            t.column :batch_file, :string, :null => false
+            t.column :created_at, :datetime, :null => false
+            t.column :completed_at, :datetime
+            t.column :status, :string
+          end
+          connection.add_column :jobs, :batch_id, :integer
+          connection.add_index :jobs, :batch_id
         end
       
         # Update the schema info table, setting the version value

data/lib/etl/generator/surrogate_key_generator.rb

@@ -4,13 +4,25 @@ module ETL #:nodoc:
   module Generator #:nodoc:
     # Surrogate key generator.
     class SurrogateKeyGenerator < Generator
+      attr_reader :table
+      attr_reader :target
+      attr_reader :column
+      attr_reader :query
+      
       # Initialize the generator
       def initialize(options={})
-        if options[:query]
-          @surrogate_key = ETL::ActiveRecord::Base.connection.select_value(options[:query])
-          @surrogate_key = 0 if @surrogate_key.blank?
-          @surrogate_key = @surrogate_key.to_i
+        @table = options[:table]
+        @target = options[:target]
+        @column = options[:column] || 'id'
+        @query = options[:query]
+        
+        if table
+          @surrogate_key = ETL::Engine.connection(target).select_value("SELECT max(#{column}) FROM #{table_name}")
+        elsif query
+          @surrogate_key = ETL::Engine.connection(target).select_value(query)
         end
+        @surrogate_key = 0 if @surrogate_key.blank?
+        @surrogate_key = @surrogate_key.to_i
       end
 
       # Get the next surrogate key
@@ -18,6 +30,10 @@ module ETL #:nodoc:
         @surrogate_key ||= 0
         @surrogate_key += 1
       end
+      
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
     end
   end
 end

data/lib/etl/http_tools.rb

@@ -8,7 +8,7 @@ module HttpTools
   def parse_user_agent(user_agent)
     if '-' == user_agent
       #raise 'Invalid User Agent'
-      puts 'Invalid User Agent'
+      #puts 'Invalid User Agent'
     end
     
     browser, browser_version_major, browser_version_minor, ostype, os, os_version = nil

data/lib/etl/processor/bulk_import_processor.rb

@@ -4,10 +4,13 @@ module ETL #:nodoc:
     # underlying database driver from ActiveRecord must support the methods
     # +bulk_load+ method.
     class BulkImportProcessor < ETL::Processor::Processor
+      
       # The file to load from
       attr_reader :file
-      # The target database information (see +initialize+)
+      # The target database
       attr_reader :target
+      # The table name
+      attr_reader :table
       # Set to true to truncate
       attr_reader :truncate
       # Array of symbols representing the column load order
@@ -23,7 +26,8 @@ module ETL #:nodoc:
       #
       # Configuration options:
       # * <tt>:file</tt>: The file to load data from
-      # * <tt>:target</tt>: The target connection information
+      # * <tt>:target</tt>: The target database
+      # * <tt>:table</tt>: The table name
       # * <tt>:truncate</tt>: Set to true to truncate before loading
       # * <tt>:columns</tt>: The columns to load in the order they appear in
       #   the bulk data file
@@ -34,23 +38,25 @@ module ETL #:nodoc:
         super
         @file = File.join(File.dirname(control.file), configuration[:file])
         @target = configuration[:target]
+        @table = configuration[:table]
         @truncate = configuration[:truncate] ||= false
         @columns = configuration[:columns]
         @field_separator = (configuration[:field_separator] || ',')
         @line_separator = (configuration[:line_separator] || "\n")
         @field_enclosure = configuration[:field_enclosure]
-        connect
+        
+        raise ControlError, "Target must be specified" unless @target
+        raise ControlError, "Table must be specified" unless @table
       end
       
       # Execute the processor
       def process
         return if ETL::Engine.skip_bulk_import
+        return if File.size(file) == 0
         
-        conn = ETL::ActiveRecord::Base.connection
+        conn = ETL::Engine.connection(target)
         conn.transaction do
-          # TODO: Support all database types
-          # Since LOCAL is used this must be allowed by both the client and server
-          conn.truncate(target[:table]) if truncate
+          conn.truncate(table_name) if truncate
           options = {}
           options[:columns] = columns
           if field_separator || field_enclosure
@@ -59,21 +65,12 @@ module ETL #:nodoc:
             options[:fields][:enclosed_by] = field_enclosure if field_enclosure
             options[:fields][:terminated_by] = line_separator if line_separator
           end
-          conn.bulk_load(file, target[:table], options)
+          conn.bulk_load(file, table_name, options)
         end
       end
       
-      private
-      # Connect to the database
-      def connect
-        Engine.logger.debug "Connecting to database #{target[:database]}"
-        ETL::ActiveRecord::Base.establish_connection(
-            :adapter  => (target[:adapter] || :mysql),
-            :username => (target[:username] || 'root'),
-            :host     => (target[:host] || 'localhost'),
-            :password => target[:password],
-            :database => target[:database] 
-        )
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
       end
     end
   end

data/lib/etl/processor/check_exist_processor.rb

@@ -6,6 +6,9 @@ module ETL #:nodoc:
       # A symbol or array of symbols representing keys that should be skipped
       attr_accessor :skip
       
+      # The target database
+      attr_accessor :target
+      
       # The name of the table to check against
       attr_accessor :table
       
@@ -26,11 +29,12 @@ module ETL #:nodoc:
       def initialize(control, configuration)
         super
         @skip = configuration[:skip] || []
-        @table = configuration[:table]
+        @target = configuration[:target] || raise(ETL::ControlError, "target must be specified")
+        @table = configuration[:table] || raise(ETL::ControlError, "table must be specified")
         @columns = configuration[:columns]
         
-        q = "SELECT COUNT(*) FROM #{table}"
-        @should_check = ETL::ActiveRecord::Base.connection.select_value(q).to_i > 0 
+        q = "SELECT COUNT(*) FROM #{table_name}"
+        @should_check = ETL::Engine.connection(target).select_value(q).to_i > 0 
       end
       
       # Return true if the given key should be skipped
@@ -51,20 +55,25 @@ module ETL #:nodoc:
       # Process the row
       def process(row)
         return row unless should_check?
-        connection = ETL::ActiveRecord::Base.connection
-        q = "SELECT * FROM #{table} WHERE "
+        conn = ETL::Engine.connection(target)
+        q = "SELECT * FROM #{table_name} WHERE "
         conditions = []
         row.each do |k,v| 
           if columns.nil? || columns.include?(k.to_sym)
-            conditions << "#{k} = #{connection.quote(v)}" unless skip?(k.to_sym)
+            conditions << "#{k} = #{conn.quote(v)}" unless skip?(k.to_sym)
           end
         end
         q << conditions.join(" AND ")
       
         #puts "query: #{q}"
-        result = connection.select_one(q)
+        result = conn.select_one(q)
         return row if result.nil?
       end
+      
+      private
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
     end
   end
 end

data/lib/etl/processor/hierarchy_exploder_processor.rb

@@ -24,8 +24,9 @@ module ETL #:nodoc:
       # Process the row expanding it into hierarchy values
       def process(row)
         rows = []
-        conn = configuration[:connection]
+        target = configuration[:target]
         table = configuration[:table]
+        conn = ETL::Engine.connection(target)
         build_rows([row[:id]], row[:id], row[:id], row[:parent_id].nil?, 0, rows, table, conn)
         rows
       end

data/lib/etl/processor/require_non_blank_processor.rb

@@ -0,0 +1,26 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A processor which requires that the particular fields are non-blank in
+    # order for the row to be retained.
+    class RequireNonBlankProcessor < ETL::Processor::RowProcessor
+      # An array of fields to check
+      attr_reader :fields
+      
+      # Initialize the processor
+      #
+      # Options:
+      # * <tt>:fields</tt>: An array of fields to check, for example:
+      #   [:first_name,:last_name]
+      def initialize(control, configuration)
+        super
+        @fields = configuration[:fields] || []
+      end
+      
+      # Process the row.
+      def process(row)
+        fields.each { |field| return if row[field].blank? }
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/surrogate_key_processor.rb

@@ -2,14 +2,29 @@ module ETL #:nodoc:
   module Processor #:nodoc:
     # A row level processor that provides surrogate keys
     class SurrogateKeyProcessor < ETL::Processor::RowProcessor
-      attr_accessor :query
       attr_accessor :destination
+      attr_accessor :table
+      attr_accessor :column
+      attr_accessor :target
       
       # Initialize the surrogate key generator
+      #
+      # Configuration options
+      # * <tt>:query</tt>: If specified it contains a query to be used to 
+      #   locate the last surrogate key. If this is specified then :target
+      #   must also be specified.
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:destination</tt>: The destination column name (defaults to :id)
       def initialize(control, configuration)
         super
+        @table = configuration[:table]
+        @column = configuration[:column] || 'id'
+        @target = configuration[:target]
         if configuration[:query]
-          @surrogate_key = ETL::ActiveRecord::Base.connection.select_value(configuration[:query])
+          raise ControlError, "Query option is no longer value, use :column and :table instead"
+        end
+        if table
+          @surrogate_key = ETL::Engine.connection(target).select_value("SELECT max(#{column}) FROM #{table_name}")
         end
         #puts "initial surrogate key: #{@surrogate_key}"
         @surrogate_key = 0 if @surrogate_key.blank?
@@ -28,6 +43,11 @@ module ETL #:nodoc:
           row
         end
       end
+      
+      private
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
     end
   end
 end

data/lib/etl/processor/truncate_processor.rb

@@ -6,29 +6,29 @@ module ETL #:nodoc:
       # Defines the table to truncate
       attr_reader :table
       
-      # Defines the database connection information
+      # Defines the database connection to use
       attr_reader :target
+      
+      # Initialize the processor
+      #
+      # Options:
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:table</tt>: The table name
       def initialize(control, configuration)
         super
         #@file = File.join(File.dirname(control.file), configuration[:file])
         @target = configuration[:target] || {}
         @table = configuration[:table]
-        connect
       end
+      
       def process
-        conn = ETL::ActiveRecord::Base.connection
-        conn.truncate(table)
+        conn = ETL::Engine.connection(target)
+        conn.truncate(table_name)
       end
       
-      # Connect to the database
-      def connect
-        ETL::ActiveRecord::Base.establish_connection(
-            :adapter  => (target[:adapter] || :mysql),
-            :username => (target[:username] || 'root'),
-            :host     => (target[:host] || 'localhost'),
-            :password => target[:password],
-            :database => target[:database] 
-        )
+      private
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
       end
     end
   end

data/lib/etl/screen.rb

@@ -0,0 +1,14 @@
+# This source file contains the ETL::Screen module and requires all of the 
+# screens
+
+module ETL #:nodoc:
+  # The ETL::Screen module contains pre-built screens useful for checking the 
+  # ETL state during execution. Screens may be fatal, which will result in 
+  # termination of the ETL process, errors, which will result in the
+  # termination of just the current ETL control file, or warnings, which will
+  # result in a warning message.
+  module Screen
+  end
+end
+
+Dir[File.dirname(__FILE__) + "/screen/*.rb"].each { |file| require(file) }

data/lib/etl/screen/row_count_screen.rb

@@ -4,11 +4,16 @@ module ETL
     # against the results from some sort of a row count query. If there 
     # is a difference then the screen will not pass
     class RowCountScreen
+      attr_accessor :control, :configuration
       def initialize(control, configuration={})
-        
+        @control = control
+        @configuration = configuration
+        execute
       end
       def execute
-        
+        unless Engine.rows_written == configuration[:rows]
+          raise "Rows written (#{Engine.rows_written}) does not match expected rows (#{configuration[:rows]})"
+        end
       end
     end
   end

data/lib/etl/transform/foreign_key_lookup_transform.rb

@@ -46,8 +46,9 @@ class ActiveRecordResolver
   # The find method to use (as a symbol)
   attr_accessor :find_method
   
-  # Initialize the resolver. The ar_class argument should extend from ActiveRecord::Base. The find_method argument
-  # must be a symbol for the finder method used. For example:
+  # Initialize the resolver. The ar_class argument should extend from 
+  # ActiveRecord::Base. The find_method argument must be a symbol for the 
+  # finder method used. For example:
   # 
   # ActiveRecordResolver.new(Person, :find_by_name)
   #
@@ -56,6 +57,7 @@ class ActiveRecordResolver
     @ar_class = ar_class
     @find_method = find_method
   end
+  
   # Resolve the value
   def resolve(value)
     rec = ar_class.__send__(find_method, value)
@@ -64,13 +66,21 @@ class ActiveRecordResolver
 end
 
 class SQLResolver
+  # Initialize the SQL resolver. Use the given table and field name to search
+  # for the appropriate foreign key. The field should be the name of a natural
+  # key that is used to locate the surrogate key for the record.
+  #
+  # The connection argument is optional. If specified it can be either a symbol
+  # referencing a connection defined in the ETL database.yml file or an actual
+  # ActiveRecord connection instance. If the connection is not specified then
+  # the ActiveRecord::Base.connection will be used.
   def initialize(table, field, connection=nil)
     @table = table
     @field = field
-    @connection = connection
+    @connection = (connection.respond_to?(:quote) ? connection : ETL::Engine.connection(connection)) if connection
+    @connection ||= ActiveRecord::Base.connection
   end
   def resolve(value)
-    conn = @connection ||= ActiveRecord::Base.connection
-    conn.select_value("SELECT id FROM #{table} WHERE #{field} = #{conn.quote(value)}")
+    @connection.select_value("SELECT id FROM #{@table} WHERE #{@field} = #{@connection.quote(value)}")
   end
 end