activewarehouse-etl 0.5.2 → 0.6.0

data/lib/etl/generator/surrogate_key_generator.rb

@@ -2,6 +2,7 @@ module ETL #:nodoc:
   module Generator #:nodoc:
     # Surrogate key generator.
     class SurrogateKeyGenerator < Generator
+      # Get the next surrogate key
       def next
         @surrogate_key ||= 0
         @surrogate_key += 1

data/lib/etl/http_tools.rb

@@ -0,0 +1,119 @@
+require 'uri'
+
+# Module which has utility methods for HTTP.
+module HttpTools
+  # Parse the given user agent string
+  #
+  # Code taken from http://gemtacular.com/gems/ParseUserAgent
+  def parse_user_agent(user_agent)
+    if '-' == user_agent
+      raise 'Invalid User Agent'
+    end
+    
+    browser, browser_version_major, browser_version_minor, ostype, os, os_version = nil
+
+    # fix Opera
+    #useragent =~ s/Opera (\d)/Opera\/$1/i;
+    useragent = user_agent.gsub(/(Opera [\d])/,'Opera\1')
+
+    # grab all Agent/version strings as 'agents'
+    agents = Array.new
+    user_agent.split(/\s+/).each {|string| 
+      if string =~ /\//
+        agents<< string
+      end
+    }
+
+    # cycle through the agents to set browser and version (MSIE is set later)
+    if agents && agents.length > 0
+        agents.each {|agent|
+          parts = agent.split('/')
+          browser = parts[0]
+          browser_version = parts[1]
+          if browser == 'Firefox'
+            browser_version_major = parts[1].slice(0,3)
+            browser_version_minor = parts[1].sub(browser_version_major,'').sub('.','')
+          elsif browser == 'Safari'
+            if parts[1].slice(0,3).to_f < 400
+              browser_version_major = '1'
+            else
+              browser_version_major = '2'
+            end
+          else
+            browser_version_major = parts[1].slice(0,1)
+          end
+        }
+    end
+
+    # grab all of the properties (within parens)
+    # should be in relation to the agent if possible  
+    detail = user_agent
+    user_agent.gsub(/\((.*)\)/,'').split(/\s/).each {|part| detail = detail.gsub(part,'')}
+    detail = detail.gsub('(','').gsub(')','').lstrip
+    properties = detail.split(/;\s+/)
+
+    # cycle through the properties to set known quantities
+    properties.each do |property| 
+      if property =~ /^Win/
+        ostype = 'Windows'
+        os = property
+        if parts = property.split(/ /,2)
+          if parts[1] =~ /^NT/
+            ostype = 'Windows'
+            subparts = parts[1].split(/ /,2)
+            if subparts[1] == '5'
+              os_version = '2000'
+            elsif subparts[1] == '5.1'
+              os_version = 'XP'
+            else
+              os_version = subparts[1]
+            end
+          end
+        end
+      end
+      if property == 'Macintosh'
+        ostype = 'Macintosh'
+        os = property
+      end
+      if property =~ /OS X/
+        ostype = 'Macintosh'
+        os_version = 'OS X'
+        os = property
+      end
+      if property =~ /^Linux/
+        ostype = 'Linux'
+        os = property
+      end
+      if property =~ /^MSIE/
+        browser = 'MSIE'
+        browser_version = property.gsub('MSIE ','').lstrip
+        browser_version_major,browser_version_minor = browser_version.split('.')
+      end
+    end
+    
+    result = {
+      :browser => browser, 
+      :browser_version_major => browser_version_major, 
+      :browser_version_minor => browser_version_minor, 
+      :ostype => ostype, 
+      :os_version => os_version,
+      :os => os,
+    }
+    result.each do |key, value|
+      result[key] = value.blank? ? nil : value.strip
+    end
+    result
+  end
+  
+  def parse_uri(uri_string)
+    if uri_string
+      uri = URI.parse(uri_string)
+      
+      results = {:scheme => uri.scheme, :host => uri.host, :port => uri.port, :uri_path => uri.path}
+      results[:domain] = $1 if uri.host =~ /\.?([^\.]+\.[^\.]+$)/
+      results
+    else
+      {:scheme => nil, :host => nil, :port => nil, :uri_path => nil, :domain => nil}
+    end
+  end
+end

data/lib/etl/parser/apache_combined_log_parser.rb

@@ -0,0 +1,47 @@
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # Parser which can parser the Apache Combined Log Format as defined at
+    # http://httpd.apache.org/docs/2.2/logs.html
+    class ApacheCombinedLogParser < ETL::Parser::Parser
+      include HttpTools
+      def initialize(source, options={})
+        super
+      end
+
+      def each
+        Dir.glob(file).each do |file|
+          File.open(file).each_line do |line|
+            yield parse(line)
+          end
+        end
+      end
+      
+      def parse(line)
+        # example line:  127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
+        line =~ /^(\S+)\s(\S+)\s(\S+)\s\[([^\]]*)\]\s"([^"]*)"\s(\d*)\s(\d*)\s"([^"]*)"\s"([^"]*)"$/
+        fields = {
+          :ip_address => $1,
+          :identd => $2,
+          :user => $3,
+          :timestamp => $4,
+          :request => $5,
+          :response_code => $6,
+          :bytes => $7,
+          :referrer => $8,
+          :user_agent => $9,
+        }
+        #fields[:timestamp] =~ r%{(\d\d)/(\w\w\w)/(\d\d\d\d):(\d\d):(\d\d):(\d\d) -(\d\d\d\d)}
+        d = Date._strptime(fields[:timestamp], '%d/%b/%Y:%H:%M:%S')
+        fields[:timestamp] = Time.mktime(d[:year], d[:mon], d[:mday], d[:hour], d[:min], d[:sec], d[:sec_fraction])
+        
+        fields.merge!(parse_user_agent(fields[:user_agent]))
+        fields.merge!(parse_uri(fields[:referrer]))
+        
+        fields.each do |key, value|
+          fields[key] = nil if value == '-'
+        end
+      end
+      
+    end
+  end
+end

data/lib/etl/parser/sax_parser.rb

@@ -1,8 +1,9 @@
 require 'rexml/parsers/sax2parser'
 require 'rexml/sax2listener'
 
-module ETL
-  module Parser
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # ETL parser implementation which uses SAX to parse XML files.
     class SaxParser < ETL::Parser::Parser
       
       # The write trigger causes whatever values are currently specified for the row to be returned.
@@ -77,7 +78,19 @@ module ETL
         
       end
       def start_element(uri, localname, qname, attributes)
-        @path.elements << XPath::Element.new(localname, attributes)
+        element = XPath::Element.new(localname, attributes)
+        @path.elements << element
+        
+        @parser.fields.each do |field|
+          #puts "#{@path} match? #{field.path}"
+          if @path.match?(field.path)
+            #puts "field.path: #{field.path}"
+            if field.path.is_attribute?
+              #puts "setting @row[#{field.name}] to #{element.attributes[field.path.attribute]}"
+              @row[field.name] = element.attributes[field.path.attribute]
+            end
+          end
+        end
       end
       def end_element(uri, localname, qname)
         element = @path.elements.last
@@ -86,13 +99,12 @@ module ETL
           #puts "#{@path} match? #{field.path}"
           if @path.match?(field.path)
             #puts "field.path: #{field.path}"
-            if field.path.is_attribute?
-              @row[field.name] = element.attributes[field.path.attribute]
-            else
+            if !field.path.is_attribute?
               @row[field.name] = @value
             end
           end
         end
+        
         #puts @path.to_s
         if @path.match?(@parser.write_trigger)
           #puts "matched: #{@path} =~ #{@parser.write_trigger}"

data/lib/etl/processor.rb

@@ -1,2 +1,3 @@
 require 'etl/processor/processor'
+require 'etl/processor/row_processor'
 Dir[File.dirname(__FILE__) + "/processor/*.rb"].each { |file| require(file) }

data/lib/etl/processor/bulk_import_processor.rb

@@ -3,12 +3,18 @@ module ETL #:nodoc:
     # Processor which is used to bulk import data into a target database
     class BulkImportProcessor < ETL::Processor::Processor
       attr_reader :file, :target, :truncate, :columns
+      attr_accessor :field_separator
+      attr_accessor :field_enclosure
+      attr_accessor :line_separator
       def initialize(control, configuration)
         super
         @file = File.join(File.dirname(control.file), configuration[:file])
         @target = configuration[:target]
         @truncate = configuration[:truncate] ||= false
         @columns = configuration[:columns]
+        @field_separator = (configuration[:field_separator] || ',')
+        @line_separator = configuration[:line_separator]
+        @field_enclosure = configuration[:field_enclosure]
         connect
       end
       def process
@@ -20,6 +26,11 @@ module ETL #:nodoc:
           conn.truncate(target[:table]) if truncate
           options = {}
           options[:columns] = columns
+          if field_separator || field_enclosure
+            options[:fields] = {}
+            options[:fields][:delimited_by] = field_separator if field_separator
+            options[:fields][:enclosed_by] = field_enclosure if field_enclosure
+          end
           conn.bulk_load(file, target[:table], options)
         end
       end
@@ -27,6 +38,7 @@ module ETL #:nodoc:
       private
       # Connect to the database
       def connect
+        Engine.logger.debug "Connecting to database #{target[:database]}"
         ETL::ActiveRecord::Base.establish_connection(
             :adapter  => (target[:adapter] || :mysql),
             :username => (target[:username] || 'root'),

data/lib/etl/processor/hierarchy_exploder_processor.rb

@@ -0,0 +1,54 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row-level processor that will convert a single row into multiple rows designed to be inserted
+    # into a hierarchy bridge table.
+    class HierarchyExploderProcessor < ETL::Processor::RowProcessor
+      attr_accessor :id_field
+      attr_accessor :parent_id_field
+      
+      # Initialize the processor
+      #
+      # Configuration options:
+      # * <tt>:connection</tt>: The ActiveRecord adapter connection
+      # * <tt>:id_field</tt>: The name of the id field (defaults to 'id')
+      # * <tt>:parent_id_field</tt>: The name of the parent id field (defaults to 'parent_id')
+      #
+      # TODO: Allow resolver to be implemented in a customizable fashion, i.e. don't rely
+      # on AR as the only resolution method.
+      def initialize(control, configuration={})
+        @id_field = configuration[:id_field] || 'id'
+        @parent_id_field = configuration[:parent_id_field] || 'parent_id'
+        super
+      end
+  
+      # Process the row expanding it into hierarchy values
+      def process(row)
+        rows = []
+        conn = configuration[:connection]
+        table = configuration[:table]
+        build_rows([row[:id]], row[:id], row[:id], row[:parent_id].nil?, 0, rows, table, conn)
+        rows
+      end
+  
+      protected
+      # Recursive function that will add a row for the current level and then call build_rows
+      # for all of the children of the current level
+      def build_rows(ids, parent_id, row_id, root, level, rows, table, conn)
+        ids.each do |id|
+          child_ids = conn.select_values("SELECT #{id_field} FROM #{table} WHERE #{parent_id_field} = #{id}")
+      
+          row = {
+            :parent_id => row_id, 
+            :child_id => id, 
+            :num_levels_from_parent => level, 
+            :is_bottom => (child_ids.empty? ? 1 : 0),
+            :is_top => (root ? 1 : 0),
+          }
+          rows << row
+      
+          build_rows(child_ids, id, row_id, false, level + 1, rows, table, conn)
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/processor.rb

@@ -14,11 +14,7 @@ module ETL #:nodoc:
         @configuration
       end
       def log
-        unless @log
-          @log = Logger.new(STDOUT)
-          @log.level = Logger::DEBUG
-        end
-        @log
+        Engine.logger
       end
     end
   end

data/lib/etl/processor/row_processor.rb

@@ -0,0 +1,17 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Processor which processes a specific row. Unlike a transformer, which deals with a specific
+    # value in the row, row processors can process an entire row at once, which can be used to 
+    # explode a single row into multiple rows (for example)
+    class RowProcessor < Processor
+      # Initialize the processor
+      def initialize(control, configuration)
+        super
+      end
+      # Process the specified row
+      def process(row)
+        raise "process_row is an abstract method"
+      end
+    end
+  end
+end

data/lib/etl/transform/date_to_string_transform.rb

@@ -11,7 +11,7 @@ module ETL #:nodoc:
         @format = configuration[:format] || "%Y-%m-%d"
       end
       # Transform the value using strftime
-      def transform(value)
+      def transform(name, value, row)
         value.strftime(@format)
       end
     end

data/lib/etl/transform/decode_transform.rb

@@ -27,7 +27,7 @@ module ETL #:nodoc:
       end
       
       # Transform the value
-      def transform(value)
+      def transform(name, value, row)
         decode_table[value] || default_value
       end
       

data/lib/etl/transform/default_transform.rb

@@ -0,0 +1,15 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which will replace nil or empty values with a specified value.
+    class DefaultTransform < Transform
+      attr_accessor :default_value
+      def initialize(control, configuration)
+        super
+        @default_value = configuration[:default_value]
+      end
+      def transform(name, value, row)
+        value.blank? ? default_value : value
+      end
+    end
+  end
+end

data/lib/etl/transform/foreign_key_lookup_transform.rb

@@ -21,7 +21,7 @@ module ETL #:nodoc:
       end
       
       # Transform the value by resolving it to a foriegn key
-      def transform(value)
+      def transform(name, value, row)
         fk = @collection[value]
         unless fk
           raise ResolverError, "Foreign key for #{value} not found and no resolver specified" unless resolver

data/lib/etl/transform/hierarchy_lookup_transform.rb

@@ -0,0 +1,56 @@
+module ETL #:nodoc: 
+  module Transform #:nodoc:
+    # Transform which walks up the hierarchy tree to find a value of the current level's value
+    # is nil.
+    #
+    # Configuration options:
+    # * <tt>:table</tt>: The name of the table to use for lookup (required)
+    # * <tt>:connection</tt>: The database adapter connection (required)
+    # * <tt>:parent_id_field</tt>: The name of the parent ID field (defaults to :parent_id)
+    #
+    # TODO: Let the resolver be implemented in a class so different resolution methods are
+    # possible.
+    class HierarchyLookupTransform < ETL::Transform::Transform
+      # The name of the field to use for the parent ID
+      attr_accessor :parent_id_field
+      
+      # Initialize the transform
+      #
+      # Configuration options:
+      # * <tt>:table</tt>: The table to search (required)
+      # * <tt>:connection</tt>: The ActiveRecord adapter (required)
+      # * <tt>:parent_id_field</tt>: The name of the field to use for the parent ID (defaults to :parent_id)
+      def initialize(control, configuration={})
+        super
+        @parent_id_field = configuration[:parent_id_field] || :parent_id
+      end
+      
+      # Transform the value.
+      def transform(name, value, row)
+        if parent_id = row[parent_id_field]
+          # TODO: should use more than just the first source out of the control
+          parent_id, value = lookup(name, 
+            control.sources.first.configuration[:table], parent_id, parent_id_field)        
+          until value || parent_id.nil?
+            # TODO: should use more than just the first source out of the control
+            parent_id, value = lookup(name, 
+              control.sources.first.configuration[:table], parent_id, parent_id_field)
+          end
+        end
+        value
+      end
+      
+      # Lookup the parent value. Note that this method requires that configuration[:connection]
+      # is specified
+      def lookup(field, table, parent_id, parent_id_field)
+        unless configuration.has_key?(:connection)
+          raise ETL::ControlError, "The configuration hash must include the database connection"
+        end
+        
+        q = "SELECT #{parent_id_field}, #{field} FROM #{table} WHERE id = #{parent_id}"
+        row = configuration[:connection].select_one(q)
+        return row[parent_id_field.to_s], row[field.to_s]
+      end
+    end
+  end
+end