activewarehouse-etl-sgonyea 0.9.6

data/lib/etl/execution.rb

@@ -0,0 +1,19 @@
+module ETL #:nodoc
+  # Classes which store information about ETL execution
+  module Execution
+    # Execution management
+    class Execution
+      class << self
+        # Migrate the data store
+        def migrate
+          ETL::Execution::Migration.migrate
+        end
+      end
+    end
+  end
+end
+
+require 'etl/execution/base'
+require 'etl/execution/batch'
+require 'etl/execution/job'
+require 'etl/execution/migration'

data/lib/etl/execution/base.rb

@@ -0,0 +1,8 @@
+module ETL #:nodoc:
+  module Execution #:nodoc:
+    # Base class for ETL execution information
+    class Base < ActiveRecord::Base
+      self.abstract_class = true
+    end
+  end
+end

data/lib/etl/execution/batch.rb

@@ -0,0 +1,10 @@
+module ETL #:nodoc:
+  module Execution #:nodoc:
+    # Persistent class representing an ETL batch
+    class Batch < Base
+      belongs_to :batch
+      has_many :batches
+      has_many :jobs
+    end
+  end
+end

data/lib/etl/execution/job.rb

@@ -0,0 +1,8 @@
+module ETL #:nodoc:
+  module Execution #:nodoc:
+    # Persistent class representing an ETL job
+    class Job < Base
+      belongs_to :batch
+    end
+  end
+end

data/lib/etl/execution/migration.rb

@@ -0,0 +1,90 @@
+module ETL #:nodoc:
+  module Execution #:nodoc
+    # Handles migration of tables required for persistent storage of meta data 
+    # for the ETL engine
+    class Migration
+      class << self
+        protected
+        # Get the schema info table name
+        def schema_info_table_name
+          ActiveRecord::Migrator.schema_migrations_table_name
+        end
+        alias :schema_migrations_table_name :schema_info_table_name
+        
+        public
+        # Execute the migrations
+        def migrate
+          connection.initialize_schema_migrations_table
+          last_migration.upto(target - 1) do |i| 
+            __send__("migration_#{i+1}".to_sym)
+            connection.assume_migrated_upto_version(i+1)
+          end
+        end
+        
+        protected
+        def last_migration
+          connection.select_values(
+            "SELECT version FROM #{schema_migrations_table_name}"
+          ).map(&:to_i).sort.last || 0
+        end
+        
+        # Get the connection to use during migration
+        def connection
+          @connection ||= ETL::Execution::Base.connection
+        end
+        
+        # Get the final target version number
+        def target
+          4
+        end
+        
+        private
+        def migration_1 #:nodoc:
+          connection.create_table :jobs do |t|
+            t.column :control_file, :string, :null => false
+            t.column :created_at, :datetime, :null => false
+            t.column :completed_at, :datetime
+            t.column :status, :string
+          end
+          connection.create_table :records do |t|
+            t.column :control_file, :string, :null => false
+            t.column :natural_key, :string, :null => false
+            t.column :crc, :string, :null => false
+            t.column :job_id, :integer, :null => false
+          end
+        end
+        
+        def migration_2 #:nodoc:
+          connection.add_index :records, :control_file
+          connection.add_index :records, :natural_key
+          connection.add_index :records, :job_id
+        end
+        
+        def migration_3 #:nodoc:
+          connection.create_table :batches do |t|
+            t.column :batch_file, :string, :null => false
+            t.column :created_at, :datetime, :null => false
+            t.column :completed_at, :datetime
+            t.column :status, :string
+          end
+          connection.add_column :jobs, :batch_id, :integer
+          connection.add_index :jobs, :batch_id
+        end
+        
+        def migration_4
+          connection.drop_table :records
+        end
+
+        def migration_5
+          connection.add_column :batches, :batch_id, :integer
+          connection.add_index :batches, :batch_id
+        end
+      
+        # Update the schema info table, setting the version value
+        def update_schema_info(version)
+          connection.update("UPDATE #{schema_info_table_name} SET version = #{version}")
+        end
+      end
+    end
+  end
+end

data/lib/etl/generator.rb

@@ -0,0 +1,2 @@
+require 'etl/generator/generator'
+Dir[File.dirname(__FILE__) + "/generator/*.rb"].each { |file| require(file) }

data/lib/etl/generator/generator.rb

@@ -0,0 +1,20 @@
+module ETL #:nodoc:
+  module Generator #:nodoc:
+    # Base class for generators.
+    class Generator
+      class << self
+        # Get the Class for the specified name.
+        #
+        # For example, if name is :surrogate_key then a SurrogateKeyGenerator class is returned
+        def class_for_name(name)
+          ETL::Generator.const_get("#{name.to_s.camelize}Generator")
+        end
+      end
+      
+      # Generate the next value. This method must be implemented by subclasses
+      def next
+        raise "Must be implemented by a subclass"
+      end
+    end
+  end
+end

data/lib/etl/generator/surrogate_key_generator.rb

@@ -0,0 +1,39 @@
+# This source file contains code for a basic sequential surrogate key generator
+
+module ETL #:nodoc:
+  module Generator #:nodoc:
+    # Surrogate key generator.
+    class SurrogateKeyGenerator < Generator
+      attr_reader :table
+      attr_reader :target
+      attr_reader :column
+      attr_reader :query
+      
+      # Initialize the generator
+      def initialize(options={})
+        @table = options[:table]
+        @target = options[:target]
+        @column = options[:column] || 'id'
+        @query = options[:query]
+        
+        if table
+          @surrogate_key = ETL::Engine.connection(target).select_value("SELECT max(#{column}) FROM #{table_name}")
+        elsif query
+          @surrogate_key = ETL::Engine.connection(target).select_value(query)
+        end
+        @surrogate_key = 0 if @surrogate_key.blank?
+        @surrogate_key = @surrogate_key.to_i
+      end
+
+      # Get the next surrogate key
+      def next
+        @surrogate_key ||= 0
+        @surrogate_key += 1
+      end
+      
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end

data/lib/etl/http_tools.rb

@@ -0,0 +1,139 @@
+require 'uri'
+
+# Module which has utility methods for HTTP.
+module HttpTools
+  # Parse the given user agent string
+  #
+  # Code taken from http://gemtacular.com/gems/ParseUserAgent
+  def parse_user_agent(user_agent)
+    if '-' == user_agent
+      #raise 'Invalid User Agent'
+      #puts 'Invalid User Agent'
+    end
+    
+    browser, browser_version_major, browser_version_minor, ostype, os, os_version = nil
+
+    # fix Opera
+    #useragent =~ s/Opera (\d)/Opera\/$1/i;
+    useragent = user_agent.gsub(/(Opera [\d])/,'Opera\1')
+
+    # grab all Agent/version strings as 'agents'
+    agents = Array.new
+    user_agent.split(/\s+/).each {|string| 
+      if string =~ /\//
+        agents<< string
+      end
+    }
+
+    # cycle through the agents to set browser and version (MSIE is set later)
+    if agents && agents.length > 0
+        agents.each {|agent|
+          parts = agent.split('/')
+          browser = parts[0]
+          browser_version = parts[1]
+          if browser == 'Firefox'
+            browser_version_major = parts[1].slice(0,3)
+            browser_version_minor = parts[1].sub(browser_version_major,'').sub('.','')
+          elsif browser == 'Safari'
+            if parts[1].slice(0,3).to_f < 400
+              browser_version_major = '1'
+            else
+              browser_version_major = '2'
+            end
+          else
+            browser_version_major = parts[1].slice(0,1)
+          end
+        }
+    end
+
+    # grab all of the properties (within parens)
+    # should be in relation to the agent if possible  
+    detail = user_agent
+    user_agent.gsub(/\((.*)\)/,'').split(/\s/).each {|part| detail = detail.gsub(part,'')}
+    detail = detail.gsub('(','').gsub(')','').lstrip
+    properties = detail.split(/;\s+/)
+
+    # cycle through the properties to set known quantities
+    properties.each do |property| 
+      if property =~ /^Win/
+        ostype = 'Windows'
+        os = property
+        if parts = property.split(/ /,2)
+          if parts[1] =~ /^NT/
+            ostype = 'Windows'
+            subparts = parts[1].split(/ /,2)
+            if subparts[1] == '5'
+              os_version = '2000'
+            elsif subparts[1] == '5.1'
+              os_version = 'XP'
+            else
+              os_version = subparts[1]
+            end
+          end
+        end
+      end
+      if property == 'Macintosh'
+        ostype = 'Macintosh'
+        os = property
+      end
+      if property =~ /OS X/
+        ostype = 'Macintosh'
+        os_version = 'OS X'
+        os = property
+      end
+      if property =~ /^Linux/
+        ostype = 'Linux'
+        os = property
+      end
+      if property =~ /^MSIE/
+        browser = 'MSIE'
+        browser_version = property.gsub('MSIE ','').lstrip
+        browser_version_major,browser_version_minor = browser_version.split('.')
+      end
+    end
+    
+    result = {
+      :browser => browser, 
+      :browser_version_major => browser_version_major, 
+      :browser_version_minor => browser_version_minor, 
+      :ostype => ostype, 
+      :os_version => os_version,
+      :os => os,
+    }
+    result.each do |key, value|
+      result[key] = value.blank? ? nil : value.strip
+    end
+    result
+  end
+  
+  # Parse a URI. If options[:prefix] is set then prepend it to the keys for the hash that
+  # is returned.
+  def parse_uri(uri_string, options={})
+    prefix = options[:prefix] ||= ''
+    empty_hash = {
+      "#{prefix}scheme".to_sym => nil, 
+      "#{prefix}host".to_sym => nil, 
+      "#{prefix}port".to_sym => nil, 
+      "#{prefix}uri_path".to_sym => nil, 
+      "#{prefix}domain".to_sym => nil
+    }
+    if uri_string
+      #attempt to parse uri --if it's a uri then catch the problem and set everything to nil
+      begin
+        uri = URI.parse(uri_string)    
+        results = {
+          "#{prefix}scheme".to_sym => uri.scheme, 
+          "#{prefix}host".to_sym => uri.host, 
+          "#{prefix}port".to_sym => uri.port, 
+          "#{prefix}uri_path".to_sym => uri.path
+        }
+        results["#{prefix}domain".to_sym] = $1 if uri.host =~ /\.?([^\.]+\.[^\.]+$)/
+        results
+      rescue
+        empty_hash
+      end
+    else
+      empty_hash
+    end
+  end
+end

data/lib/etl/parser.rb

@@ -0,0 +1,11 @@
+# This source file contains the ETL::Parser module and requires all of the files
+# in the parser directory ending with .rb
+
+module ETL #:nodoc:
+  # The ETL::Parser module provides various text parsers.
+  module Parser
+  end
+end
+
+require 'etl/parser/parser'
+Dir[File.dirname(__FILE__) + "/parser/*.rb"].each { |file| require(file) }

data/lib/etl/parser/apache_combined_log_parser.rb

@@ -0,0 +1,49 @@
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # Parser which can parser the Apache Combined Log Format as defined at
+    # http://httpd.apache.org/docs/2.2/logs.html
+    class ApacheCombinedLogParser < ETL::Parser::Parser
+      include HttpTools
+      def initialize(source, options={})
+        super
+      end
+
+      def each
+        Dir.glob(file).each do |file|
+          File.open(file).each_line do |line|
+            yield parse(line)
+          end
+        end
+      end
+      
+      def parse(line)
+        # example line:  127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
+        line =~ /^(\S+)\s(\S+)\s(\S+)\s\[([^\]]*)\]\s"([^"]*)"\s(\d*)\s(\d*)\s"([^"]*)"\s"([^"]*)"$/
+        fields = {
+          :ip_address => $1,
+          :identd => $2,
+          :user => $3,
+          :timestamp => $4,
+          :request => $5,
+          :response_code => $6,
+          :bytes => $7,
+          :referrer => $8,
+          :user_agent => $9,
+        }
+        #fields[:timestamp] =~ r%{(\d\d)/(\w\w\w)/(\d\d\d\d):(\d\d):(\d\d):(\d\d) -(\d\d\d\d)}
+        d = Date._strptime(fields[:timestamp], '%d/%b/%Y:%H:%M:%S') unless fields[:timestamp].nil?
+        fields[:timestamp] = Time.mktime(d[:year], d[:mon], d[:mday], d[:hour], d[:min], d[:sec], d[:sec_fraction]) unless d.nil?
+        
+        fields[:method], fields[:path] = fields[:request].split(/\s/)
+
+        fields.merge!(parse_user_agent(fields[:user_agent])) unless fields[:user_agent].nil?
+        fields.merge!(parse_uri(fields[:referrer], :prefix => 'referrer_'))
+        
+        fields.each do |key, value|
+          fields[key] = nil if value == '-'
+        end
+      end
+      
+    end
+  end
+end

data/lib/etl/parser/csv_parser.rb

@@ -0,0 +1,93 @@
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # Parses CSV files
+    class CsvParser < ETL::Parser::Parser
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Hash of options for the parser, defaults to an empty hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      def get_fields_names(file)
+        File.open(file) do |input|
+          fields = CSV.parse(input.readline, options).first
+          new_fields = []
+          fields.each_with_index do |field,index|
+            # compute the index of occurrence of this specific occurrence of the field (usually, will be 1)
+            occurrence_index = fields[0..index].find_all { |e| e == field }.size
+            number_of_occurrences = fields.find_all { |e| e == field }.size
+            new_field = field + (number_of_occurrences > 1 ? "_#{occurrence_index}" : "")
+            new_fields << Field.new(new_field.to_sym)
+          end
+          return new_fields
+        end
+      end
+
+      # Returns each row.
+      def each
+        Dir.glob(file).each do |file|
+          ETL::Engine.logger.debug "parsing #{file}"
+          if fields.length == 0
+            ETL::Engine.logger.debug "no columns specified so reading names from first line of #{file}"
+            @fields = get_fields_names(file)
+          end
+          line = 0
+          lines_skipped = 0
+          CSV.foreach(file, options) do |raw_row|
+            if lines_skipped < source.skip_lines
+              ETL::Engine.logger.debug "skipping line"
+              lines_skipped += 1
+              next
+            end
+            line += 1
+            row = {}
+            validate_row(raw_row, line, file)
+            raw_row.each_with_index do |value, index|
+              f = fields[index]
+              row[f.name] = value
+            end
+            yield row
+          end
+        end
+      end
+      
+      # Get an array of defined fields
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def validate_row(row, line, file)
+        ETL::Engine.logger.debug "validating line #{line} in file #{file}"
+        if row.length != fields.length
+          raise_with_info( MismatchError, 
+            "The number of columns from the source (#{row.length}) does not match the number of columns in the definition (#{fields.length})", 
+            line, file
+          )
+        end
+      end
+      
+      def configure
+        source.definition.each do |options|
+          case options
+          when Symbol
+            fields << Field.new(options)
+          when Hash
+            fields << Field.new(options[:name])
+          else
+            raise DefinitionError, "Each field definition must either be a symbol or a hash"
+          end
+        end
+      end
+      
+      class Field #:nodoc:
+        attr_reader :name
+        def initialize(name)
+          @name = name
+        end
+      end
+    end
+  end
+end