universal-access-log-parser 1.0.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,13 @@
+source "http://rubygems.org"
+
+gem "ruby-ip", ">= 0"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "rspec", "~> 2.3.0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  gem "rcov", ">= 0"
+end
+

data/Gemfile.lock

@@ -0,0 +1,30 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    git (1.2.5)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    rake (0.9.2)
+    rcov (0.9.10)
+    rspec (2.3.0)
+      rspec-core (~> 2.3.0)
+      rspec-expectations (~> 2.3.0)
+      rspec-mocks (~> 2.3.0)
+    rspec-core (2.3.1)
+    rspec-expectations (2.3.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.3.0)
+    ruby-ip (0.9.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.6.4)
+  rcov
+  rspec (~> 2.3.0)
+  ruby-ip

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Jakub Pastuszek
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,98 @@
+= universal-access-log-parser
+
+== Installation
+
+    gem install universal-access-log-parser
+
+== Usage
+
+    require 'universal-access-log-parser'
+
+    # use predefined parser
+    parser = UniversalAccessLogParser.apache_combined
+
+    # or extend it inline
+    parser = UniversalAccessLogParser.new do
+        # reuse predefined element set
+        apache_combined
+
+        # add your own
+        string :varnish
+        string :varnish_status, :nil_on => '-'
+        string :initial_varnish_status, :nil_on => '-'
+        integer :cache_hits
+        integer :cache_ttl, :nil_on => '-'
+        integer :cache_age
+    end
+
+    # or define new parser
+    UniversalAccessLogParser.parser(:iis) do
+        skip_line '^#'
+        date_iis :time
+        ip :server_ip
+        string :method
+        string :url
+        string :query, :nil_on => '-'
+        integer :port
+        string :username, :nil_on => '-'
+        ip :client_ip
+        string :user_agent, :nil_on => '-', :process => lambda{|s| s.tr('+', ' ')}
+        integer :status
+        integer :substatus
+        integer :win32_status
+        integer :duration, :process => lambda{|i| i.to_f / 1000}
+    end
+    parser = UniversalAccessLogParser.iis
+
+    # and iterate entries with #each - won't raise errors
+    stats = parser.parse_file('access.log').each |entry|
+        puts entry.time
+        puts entry.cache_age
+    end
+
+    # and get parsing stats
+    puts stats.failures
+    puts stats.successes	
+
+    # or wait for exception with #each!
+    parser.parse_file('access.log').each! |entry|
+        puts entry.time
+        puts entry.cache_age
+    end # will raise UniversalAccessLogParser::ParsingError on line parsing error
+
+    # data elements wont be parsed until accessed, so if you are not interested in some elements you won't waste time
+    stats = parser.parse_file('access.log').each |entry|
+        # entry.time not parsed yet - Time object is not created
+        puts entry.time # this will parse time and create Time object - this may raise UniversalAccessLogParser::ElementParsingError!
+        puts entry.time # now Time object is returned from cache
+        puts entry.cache_age
+
+        # parse all elements
+        entry.parse!
+
+        # this will also parse all elements and return hash map of them
+        entry.to_hash 
+    end
+
+    # iterate and parse all data with #each_parsed! - if this won't raise, all log lines are parsing fine including elements
+    parser.parse_file('access.log').each_parsed! |entry|
+        puts entry.time # already in cache
+        puts entry.cache_age # already in cache
+    end # will raise on line and element parsing error - try rescuing UniversalAccessLogParser::ParserError to catch both
+
+== Contributing to universal-access-log-parser
+
+* Please add more common parsers to lib/common_parsers.rb and rspec in spec/common_parsers_spec.rb or send me a gist
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Jakub Pastuszek. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,49 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "universal-access-log-parser"
+  gem.homepage = "http://github.com/jpastuszek/universal-access-log-parser"
+  gem.license = "MIT"
+  gem.summary = 'Define your own access log parser of reuse/extend predefined common parsers'
+  gem.description = 'Meta parser that allows you to define new parser with ruby DSL to match access log file format you are willing to parse or use one of the predefined parsers like Apache common, IIS and more'
+  gem.email = "jpastuszek@gmail.com"
+  gem.authors = ["Jakub Pastuszek"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "universal-access-log-parser #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/lib/common_parsers.rb

@@ -0,0 +1,66 @@
+UniversalAccessLogParser.parser(:apache_common) do
+	ip :remote_host
+	string :logname, :nil_on => '-'
+	string :user, :nil_on => '-'
+	surrounded_by '\[', '\]' do
+		date_ncsa :time
+	end
+	double_quoted do
+		optional :first_request_line do
+			string :method, :nil_on => ''
+			string :uri, :nil_on => ''
+			string :protocol, :nil_on => ''
+		end
+	end
+	integer :status
+	integer :response_size, :nil_on => '-'
+end
+
+UniversalAccessLogParser.parser(:apache_vhost_common) do
+	string :vhost
+	apache_common
+end
+
+UniversalAccessLogParser.parser(:apache_combined) do
+	apache_common
+	double_quoted do
+		string :referer, :nil_on => '-'
+	end
+	double_quoted do
+		string :user_agent, :nil_on => '-'
+	end
+end
+
+UniversalAccessLogParser.parser(:apache_referer) do
+	separated_with ' -> ' do
+		string :referer, :nil_on => '-'
+		string :url
+	end
+end
+
+UniversalAccessLogParser.parser(:apache_user_agent) do
+	string :user_agent, :nil_on => '-', :greedy => false
+end
+
+UniversalAccessLogParser.parser(:icecast) do
+	apache_combined
+	integer :duration, :nil_on => '-'
+end
+
+UniversalAccessLogParser.parser(:iis) do
+	skip_line '^#'
+	date_iis :time
+	ip :server_ip
+	string :method
+	string :url
+	string :query, :nil_on => '-'
+	integer :port
+	string :username, :nil_on => '-'
+	ip :client_ip
+	string :user_agent, :nil_on => '-', :process => lambda{|s| s.tr('+', ' ')}
+	integer :status
+	integer :substatus
+	integer :win32_status
+	integer :duration, :process => lambda{|i| i.to_f / 1000}
+end
+

data/lib/universal-access-log-parser.rb

@@ -0,0 +1,418 @@
+require 'ip'
+
+class UniversalAccessLogParser
+	class ParserError < ArgumentError
+	end
+
+	class ParsingError < ParserError
+		def initialize(msg, parser, line)
+			@parser = parser
+			@line = line
+			super(msg)
+		end
+
+		attr_reader :parser, :line
+	end
+
+	class ElementParsingError < ParserError
+		def initialize(e)
+			@error = e
+			super("argument parsing error: #{e}")
+		end
+
+		attr_reader :error
+	end
+
+	class ElementGroup < Array
+		class Element
+			def initialize(name, regexp, nil_on = nil)
+				@name = name
+				@regexp = regexp
+				@nil_on = nil_on
+				@parser = lambda{|s|
+					return nil if @nil_on and s == @nil_on
+					yield s if block_given?
+				}
+			end
+
+			attr_reader :name, :parser
+
+			def regexp
+				return "(#{@nil_on}|#{@regexp})" if @nil_on
+				"(#{@regexp})"
+			end
+		end
+
+		class Integrating < ElementGroup
+			def initialize(parent, separator, &block)
+				@separator = separator
+				super(parent, &block)
+			end
+
+			attr_reader :separator
+		end
+
+		class Root < Integrating
+			def initialize(separator, &block)
+				@skip_lines = []
+				super(nil, separator, &block)
+			end
+
+			attr_reader :skip_lines
+
+			def regexp
+				super + "(|#{separator}.*)"
+			end
+
+			def names
+				super << :other
+			end
+
+			def parsers
+				super << lambda{ |s|
+					return nil if s.empty?
+					s.sub(Regexp.new("^#{separator}"), '')
+				}
+			end
+
+			# root specific DSL
+			def skip_line(regexp)
+				@skip_lines << regexp
+			end
+		end
+
+		class Surrounding < ElementGroup
+			def initialize(parent, left, right, &block)
+				@left = left
+				@right = right
+				super(parent, &block)
+			end
+
+			def regexp
+				@left + super + @right
+			end
+		end
+
+		class Optional < ElementGroup
+			def initialize(parent, name, options = {}, &block)
+				@group_name = name
+				@nil_on = options[:nil_on]
+				super(parent, &block)
+			end
+
+			def regexp
+				'(' + super + '|.*?)'
+			end
+
+			def names
+				super.unshift @group_name
+			end
+
+			def parsers
+				if @nil_on
+					super.unshift lambda{ |s| s == @nil_on ? nil : s }
+				else
+					super.unshift lambda{ |s| s}
+				end
+			end
+		end
+
+		def initialize(parent, &block)
+			@parent = parent
+			instance_eval &block
+		end
+
+		# custom parser definition
+		def self.parser(name, &block)
+			define_method(name, &block)
+		end
+
+		def separator
+			raise ParsingError, 'Integrating ElementGroup not defined in ElementGroup hierarhy' unless @parent
+			@parent.separator
+		end
+
+		def regexp
+			map{|e| e.regexp}.join(separator)
+		end
+		
+		def names
+			map do |e|
+				if e.kind_of? ElementGroup
+					e.names
+				else
+					e.name
+				end
+			end.flatten
+		end
+
+		def parsers
+			map do |e|
+				if e.kind_of? ElementGroup
+					e.parsers
+				else
+					e.parser
+				end
+			end.flatten
+		end
+
+		# core DSL
+		def integratin_group(separator, &block)
+			push ElementGroup::Integrating.new(self, separator, &block)
+		end
+
+		def surrounding_group(left, right, &block)
+			push ElementGroup::Surrounding.new(self, left, right, &block)
+		end
+
+		def optional(name, options = {}, &block)
+			push ElementGroup::Optional.new(self, name, options, &block)
+		end
+
+		def element(name, regexp, options = {}, &parser)
+			nil_on = options[:nil_on]
+			process = options[:process]
+			if process
+				p = lambda{|s| process.call(parser.call(s))}
+			else
+				p = parser 
+			end
+			push Element.new(name, regexp, nil_on, &p)
+		end
+
+		# DSL
+		def separated_with(separator, &block)
+			integratin_group(separator, &block)
+		end
+
+		def surrounded_by(left, right, &block)
+			surrounding_group(left, right, &block)
+		end
+
+		def single_quoted(&block)
+			surrounded_by("'", "'", &block)
+		end
+
+		def double_quoted(&block)
+			surrounded_by('"', '"', &block)
+		end
+
+		def date_ncsa(name, options = {})
+			date(name, '%d/%b/%Y:%H:%M:%S %z', options)
+		end
+
+		def date_iis(name, options = {})
+			date(name, '%Y-%m-%d %H:%M:%S', options)
+		end
+
+		def date(name, format = '%d/%b/%Y:%H:%M:%S %z', options = {})
+			regex = Regexp.escape(format).gsub(/%./, '.+').gsub(/\//, '\\/') + '?'
+			element(name, regex, options) do |match|
+				DateTime.strptime(match, format).new_offset(0).instance_eval do
+					Time.utc(year, mon, mday, hour, min, sec + sec_fraction)
+				end
+			end
+		end
+
+		def ip(name, options = {})
+			greedy = true
+			greedy = options[:greedy] if options.member? :greedy
+			element(name, ".*#{greedy ? '?' : ''}", options){|s| IP.new(s)}
+		end
+
+		def integer(name, options = {})
+			element(name, '[\+|-]?\d+', options){|s| s.to_i}
+		end
+
+		def float(name, options = {})
+			element(name, '[\+|-]?\d+\.?\d*', options){|s| s.to_f}
+		end
+
+		def string(name, options = {})
+			greedy = true
+			greedy = options[:greedy] if options.member? :greedy
+			element(name, ".*#{greedy ? '?' : ''}", options){|s| s}
+		end
+	end
+
+	class EntryIterator
+		class Stats < Struct.new(:failures, :successes)
+		end
+
+		def initialize(parser, io, close_io)
+			@parser = parser
+			@io = io
+			@close_io = close_io
+		end
+
+		def each
+			failures = 0
+			successes = 0
+
+			@io.each_line do |line|
+				begin
+					line.strip!
+					next if @parser.skip?(line)
+					yield @parser.parse(line.strip)
+					successes += 1
+				rescue ParsingError
+					failures += 1
+				end
+			end
+			@io.close if @close_io
+			Stats.new(failures, successes)
+		end
+
+		def each!
+			begin
+				@io.each_line do |line|
+					line.strip!
+					next if @parser.skip?(line)
+					yield  @parser.parse(line.strip)
+				end
+			ensure
+				@io.close if @close_io
+			end
+		end
+
+		def each_parsed!
+			begin
+				@io.each_line do |line|
+					line.strip!
+					next if @parser.skip?(line)
+					yield @parser.parse(line.strip).parse!
+				end
+			ensure
+				@io.close if @close_io
+			end
+		end
+
+		def close
+			@io.close
+		end
+	end
+
+	# just so parsed log line class can be tested and named
+	class ParsedLogLine
+	end
+
+	def initialize(&block)
+		@@parser_id ||= 0
+		@@parser_id += 1
+
+		@elements = ElementGroup::Root.new(' ', &block)
+
+		@skip_lines = @elements.skip_lines.map{|s| Regexp.new(s)}
+		@regexp = Regexp.new('^' + @elements.regexp + '$')
+
+		@names = @elements.names
+
+		@parsers = {}
+		@names.zip(@elements.parsers).each do |name, parser|
+			@parsers[name] = parser
+		end
+
+		@parsed_log_entry_class = Class.new(ParsedLogLine) do
+			def self.name
+				superclass.name
+			end
+
+			def self.make_metods(names)
+				names.each do |name|
+					class_eval """
+						def #{name}
+							return @cache[:#{name}] if @cache.member? :#{name}
+							begin
+								value = @parsers[:#{name}].call(@strings[:#{name}])
+							rescue => e
+								raise ElementParsingError.new(e)
+							end
+							@cache[:#{name}] = value
+							value
+						end
+					"""
+				end
+			end
+
+			def initialize(names, parsers, strings)
+				@parsers = parsers
+
+				@strings = {}
+				names.zip(strings).each do |name, string|
+					@strings[name] = string
+				end
+
+				@cache = {}
+			end
+
+			def parse!
+				@strings.keys.each do |name|
+					send(name)
+				end
+				self
+			end
+
+			def to_hash
+				parse!
+				@cache
+			end
+
+			def inspect
+				hash = @cache.dup
+				@strings.keys.each do |name|
+					hash[name] = '<unparsed>' unless hash.member? name
+				end
+				"#<#{self.class.name}: #{hash.keys.map{|s| s.to_s}.sort.map{|name| "#{name}: #{hash[name.to_sym].inspect}"}.join(', ')}>"
+			end
+
+			def to_s
+				"#<#{self.class.name}:#{object_id}>"
+			end
+		end
+
+		@parsed_log_entry_class.make_metods(@names)
+	end
+
+	# custom parser definition
+	def self.parser(name, &block)
+		ElementGroup.parser(name, &block)
+
+		eval """
+			def self.#{name}
+					self.new{ #{name} }
+			end
+		"""
+	end
+
+	def skip?(line)
+		@skip_lines.each do |regexp|
+			return true if line =~ regexp
+		end
+		return false
+	end
+
+	def parse(line)
+		matched, *strings = @regexp.match(line).to_a
+
+		raise ParsingError.new('parser regexp did not match log line', self, line) if strings.empty?
+
+		@parsed_log_entry_class.new(@names, @parsers, strings)
+	end
+
+	def parse_io(io, close_io = false)
+		EntryIterator.new(self, io, close_io)
+	end
+
+	def parse_file(file_path)
+		io = File.open(file_path)
+		# io will be closed after each
+		parse_io(io, true)
+	end
+
+	def inspect
+		"#<#{self.class.name}:#{@regexp.inspect} => #{@elements.names.join(' ')}>"
+	end
+end
+
+require 'common_parsers'
+