iis-access-log-parser 1.1.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,16 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "rspec", ">= 2.3.0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.2"
+  gem "rcov", ">= 0"
+end
+
+gem "ruby-ip", ">= 0"
+

data/Gemfile.lock

@@ -0,0 +1,30 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.2)
+    git (1.2.5)
+    jeweler (1.6.2)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    rake (0.9.2)
+    rcov (0.9.9)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    ruby-ip (0.9.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.6.2)
+  rcov
+  rspec (>= 2.3.0)
+  ruby-ip

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Jakub Pastuszek
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,19 @@
+= iis-access-log-parser
+
+Description goes here.
+
+== Contributing to iis-access-log-parser
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Jakub Pastuszek. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,49 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "iis-access-log-parser"
+  gem.homepage = "http://github.com/jpastuszek/iis-access-log-parser"
+  gem.license = "MIT"
+  gem.summary = "Parser for IIS access logs"
+  gem.description = "Class that can read ISS log file and iterate over parsed log entries."
+  gem.email = "jpastuszek@gmail.com"
+  gem.authors = ["Jakub Pastuszek"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "iis-access-log-parser #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+1.1.0

data/iis-access-log-parser.gemspec

@@ -0,0 +1,65 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{iis-access-log-parser}
+  s.version = "1.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Jakub Pastuszek"]
+  s.date = %q{2011-06-22}
+  s.description = %q{Class that can read ISS log file and iterate over parsed log entries.}
+  s.email = %q{jpastuszek@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "iis-access-log-parser.gemspec",
+    "lib/iis-access-log-parser.rb",
+    "spec/data/short.log",
+    "spec/iis-access-log-parser_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/jpastuszek/iis-access-log-parser}
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Parser for IIS access logs}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<ruby-ip>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, [">= 2.3.0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.2"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<ruby-ip>, [">= 0"])
+      s.add_dependency(%q<rspec>, [">= 2.3.0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.2"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<ruby-ip>, [">= 0"])
+    s.add_dependency(%q<rspec>, [">= 2.3.0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.2"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+

data/lib/iis-access-log-parser.rb

@@ -0,0 +1,51 @@
+require 'ip'
+require 'time'
+
+class IISAccessLogParser
+	#Fields: date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) sc-status sc-substatus sc-win32-status time-taken
+	Entry = Struct.new(:date, :server_ip, :method, :url, :query, :port, :username, :client_ip, :user_agent, :status, :substatus, :win32_status, :time_taken, :other)
+	class Entry
+		def self.from_string(line)
+			# 2011-06-20 00:00:00 83.222.242.43 GET /SharedControls/getListingThumbs.aspx img=48,13045,27801,25692,35,21568,21477,21477,10,18,46,8&premium=0|1|0|0|0|0|0|0|0|0|0|0&h=100&w=125&pos=175&scale=true 80 - 92.20.10.104 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+6.1;+Trident/4.0;+GTB6.6;+SLCC2;+.NET+CLR+2.0.50727;+.NET+CLR+3.5.30729;+.NET+CLR+3.0.30729;+Media+Center+PC+6.0;+aff-kingsoft-ciba;+.NET4.0C;+MASN;+AskTbSTC/5.8.0.12304) 200 0 0 609
+
+			x, date, server_ip, method, url, query, port, username, client_ip, user_agent, status, substatus, win32_status, time_taken, y, other = *line.match(/^([^ ]* [^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*)($| )(.*)/)
+			raise ArgumentError, "bad format: '#{line}'" unless x
+
+			date = Time.parse(date + ' UTC')
+
+			server_ip = IP.new(server_ip)
+			client_ip = IP.new(client_ip)
+
+			port = port.to_i
+			status = status.to_i
+			substatus = substatus.to_i
+			win32_status = win32_status.to_i
+
+			time_taken = time_taken.to_f / 1000
+
+			query = nil if query == '-'
+			username = nil if username == '-'
+			user_agent = nil if user_agent == '-'
+
+			user_agent.tr!('+', ' ') unless user_agent.nil?
+
+			self.new(date, server_ip, method, url, query, port, username, client_ip, user_agent, status, substatus, win32_status, time_taken, other)
+		end
+	end
+
+	def self.from_file(log_file)
+		File.open(log_file, 'r') do |io|
+			self.new(io) do |entry|
+				yield entry
+			end
+		end
+	end
+
+	def initialize(io)
+		io.each_line do |line|
+			next if line[0,1] == '#'
+			yield Entry.from_string(line)
+		end
+	end
+end
+

data/spec/data/short.log

@@ -0,0 +1,10 @@
+#Software: Microsoft Internet Information Services 7.5
+#Version: 1.0
+#Date: 2011-06-18 00:00:00
+#Fields: date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) sc-status sc-substatus sc-win32-status time-taken
+2011-06-18 00:00:00 38.111.242.43 GET /blahs/bulgaria/teeth-whitening/map/index.aspx - 80 - 66.249.72.231 Mediapartners-Google 200 0 0 343
+2011-06-18 00:00:00 38.111.242.43 GET /physiotherapy/uk/lancashire/bryn/peripheral-joint-manipulation/index.aspx - 80 - 66.249.72.16 Mozilla/5.0+(compatible;+Googlebot/2.1;++http://www.google.com/bot.html) 200 0 0 609
+2011-06-18 00:00:01 38.111.242.43 GET /blahs/canada/ontario/kawartha-lakes/bone-graft/index.aspx - 80 - 66.249.72.231 Mozilla/5.0+(compatible;+Googlebot/2.1;++http://www.google.com/bot.html) 200 0 0 4296
+2011-06-18 00:00:01 38.111.242.43 GET /traditional-chinese-medicine-practitioners/ireland/county-westmeath/traditional-chinese-medicine-consultation/index.aspx - 80 - 66.249.72.204 Mozilla/5.0+(compatible;+Googlebot/2.1;++http://www.google.com/bot.html) 301 0 0 187
+2011-06-18 00:00:01 38.111.242.43 GET /blahs/canada/ontario/pickering/sedation-dental/index.aspx - 80 - 66.249.72.16 Mozilla/5.0+(compatible;+Googlebot/2.1;++http://www.google.com/bot.html) 200 0 0 2640
+2011-06-18 00:00:01 38.111.242.43 GET /blahs/uk/leicestershire/little-bowden/invisalign/index.aspx - 80 - 87.250.252.241 Mozilla/5.0+(compatible;+YandexBot/3.0;++http://yandex.com/bots) 200 0 0 5656

data/spec/iis-access-log-parser_spec.rb

@@ -0,0 +1,68 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "IISAccessLogParser::Entry" do
+	it "can be constructed from string" do
+		IISAccessLogParser::Entry.from_string("2011-06-20 00:00:00 38.111.242.43 GET /SharedControls/getListingThumbs.aspx img=48,13045,27801,25692,35,21568,21477,21477,10,18,46,8&premium=0|1|0|0|0|0|0|0|0|0|0|0&h=100&w=125&pos=175&scale=true 80 - 92.20.10.104 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+6.1;+Trident/4.0;+GTB6.6;+SLCC2;+.NET+CLR+2.0.50727;+.NET+CLR+3.5.30729;+.NET+CLR+3.0.30729;+Media+Center+PC+6.0;+aff-kingsoft-ciba;+.NET4.0C;+MASN;+AskTbSTC/5.8.0.12304) 200 0 0 609")
+
+		IISAccessLogParser::Entry.from_string("2011-06-20 00:00:00 38.111.242.43 GET /SharedControls/getListingThumbs.aspx img=48,13045,27801,25692,35,21568,21477,21477,10,18,46,8&premium=0|1|0|0|0|0|0|0|0|0|0|0&h=100&w=125&pos=175&scale=true 80 - 92.20.10.104 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+6.1;+Trident/4.0;+GTB6.6;+SLCC2;+.NET+CLR+2.0.50727;+.NET+CLR+3.5.30729;+.NET+CLR+3.0.30729;+Media+Center+PC+6.0;+aff-kingsoft-ciba;+.NET4.0C;+MASN;+AskTbSTC/5.8.0.12304) 200 0 0 609 test oter now")
+
+		lambda{ IISAccessLogParser::Entry.from_string("2011-06-20 00:00:00 38.111.242.43 GET /SharedControls/getListingThumbs.aspx img=48,13045,27801,25692,35,21568,21477,21477,10,18,46,8&premium=0|1|0|0|0|0|0|0|0|0|0|0&h=100&w=125&pos=175&scale=true 80 - 92.20.10.104 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+6.1;+Trident/4.0;+GTB6.6;+SLCC2;+.NET+CLR+2.0.50727;+.NET+CLR+3.5.30729;+.NET+CLR+3.0.30729;+Media+Center+PC+6.0;+aff-kingsoft-ciba;+.NET4.0C;+MASN;+AskTbSTC/5.") }.should raise_error ArgumentError
+	end
+
+	it "should contain properly parsed date types" do
+		e = IISAccessLogParser::Entry.from_string("2011-06-20 00:01:02 38.111.242.43 GET /SharedControls/getListingThumbs.aspx img=48,13045,27801,25692,35,21568,21477,21477,10,18,46,8&premium=0|1|0|0|0|0|0|0|0|0|0|0&h=100&w=125&pos=175&scale=true 80 - 92.20.10.104 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+6.1;+Trident/4.0;+GTB6.6;+SLCC2;+.NET+CLR+2.0.50727;+.NET+CLR+3.5.30729;+.NET+CLR+3.0.30729;+Media+Center+PC+6.0;+aff-kingsoft-ciba;+.NET4.0C;+MASN;+AskTbSTC/5.8.0.12304) 200 0 0 609 test oter now")
+
+		e.date.should be_kind_of Time
+		e.date.day.should == 20
+		e.date.month.should == 6
+		e.date.year.should == 2011
+		e.date.hour.should == 0
+		e.date.min.should == 1
+		e.date.sec.should == 2
+		e.date.zone == 'UTC'
+
+		e.server_ip.should be_kind_of IP::V4
+		e.client_ip.should be_kind_of IP::V4
+
+		e.port.should be_kind_of Fixnum
+		e.status.should be_kind_of Fixnum
+		e.substatus.should be_kind_of Fixnum
+		e.win32_status.should be_kind_of Fixnum
+
+		e.time_taken.should be_kind_of Float
+		e.time_taken.should == 0.609
+
+		e.username.should be_nil
+
+		e.user_agent.should == "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB6.6; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; aff-kingsoft-ciba; .NET4.0C; MASN; AskTbSTC/5.8.0.12304)"
+	end
+
+	it "should handle correctly nil useragent" do
+		e = IISAccessLogParser::Entry.from_string("2011-06-20 00:01:02 38.111.242.43 GET /SharedControls/getListingThumbs.aspx img=48,13045,27801,25692,35,21568,21477,21477,10,18,46,8&premium=0|1|0|0|0|0|0|0|0|0|0|0&h=100&w=125&pos=175&scale=true 80 - 92.20.10.104 - 200 0 0 609 test oter now")
+
+		e.user_agent.should be_nil
+	end
+end
+
+describe "IISAccessLogParser" do
+	it "should read entries for IO" do
+		last = nil
+		File.open('spec/data/short.log', 'r') do |io|
+			IISAccessLogParser.new(io) do |entry|
+				last = entry
+			end
+		end
+
+		last.url.should == '/blahs/uk/leicestershire/little-bowden/invisalign/index.aspx'
+	end
+
+	it "should allow reading entries from file" do
+		last = nil
+		IISAccessLogParser.from_file('spec/data/short.log') do |entry|
+			last = entry
+		end
+
+		last.url.should == '/blahs/uk/leicestershire/little-bowden/invisalign/index.aspx'
+	end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'iis-access-log-parser'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+  
+end

metadata

@@ -0,0 +1,155 @@
+--- !ruby/object:Gem::Specification 
+name: iis-access-log-parser
+version: !ruby/object:Gem::Version 
+  hash: 19
+  prerelease: false
+  segments: 
+  - 1
+  - 1
+  - 0
+  version: 1.1.0
+platform: ruby
+authors: 
+- Jakub Pastuszek
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-06-22 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  name: ruby-ip
+  requirement: *id001
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 2
+        - 3
+        - 0
+        version: 2.3.0
+  name: rspec
+  requirement: *id002
+  type: :development
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  name: bundler
+  requirement: *id003
+  type: :development
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 1
+        - 6
+        - 2
+        version: 1.6.2
+  name: jeweler
+  requirement: *id004
+  type: :development
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  name: rcov
+  requirement: *id005
+  type: :development
+description: Class that can read ISS log file and iterate over parsed log entries.
+email: jpastuszek@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- .document
+- .rspec
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- iis-access-log-parser.gemspec
+- lib/iis-access-log-parser.rb
+- spec/data/short.log
+- spec/iis-access-log-parser_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/jpastuszek/iis-access-log-parser
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Parser for IIS access logs
+test_files: []
+