vizi_tracker 0.1.0

data/log/system.log

@@ -0,0 +1,66 @@
+# Logfile created on 2011-06-05 21:46:30 -0400 by logger.rb/25413
+I, [2011-06-05T21:46:30.867462 #3444]  INFO -- : starting ... >>> 2011-06-05 21:46:30 -0400
+I, [2011-06-05T21:46:32.576065 #3444]  INFO -- : Record count is 5458
+I, [2011-06-05T21:46:32.576065 #3444]  INFO -- : Hit count is 5438
+I, [2011-06-05T21:46:32.576065 #3444]  INFO -- : Page count is 506
+I, [2011-06-05T21:46:32.576065 #3444]  INFO -- : Total visit count is 461
+I, [2011-06-05T21:46:32.576065 #3444]  INFO -- : Human visit count is 119
+I, [2011-06-05T21:46:32.576065 #3444]  INFO -- : Drop visit count is 78
+I, [2011-06-05T21:46:32.576065 #3444]  INFO -- : Spider visit count is 264
+I, [2011-06-05T21:46:32.576065 #3444]  INFO -- : Batch processing time 1.677403
+I, [2011-06-05T21:46:32.576065 #3444]  INFO -- : ending ... >>> 2011-06-05 21:46:32 -0400
+I, [2011-06-05T21:46:53.028704 #5108]  INFO -- : starting ... >>> 2011-06-05 21:46:53 -0400
+I, [2011-06-05T21:47:06.719429 #5108]  INFO -- : Record count is 30474
+I, [2011-06-05T21:47:06.719429 #5108]  INFO -- : Hit count is 30474
+I, [2011-06-05T21:47:06.719429 #5108]  INFO -- : Page count is 0
+I, [2011-06-05T21:47:06.719429 #5108]  INFO -- : Total visit count is 4484
+I, [2011-06-05T21:47:06.719429 #5108]  INFO -- : Human visit count is 4227
+I, [2011-06-05T21:47:06.719429 #5108]  INFO -- : Drop visit count is 0
+I, [2011-06-05T21:47:06.719429 #5108]  INFO -- : Spider visit count is 257
+I, [2011-06-05T21:47:06.719429 #5108]  INFO -- : Batch processing time 13.675125
+I, [2011-06-05T21:47:06.719429 #5108]  INFO -- : ending ... >>> 2011-06-05 21:47:06 -0400
+I, [2011-06-05T21:47:44.613806 #1112]  INFO -- : starting ... >>> 2011-06-05 21:47:44 -0400
+I, [2011-06-05T21:48:01.451639 #4252]  INFO -- : starting ... >>> 2011-06-05 21:48:01 -0400
+I, [2011-06-05T21:48:05.195847 #4252]  INFO -- : Record count is 11870
+I, [2011-06-05T21:48:05.195847 #4252]  INFO -- : Hit count is 11870
+I, [2011-06-05T21:48:05.195847 #4252]  INFO -- : Page count is 0
+I, [2011-06-05T21:48:05.195847 #4252]  INFO -- : Total visit count is 1
+I, [2011-06-05T21:48:05.195847 #4252]  INFO -- : Human visit count is 1
+I, [2011-06-05T21:48:05.195847 #4252]  INFO -- : Drop visit count is 0
+I, [2011-06-05T21:48:05.195847 #4252]  INFO -- : Spider visit count is 0
+I, [2011-06-05T21:48:05.195847 #4252]  INFO -- : Batch processing time 3.744208
+I, [2011-06-05T21:48:05.195847 #4252]  INFO -- : ending ... >>> 2011-06-05 21:48:05 -0400
+I, [2011-06-05T21:48:20.829574 #4544]  INFO -- : starting ... >>> 2011-06-05 21:48:20 -0400
+I, [2011-06-05T21:50:44.333188 #960]  INFO -- : starting ... >>> 2011-06-05 21:50:44 -0400
+I, [2011-06-05T21:51:56.322305 #4712]  INFO -- : starting ... >>> 2011-06-05 21:51:56 -0400
+I, [2011-06-05T21:53:42.440375 #4832]  INFO -- : starting ... >>> 2011-06-05 21:53:42 -0400
+I, [2011-06-05T22:07:49.079943 #4560]  INFO -- : starting ... >>> 2011-06-05 22:07:49 -0400
+I, [2011-06-05T22:07:49.184949 #4560]  INFO -- : Record count is 47
+I, [2011-06-05T22:07:49.184949 #4560]  INFO -- : Hit count is 42
+I, [2011-06-05T22:07:49.184949 #4560]  INFO -- : Page count is 0
+I, [2011-06-05T22:07:49.184949 #4560]  INFO -- : Total visit count is 1
+I, [2011-06-05T22:07:49.184949 #4560]  INFO -- : Human visit count is 1
+I, [2011-06-05T22:07:49.185949 #4560]  INFO -- : Drop visit count is 0
+I, [2011-06-05T22:07:49.185949 #4560]  INFO -- : Spider visit count is 0
+I, [2011-06-05T22:07:49.185949 #4560]  INFO -- : Batch processing time 0.105006
+I, [2011-06-05T22:07:49.185949 #4560]  INFO -- : ending ... >>> 2011-06-05 22:07:49 -0400
+I, [2011-06-05T22:08:53.515628 #560]  INFO -- : starting ... >>> 2011-06-05 22:08:53 -0400
+I, [2011-06-05T22:08:53.529629 #560]  INFO -- : Record count is 47
+I, [2011-06-05T22:08:53.529629 #560]  INFO -- : Hit count is 42
+I, [2011-06-05T22:08:53.529629 #560]  INFO -- : Page count is 0
+I, [2011-06-05T22:08:53.530629 #560]  INFO -- : Total visit count is 1
+I, [2011-06-05T22:08:53.530629 #560]  INFO -- : Human visit count is 1
+I, [2011-06-05T22:08:53.530629 #560]  INFO -- : Drop visit count is 0
+I, [2011-06-05T22:08:53.530629 #560]  INFO -- : Spider visit count is 0
+I, [2011-06-05T22:08:53.530629 #560]  INFO -- : Batch processing time 0.014
+I, [2011-06-05T22:08:53.531629 #560]  INFO -- : ending ... >>> 2011-06-05 22:08:53 -0400
+I, [2011-06-05T22:37:50.150958 #3476]  INFO -- : starting ... >>> 2011-06-05 22:37:50 -0400
+I, [2011-06-05T22:37:51.664045 #3476]  INFO -- : Record count is 5458
+I, [2011-06-05T22:37:51.665045 #3476]  INFO -- : Hit count is 5438
+I, [2011-06-05T22:37:51.665045 #3476]  INFO -- : Page count is 506
+I, [2011-06-05T22:37:51.665045 #3476]  INFO -- : Total visit count is 461
+I, [2011-06-05T22:37:51.665045 #3476]  INFO -- : Human visit count is 119
+I, [2011-06-05T22:37:51.665045 #3476]  INFO -- : Drop visit count is 78
+I, [2011-06-05T22:37:51.665045 #3476]  INFO -- : Spider visit count is 264
+I, [2011-06-05T22:37:51.665045 #3476]  INFO -- : Batch processing time 1.513087
+I, [2011-06-05T22:37:51.665045 #3476]  INFO -- : ending ... >>> 2011-06-05 22:37:51 -0400

data/test/parser_test.rb

@@ -0,0 +1,48 @@
+require File.expand_path(File.dirname(__FILE__) + '/test_helper')
+
+require 'uri'
+
+class ParserTest < Test::Unit::TestCase
+  
+  def test_parser_creation
+
+    parser = HttpLogParser.new
+    assert_not_nil parser
+    assert_equal 5, parser.formats.size
+
+    parser = HttpLogParser.new('%h %l %u %t \"%r\" %>s %b')    
+    assert_not_nil parser
+    assert_equal 1, parser.formats.size
+
+    parser = HttpLogParser.new({
+      :common => '%h %l %u %t \"%r\" %>s %b',
+      :common_with_virtual => '%v %h %l %u %t \"%r\" %>s %b',
+    })   
+    assert_not_nil parser
+    assert_equal 2, parser.formats.size
+
+  end
+
+  def test_simple_parsing
+
+    parser = HttpLogParser.new
+    assert_not_nil parser
+
+    parsed = parser.parse_line('111.222.333.444 - - [21/Apr/2010:01:02:03 +0000] "GET /some/url?some=parameter HTTP/1.1" 302 123 "http://somewhere.com" "Browser (Version 1.0)"')
+
+    assert_equal '111.222.333.444', parsed[:ip]
+    assert_equal '111.222.333.444', parsed[:domain]
+    assert_equal '-', parsed[:auth]
+    assert_equal '-', parsed[:username]
+    assert_equal '21/Apr/2010:01:02:03 +0000', parsed[:datetime]
+    assert_equal 'GET /some/url?some=parameter HTTP/1.1', parsed[:request]
+    assert_equal '302', parsed[:status]
+    assert_equal '123', parsed[:bytecount]
+    assert_equal 'http://somewhere.com', parsed[:referer]
+    assert_equal 'Browser (Version 1.0)', parsed[:user_agent]
+
+    assert_equal 11, parsed.size
+
+  end
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'test/unit'
+require "#{File.expand_path(File.dirname(__FILE__))}/../lib/http_log_parser.rb"
+begin; require 'ruby-debug'; rescue LoadError; end

data/testit.rb

@@ -0,0 +1,105 @@
+# This is a sample application that uses the Vizitracker gem classes
+# 
+# This application will open a web log file (with either an IIS or Apache format).
+# Each record will be parser, and Visit records will be created. When the visit
+# timeout duration has been reached, an output record will be generated.
+#
+# Author::    Al Kivi <al.kivi@vizitrax.com>
+
+# Use either the library version of the code
+require './lib/vizi_tracker'
+# Or replace the above line to reference the gem version, if installed
+# require 'vizi_tracker'
+
+require 'yaml'
+require 'logger'
+
+config = YAML.load_file("config/logger.yml")
+
+@@download_page_number = config["download_page_number"]
+@@visit_timeout = config["visit_timeout"]
+
+# Initialize the log parser
+parser = Vizi::LogParser.new(config["drop_ips"], config["spider_ips"], 
+  config["spider_names"], config["page_urls"], config["hide_urls"], 
+  config["homepage"], config["accept_only_homepage"],config["hostname"], 
+  config["drop_refers_by_hostname"], config["use_local_time"],
+  config["assigned_numbers"], config["match_page_numbers"])
+ 
+syslog = Logger.new('./log/system.log',shift_age = 'weekly')
+case config["log_level"]
+when "info"
+  syslog.level = Logger::INFO
+when "warn"
+  syslog.level = Logger::WARN  
+when "error"
+  syslog.level = Logger::ERROR  
+when "fatal"
+  syslog.level = Logger::FATAL   
+else
+  syslog.level = Logger::DEBUG 
+end
+syslog.info "starting ... >>> "+Time.now.to_s
+
+# Open log file for reading
+File.open('./data/exlog.log', 'r') do |file|
+  vlist = Vizi::VisitList.new
+  rec_count = 0
+  hit_count = 0
+  max_rec_count = 99999
+  max_rec_count = config["max_rec_count"] if config["max_rec_count"]
+  visit_count = 0
+  page_count = 0
+  human_count = 0
+  drop_count = 0
+  spider_count = 0
+  start_time = Time.now
+  logformat = nil
+  # Begin to parse each record
+  while(line = file.gets)
+    parsed_data = parser.parse_line(line, logformat)
+    logformat = parsed_data[:p_logformat]
+    rec_count = rec_count + 1 
+    next if parsed_data[:p_linetype] != "V"    
+    hit_count = hit_count + 1
+    page_count = page_count + 1 if parsed_data[:p_pageflag]
+    @visit=vlist.find_by_ip(parsed_data[:ip])
+	if @visit.nil?
+	  vlist.append(Vizi::Visit.new(parsed_data[:ip],parsed_data[:datetime],parsed_data[:csuristem],parsed_data[:csuriquery], parsed_data[:timetaken],
+	    parsed_data[:p_visitortype],parsed_data[:p_pageflag],parsed_data[:p_searchphrase],parsed_data[:p_pageid]))    
+      @visit=vlist.find_by_ip(parsed_data[:ip])     
+	  visit_count = visit_count + 1 
+	else
+	  @visit.update(parsed_data[:datetime],parsed_data[:csuriquery],parsed_data[:timetaken],
+	    parsed_data[:p_visitortype],parsed_data[:p_pageflag],parsed_data[:p_searchphrase], parsed_data[:p_pageid])
+    end
+    @visits = vlist.find_expired(@visit.start_dt)
+    if @visits 
+      @visits.sendoutput
+      vlist.delete(@visits) 
+      human_count = human_count + 1 if @visits.visitortype == "H" 
+      drop_count = drop_count + 1 if @visits.visitortype == "D"
+      spider_count = spider_count + 1 if @visits.visitortype == "S"                   
+    end
+    break if rec_count == max_rec_count
+  end
+  @visits = vlist.find_all
+  @visits.each {|v|
+    v.sendoutput
+    human_count = human_count + 1 if v.visitortype == "H"
+    drop_count = drop_count + 1 if v.visitortype == "D"
+    spider_count = spider_count + 1 if v.visitortype == "S"      
+  }  
+  if config["summary_flag"]
+    syslog.info "Record count is "+rec_count.to_s
+    syslog.info "Hit count is "+hit_count.to_s 
+    syslog.info "Page count is "+page_count.to_s      
+    syslog.info "Total visit count is "+visit_count.to_s  
+    syslog.info "Human visit count is "+human_count.to_s
+    syslog.info "Drop visit count is "+drop_count.to_s
+    syslog.info "Spider visit count is "+spider_count.to_s            
+    syslog.info "Batch processing time "+(Time.now-start_time).to_s
+  end
+  syslog.info "ending ... >>> "+Time.now.to_s
+end
+

data/vizi_tracker.gemspec

@@ -0,0 +1,21 @@
+require 'rubygems'
+
+spec = Gem::Specification.new do |s|
+  s.name = 'vizi_tracker'
+  s.version = '0.1.0'
+  s.summary = "Visit tracking from Apache or IIS web log files"
+  s.description = "This module provides a set of classes to support the parsing of web log files and
+    the creation of visits from the individual parsed web log records.
+    
+    The LogFormat and LogParser classes were derived in part from an Apache logger application
+    developed by Jan Wikholm. These two classes were extended to support both Apache and IIS
+    web logs. The details from the web logs are assembled to compose Visit objects and Visit 
+    history detail"
+  s.files = Dir.glob("**/**/**")
+  s.test_files = Dir.glob("test/*_test.rb")
+  s.author = "Al Kivi"
+  s.homepage = "http://www.vizitrax.com"
+  s.email = "al.kivi@vizitrax.com"
+  s.has_rdoc = true
+  s.required_ruby_version = '>= 1.8.2'
+end

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification 
+name: vizi_tracker
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Al Kivi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-06-23 00:00:00 Z
+dependencies: []
+
+description: |-
+  This module provides a set of classes to support the parsing of web log files and
+      the creation of visits from the individual parsed web log records.
+      
+      The LogFormat and LogParser classes were derived in part from an Apache logger application
+      developed by Jan Wikholm. These two classes were extended to support both Apache and IIS
+      web logs. The details from the web logs are assembled to compose Visit objects and Visit 
+      history detail
+email: al.kivi@vizitrax.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- doc/Visit.html
+- doc/testit_rb.html
+- doc/Rakefile.html
+- doc/formats/apache.rtf
+- doc/formats/w3c-extended.rtf
+- doc/formats/apache-custom-log.pdf
+- doc/formats/format-descriptions.xls
+- doc/formats/w3c-extended.pdf
+- doc/Vizi/Visit.html
+- doc/Vizi/VisitList.html
+- doc/Vizi/LogFormat.html
+- doc/Vizi/LogParser.html
+- doc/images/bullet_black.png
+- doc/images/bullet_toggle_plus.png
+- doc/images/zoom.png
+- doc/images/bug.png
+- doc/images/wrench_orange.png
+- doc/images/bullet_toggle_minus.png
+- doc/images/loadingAnimation.gif
+- doc/images/macFFBgHack.png
+- doc/images/date.png
+- doc/images/brick_link.png
+- doc/images/tag_green.png
+- doc/images/ruby.png
+- doc/images/page_white_width.png
+- doc/images/find.png
+- doc/images/wrench.png
+- doc/images/page_green.png
+- doc/images/package.png
+- doc/images/plugin.png
+- doc/images/brick.png
+- doc/images/page_white_text.png
+- doc/VisitList.html
+- doc/ViziLogParser.html
+- doc/Object.html
+- doc/Vizi.html
+- doc/created.rid
+- doc/lib/vizi_tracker_rb.html
+- doc/lib/vizi/vizi_tracker_rb.html
+- doc/lib/vizi/parser_rb.html
+- doc/lib/vizi_log_parser_rb.html
+- doc/ViziLogFormat.html
+- doc/js/thickbox-compressed.js
+- doc/js/jquery.js
+- doc/js/darkfish.js
+- doc/js/quicksearch.js
+- doc/ParserTest.html
+- doc/index.html
+- doc/test/test_helper_rb.html
+- doc/test/parser_test_rb.html
+- doc/README_rdoc.html
+- doc/rdoc.css
+- data/sample-wle.log
+- data/testlog.log
+- data/exlog.log
+- data/sample-surf.log
+- data/sample-alter.log
+- data/vizitrax.log
+- lib/vizi/vizi_tracker.rb
+- lib/vizi_tracker.rb
+- README.rdoc
+- vizi_tracker-0.1.0.gem
+- Rakefile
+- log/parse.log
+- log/system.log
+- test/parser_test.rb
+- test/test_helper.rb
+- config/logger.yml
+- config/logger-backup.yml
+- config/logger_sample.yml
+- config/logger_apache.yml
+- testit.rb
+- vizi_tracker.gemspec
+homepage: http://www.vizitrax.com
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 51
+      segments: 
+      - 1
+      - 8
+      - 2
+      version: 1.8.2
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.5
+signing_key: 
+specification_version: 3
+summary: Visit tracking from Apache or IIS web log files
+test_files: 
+- test/parser_test.rb