vizi_tracker 0.1.0

data/README.rdoc

@@ -0,0 +1,116 @@
+= ViziTracker
+
+== Introduction
+
+This gem provides a set of classes to support the parsing of web log files and
+the creation of Visit records from the individual parsed web log records
+
+== Installation
+
+Just run:
+
+    gem install vizi_tracker
+
+The following code uses a YAML file to store the configuration attributes that wil
+drive the logging activity. This file (logger.yml) should be setup and stored in the 
+config sub-directory. Refer also to logger_sample.yml for more details on the 
+configuration values.
+
+== Usage
+
+	require 'vizi_tracker'
+	require 'yaml'
+	require 'logger'
+	
+	config = YAML.load_file("config/logger.yml")
+	
+	@@download_page_number = config["download_page_number"]
+	@@visit_timeout = config["visit_timeout"]
+	
+	# Initialize the log parser
+	parser = Vizi::LogParser.new(config["drop_ips"], config["spider_ips"], 
+	  config["spider_names"], config["page_urls"], config["hide_urls"], 
+	  config["homepage"], config["accept_only_homepage"],config["hostname"], 
+	  config["drop_refers_by_hostname"], config["use_local_time"],
+	  config["assigned_numbers"], config["match_page_numbers"])
+	 
+	syslog = Logger.new('./log/system.log',shift_age = 'weekly')
+	case config["log_level"]
+	when "info"
+	  syslog.level = Logger::INFO
+	when "warn"
+	  syslog.level = Logger::WARN  
+	when "error"
+	  syslog.level = Logger::ERROR  
+	when "fatal"
+	  syslog.level = Logger::FATAL   
+	else
+	  syslog.level = Logger::DEBUG 
+	end
+	syslog.info "starting ... >>> "+Time.now.to_s
+	
+	# Open log file for reading
+	File.open('./data/exlog.log', 'r') do |file|
+	  vlist = Vizi::VisitList.new
+	  rec_count = 0
+	  hit_count = 0
+	  max_rec_count = 99999
+	  max_rec_count = config["max_rec_count"] if config["max_rec_count"]
+	  visit_count = 0
+	  page_count = 0
+	  human_count = 0
+	  drop_count = 0
+	  spider_count = 0
+	  start_time = Time.now
+	  logformat = nil
+	  # Begin to parse each record
+	  while(line = file.gets)
+	    parsed_data = parser.parse_line(line, logformat)
+	    logformat = parsed_data[:p_logformat]
+	    rec_count = rec_count + 1 
+	    next if parsed_data[:p_linetype] != "V"    
+	    hit_count = hit_count + 1
+	    page_count = page_count + 1 if parsed_data[:p_pageflag]
+	    @visit=vlist.find_by_ip(parsed_data[:ip])
+		if @visit.nil?
+		  vlist.append(Vizi::Visit.new(parsed_data[:ip],parsed_data[:datetime],parsed_data[:csuristem],parsed_data[:csuriquery], parsed_data[:timetaken],
+		    parsed_data[:p_visitortype],parsed_data[:p_pageflag],parsed_data[:p_searchphrase],parsed_data[:p_pageid]))    
+	      @visit=vlist.find_by_ip(parsed_data[:ip])     
+		  visit_count = visit_count + 1 
+		else
+		  @visit.update(parsed_data[:datetime],parsed_data[:csuriquery],parsed_data[:timetaken],
+		    parsed_data[:p_visitortype],parsed_data[:p_pageflag],parsed_data[:p_searchphrase], parsed_data[:p_pageid])
+	    end
+	    @visits = vlist.find_expired(@visit.start_dt)
+	    if @visits 
+	      @visits.sendoutput
+	      vlist.delete(@visits) 
+	      human_count = human_count + 1 if @visits.visitortype == "H" 
+	      drop_count = drop_count + 1 if @visits.visitortype == "D"
+	      spider_count = spider_count + 1 if @visits.visitortype == "S"                   
+	    end
+	    break if rec_count == max_rec_count
+	  end
+	  @visits = vlist.find_all
+	  @visits.each {|v|
+	    v.sendoutput
+	    human_count = human_count + 1 if v.visitortype == "H"
+	    drop_count = drop_count + 1 if v.visitortype == "D"
+	    spider_count = spider_count + 1 if v.visitortype == "S"      
+	  }  
+	  if config["summary_flag"]
+	    syslog.info "Record count is "+rec_count.to_s
+	    syslog.info "Hit count is "+hit_count.to_s 
+	    syslog.info "Page count is "+page_count.to_s      
+	    syslog.info "Total visit count is "+visit_count.to_s  
+	    syslog.info "Human visit count is "+human_count.to_s
+	    syslog.info "Drop visit count is "+drop_count.to_s
+	    syslog.info "Spider visit count is "+spider_count.to_s            
+	    syslog.info "Batch processing time "+(Time.now-start_time).to_s
+	  end
+	  syslog.info "ending ... >>> "+Time.now.to_s
+	end
+
+== License
+
+This code is made available under the MIT license. It is based on original parser code from Jan Wikholm.

data/Rakefile

@@ -0,0 +1,44 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/testtask'
+
+spec = Gem::Specification.new do |s|
+    s.name      =   "vizi_tracker"
+    s.version   =   "0.1.0"
+    s.author    =   "Al Kivi"
+    s.email     =   "al.kivi at vizitrax.com"
+    s.homepage  =   "http://github.com/al-kivi/vizi_tracker"
+    s.description = "Univeral web log file parser and visit analyzer"
+    s.summary   =   "A package for parsing web server logs and creating visit records"
+
+    s.platform  =   Gem::Platform::RUBY
+    s.has_rdoc  =   true
+    s.extra_rdoc_files  =   ["README.rdoc"]
+
+    s.require_path  =   "lib"
+    s.files     =   %w(README.rdoc Rakefile) + Dir.glob("lib/**/*")
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+    pkg.need_tar = true
+end
+
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'ViziTracker'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.verbose = true
+end
+
+task :default => "pkg/#{spec.name}-#{spec.version}.gem" do
+    puts "generated latest version"
+end
+

data/config/logger-backup.yml

@@ -0,0 +1,49 @@
+# Configuration data for logger
+visit_timeout: 1200
+summary_flag: true
+convert_to_lower_case: true
+#max_rec_count: 100
+homepage: home.aspx
+accept_only_homepage: true
+page_extensions:
+- aspx
+- asp
+hostname: www.sigma-systems.com
+drop_refers_by_hostname: true 
+drop_ips:
+- 76.12.185.100
+spider_ips:
+- 66.98.254.55
+- 64.208.168.252
+- 64.235.108.183
+- 76.2.144.115
+- 66.98.254.236
+- 202.108.22.132
+- 89.122.29.77
+- 95.174.93.222
+- 66.55.37.179
+- 198.45.18.20
+- 38.104.227.3
+spider_names:
+- bot
+- spider
+- slurp
+- root.exe
+- .dll
+- slurp
+- looksmart
+- nutchsvc
+- iconsurf
+- objectsearch
+- openfind
+- iltovatore
+- mozdex
+- netresearch
+- konsqueror
+- crawler	
+- searchme
+- java/1.6.0_04
+- scoutjet
+- yeti
+- yandex
+drop_spiders: true	

data/config/logger.yml

@@ -0,0 +1,69 @@
+# Configuration data for logger
+#max_rec_count: 100
+visit_timeout: 1200
+summary_flag: true
+convert_to_lower_case: true
+log_level: warn 
+#log_level options are debug, info, warn, error, fatal
+homepage: /home.aspx
+#homepage: /
+accept_only_homepage: true
+page_urls:
+- aspx
+- asp
+hide_urls:
+- css
+- js
+- gif
+- swf
+- ico
+- chart
+- robots
+hostname: www.sigma-systems.com
+drop_refers_by_hostname: true 
+use_local_time: true
+download_page_number: 45
+drop_ips:
+- 76.12.185.100
+spider_ips:
+- 66.98.254.55
+- 64.208.168.252
+- 64.235.108.183
+- 76.2.144.115
+- 66.98.254.236
+- 202.108.22.132
+- 89.122.29.77
+- 95.174.93.222
+- 66.55.37.179
+- 198.45.18.20
+- 38.104.227.3
+spider_names:
+- bot
+- spider
+- slurp
+- root.exe
+- .dll
+- slurp
+- looksmart
+- nutchsvc
+- iconsurf
+- objectsearch
+- openfind
+- iltovatore
+- mozdex
+- netresearch
+- konsqueror
+- crawler	
+- searchme
+- java/1.6.0_04
+- scoutjet
+- yeti
+- yandex
+# convert urls to assigned numbers where numbers cannot be parsed from url
+assigned_numbers:
+- /visit/index,1
+- /visit/show/,2
+- /visit/vcardedit/,3
+- /visit/showmap_na,4
+- /visit/showmap_row,5
+match_page_numbers: false

data/config/logger_apache.yml

@@ -0,0 +1,65 @@
+# Configuration data for logger
+#max_rec_count: 100
+visit_timeout: 1200
+summary_flag: true
+convert_to_lower_case: true
+#homepage: /home.aspx
+homepage: /
+accept_only_homepage: true
+page_urls:
+- aspx
+- asp
+hide_urls:
+- css
+- js
+- gif
+- swf
+- ico
+- chart
+- robots
+hostname: www.sigma-systems.com
+drop_refers_by_hostname: true 
+getlocal_time: true
+drop_ips:
+- 76.12.185.100
+spider_ips:
+- 66.98.254.55
+- 64.208.168.252
+- 64.235.108.183
+- 76.2.144.115
+- 66.98.254.236
+- 202.108.22.132
+- 89.122.29.77
+- 95.174.93.222
+- 66.55.37.179
+- 198.45.18.20
+- 38.104.227.3
+spider_names:
+- bot
+- spider
+- slurp
+- root.exe
+- .dll
+- slurp
+- looksmart
+- nutchsvc
+- iconsurf
+- objectsearch
+- openfind
+- iltovatore
+- mozdex
+- netresearch
+- konsqueror
+- crawler	
+- searchme
+- java/1.6.0_04
+- scoutjet
+- yeti
+- yandex
+page_numbers:
+- /visit/index,1
+- /visit/show/,2
+- /visit/vcardedit/,3
+- /visit/showmap_na,4
+- /visit/showmap_row,5
+match_page_numbers: true

data/config/logger_sample.yml

@@ -0,0 +1,69 @@
+# Configuration data for logger
+#max_rec_count: 100
+visit_timeout: 1200
+summary_flag: true
+convert_to_lower_case: true
+log_level: warn 
+#log_level options are debug, info, warn, error, fatal
+homepage: /home.aspx
+#homepage: /
+accept_only_homepage: true
+page_urls:
+- aspx
+- asp
+hide_urls:
+- css
+- js
+- gif
+- swf
+- ico
+- chart
+- robots
+hostname: www.sigma-systems.com
+drop_refers_by_hostname: true 
+use_local_time: true
+download_page_number: 45
+drop_ips:
+- 76.12.185.100
+spider_ips:
+- 66.98.254.55
+- 64.208.168.252
+- 64.235.108.183
+- 76.2.144.115
+- 66.98.254.236
+- 202.108.22.132
+- 89.122.29.77
+- 95.174.93.222
+- 66.55.37.179
+- 198.45.18.20
+- 38.104.227.3
+spider_names:
+- bot
+- spider
+- slurp
+- root.exe
+- .dll
+- slurp
+- looksmart
+- nutchsvc
+- iconsurf
+- objectsearch
+- openfind
+- iltovatore
+- mozdex
+- netresearch
+- konsqueror
+- crawler	
+- searchme
+- java/1.6.0_04
+- scoutjet
+- yeti
+- yandex
+# convert urls to assigned numbers where numbers cannot be parsed from url
+assigned_numbers:
+- /visit/index,1
+- /visit/show/,2
+- /visit/vcardedit/,3
+- /visit/showmap_na,4
+- /visit/showmap_row,5
+match_page_numbers: false