vizi_logger 0.1.0

data/README.rdoc

@@ -0,0 +1,52 @@
+= ViziLogger
+
+== Introduction
+
+This gem module creates a pagelogger that can used to log the visit to each
+web page in a Ruby based web application (e.g., Ramaze, Rails). Each web page is
+logged in Common Log Format to support analysis by an external Web Log Analyzer
+
+== Installation
+
+Just run:
+
+    gem install vizi_logger
+
+The following code will simulate the implementation of the software in a Ruby 
+based web application. When used in a web application, the pagelogger should be
+called with the following:
+
+	weblog.page page_url  # where page_url provides the url information
+
+The following example will allow for the testing of the gem in your development
+environment.
+
+You will need a Web Log Analyzer program to review the results of your application
+logging. The output of this gem has been tested with the Summary web log analyzer.
+Check it out at www.summary.net.
+
+== Usage
+	 
+	require 'rubygems'   # needed for ruby 1.8.7
+	require 'vizi_logger'
+
+	# weblog = Vizi::PageLogger.new(STDOUT)
+
+	weblog = Vizi::PageLogger.new('./log/vizidemo.log',shift_age = 'weekly')
+
+	weblog.addremote '127.0.0.1'  # for batch testing
+	# weblog.addremote request.remote_addr  # use this in live mode
+
+	weblog.formatter=Vizi::LogFormatter.new
+
+	urlist = ["/","/users/login","/users/list","/posts","/posts/view/1"]
+
+	25.times do
+		sleep rand*2  # create random interval
+		page_url = urlist[(5*rand).to_i]  # select a random page url
+		weblog.page page_url
+	end
+
+== License
+
+This code is made available under the MIT license.

data/Rakefile

@@ -0,0 +1,45 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/testtask'
+
+spec = Gem::Specification.new do |s|
+    s.name      =   "vizi_whois"
+    s.version   =   "0.1.0"
+    s.author    =   "Al Kivi"
+    s.email     =   "al.kivi at vizitrax.com"
+    s.homepage  =   "http://github.com/al-kivi/Vizi_Whois"
+    s.summary = "Global whois module to select the right whois server and get response data"
+    s.description = "This gem module provides a classes to find the right Regional Internet Registry
+      for a given IP Address. The query method will navigate each major RIR until a response is found"
+    
+    s.platform  =   Gem::Platform::RUBY
+    s.has_rdoc  =   true
+    s.extra_rdoc_files  =   ["README.rdoc"]
+
+    s.require_path  =   "lib"
+    s.files     =   %w(README.rdoc Rakefile) + Dir.glob("lib/**/*")
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+    pkg.need_tar = true
+end
+
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'HttpLogParser'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.verbose = true
+end
+
+task :default => "pkg/#{spec.name}-#{spec.version}.gem" do
+    puts "generated latest version"
+end
+

data/build_gem.sh

@@ -0,0 +1,9 @@
+# run with ... sudo bash build_gem.sh
+
+gem uninstall vizi_logger
+
+chmod -R a+r ./lib
+
+gem build vizi_logger.gemspec
+
+gem install -l vizi_logger-0.1.0.gem

data/build_gem1.sh

@@ -0,0 +1 @@
+sudo gem uninstall vizi_logger

data/build_gem2.sh

@@ -0,0 +1 @@
+sudo gem build vizi_logger.gemspec

data/build_gem3.sh

@@ -0,0 +1,3 @@
+## sudo gem install --local vizi_logger-0.1.0.gem
+
+sudo gem install -l vizi_logger-0.1.0.gem

data/geany_run_script.bat

@@ -0,0 +1,6 @@
+ruby "testit.rb"
+
+pause
+del "%0"
+
+pause

data/lib/vizi/geany_run_script.bat

@@ -0,0 +1,6 @@
+ruby "vizi_whois.rb"
+
+pause
+del "%0"
+
+pause

data/lib/vizi/vizi_logger.rb

@@ -0,0 +1,55 @@
+# This gem module creates a pagelogger that can used to log the visit to each
+# web page in a Ruby based web application (e.g., Ramaze, Rails). Each web page is
+# logged in Common Log Format to support analysis by an external Web Log Analyzer
+#
+# Author::    Al Kivi <al.kivi@vizitrax.com>
+# License::   MIT
+
+require 'logger'
+
+module Vizi
+# This class includes a set of methods that extend the Ruby Logger
+  class PageLogger < ::Logger
+
+    attr_accessor :ip
+    
+# This method will add the remote ip address to the Logger class 
+    def initialize (*args)
+      super
+      ip = ''
+    end
+    
+# This method will add the ip address to the logger class
+    def addremote(ipaddr)
+      self.ip = ipaddr
+    end
+    
+# This method will create the log entry adding the request information
+# Calling format is ... log.page '/admin/login.htm' or similar
+    def page(*args)
+      request = args.first
+      self.info([self.ip, request])
+    end  
+    
+  end # Pagelogger
+  
+# This class customizes the Logger::Formatter to match the Common Log Format
+  class LogFormatter < Logger::Formatter
+    # Provide a call() method that returns the formatted message.
+    def call(severity, time, program_name, message)
+      if severity == "INFO"
+        message.insert(1, ' -')
+        message.insert(2, ' - ')
+        message.insert(3, "["+Time.now.strftime("%d/%b/%Y:%H:%M:%S %z")+"] ")
+        request = message[4]
+        message[4] = '"'+request+'"'
+        message.insert(5, ' 200')
+        message.insert(7, ' 0')
+        print_message = "#{String(message)}\n"
+      else
+        super
+      end
+    end
+  end
+  
+end # Vizi

data/lib/vizi_logger.rb

@@ -0,0 +1,4 @@
+# module Vizi # :doc:
+  require 'vizi/vizi_logger'   # use for the gem
+## load './lib/vizi/vizi_logger.rb'  # use for testing
+# end

data/log/apache.log

@@ -0,0 +1,7 @@
+192.168.198.92 - - [22/Jul/2012:23:08:37 -0400] "GET / HTTP/1.1" 200 6394 www.yahoo.com "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1...)" "-"
+192.168.198.92 - - [22/Jul/2012:23:08:38 -0400] "GET /images/logo.gif HTTP/1.1" 200 807 www.yahoo.com "http://www.some.com/" "Mozilla/4.0 (compatible; MSIE 6...)" "-"
+192.168.72.177 - - [22/Jul/2012:23:32:14 -0400] "GET /news/sports.html HTTP/1.1" 200 3500 www.yahoo.com "http://www.some.com/" "Mozilla/4.0 (compatible; MSIE ...)" "-"
+192.168.72.177 - - [22/Jul/2012:23:32:14 -0400] "GET /favicon.ico HTTP/1.1" 404 1997 www.yahoo.com "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3)..." "-"
+192.168.72.177 - - [22/Jul/2012:23:32:15 -0400] "GET /style.css HTTP/1.1" 200 4138 www.yahoo.com "http://www.yahoo.com/index.html" "Mozilla/5.0 (Windows..." "-"
+192.168.72.177 - - [22/Jul/2012:23:32:16 -0400] "GET /js/ads.js HTTP/1.1" 200 10229 www.yahoo.com "http://www.search.com/index.html" "Mozilla/5.0 (Windows..." "-"
+192.168.72.177 - - [22/Jul/2012:23:32:19 -0400] "GET /search.php HTTP/1.1" 400 1997 www.yahoo.com "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; ...)" "-"

data/log/vizidemo.log

@@ -0,0 +1,26 @@
+# Logfile created on Sun Jul 22 12:14:42 -0400 2012 by logger.rb/1.2.6
+127.0.0.1 - - [22/Jul/2012:12:14:42 -0400] "/users/list" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:42 -0400] "/users/login" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:43 -0400] "/users/login" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:43 -0400] "/users/login" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:43 -0400] "/" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:44 -0400] "/users/login" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:46 -0400] "/users/login" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:47 -0400] "/users/list" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:49 -0400] "/users/login" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:49 -0400] "/posts" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:50 -0400] "/posts" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:50 -0400] "/users/login" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:51 -0400] "/posts/view/1" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:53 -0400] "/posts" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:55 -0400] "/users/list" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:55 -0400] "/posts/view/1" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:57 -0400] "/posts/view/1" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:59 -0400] "/users/login" 200 0
+127.0.0.1 - - [22/Jul/2012:12:14:59 -0400] "/" 200 0
+127.0.0.1 - - [22/Jul/2012:12:15:00 -0400] "/users/list" 200 0
+127.0.0.1 - - [22/Jul/2012:12:15:00 -0400] "/users/login" 200 0
+127.0.0.1 - - [22/Jul/2012:12:15:01 -0400] "/users/list" 200 0
+127.0.0.1 - - [22/Jul/2012:12:15:02 -0400] "/" 200 0
+127.0.0.1 - - [22/Jul/2012:12:15:03 -0400] "/users/login" 200 0
+127.0.0.1 - - [22/Jul/2012:12:15:04 -0400] "/users/list" 200 0

data/test/parser_test.rb

@@ -0,0 +1,48 @@
+require File.expand_path(File.dirname(__FILE__) + '/test_helper')
+
+require 'uri'
+
+class ParserTest < Test::Unit::TestCase
+  
+  def test_parser_creation
+
+    parser = HttpLogParser.new
+    assert_not_nil parser
+    assert_equal 5, parser.formats.size
+
+    parser = HttpLogParser.new('%h %l %u %t \"%r\" %>s %b')    
+    assert_not_nil parser
+    assert_equal 1, parser.formats.size
+
+    parser = HttpLogParser.new({
+      :common => '%h %l %u %t \"%r\" %>s %b',
+      :common_with_virtual => '%v %h %l %u %t \"%r\" %>s %b',
+    })   
+    assert_not_nil parser
+    assert_equal 2, parser.formats.size
+
+  end
+
+  def test_simple_parsing
+
+    parser = HttpLogParser.new
+    assert_not_nil parser
+
+    parsed = parser.parse_line('111.222.333.444 - - [21/Apr/2010:01:02:03 +0000] "GET /some/url?some=parameter HTTP/1.1" 302 123 "http://somewhere.com" "Browser (Version 1.0)"')
+
+    assert_equal '111.222.333.444', parsed[:ip]
+    assert_equal '111.222.333.444', parsed[:domain]
+    assert_equal '-', parsed[:auth]
+    assert_equal '-', parsed[:username]
+    assert_equal '21/Apr/2010:01:02:03 +0000', parsed[:datetime]
+    assert_equal 'GET /some/url?some=parameter HTTP/1.1', parsed[:request]
+    assert_equal '302', parsed[:status]
+    assert_equal '123', parsed[:bytecount]
+    assert_equal 'http://somewhere.com', parsed[:referer]
+    assert_equal 'Browser (Version 1.0)', parsed[:user_agent]
+
+    assert_equal 11, parsed.size
+
+  end
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'test/unit'
+require "#{File.expand_path(File.dirname(__FILE__))}/../lib/http_log_parser.rb"
+begin; require 'ruby-debug'; rescue LoadError; end

data/testit.rb

@@ -0,0 +1,30 @@
+# This is a sample application that uses the Vizi_logger gem classes
+# 
+# This application will create a log file with a set of simulated web log records
+# You will need a web log analyzer (e.g., Analog) to confirm the readability of this file
+#
+# Author::    Al Kivi <al.kivi@vizitrax.com>
+
+# require '~/appgems/vizi_logger/lib/vizi_logger.rb'  # use this line for testing
+
+require 'rubygems'   # needed for ruby 1.8.7
+require 'vizi_logger'
+
+# weblog = Vizi::PageLogger.new(STDOUT)
+
+weblog = Vizi::PageLogger.new('./log/vizidemo.log',shift_age = 'weekly')
+
+weblog.addremote '127.0.0.1'  # for batch testing
+# weblog.addremote request.remote_addr  # use this in live mode
+
+weblog.formatter=Vizi::LogFormatter.new
+
+urlist = ["/","/users/login","/users/list","/posts","/posts/view/1"]
+
+25.times do
+  sleep rand*2  # create random interval
+  page_url = urlist[(5*rand).to_i]  # select a random page url
+  weblog.page page_url
+end
+
+#weblog.page 

data/testit_old.rb

@@ -0,0 +1,49 @@
+# This is a sample application that uses the Vizi_whois gem classes
+# 
+# This application will read a text file with IP addresses.
+# For each IP address, the whois method will be called and responses received.
+#
+# Author::    Al Kivi <al.kivi@vizitrax.com>
+
+#require 'c:\rails\vizi_whois\lib\vizi_whois'
+require 'rubygems'   # needed for ruby 1.8.7
+require 'vizi_whois'
+
+require 'logger'
+require 'socket'
+require 'yaml'
+ 
+syslog = Logger.new('./log/system.log',shift_age = 'weekly')
+syslog.info "Starting IP address test file ... >>> "+Time.now.to_s
+
+out_file = File.new('./log/output.log', 'w')
+
+File.delete('./log/formatted.log') if File.exist?('./log/formatted.log') 
+parse_file = File.new('./log/formatted.log', 'w')
+
+p 'starting'
+
+# Open test file for reading
+File.open('./data/testfile.txt', 'r') do |file|
+  rec_count = 0
+  while(line = file.gets) # Read each line of the test file, one IP address per line
+  p line
+    @whoisresult = Vizi::Gowhois.new
+    p line.chomp
+    rarray = @whoisresult.query(line.chomp)
+    @contents = rarray[0]
+    out_file.puts '----------------------------------------------------------------'
+    out_file.puts '> ' + line 
+    out_file.puts '>> ' + rarray[1]
+    out_file.puts '>>> ' + rarray[2]     
+    out_file.puts '----------------------------------------------------------------'    
+    out_file.puts @contents
+    @result = Vizi::Formatter.new
+    @formatted = @result.parse(@contents, rarray[1]) 
+    @formatted = @formatted.gsub('\n','<br/>') 
+    p @formatted   
+    rec_count = rec_count + 1 
+  end
+  syslog.info "Record count is "+rec_count.to_s
+  syslog.info "Ending ... >>> "+Time.now.to_s
+end

data/vizi_logger.gemspec

@@ -0,0 +1,20 @@
+require 'rubygems'
+
+spec = Gem::Specification.new do |s|
+  s.name = 'vizi_logger'
+  s.version = '0.1.0'
+  s.summary = "Ruby based logger that will record web page visits in the common log format"
+  s.description = "This gem module provides a logger class and a log formatter class that will create
+    a log file in a common log format.
+    
+    A ruby based application can generate a log file record with each page visit. This capability provides
+    logging that can be highly customized by the ruby developer. A web log analyzer will be needed to
+    summarize and analyze the web log records"
+  s.files = Dir.glob("**/**/**")
+  s.test_files = Dir.glob("test/*_test.rb")
+  s.author = "Al Kivi"
+  s.homepage = "http://www.vizitrax.com"
+  s.email = "al.kivi@yahoo.com"
+  s.has_rdoc = true
+  s.required_ruby_version = '>= 1.8.7'
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification 
+name: vizi_logger
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Al Kivi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-07-22 00:00:00 Z
+dependencies: []
+
+description: |-
+  This gem module provides a logger class and a log formatter class that will create
+      a log file in a common log format.
+      
+      A ruby based application can generate a log file record with each page visit. This capability provides
+      logging that can be highly customized by the ruby developer. A web log analyzer will be needed to
+      summarize and analyze the web log records
+email: al.kivi@yahoo.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- log/vizidemo.log
+- log/apache.log
+- testit.rb
+- test/test_helper.rb
+- test/parser_test.rb
+- build_gem.sh
+- Rakefile
+- build_gem3.sh
+- geany_run_script.bat
+- build_gem1.sh
+- testit_old.rb
+- build_gem2.sh
+- vizi_logger.gemspec
+- lib/vizi/geany_run_script.bat
+- lib/vizi/vizi_logger.rb
+- lib/vizi_logger.rb
+- README.rdoc
+- vizi_logger-0.1.0.gem
+homepage: http://www.vizitrax.com
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 57
+      segments: 
+      - 1
+      - 8
+      - 7
+      version: 1.8.7
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Ruby based logger that will record web page visits in the common log format
+test_files: 
+- test/parser_test.rb