vizi_tracker 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. data/README.rdoc +92 -92
  2. data/build_gem.sh +9 -0
  3. data/lib/vizi/vizi_tracker.rb +1 -1
  4. data/log/parse.log +24 -0
  5. data/log/system.log +4 -0
  6. data/testit.rb +1 -1
  7. data/vizi_tracker-0.4.0.gem +0 -0
  8. data/vizi_tracker.gemspec +1 -1
  9. metadata +66 -67
data/README.rdoc CHANGED
@@ -18,98 +18,98 @@ configuration values.
18
18
 
19
19
  == Usage
20
20
 
21
- require 'vizi_tracker'
22
- require 'yaml'
23
- require 'logger'
24
-
25
- config = YAML.load_file("config/logger.yml")
26
-
27
- @@download_page_number = config["download_page_number"]
28
- @@visit_timeout = config["visit_timeout"]
29
-
30
- # Initialize the log parser
31
- parser = Vizi::LogParser.new(config["drop_ips"], config["spider_ips"],
32
- config["spider_names"], config["page_urls"], config["hide_urls"],
33
- config["homepage"], config["accept_only_homepage"],config["hostname"],
34
- config["drop_refers_by_hostname"], config["use_local_time"],
35
- config["assigned_numbers"], config["match_page_numbers"])
36
-
37
- syslog = Logger.new('./log/system.log',shift_age = 'weekly')
38
- case config["log_level"]
39
- when "info"
40
- syslog.level = Logger::INFO
41
- when "warn"
42
- syslog.level = Logger::WARN
43
- when "error"
44
- syslog.level = Logger::ERROR
45
- when "fatal"
46
- syslog.level = Logger::FATAL
47
- else
48
- syslog.level = Logger::DEBUG
49
- end
50
- syslog.info "starting ... >>> "+Time.now.to_s
51
-
52
- # Open log file for reading
53
- File.open('./data/exlog.log', 'r') do |file|
54
- vlist = Vizi::VisitList.new
55
- rec_count = 0
56
- hit_count = 0
57
- max_rec_count = 99999
58
- max_rec_count = config["max_rec_count"] if config["max_rec_count"]
59
- visit_count = 0
60
- page_count = 0
61
- human_count = 0
62
- drop_count = 0
63
- spider_count = 0
64
- start_time = Time.now
65
- logformat = nil
66
- # Begin to parse each record
67
- while(line = file.gets)
68
- parsed_data = parser.parse_line(line, logformat)
69
- logformat = parsed_data[:p_logformat]
70
- rec_count = rec_count + 1
71
- next if parsed_data[:p_linetype] != "V"
72
- hit_count = hit_count + 1
73
- page_count = page_count + 1 if parsed_data[:p_pageflag]
74
- @visit=vlist.find_by_ip(parsed_data[:ip])
75
- if @visit.nil?
76
- vlist.append(Vizi::Visit.new(parsed_data[:ip],parsed_data[:datetime],parsed_data[:csuristem],parsed_data[:csuriquery], parsed_data[:timetaken],
77
- parsed_data[:p_visitortype],parsed_data[:p_pageflag],parsed_data[:p_searchphrase],parsed_data[:p_pageid]))
78
- @visit=vlist.find_by_ip(parsed_data[:ip])
79
- visit_count = visit_count + 1
80
- else
81
- @visit.update(parsed_data[:datetime],parsed_data[:csuriquery],parsed_data[:timetaken],
82
- parsed_data[:p_visitortype],parsed_data[:p_pageflag],parsed_data[:p_searchphrase], parsed_data[:p_pageid])
83
- end
84
- @visits = vlist.find_expired(@visit.start_dt)
85
- if @visits
86
- @visits.sendoutput
87
- vlist.delete(@visits)
88
- human_count = human_count + 1 if @visits.visitortype == "H"
89
- drop_count = drop_count + 1 if @visits.visitortype == "D"
90
- spider_count = spider_count + 1 if @visits.visitortype == "S"
91
- end
92
- break if rec_count == max_rec_count
93
- end
94
- @visits = vlist.find_all
95
- @visits.each {|v|
96
- v.sendoutput
97
- human_count = human_count + 1 if v.visitortype == "H"
98
- drop_count = drop_count + 1 if v.visitortype == "D"
99
- spider_count = spider_count + 1 if v.visitortype == "S"
100
- }
101
- if config["summary_flag"]
102
- syslog.info "Record count is "+rec_count.to_s
103
- syslog.info "Hit count is "+hit_count.to_s
104
- syslog.info "Page count is "+page_count.to_s
105
- syslog.info "Total visit count is "+visit_count.to_s
106
- syslog.info "Human visit count is "+human_count.to_s
107
- syslog.info "Drop visit count is "+drop_count.to_s
108
- syslog.info "Spider visit count is "+spider_count.to_s
109
- syslog.info "Batch processing time "+(Time.now-start_time).to_s
110
- end
111
- syslog.info "ending ... >>> "+Time.now.to_s
112
- end
21
+ require 'vizi_tracker'
22
+ require 'yaml'
23
+ require 'logger'
24
+
25
+ config = YAML.load_file("config/logger.yml")
26
+
27
+ @@download_page_number = config["download_page_number"]
28
+ @@visit_timeout = config["visit_timeout"]
29
+
30
+ # Initialize the log parser
31
+ parser = Vizi::LogParser.new(config["drop_ips"], config["spider_ips"],
32
+ config["spider_names"], config["page_urls"], config["hide_urls"],
33
+ config["homepage"], config["accept_only_homepage"],config["hostname"],
34
+ config["drop_refers_by_hostname"], config["use_local_time"],
35
+ config["assigned_numbers"], config["match_page_numbers"])
36
+
37
+ syslog = Logger.new('./log/system.log',shift_age = 'weekly')
38
+ case config["log_level"]
39
+ when "info"
40
+ syslog.level = Logger::INFO
41
+ when "warn"
42
+ syslog.level = Logger::WARN
43
+ when "error"
44
+ syslog.level = Logger::ERROR
45
+ when "fatal"
46
+ syslog.level = Logger::FATAL
47
+ else
48
+ syslog.level = Logger::DEBUG
49
+ end
50
+ syslog.info "starting ... >>> "+Time.now.to_s
51
+
52
+ # Open log file for reading
53
+ File.open('./data/exlog.log', 'r') do |file|
54
+ vlist = Vizi::VisitList.new
55
+ rec_count = 0
56
+ hit_count = 0
57
+ max_rec_count = 99999
58
+ max_rec_count = config["max_rec_count"] if config["max_rec_count"]
59
+ visit_count = 0
60
+ page_count = 0
61
+ human_count = 0
62
+ drop_count = 0
63
+ spider_count = 0
64
+ start_time = Time.now
65
+ logformat = nil
66
+ # Begin to parse each record
67
+ while(line = file.gets)
68
+ parsed_data = parser.parse_line(line, logformat)
69
+ logformat = parsed_data[:p_logformat]
70
+ rec_count = rec_count + 1
71
+ next if parsed_data[:p_linetype] != "V"
72
+ hit_count = hit_count + 1
73
+ page_count = page_count + 1 if parsed_data[:p_pageflag]
74
+ @visit=vlist.find_by_ip(parsed_data[:ip])
75
+ if @visit.nil?
76
+ vlist.append(Vizi::Visit.new(parsed_data[:ip],parsed_data[:datetime],parsed_data[:csuristem],parsed_data[:csuriquery], parsed_data[:timetaken],
77
+ parsed_data[:p_visitortype],parsed_data[:p_pageflag],parsed_data[:p_searchphrase],parsed_data[:p_pageid]))
78
+ @visit=vlist.find_by_ip(parsed_data[:ip])
79
+ visit_count = visit_count + 1
80
+ else
81
+ @visit.update(parsed_data[:datetime],parsed_data[:csuriquery],parsed_data[:timetaken],
82
+ parsed_data[:p_visitortype],parsed_data[:p_pageflag],parsed_data[:p_searchphrase], parsed_data[:p_pageid])
83
+ end
84
+ @visits = vlist.find_expired(@visit.start_dt)
85
+ if @visits
86
+ @visits.sendoutput
87
+ vlist.delete(@visits)
88
+ human_count = human_count + 1 if @visits.visitortype == "H"
89
+ drop_count = drop_count + 1 if @visits.visitortype == "D"
90
+ spider_count = spider_count + 1 if @visits.visitortype == "S"
91
+ end
92
+ break if rec_count == max_rec_count
93
+ end
94
+ @visits = vlist.find_all
95
+ @visits.each {|v|
96
+ v.sendoutput
97
+ human_count = human_count + 1 if v.visitortype == "H"
98
+ drop_count = drop_count + 1 if v.visitortype == "D"
99
+ spider_count = spider_count + 1 if v.visitortype == "S"
100
+ }
101
+ if config["summary_flag"]
102
+ syslog.info "Record count is "+rec_count.to_s
103
+ syslog.info "Hit count is "+hit_count.to_s
104
+ syslog.info "Page count is "+page_count.to_s
105
+ syslog.info "Total visit count is "+visit_count.to_s
106
+ syslog.info "Human visit count is "+human_count.to_s
107
+ syslog.info "Drop visit count is "+drop_count.to_s
108
+ syslog.info "Spider visit count is "+spider_count.to_s
109
+ syslog.info "Batch processing time "+(Time.now-start_time).to_s
110
+ end
111
+ syslog.info "ending ... >>> "+Time.now.to_s
112
+ end
113
113
 
114
114
  == License
115
115
 
data/build_gem.sh ADDED
@@ -0,0 +1,9 @@
1
+ # run with ... sudo bash build_gem.sh
2
+
3
+ gem uninstall vizi_tracker
4
+
5
+ chmod -R a+r ./lib
6
+
7
+ gem build vizi_tracker.gemspec
8
+
9
+ gem install -l vizi_tracker-0.5.0.gem
data/lib/vizi/vizi_tracker.rb CHANGED
@@ -496,7 +496,7 @@ module Vizi
496
496
  @vzvisit[:country] = @country
497
497
  @vzvisit[:region] = @region
498
498
  @vzvisit[:returnhit] = @returnhit
499
- @vzvisit[:group] = @group
499
+ @vzvisit[:grp] = @group
500
500
  @vzvisit[:groupcount] = @groupcount
501
501
  @vzvisit[:persona] = @persona
502
502
  @vzvisit[:orgmatch] = @orgmatch
data/log/parse.log CHANGED
@@ -23,3 +23,27 @@ W, [2012-05-05T17:30:46.357847 #2593] WARN -- : Found comment lines embedded in
23
23
  W, [2012-05-05T17:30:47.451169 #2593] WARN -- : #Software: Microsoft Internet Information Services 7.0
24
24
 
25
25
  W, [2012-05-05T17:30:47.451274 #2593] WARN -- : Found comment lines embedded in the log file ... resetting to nil
26
+ W, [2012-05-05T17:31:41.618420 #2597] WARN -- : #Software: Microsoft Internet Information Services 7.0
27
+
28
+ W, [2012-05-05T17:31:41.618578 #2597] WARN -- : Found comment lines embedded in the log file ... resetting to nil
29
+ W, [2012-05-05T17:31:41.928803 #2597] WARN -- : #Software: Microsoft Internet Information Services 7.0
30
+
31
+ W, [2012-05-05T17:31:41.928902 #2597] WARN -- : Found comment lines embedded in the log file ... resetting to nil
32
+ W, [2012-05-05T17:31:42.065349 #2597] WARN -- : #Software: Microsoft Internet Information Services 7.0
33
+
34
+ W, [2012-05-05T17:31:42.065445 #2597] WARN -- : Found comment lines embedded in the log file ... resetting to nil
35
+ W, [2012-05-05T17:31:42.266731 #2597] WARN -- : #Software: Microsoft Internet Information Services 7.0
36
+
37
+ W, [2012-05-05T17:31:42.266860 #2597] WARN -- : Found comment lines embedded in the log file ... resetting to nil
38
+ W, [2012-05-05T17:32:19.666611 #2600] WARN -- : #Software: Microsoft Internet Information Services 7.0
39
+
40
+ W, [2012-05-05T17:32:19.666713 #2600] WARN -- : Found comment lines embedded in the log file ... resetting to nil
41
+ W, [2012-05-05T17:32:19.923234 #2600] WARN -- : #Software: Microsoft Internet Information Services 7.0
42
+
43
+ W, [2012-05-05T17:32:19.923364 #2600] WARN -- : Found comment lines embedded in the log file ... resetting to nil
44
+ W, [2012-05-05T17:32:20.038568 #2600] WARN -- : #Software: Microsoft Internet Information Services 7.0
45
+
46
+ W, [2012-05-05T17:32:20.038665 #2600] WARN -- : Found comment lines embedded in the log file ... resetting to nil
47
+ W, [2012-05-05T17:32:20.172938 #2600] WARN -- : #Software: Microsoft Internet Information Services 7.0
48
+
49
+ W, [2012-05-05T17:32:20.173061 #2600] WARN -- : Found comment lines embedded in the log file ... resetting to nil
data/log/system.log CHANGED
@@ -10,3 +10,7 @@ I, [2012-05-05T17:30:00.755342 #2592] INFO -- : starting ... >>> Sat May 05 17:
10
10
  I, [2012-05-05T17:30:10.734129 #2592] INFO -- : ending ... >>> Sat May 05 17:30:10 -0400 2012
11
11
  I, [2012-05-05T17:30:37.881134 #2593] INFO -- : starting ... >>> Sat May 05 17:30:37 -0400 2012
12
12
  I, [2012-05-05T17:30:47.799064 #2593] INFO -- : ending ... >>> Sat May 05 17:30:47 -0400 2012
13
+ I, [2012-05-05T17:31:40.484204 #2597] INFO -- : starting ... >>> Sat May 05 17:31:40 -0400 2012
14
+ I, [2012-05-05T17:31:42.336851 #2597] INFO -- : ending ... >>> Sat May 05 17:31:42 -0400 2012
15
+ I, [2012-05-05T17:32:18.802934 #2600] INFO -- : starting ... >>> Sat May 05 17:32:18 -0400 2012
16
+ I, [2012-05-05T17:32:20.231211 #2600] INFO -- : ending ... >>> Sat May 05 17:32:20 -0400 2012
data/testit.rb CHANGED
@@ -56,7 +56,7 @@ File.open('./data/exlog.log', 'r') do |file|
56
56
  logformat = nil
57
57
  # Begin to parse each record
58
58
  while(line = file.gets)
59
- p line
59
+ # p line
60
60
  parsed_data = parser.parse_line(line, logformat)
61
61
  logformat = parsed_data[:p_logformat]
62
62
  rec_count = rec_count + 1
data/vizi_tracker-0.4.0.gem ADDED
Binary file
data/vizi_tracker.gemspec CHANGED
@@ -2,7 +2,7 @@ require 'rubygems'
2
2
 
3
3
  spec = Gem::Specification.new do |s|
4
4
  s.name = 'vizi_tracker'
5
- s.version = '0.4.0'
5
+ s.version = '0.5.0'
6
6
  s.summary = "Visit tracking from Apache or IIS web log files"
7
7
  s.description = "This module provides a set of classes to support the parsing of web log files and
8
8
  the creation of visits from the individual parsed web log records.
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: vizi_tracker
3
3
  version: !ruby/object:Gem::Version
4
- hash: 15
4
+ hash: 11
5
5
  prerelease:
6
6
  segments:
7
7
  - 0
8
- - 4
8
+ - 5
9
9
  - 0
10
- version: 0.4.0
10
+ version: 0.5.0
11
11
  platform: ruby
12
12
  authors:
13
13
  - Al Kivi
@@ -15,8 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2012-05-05 00:00:00 -04:00
19
- default_executable:
18
+ date: 2012-12-28 00:00:00 Z
20
19
  dependencies: []
21
20
 
22
21
  description: |-
@@ -35,85 +34,85 @@ extensions: []
35
34
  extra_rdoc_files: []
36
35
 
37
36
  files:
38
- - doc/Visit.html
39
- - doc/testit_rb.html
40
- - doc/Rakefile.html
41
- - doc/formats/apache.rtf
42
- - doc/formats/w3c-extended.rtf
43
- - doc/formats/apache-custom-log.pdf
44
- - doc/formats/format-descriptions.xls
37
+ - log/parse.log.20120428
38
+ - log/system.log.20120428
39
+ - log/parse.log
40
+ - log/system.log
41
+ - testit.rb
42
+ - test/test_helper.rb
43
+ - test/parser_test.rb
44
+ - build_gem.sh
45
+ - Rakefile
46
+ - vizi_tracker-0.2.0.gem
47
+ - config/logger.yml
48
+ - config/logger-backup.yml
49
+ - config/logger_sample.yml
50
+ - config/logger_apache.yml
51
+ - data/exlog.log
52
+ - data/vizitrax.log
53
+ - data/sample-wle.log
54
+ - data/sample-alter.log
55
+ - data/sample-surf.log
56
+ - data/testlog.log
57
+ - doc/Object.html
58
+ - doc/ViziLogFormat.html
45
59
  - doc/formats/w3c-extended.pdf
46
- - doc/Vizi/Visit.html
47
- - doc/Vizi/VisitList.html
48
- - doc/Vizi/LogFormat.html
49
- - doc/Vizi/LogParser.html
60
+ - doc/formats/format-descriptions.xls
61
+ - doc/formats/apache-custom-log.pdf
62
+ - doc/formats/w3c-extended.rtf
63
+ - doc/formats/apache.rtf
64
+ - doc/VisitList.html
65
+ - doc/Rakefile.html
66
+ - doc/README_rdoc.html
67
+ - doc/test/parser_test_rb.html
68
+ - doc/test/test_helper_rb.html
69
+ - doc/Vizi.html
70
+ - doc/ParserTest.html
71
+ - doc/Visit.html
72
+ - doc/ViziLogParser.html
73
+ - doc/images/ruby.png
74
+ - doc/images/date.png
50
75
  - doc/images/bullet_black.png
76
+ - doc/images/brick_link.png
77
+ - doc/images/plugin.png
78
+ - doc/images/wrench.png
51
79
  - doc/images/bullet_toggle_plus.png
52
- - doc/images/zoom.png
53
80
  - doc/images/bug.png
54
- - doc/images/wrench_orange.png
55
- - doc/images/bullet_toggle_minus.png
56
- - doc/images/loadingAnimation.gif
81
+ - doc/images/page_green.png
82
+ - doc/images/zoom.png
57
83
  - doc/images/macFFBgHack.png
58
- - doc/images/date.png
59
- - doc/images/brick_link.png
84
+ - doc/images/page_white_text.png
60
85
  - doc/images/tag_green.png
61
- - doc/images/ruby.png
62
86
  - doc/images/page_white_width.png
87
+ - doc/images/brick.png
88
+ - doc/images/wrench_orange.png
63
89
  - doc/images/find.png
64
- - doc/images/wrench.png
65
- - doc/images/page_green.png
90
+ - doc/images/bullet_toggle_minus.png
91
+ - doc/images/loadingAnimation.gif
66
92
  - doc/images/package.png
67
- - doc/images/plugin.png
68
- - doc/images/brick.png
69
- - doc/images/page_white_text.png
70
- - doc/VisitList.html
71
- - doc/ViziLogParser.html
72
- - doc/Object.html
73
- - doc/Vizi.html
74
- - doc/created.rid
75
- - doc/lib/vizi_tracker_rb.html
76
- - doc/lib/vizi/vizi_tracker_rb.html
77
- - doc/lib/vizi/parser_rb.html
78
- - doc/lib/vizi_log_parser_rb.html
79
- - doc/ViziLogFormat.html
93
+ - doc/testit_rb.html
94
+ - doc/js/quicksearch.js
80
95
  - doc/js/thickbox-compressed.js
81
96
  - doc/js/jquery.js
82
97
  - doc/js/darkfish.js
83
- - doc/js/quicksearch.js
84
- - doc/ParserTest.html
98
+ - doc/created.rid
99
+ - doc/lib/vizi/parser_rb.html
100
+ - doc/lib/vizi/vizi_tracker_rb.html
101
+ - doc/lib/vizi_tracker_rb.html
102
+ - doc/lib/vizi_log_parser_rb.html
85
103
  - doc/index.html
86
- - doc/test/test_helper_rb.html
87
- - doc/test/parser_test_rb.html
88
- - doc/README_rdoc.html
89
104
  - doc/rdoc.css
90
- - data/sample-wle.log
91
- - data/testlog.log
92
- - data/exlog.log
93
- - data/sample-surf.log
94
- - data/sample-alter.log
95
- - data/vizitrax.log
96
- - vizi_tracker-0.2.0.gem
105
+ - doc/Vizi/VisitList.html
106
+ - doc/Vizi/Visit.html
107
+ - doc/Vizi/LogFormat.html
108
+ - doc/Vizi/LogParser.html
97
109
  - vizi_tracker-0.3.0.gem
110
+ - vizi_tracker.gemspec
111
+ - vizi_tracker-0.1.0.gem
112
+ - vizi_tracker-0.4.0.gem
98
113
  - lib/vizi/vizi_tracker.rb
99
114
  - lib/vizi_tracker.rb
100
115
  - README.rdoc
101
- - vizi_tracker-0.1.0.gem
102
- - Rakefile
103
- - log/parse.log
104
- - log/system.log
105
- - log/system.log.20120428
106
- - log/parse.log.20120428
107
- - test/parser_test.rb
108
- - test/test_helper.rb
109
- - config/logger.yml
110
- - config/logger-backup.yml
111
- - config/logger_sample.yml
112
- - config/logger_apache.yml
113
- - testit.rb
114
- - vizi_tracker-0.4.0.gem
115
- - vizi_tracker.gemspec
116
- has_rdoc: true
117
116
  homepage: http://www.vizitrax.com
118
117
  licenses: []
119
118
 
@@ -145,7 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
145
144
  requirements: []
146
145
 
147
146
  rubyforge_project:
148
- rubygems_version: 1.6.2
147
+ rubygems_version: 1.8.24
149
148
  signing_key:
150
149
  specification_version: 3
151
150
  summary: Visit tracking from Apache or IIS web log files