vizi_tracker 0.4.0 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (9) hide show
  1. data/README.rdoc +92 -92
  2. data/build_gem.sh +9 -0
  3. data/lib/vizi/vizi_tracker.rb +1 -1
  4. data/log/parse.log +24 -0
  5. data/log/system.log +4 -0
  6. data/testit.rb +1 -1
  7. data/vizi_tracker-0.4.0.gem +0 -0
  8. data/vizi_tracker.gemspec +1 -1
  9. metadata +66 -67
data/README.rdoc CHANGED
@@ -18,98 +18,98 @@ configuration values.
18
18
 
19
19
  == Usage
20
20
 
21
- require 'vizi_tracker'
22
- require 'yaml'
23
- require 'logger'
24
-
25
- config = YAML.load_file("config/logger.yml")
26
-
27
- @@download_page_number = config["download_page_number"]
28
- @@visit_timeout = config["visit_timeout"]
29
-
30
- # Initialize the log parser
31
- parser = Vizi::LogParser.new(config["drop_ips"], config["spider_ips"],
32
- config["spider_names"], config["page_urls"], config["hide_urls"],
33
- config["homepage"], config["accept_only_homepage"],config["hostname"],
34
- config["drop_refers_by_hostname"], config["use_local_time"],
35
- config["assigned_numbers"], config["match_page_numbers"])
36
-
37
- syslog = Logger.new('./log/system.log',shift_age = 'weekly')
38
- case config["log_level"]
39
- when "info"
40
- syslog.level = Logger::INFO
41
- when "warn"
42
- syslog.level = Logger::WARN
43
- when "error"
44
- syslog.level = Logger::ERROR
45
- when "fatal"
46
- syslog.level = Logger::FATAL
47
- else
48
- syslog.level = Logger::DEBUG
49
- end
50
- syslog.info "starting ... >>> "+Time.now.to_s
51
-
52
- # Open log file for reading
53
- File.open('./data/exlog.log', 'r') do |file|
54
- vlist = Vizi::VisitList.new
55
- rec_count = 0
56
- hit_count = 0
57
- max_rec_count = 99999
58
- max_rec_count = config["max_rec_count"] if config["max_rec_count"]
59
- visit_count = 0
60
- page_count = 0
61
- human_count = 0
62
- drop_count = 0
63
- spider_count = 0
64
- start_time = Time.now
65
- logformat = nil
66
- # Begin to parse each record
67
- while(line = file.gets)
68
- parsed_data = parser.parse_line(line, logformat)
69
- logformat = parsed_data[:p_logformat]
70
- rec_count = rec_count + 1
71
- next if parsed_data[:p_linetype] != "V"
72
- hit_count = hit_count + 1
73
- page_count = page_count + 1 if parsed_data[:p_pageflag]
74
- @visit=vlist.find_by_ip(parsed_data[:ip])
75
- if @visit.nil?
76
- vlist.append(Vizi::Visit.new(parsed_data[:ip],parsed_data[:datetime],parsed_data[:csuristem],parsed_data[:csuriquery], parsed_data[:timetaken],
77
- parsed_data[:p_visitortype],parsed_data[:p_pageflag],parsed_data[:p_searchphrase],parsed_data[:p_pageid]))
78
- @visit=vlist.find_by_ip(parsed_data[:ip])
79
- visit_count = visit_count + 1
80
- else
81
- @visit.update(parsed_data[:datetime],parsed_data[:csuriquery],parsed_data[:timetaken],
82
- parsed_data[:p_visitortype],parsed_data[:p_pageflag],parsed_data[:p_searchphrase], parsed_data[:p_pageid])
83
- end
84
- @visits = vlist.find_expired(@visit.start_dt)
85
- if @visits
86
- @visits.sendoutput
87
- vlist.delete(@visits)
88
- human_count = human_count + 1 if @visits.visitortype == "H"
89
- drop_count = drop_count + 1 if @visits.visitortype == "D"
90
- spider_count = spider_count + 1 if @visits.visitortype == "S"
91
- end
92
- break if rec_count == max_rec_count
93
- end
94
- @visits = vlist.find_all
95
- @visits.each {|v|
96
- v.sendoutput
97
- human_count = human_count + 1 if v.visitortype == "H"
98
- drop_count = drop_count + 1 if v.visitortype == "D"
99
- spider_count = spider_count + 1 if v.visitortype == "S"
100
- }
101
- if config["summary_flag"]
102
- syslog.info "Record count is "+rec_count.to_s
103
- syslog.info "Hit count is "+hit_count.to_s
104
- syslog.info "Page count is "+page_count.to_s
105
- syslog.info "Total visit count is "+visit_count.to_s
106
- syslog.info "Human visit count is "+human_count.to_s
107
- syslog.info "Drop visit count is "+drop_count.to_s
108
- syslog.info "Spider visit count is "+spider_count.to_s
109
- syslog.info "Batch processing time "+(Time.now-start_time).to_s
110
- end
111
- syslog.info "ending ... >>> "+Time.now.to_s
112
- end
21
+ require 'vizi_tracker'
22
+ require 'yaml'
23
+ require 'logger'
24
+
25
+ config = YAML.load_file("config/logger.yml")
26
+
27
+ @@download_page_number = config["download_page_number"]
28
+ @@visit_timeout = config["visit_timeout"]
29
+
30
+ # Initialize the log parser
31
+ parser = Vizi::LogParser.new(config["drop_ips"], config["spider_ips"],
32
+ config["spider_names"], config["page_urls"], config["hide_urls"],
33
+ config["homepage"], config["accept_only_homepage"],config["hostname"],
34
+ config["drop_refers_by_hostname"], config["use_local_time"],
35
+ config["assigned_numbers"], config["match_page_numbers"])
36
+
37
+ syslog = Logger.new('./log/system.log',shift_age = 'weekly')
38
+ case config["log_level"]
39
+ when "info"
40
+ syslog.level = Logger::INFO
41
+ when "warn"
42
+ syslog.level = Logger::WARN
43
+ when "error"
44
+ syslog.level = Logger::ERROR
45
+ when "fatal"
46
+ syslog.level = Logger::FATAL
47
+ else
48
+ syslog.level = Logger::DEBUG
49
+ end
50
+ syslog.info "starting ... >>> "+Time.now.to_s
51
+
52
+ # Open log file for reading
53
+ File.open('./data/exlog.log', 'r') do |file|
54
+ vlist = Vizi::VisitList.new
55
+ rec_count = 0
56
+ hit_count = 0
57
+ max_rec_count = 99999
58
+ max_rec_count = config["max_rec_count"] if config["max_rec_count"]
59
+ visit_count = 0
60
+ page_count = 0
61
+ human_count = 0
62
+ drop_count = 0
63
+ spider_count = 0
64
+ start_time = Time.now
65
+ logformat = nil
66
+ # Begin to parse each record
67
+ while(line = file.gets)
68
+ parsed_data = parser.parse_line(line, logformat)
69
+ logformat = parsed_data[:p_logformat]
70
+ rec_count = rec_count + 1
71
+ next if parsed_data[:p_linetype] != "V"
72
+ hit_count = hit_count + 1
73
+ page_count = page_count + 1 if parsed_data[:p_pageflag]
74
+ @visit=vlist.find_by_ip(parsed_data[:ip])
75
+ if @visit.nil?
76
+ vlist.append(Vizi::Visit.new(parsed_data[:ip],parsed_data[:datetime],parsed_data[:csuristem],parsed_data[:csuriquery], parsed_data[:timetaken],
77
+ parsed_data[:p_visitortype],parsed_data[:p_pageflag],parsed_data[:p_searchphrase],parsed_data[:p_pageid]))
78
+ @visit=vlist.find_by_ip(parsed_data[:ip])
79
+ visit_count = visit_count + 1
80
+ else
81
+ @visit.update(parsed_data[:datetime],parsed_data[:csuriquery],parsed_data[:timetaken],
82
+ parsed_data[:p_visitortype],parsed_data[:p_pageflag],parsed_data[:p_searchphrase], parsed_data[:p_pageid])
83
+ end
84
+ @visits = vlist.find_expired(@visit.start_dt)
85
+ if @visits
86
+ @visits.sendoutput
87
+ vlist.delete(@visits)
88
+ human_count = human_count + 1 if @visits.visitortype == "H"
89
+ drop_count = drop_count + 1 if @visits.visitortype == "D"
90
+ spider_count = spider_count + 1 if @visits.visitortype == "S"
91
+ end
92
+ break if rec_count == max_rec_count
93
+ end
94
+ @visits = vlist.find_all
95
+ @visits.each {|v|
96
+ v.sendoutput
97
+ human_count = human_count + 1 if v.visitortype == "H"
98
+ drop_count = drop_count + 1 if v.visitortype == "D"
99
+ spider_count = spider_count + 1 if v.visitortype == "S"
100
+ }
101
+ if config["summary_flag"]
102
+ syslog.info "Record count is "+rec_count.to_s
103
+ syslog.info "Hit count is "+hit_count.to_s
104
+ syslog.info "Page count is "+page_count.to_s
105
+ syslog.info "Total visit count is "+visit_count.to_s
106
+ syslog.info "Human visit count is "+human_count.to_s
107
+ syslog.info "Drop visit count is "+drop_count.to_s
108
+ syslog.info "Spider visit count is "+spider_count.to_s
109
+ syslog.info "Batch processing time "+(Time.now-start_time).to_s
110
+ end
111
+ syslog.info "ending ... >>> "+Time.now.to_s
112
+ end
113
113
 
114
114
  == License
115
115
 
data/build_gem.sh ADDED
@@ -0,0 +1,9 @@
1
+ # run with ... sudo bash build_gem.sh
2
+
3
+ gem uninstall vizi_tracker
4
+
5
+ chmod -R a+r ./lib
6
+
7
+ gem build vizi_tracker.gemspec
8
+
9
+ gem install -l vizi_tracker-0.5.0.gem
data/lib/vizi/vizi_tracker.rb CHANGED
@@ -496,7 +496,7 @@ module Vizi
496
496
  @vzvisit[:country] = @country
497
497
  @vzvisit[:region] = @region
498
498
  @vzvisit[:returnhit] = @returnhit
499
- @vzvisit[:group] = @group
499
+ @vzvisit[:grp] = @group
500
500
  @vzvisit[:groupcount] = @groupcount
501
501
  @vzvisit[:persona] = @persona
502
502
  @vzvisit[:orgmatch] = @orgmatch
data/log/parse.log CHANGED
@@ -23,3 +23,27 @@ W, [2012-05-05T17:30:46.357847 #2593] WARN -- : Found comment lines embedded in
23
23
  W, [2012-05-05T17:30:47.451169 #2593] WARN -- : #Software: Microsoft Internet Information Services 7.0
24
24
 
25
25
  W, [2012-05-05T17:30:47.451274 #2593] WARN -- : Found comment lines embedded in the log file ... resetting to nil
26
+ W, [2012-05-05T17:31:41.618420 #2597] WARN -- : #Software: Microsoft Internet Information Services 7.0
27
+
28
+ W, [2012-05-05T17:31:41.618578 #2597] WARN -- : Found comment lines embedded in the log file ... resetting to nil
29
+ W, [2012-05-05T17:31:41.928803 #2597] WARN -- : #Software: Microsoft Internet Information Services 7.0
30
+
31
+ W, [2012-05-05T17:31:41.928902 #2597] WARN -- : Found comment lines embedded in the log file ... resetting to nil
32
+ W, [2012-05-05T17:31:42.065349 #2597] WARN -- : #Software: Microsoft Internet Information Services 7.0
33
+
34
+ W, [2012-05-05T17:31:42.065445 #2597] WARN -- : Found comment lines embedded in the log file ... resetting to nil
35
+ W, [2012-05-05T17:31:42.266731 #2597] WARN -- : #Software: Microsoft Internet Information Services 7.0
36
+
37
+ W, [2012-05-05T17:31:42.266860 #2597] WARN -- : Found comment lines embedded in the log file ... resetting to nil
38
+ W, [2012-05-05T17:32:19.666611 #2600] WARN -- : #Software: Microsoft Internet Information Services 7.0
39
+
40
+ W, [2012-05-05T17:32:19.666713 #2600] WARN -- : Found comment lines embedded in the log file ... resetting to nil
41
+ W, [2012-05-05T17:32:19.923234 #2600] WARN -- : #Software: Microsoft Internet Information Services 7.0
42
+
43
+ W, [2012-05-05T17:32:19.923364 #2600] WARN -- : Found comment lines embedded in the log file ... resetting to nil
44
+ W, [2012-05-05T17:32:20.038568 #2600] WARN -- : #Software: Microsoft Internet Information Services 7.0
45
+
46
+ W, [2012-05-05T17:32:20.038665 #2600] WARN -- : Found comment lines embedded in the log file ... resetting to nil
47
+ W, [2012-05-05T17:32:20.172938 #2600] WARN -- : #Software: Microsoft Internet Information Services 7.0
48
+
49
+ W, [2012-05-05T17:32:20.173061 #2600] WARN -- : Found comment lines embedded in the log file ... resetting to nil
data/log/system.log CHANGED
@@ -10,3 +10,7 @@ I, [2012-05-05T17:30:00.755342 #2592] INFO -- : starting ... >>> Sat May 05 17:
10
10
  I, [2012-05-05T17:30:10.734129 #2592] INFO -- : ending ... >>> Sat May 05 17:30:10 -0400 2012
11
11
  I, [2012-05-05T17:30:37.881134 #2593] INFO -- : starting ... >>> Sat May 05 17:30:37 -0400 2012
12
12
  I, [2012-05-05T17:30:47.799064 #2593] INFO -- : ending ... >>> Sat May 05 17:30:47 -0400 2012
13
+ I, [2012-05-05T17:31:40.484204 #2597] INFO -- : starting ... >>> Sat May 05 17:31:40 -0400 2012
14
+ I, [2012-05-05T17:31:42.336851 #2597] INFO -- : ending ... >>> Sat May 05 17:31:42 -0400 2012
15
+ I, [2012-05-05T17:32:18.802934 #2600] INFO -- : starting ... >>> Sat May 05 17:32:18 -0400 2012
16
+ I, [2012-05-05T17:32:20.231211 #2600] INFO -- : ending ... >>> Sat May 05 17:32:20 -0400 2012
data/testit.rb CHANGED
@@ -56,7 +56,7 @@ File.open('./data/exlog.log', 'r') do |file|
56
56
  logformat = nil
57
57
  # Begin to parse each record
58
58
  while(line = file.gets)
59
- p line
59
+ # p line
60
60
  parsed_data = parser.parse_line(line, logformat)
61
61
  logformat = parsed_data[:p_logformat]
62
62
  rec_count = rec_count + 1
data/vizi_tracker-0.4.0.gem ADDED
Binary file
data/vizi_tracker.gemspec CHANGED
@@ -2,7 +2,7 @@ require 'rubygems'
2
2
 
3
3
  spec = Gem::Specification.new do |s|
4
4
  s.name = 'vizi_tracker'
5
- s.version = '0.4.0'
5
+ s.version = '0.5.0'
6
6
  s.summary = "Visit tracking from Apache or IIS web log files"
7
7
  s.description = "This module provides a set of classes to support the parsing of web log files and
8
8
  the creation of visits from the individual parsed web log records.
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: vizi_tracker
3
3
  version: !ruby/object:Gem::Version
4
- hash: 15
4
+ hash: 11
5
5
  prerelease:
6
6
  segments:
7
7
  - 0
8
- - 4
8
+ - 5
9
9
  - 0
10
- version: 0.4.0
10
+ version: 0.5.0
11
11
  platform: ruby
12
12
  authors:
13
13
  - Al Kivi
@@ -15,8 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2012-05-05 00:00:00 -04:00
19
- default_executable:
18
+ date: 2012-12-28 00:00:00 Z
20
19
  dependencies: []
21
20
 
22
21
  description: |-
@@ -35,85 +34,85 @@ extensions: []
35
34
  extra_rdoc_files: []
36
35
 
37
36
  files:
38
- - doc/Visit.html
39
- - doc/testit_rb.html
40
- - doc/Rakefile.html
41
- - doc/formats/apache.rtf
42
- - doc/formats/w3c-extended.rtf
43
- - doc/formats/apache-custom-log.pdf
44
- - doc/formats/format-descriptions.xls
37
+ - log/parse.log.20120428
38
+ - log/system.log.20120428
39
+ - log/parse.log
40
+ - log/system.log
41
+ - testit.rb
42
+ - test/test_helper.rb
43
+ - test/parser_test.rb
44
+ - build_gem.sh
45
+ - Rakefile
46
+ - vizi_tracker-0.2.0.gem
47
+ - config/logger.yml
48
+ - config/logger-backup.yml
49
+ - config/logger_sample.yml
50
+ - config/logger_apache.yml
51
+ - data/exlog.log
52
+ - data/vizitrax.log
53
+ - data/sample-wle.log
54
+ - data/sample-alter.log
55
+ - data/sample-surf.log
56
+ - data/testlog.log
57
+ - doc/Object.html
58
+ - doc/ViziLogFormat.html
45
59
  - doc/formats/w3c-extended.pdf
46
- - doc/Vizi/Visit.html
47
- - doc/Vizi/VisitList.html
48
- - doc/Vizi/LogFormat.html
49
- - doc/Vizi/LogParser.html
60
+ - doc/formats/format-descriptions.xls
61
+ - doc/formats/apache-custom-log.pdf
62
+ - doc/formats/w3c-extended.rtf
63
+ - doc/formats/apache.rtf
64
+ - doc/VisitList.html
65
+ - doc/Rakefile.html
66
+ - doc/README_rdoc.html
67
+ - doc/test/parser_test_rb.html
68
+ - doc/test/test_helper_rb.html
69
+ - doc/Vizi.html
70
+ - doc/ParserTest.html
71
+ - doc/Visit.html
72
+ - doc/ViziLogParser.html
73
+ - doc/images/ruby.png
74
+ - doc/images/date.png
50
75
  - doc/images/bullet_black.png
76
+ - doc/images/brick_link.png
77
+ - doc/images/plugin.png
78
+ - doc/images/wrench.png
51
79
  - doc/images/bullet_toggle_plus.png
52
- - doc/images/zoom.png
53
80
  - doc/images/bug.png
54
- - doc/images/wrench_orange.png
55
- - doc/images/bullet_toggle_minus.png
56
- - doc/images/loadingAnimation.gif
81
+ - doc/images/page_green.png
82
+ - doc/images/zoom.png
57
83
  - doc/images/macFFBgHack.png
58
- - doc/images/date.png
59
- - doc/images/brick_link.png
84
+ - doc/images/page_white_text.png
60
85
  - doc/images/tag_green.png
61
- - doc/images/ruby.png
62
86
  - doc/images/page_white_width.png
87
+ - doc/images/brick.png
88
+ - doc/images/wrench_orange.png
63
89
  - doc/images/find.png
64
- - doc/images/wrench.png
65
- - doc/images/page_green.png
90
+ - doc/images/bullet_toggle_minus.png
91
+ - doc/images/loadingAnimation.gif
66
92
  - doc/images/package.png
67
- - doc/images/plugin.png
68
- - doc/images/brick.png
69
- - doc/images/page_white_text.png
70
- - doc/VisitList.html
71
- - doc/ViziLogParser.html
72
- - doc/Object.html
73
- - doc/Vizi.html
74
- - doc/created.rid
75
- - doc/lib/vizi_tracker_rb.html
76
- - doc/lib/vizi/vizi_tracker_rb.html
77
- - doc/lib/vizi/parser_rb.html
78
- - doc/lib/vizi_log_parser_rb.html
79
- - doc/ViziLogFormat.html
93
+ - doc/testit_rb.html
94
+ - doc/js/quicksearch.js
80
95
  - doc/js/thickbox-compressed.js
81
96
  - doc/js/jquery.js
82
97
  - doc/js/darkfish.js
83
- - doc/js/quicksearch.js
84
- - doc/ParserTest.html
98
+ - doc/created.rid
99
+ - doc/lib/vizi/parser_rb.html
100
+ - doc/lib/vizi/vizi_tracker_rb.html
101
+ - doc/lib/vizi_tracker_rb.html
102
+ - doc/lib/vizi_log_parser_rb.html
85
103
  - doc/index.html
86
- - doc/test/test_helper_rb.html
87
- - doc/test/parser_test_rb.html
88
- - doc/README_rdoc.html
89
104
  - doc/rdoc.css
90
- - data/sample-wle.log
91
- - data/testlog.log
92
- - data/exlog.log
93
- - data/sample-surf.log
94
- - data/sample-alter.log
95
- - data/vizitrax.log
96
- - vizi_tracker-0.2.0.gem
105
+ - doc/Vizi/VisitList.html
106
+ - doc/Vizi/Visit.html
107
+ - doc/Vizi/LogFormat.html
108
+ - doc/Vizi/LogParser.html
97
109
  - vizi_tracker-0.3.0.gem
110
+ - vizi_tracker.gemspec
111
+ - vizi_tracker-0.1.0.gem
112
+ - vizi_tracker-0.4.0.gem
98
113
  - lib/vizi/vizi_tracker.rb
99
114
  - lib/vizi_tracker.rb
100
115
  - README.rdoc
101
- - vizi_tracker-0.1.0.gem
102
- - Rakefile
103
- - log/parse.log
104
- - log/system.log
105
- - log/system.log.20120428
106
- - log/parse.log.20120428
107
- - test/parser_test.rb
108
- - test/test_helper.rb
109
- - config/logger.yml
110
- - config/logger-backup.yml
111
- - config/logger_sample.yml
112
- - config/logger_apache.yml
113
- - testit.rb
114
- - vizi_tracker-0.4.0.gem
115
- - vizi_tracker.gemspec
116
- has_rdoc: true
117
116
  homepage: http://www.vizitrax.com
118
117
  licenses: []
119
118
 
@@ -145,7 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
145
144
  requirements: []
146
145
 
147
146
  rubyforge_project:
148
- rubygems_version: 1.6.2
147
+ rubygems_version: 1.8.24
149
148
  signing_key:
150
149
  specification_version: 3
151
150
  summary: Visit tracking from Apache or IIS web log files