logbox 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.bundle/config +3 -0
- data/.rvmrc +2 -0
- data/Gemfile +17 -0
- data/Gemfile.lock +30 -0
- data/README +14 -0
- data/Rakefile +74 -0
- data/VERSION +1 -0
- data/bin/download_logs +20 -0
- data/bin/obsstats +39 -0
- data/bin/rotate +17 -0
- data/bin/viewobs +198 -0
- data/lib/logbox.rb +9 -0
- data/lib/logbox/ansi_colors.rb +28 -0
- data/lib/logbox/log_parser.rb +79 -0
- data/lib/logbox/mockup_log.rb +44 -0
- data/lib/logbox/observation.rb +162 -0
- data/lib/logbox/observation_compiler.rb +311 -0
- data/lib/logbox/observation_mover.rb +142 -0
- data/lib/logbox/stream_wrapper.rb +20 -0
- data/lib/logbox/stream_wrapper/gzip_multi_file.rb +90 -0
- data/lib/logbox/stream_wrapper/observation_filter.rb +113 -0
- data/lib/logbox/stream_wrapper/order_blob_splitter.rb +96 -0
- data/lib/setup_environment.rb +15 -0
- data/logbox.gemspec +110 -0
- data/test/bin_viewobs_test.rb +42 -0
- data/test/fixtures/aws_keys_yaml.txt +3 -0
- data/test/fixtures/double-obs.log +1 -0
- data/test/fixtures/error_line.log +1 -0
- data/test/fixtures/log-for-md5.log +1 -0
- data/test/fixtures/log0.log +0 -0
- data/test/fixtures/log1.log +1 -0
- data/test/fixtures/log1.log.gz +0 -0
- data/test/fixtures/log2.log +2 -0
- data/test/fixtures/log2.log.gz +0 -0
- data/test/fixtures/log_invalid_mixed_encoding.log +1 -0
- data/test/fixtures/observation_filter.log +5 -0
- data/test/fixtures/unquoted_ugliness.log +2 -0
- data/test/log_parser_test.rb +84 -0
- data/test/observation_compiler_test.rb +216 -0
- data/test/observation_mover_test.rb +135 -0
- data/test/observation_test.rb +114 -0
- data/test/stream_wrapper/gzip_multi_file_test.rb +147 -0
- data/test/stream_wrapper/observation_filter_test.rb +171 -0
- data/test/stream_wrapper/order_blob_splitter_test.rb +129 -0
- data/test/test_helper.rb +23 -0
- metadata +177 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
require "rubygems"
|
|
2
|
+
require "bundler"
|
|
3
|
+
|
|
4
|
+
# Change directory so Bundler finds the gemfile regardless.
|
|
5
|
+
Dir.chdir(File.dirname(__FILE__)) do
|
|
6
|
+
if defined?(TEST_RUN)
|
|
7
|
+
Bundler.setup(:default, :test)
|
|
8
|
+
else
|
|
9
|
+
Bundler.setup(:default)
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
|
14
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..'))
|
|
15
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), 'logbox'))
|
data/logbox.gemspec
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# Generated by jeweler
|
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
|
4
|
+
# -*- encoding: utf-8 -*-
|
|
5
|
+
|
|
6
|
+
Gem::Specification.new do |s|
|
|
7
|
+
s.name = %q{logbox}
|
|
8
|
+
s.version = "0.2.10"
|
|
9
|
+
|
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
11
|
+
s.authors = ["dvrensk", "enarsson", "Jell"]
|
|
12
|
+
s.date = %q{2011-12-22}
|
|
13
|
+
s.description = %q{Log-related code and tools that are used cross different applications}
|
|
14
|
+
s.email = %q{dev@icehouse.se}
|
|
15
|
+
s.executables = ["download_logs", "obsstats", "rotate", "viewobs"]
|
|
16
|
+
s.extra_rdoc_files = [
|
|
17
|
+
"README"
|
|
18
|
+
]
|
|
19
|
+
s.files = [
|
|
20
|
+
".bundle/config",
|
|
21
|
+
".rvmrc",
|
|
22
|
+
"Gemfile",
|
|
23
|
+
"Gemfile.lock",
|
|
24
|
+
"README",
|
|
25
|
+
"Rakefile",
|
|
26
|
+
"VERSION",
|
|
27
|
+
"bin/download_logs",
|
|
28
|
+
"bin/obsstats",
|
|
29
|
+
"bin/rotate",
|
|
30
|
+
"bin/viewobs",
|
|
31
|
+
"lib/logbox.rb",
|
|
32
|
+
"lib/logbox/ansi_colors.rb",
|
|
33
|
+
"lib/logbox/log_parser.rb",
|
|
34
|
+
"lib/logbox/mockup_log.rb",
|
|
35
|
+
"lib/logbox/observation.rb",
|
|
36
|
+
"lib/logbox/observation_compiler.rb",
|
|
37
|
+
"lib/logbox/observation_mover.rb",
|
|
38
|
+
"lib/logbox/stream_wrapper.rb",
|
|
39
|
+
"lib/logbox/stream_wrapper/gzip_multi_file.rb",
|
|
40
|
+
"lib/logbox/stream_wrapper/observation_filter.rb",
|
|
41
|
+
"lib/logbox/stream_wrapper/order_blob_splitter.rb",
|
|
42
|
+
"lib/setup_environment.rb",
|
|
43
|
+
"logbox.gemspec",
|
|
44
|
+
"test/bin_viewobs_test.rb",
|
|
45
|
+
"test/fixtures/aws_keys_yaml.txt",
|
|
46
|
+
"test/fixtures/double-obs.log",
|
|
47
|
+
"test/fixtures/error_line.log",
|
|
48
|
+
"test/fixtures/log-for-md5.log",
|
|
49
|
+
"test/fixtures/log0.log",
|
|
50
|
+
"test/fixtures/log1.log",
|
|
51
|
+
"test/fixtures/log1.log.gz",
|
|
52
|
+
"test/fixtures/log2.log",
|
|
53
|
+
"test/fixtures/log2.log.gz",
|
|
54
|
+
"test/fixtures/log_invalid_mixed_encoding.log",
|
|
55
|
+
"test/fixtures/observation_filter.log",
|
|
56
|
+
"test/fixtures/unquoted_ugliness.log",
|
|
57
|
+
"test/log_parser_test.rb",
|
|
58
|
+
"test/observation_compiler_test.rb",
|
|
59
|
+
"test/observation_mover_test.rb",
|
|
60
|
+
"test/observation_test.rb",
|
|
61
|
+
"test/stream_wrapper/gzip_multi_file_test.rb",
|
|
62
|
+
"test/stream_wrapper/observation_filter_test.rb",
|
|
63
|
+
"test/stream_wrapper/order_blob_splitter_test.rb",
|
|
64
|
+
"test/test_helper.rb"
|
|
65
|
+
]
|
|
66
|
+
s.homepage = %q{https://github.com/icehouse/logbox}
|
|
67
|
+
s.licenses = ["MIT"]
|
|
68
|
+
s.require_paths = ["lib"]
|
|
69
|
+
s.rubygems_version = %q{1.6.2}
|
|
70
|
+
s.summary = %q{A Toolbox for Logs and Observations}
|
|
71
|
+
s.test_files = [
|
|
72
|
+
"test/bin_viewobs_test.rb",
|
|
73
|
+
"test/log_parser_test.rb",
|
|
74
|
+
"test/observation_compiler_test.rb",
|
|
75
|
+
"test/observation_mover_test.rb",
|
|
76
|
+
"test/observation_test.rb",
|
|
77
|
+
"test/stream_wrapper/gzip_multi_file_test.rb",
|
|
78
|
+
"test/stream_wrapper/observation_filter_test.rb",
|
|
79
|
+
"test/stream_wrapper/order_blob_splitter_test.rb",
|
|
80
|
+
"test/test_helper.rb"
|
|
81
|
+
]
|
|
82
|
+
|
|
83
|
+
if s.respond_to? :specification_version then
|
|
84
|
+
s.specification_version = 3
|
|
85
|
+
|
|
86
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
|
87
|
+
s.add_runtime_dependency(%q<right_aws>, ["~> 2.0"])
|
|
88
|
+
s.add_runtime_dependency(%q<single_instance>, [">= 0"])
|
|
89
|
+
s.add_runtime_dependency(%q<rake>, [">= 0"])
|
|
90
|
+
s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
|
|
91
|
+
s.add_development_dependency(%q<jeweler>, ["~> 1.5.1"])
|
|
92
|
+
s.add_development_dependency(%q<rcov>, [">= 0"])
|
|
93
|
+
else
|
|
94
|
+
s.add_dependency(%q<right_aws>, ["~> 2.0"])
|
|
95
|
+
s.add_dependency(%q<single_instance>, [">= 0"])
|
|
96
|
+
s.add_dependency(%q<rake>, [">= 0"])
|
|
97
|
+
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
|
98
|
+
s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
|
|
99
|
+
s.add_dependency(%q<rcov>, [">= 0"])
|
|
100
|
+
end
|
|
101
|
+
else
|
|
102
|
+
s.add_dependency(%q<right_aws>, ["~> 2.0"])
|
|
103
|
+
s.add_dependency(%q<single_instance>, [">= 0"])
|
|
104
|
+
s.add_dependency(%q<rake>, [">= 0"])
|
|
105
|
+
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
|
106
|
+
s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
|
|
107
|
+
s.add_dependency(%q<rcov>, [">= 0"])
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
require 'test_helper'
|
|
2
|
+
# Cannot use require since file name has no extension.
|
|
3
|
+
load File.join(File.dirname(__FILE__), '../bin/viewobs')
|
|
4
|
+
|
|
5
|
+
class ViewobsTest < Test::Unit::TestCase
|
|
6
|
+
context "Viewobs#view_line" do
|
|
7
|
+
setup do
|
|
8
|
+
@vo = Viewobs.new
|
|
9
|
+
@vo.options = {}
|
|
10
|
+
@line = File.readlines(fixture_file("double-obs.log")).first
|
|
11
|
+
@attributes = LogParser.parse_line(@line)
|
|
12
|
+
@observation = Observation.new(@attributes)
|
|
13
|
+
end
|
|
14
|
+
should "raise NoMethodError if there are two observations in one log line" do
|
|
15
|
+
assert_raises(NoMethodError) { @vo.view_line(@attributes, @observation) }
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
context "Viewobs when encountering a line with two observations" do
|
|
20
|
+
setup do
|
|
21
|
+
@argv = ARGV.dup
|
|
22
|
+
ARGV.replace [fixture_file("double-obs.log")]
|
|
23
|
+
@vo = Viewobs.new
|
|
24
|
+
@vo.options = {};
|
|
25
|
+
@buffer = ""
|
|
26
|
+
@stderr, $stderr = $stderr, StringIO.new(@buffer)
|
|
27
|
+
end
|
|
28
|
+
teardown do
|
|
29
|
+
ARGV.replace @argv
|
|
30
|
+
$stderr = @stderr if @stderr
|
|
31
|
+
end
|
|
32
|
+
should "not crash" do
|
|
33
|
+
assert_nothing_raised { @vo.view_until_end }
|
|
34
|
+
end
|
|
35
|
+
should "report on stderr" do
|
|
36
|
+
assert @buffer.empty?
|
|
37
|
+
@vo.view_until_end
|
|
38
|
+
assert ! @buffer.empty?
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
end
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
81.231.246.117 - - [18/Feb/2009:11:27:24 +0100] "GET /log.gif?a=Mozilla%2F5.0%20(Macintosh%3B%20U%3B%20Intel%20Mac%20OS%20X%2010.5%3B%20en-US%3B%20rv%3A1.9.0.6)%20Gecko%2F2009011912%20Firefox%2F3.0.6&a=Mozilla%2F5.0%20(Macintosh%3B%20U%3B%20Intel%20Mac%20OS%20X%2010.5%3B%20en-US%3B%20rv%3A1.9.0.6)%20Gecko%2F2009011912%20Firefox%2F3.0.6&aid=test&aid=test&d=debug&h=5&h=5&l=en-us&l=en-us&n=netscape&n=netscape&o=view_item&o=view_item&p=macintel&p=macintel&r=http%3A%2F%2Freleware%2Freleware%2Fjavascript%2Fv1%2F&r=http%3A%2F%2Freleware%2Freleware%2Fjavascript%2Fv1%2F&s=1920x1200&s=1920x1200&sid=books&sid=books&t=observer.test&t=observer.test&u=http%3A%2F%2Freleware%2Freleware%2Fjavascript%2Fv1%2Fobserver.test.html&u=http%3A%2F%2Freleware%2Freleware%2Fjavascript%2Fv1%2Fobserver.test.html&uid=1234952844614067497&uid=1234952844614067497&x=2511&x=90891&z=-60&z=-60& HTTP/1.1" 200 35 "http://releware/releware/javascript/v1/observer.test.html" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
218.10.51.78 - - [19/Dec/2011:20:15:49 +0000] "1?pnW6\x22b+P-Ɂ'ii\x15RN$Q\x15\x1B\x15k\x1F\x05R]\x1E&mUo\x0E߈%Dc֍\x16$e5.Mz\x11sbI5D)-\x145o\x13\x10e\x0B?5I;1mʡd䌫b#(cX\x16b\x1E RW\x17Ȉ\x03ѣGP\x08R{iZ\x01\x1DC3tUutP]r\x09\x11q+\x13G:-\x0E8\x07C\x1E\x17Vf\x1C\x1BfEy'\x0B͟\x15\x1AL\x1B6cs\x05\x22&\x1D\x1ED\x17bZcv|Az{\x13\x14=\x01I\x1F\x05#\x5CО\x19\x19C5%`Z,='\x116!I^\x22'\x1F'n\x06Cjd\x0Cv\x13ܢ0@\x0B\x0Bs\x0Fof\x10Ua\x0EhAo1h\x1Dz+\x19Vz\x16ERqX~1l4\x1CxP\x0C:p`B,1L{=W\x02" 400 173 "-" "-" "rwcdid=1755EC0A759BEF4E59108D2D024BB606" "-"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
88.80.10.1 - - [17/Apr/2010:23:13:37 +0000] "GET http://88.80.10.1/pp/anp.php?a=S%5DJWTS_FCM%5CUCS&b=1155&c=4dd9 HTTP/1.1" 404 199 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" "rwcdid=C367E30AA140CA4B5619740B02110303" "-"
|
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Log 1 line 1
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
193.10.32.222 - - [10/Feb/2011:01:50:13 +0000] "GET /log.gif?a=Mozilla%2F4.0%20(compatible%3B%20MSIE%206.0%3B%20Windows%20NT%205.1%3B%20SV1%3B%20.NET%20CLR%201.1.4322%3B%20.NET%20CLR%202.0.50727)&aid=jetshop&l=sv&n=microsoft%20internet%20explorer&o=visit_page&p=win32&s=1024x768&sid=www.kyssjohanna.se&u=http%3A%2F%2Fwww.kyssjohanna.se%2FProductPopup.aspx%3Ffilename%3Dfriendp%C3%A4rla_3968.jpg&uid=1297291298745007919&x=8149&z=-60& HTTP/1.1" 200 35 "http://www.kyssjohanna.se/ProductPopup.aspx?filename=friendp�rla_3968.jpg" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)" "-" "rwcdid=B566E00ADB6A504D8108F42D02421E03"
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
213.88.235.250 - - [16/Feb/2009:07:43:58 +0100] "GET /log.gif?_image=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FFortum%2F14681_stor.jpg&_item_id=7611&_thumbnail=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FFortum%2F14681_liten.jpg&_title=%C2%A0Upplev%20en%20%C3%B6kad%20k%C3%A4nsla%20av%20trygghet%20med%20bostadslarm&_url=http%3A%2F%2Fwww.xroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D7611&a=Mozilla%2F4.0%20(compatible%3B%20MSIE%206.0%3B%20Windows%20NT%205.0)&aid=xroads&h=21&l=sv&n=microsoft%20internet%20explorer&o=view_item&p=win32&r=http%3A%2F%2Fwww.xroads.se%2FPortal%2FProducts%2FProductGroup.asp%3FsectionId%3D3055&s=1024x768&sid=Fortum&t=Product&u=http%3A%2F%2Fwww.xroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D7611&uid=1234766663495089586&x=41227&z=-60&d=debug& HTTP/1.1" 200 35 "http://www.xroads.se/Portal/Products/Product.asp?itemId=7611" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)"
|
|
2
|
+
81.231.246.116 - - [16/Feb/2009:07:44:51 +0100] "GET /log.gif?_image=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FVKG%2F14766_stor.jpg&_item_id=7712&_thumbnail=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FVKG%2F14766_liten.jpg&_title=%C2%A0Luft%2Fvattenv%C3%A4rmepump%20-%20Alpha-Innotec%20LW&_url=http%3A%2F%2Fwww.xroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D7712&a=Mozilla%2F4.0%20(compatible%3B%20MSIE%206.0%3B%20Windows%20NT%205.0)&aid=xroads&h=30&l=sv&n=microsoft%20internet%20explorer&o=pick_item&p=win32&r=http%3A%2F%2Fwww.xroads.se%2FPortal%2FProducts%2FProductGroup.asp%3FsectionId%3D3067&s=1024x768&sid=Fortum&t=Product&u=http%3A%2F%2Fwww.xroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D7712&uid=1234766663495089586&x=23072&z=-60& HTTP/1.1" 200 35 "http://www.xroads.se/Portal/Products/Product.asp?itemId=7712" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)"
|
|
3
|
+
81.231.246.117 - - [16/Feb/2009:07:49:13 +0100] "GET /log.gif?_image=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FFortum%2F14682_stor.jpg&_item_id=7612&_thumbnail=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FFortum%2F14682_liten.jpg&_title=Anticimex&_url=http%3A%2F%2Fwww.xroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D7612&a=Mozilla%2F5.0%20(Windows%3B%20U%3B%20Windows%20NT%205.1%3B%20sv-SE%3B%20rv%3A1.9.0.6)%20Gecko%2F2009011913%20Firefox%2F3.0.6&aid=xroads&h=18&l=sv-se&n=netscape&o=view_item&p=win32&r=http%3A%2F%2Fwww.xroads.se%2FPortal%2FProducts%2FProductGroup.asp%3FsectionId%3D3073&s=1920x1200&sid=Fortum&t=Product&u=http%3A%2F%2Fwww.xroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D7612&uid=1234766953513569566&x=20034&z=-60& HTTP/1.1" 200 35 "http://www.xroads.se/Portal/Products/Product.asp?itemId=7612" "Mozilla/5.0 (Windows; U; Windows NT 5.1; sv-SE; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6"
|
|
4
|
+
81.231.246.117 - - [16/Feb/2009:08:27:18 +0100] "GET /log.gif?_current_price=899%20SEK&_image=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FBlackDecker%2FGW350_stor.jpg&_item_id=2370&_thumbnail=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FBlackDecker%2FGW350_liten.jpg&_title=%C2%A0GW2600%20-%20Tr%C3%A4dg%C3%A5rdsst%C3%A4dare%202600%20W&_url=http%3A%2F%2Fwww.xroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D2370&_user1=..%2Fimages%2FBlackDecker%20Concept%20Store%2Flagerstatus_1.gif&_user2=Finns%20i%20lager%20f%C3%B6r%20omg%C3%A5ende%20leverans%C2%A0&a=Mozilla%2F4.0%20(compatible%3B%20MSIE%207.0%3B%20Windows%20NT%205.1%3B%20GTB5%3B%20.NET%20CLR%201.1.4322%3B%20.NET%20CLR%202.0.50727%3B%20.NET%20CLR%203.0.04506.30%3B%20.NET%20CLR%203.0.04506.648)&aid=xroads&h=5&l=sv&n=microsoft%20internet%20explorer&o=view_item&p=win32&r=http%3A%2F%2Fwww.xroads.se%2Fblackdecker%2FProducts%2FProductGroup.asp%3FtxtSearchName%3Ds%25E4ck&s=1280x1024&sid=BlackDecker%20Concept%20Store&t=Product&u=http%3A%2F%2Fwww.xroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D2370&uid=1234440861774400309&x=32137&z=-60& HTTP/1.1" 200 35 "http://www.xroads.se/Portal/Products/Product.asp?itemId=2370" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; GTB5; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648)"
|
|
5
|
+
81.233.91.34 - - [16/Feb/2009:08:36:36 +0100] "GET /log.gif?_current_price=449%20SEK&_image=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FTrygghansa%2FPresto%2F14067_stor.jpg&_item_id=6971&_thumbnail=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FTrygghansa%2FPresto%2F14067_liten.jpg&_title=Pulverbrandsl%C3%A4ckare%206%20kg%20Nordic&_url=http%3A%2F%2Fwww.xroads.se%2Ftrygghansa%2FProducts%2FProduct.asp%3FitemId%3D6971&_user1=..%2Fimages%2Flagerstatus_1.gif&_user2=Finns%20i%20lager%20f%C3%B6r%20omg%C3%A5ende%20leverans&a=Mozilla%2F5.0%20(Windows%3B%20U%3B%20Windows%20NT%205.1%3B%20sv-SE%3B%20rv%3A1.9.0.6)%20Gecko%2F2009011913%20Firefox%2F3.0.6&aid=xroads&h=6&l=sv-se&n=netscape&o=view_item&p=win32&r=http%3A%2F%2Fwww.xroads.se%2Ftrygghansa%2Fstart.asp%3FLink%3DProducts%252FProduct%252Easp%253FitemId%253D6971&s=1680x1050&sid=Trygg%20Hansa&t=Product&u=http%3A%2F%2Fwww.xroads.se%2Ftrygghansa%2FProducts%2FProduct.asp%3FitemId%3D6971&uid=1234769853330005978&x=44757&z=-60& HTTP/1.1" 200 35 "http://www.xroads.se/trygghansa/Products/Product.asp?itemId=6971" "Mozilla/5.0 (Windows; U; Windows NT 5.1; sv-SE; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6"
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
require 'test_helper'
|
|
2
|
+
require 'log_parser'
|
|
3
|
+
|
|
4
|
+
class LogParserTest < Test::Unit::TestCase
|
|
5
|
+
context "The log line-parser" do
|
|
6
|
+
setup do
|
|
7
|
+
logline = "213.47.64.53 - - [26/Dec/2008:19:54:28 -0800] \"GET /log.gif?_mykey=myvalue&_stars%2A=%2A%2A%2A&_tags=bok&_tags=referens&_title=Stoke%20City%20-%20Manchester%20City&_type=viewitem&_url=%2Fsearch%2F%3Flanguage%3Des%26q%3Devent%253A1466569%2Bvendor%253Abwin&_user%3Adate=31%20Enero%2C%202009%2013%3A45&a=Mozilla%2F5.0%20(Macintosh%3B%20U%3B%20Intel%20Mac%20OS%20X%2010_5_6%3B%20en-us)%20AppleWebKit%2F525.27.1%20(KHTML%2C%20like%20Gecko)%20Version%2F3.2.1%20Safari%2F525.27.1&aid=bwintest&h=2&l=en-us&n=netscape&p=macintel&r=https%3A%2F%2Fsearch.bwinlabs.com%2F&s=1280x800&t=bwin%20search&u=https%3A%2F%2Fsearch.bwinlabs.com%2Fsearch%2F%3Fq%3D(%2B%2Bcategory%3Alive_events%2Cevents)%2BAND%2B(manchester)%26query%3Dmanchester&uid=1232996053305816134&x=95757&z=-60& HTTP/1.1\" 200 35 \"https://search.bwinlabs.com/search/?q=%28+%2Bcategory%3Alive_events%2Cevents%29+AND+%28manchester%29&query=manchester\" \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1\""
|
|
8
|
+
@parsed_line = LogParser.parse_line(logline)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
should "return the IP" do
|
|
12
|
+
assert_equal '213.47.64.53', @parsed_line[:ip]
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
should "return the timestamp" do
|
|
16
|
+
assert_equal '2008-12-26T19:54:28-08:00', @parsed_line[:timestamp].to_s
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
should "return the full request" do
|
|
20
|
+
assert @parsed_line[:request] =~ /^GET \/log.gif\?/
|
|
21
|
+
assert @parsed_line[:request] =~ /HTTP\/1\.1$/
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
should "return the status" do
|
|
25
|
+
assert_equal '200', @parsed_line[:status].to_s
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
should "return keys and values" do
|
|
29
|
+
assert_equal 'myvalue', @parsed_line[:_mykey]
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
should "decode keys and values" do
|
|
33
|
+
assert_equal '***', @parsed_line["_stars*".to_sym]
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
should "support multiple values for a key" do
|
|
37
|
+
assert_equal Array, @parsed_line[:_tags].class
|
|
38
|
+
assert_equal 2, @parsed_line[:_tags].length
|
|
39
|
+
assert_equal "bok", @parsed_line[:_tags][0]
|
|
40
|
+
assert_equal "referens", @parsed_line[:_tags][1]
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
should "parse all web server specific attributes that it knows of" do
|
|
44
|
+
LogParser::SERVER_ATTRIBUTES.each do |key|
|
|
45
|
+
assert @parsed_line.has_key?(key)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
should "return nil for empty lines" do
|
|
50
|
+
assert_equal nil, LogParser.parse_line("")
|
|
51
|
+
assert_equal nil, LogParser.parse_line("\n")
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
should "raise ParseError for unparseable lines" do
|
|
55
|
+
assert_raise LogParser::ParseError do
|
|
56
|
+
of = LogParser.parse_line("0.47.64.53 - - [NOTATIMESTAMP] \"GET /log.gif?_mykey=myvalue\" 200\n")
|
|
57
|
+
end
|
|
58
|
+
assert_raise LogParser::ParseError do
|
|
59
|
+
file = File.open(fixture_file('error_line.log'))
|
|
60
|
+
file.set_encoding("UTF-8")
|
|
61
|
+
line = file.readline()
|
|
62
|
+
of = LogParser.parse_line(line)
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
context "The log parser" do
|
|
68
|
+
setup do
|
|
69
|
+
@log = "0.47.64.53 - - [26/Dec/2008:19:54:28 -0800] \"GET /log.gif?_mykey=myvalue\" 200\n" +
|
|
70
|
+
"1.47.64.53 - - [26/Dec/2008:19:54:28 -0800] \"GET /log.gif?_mykey=myvalue\" 200\n" +
|
|
71
|
+
"2.47.64.53 - - [26/Dec/2008:19:54:28 -0800] \"GET /log.gif?_mykey=myvalue\" 200"
|
|
72
|
+
@parsed_log = LogParser.new(StringIO.new(@log)).to_a
|
|
73
|
+
end
|
|
74
|
+
should "parse several lines" do
|
|
75
|
+
assert_equal 3, @parsed_log.length
|
|
76
|
+
assert_equal "0.47.64.53", @parsed_log[0][:ip]
|
|
77
|
+
assert_equal "1.47.64.53", @parsed_log[1][:ip]
|
|
78
|
+
assert_equal "2.47.64.53", @parsed_log[2][:ip]
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
end
|
|
84
|
+
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
|
2
|
+
require 'test_helper'
|
|
3
|
+
require 'observation_compiler'
|
|
4
|
+
include ObservationCompiler
|
|
5
|
+
|
|
6
|
+
class ObservationCompilerTest < Test::Unit::TestCase
|
|
7
|
+
context "LogLine" do
|
|
8
|
+
setup do
|
|
9
|
+
@ugly_cdt, @ugly_utc = File.readlines(fixture_file('unquoted_ugliness.log'))
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
context "#initialize" do
|
|
13
|
+
if RUBY_VERSION >= "1.9"
|
|
14
|
+
should "remove invalid characters" do
|
|
15
|
+
@mixed_encoding = File.readlines(fixture_file('log_invalid_mixed_encoding.log'), :encoding => "utf-8").first
|
|
16
|
+
assert LogLine.new(@mixed_encoding).to_s.valid_encoding?
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
context "#valid?" do
|
|
22
|
+
should "return true for a valid line" do
|
|
23
|
+
line = %Q{195.178.165.141 - - [12/Apr/2010:09:07:20 +0200] "GET /log.gif?a=Mozilla...\n}
|
|
24
|
+
assert LogLine.new(line).valid?
|
|
25
|
+
end
|
|
26
|
+
should "return false for a non-valid line" do
|
|
27
|
+
line = %Q{195.178.165.141 - - [] "GET /log.gif?a=Mozilla...\n}
|
|
28
|
+
assert ! LogLine.new(line).valid?
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
context "#normalize_timestamp" do
|
|
33
|
+
|
|
34
|
+
should "convert Apache common log format dates to UTC" do
|
|
35
|
+
line = %Q{195.178.165.141 - - [12/Apr/2010:09:07:20 +0200] "GET /log.gif?a=Mozilla...\n}
|
|
36
|
+
normalized_line = %Q{195.178.165.141 - - [12/Apr/2010:07:07:20 +0000] "GET /log.gif?a=Mozilla...\n}
|
|
37
|
+
assert_equal normalized_line, LogLine.new(line).send(:normalize_timestamp)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
should "convert Apache common log format dates to UTC even for ugly requests" do
|
|
41
|
+
assert_equal @ugly_utc, LogLine.new(@ugly_cdt).send(:normalize_timestamp)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
should "not stumble on S3 log format" do
|
|
45
|
+
line = %Q{044075be8905af794e579cfb92edacca4f1bfb4de13a87e51a163bd7ffd442bc rw-obs-1 [16/Mar/2010:16:00:00 +0000] 85.225.221.221 65a011a29cdf8ec533ec3d1ccaae921c...\n}
|
|
46
|
+
assert_equal line, LogLine.new(line).send(:normalize_timestamp)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
context "#normalize_s3_format" do
|
|
52
|
+
|
|
53
|
+
should "not change log lines of the right format" do
|
|
54
|
+
line = %Q{83.255.102.181 - - [22/Apr/2010:16:19:52 +0200] "GET /log.gif?a=Mozilla%2F4.0%20(compatible%3B%20MSIE%208.0%3B%20Windows%20NT%206.0%3B%20Trident%2F4.0%3B%20SLCC1%3B%20.NET%20CLR%202.0.50727%3B%20OfficeLiveConnector.1.3%3B%20OfficeLivePatch.0.0%3B%20.NET%20CLR%203.5.30729%3B%20.NET%20CLR%203.0.30618)&aid=jetshop&l=sv&n=microsoft%20internet%20explorer&o=visit_page&p=win32&s=1024x768&sid=www.24.se&t=Mobiltillbeh%C3%B6r%2C%20batteri%20laptop%20%2C%20datortillbeh%C3%B6r%20och%20mobilskal%20hos%2024.se&u=http%3A%2F%2Fwww.24.se%2FShowSearchResult.aspx%3F%26pagesize%3D50%26searchstring%3Dfl%C3%A4kt&uid=1271105199151801502&x=30270&z=-120& HTTP/1.1" 200 35 "http://www.24.se/ShowSearchResult.aspx?&pagesize=50&searchstring=fläkt" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; SLCC1; .NET CLR 2.0.50727; OfficeLiveConnector.1.3; OfficeLivePatch.0.0; .NET CLR 3.5.30729; .NET CLR 3.0.30618)" "-" "rwcdid=0A000104B386C34BD146BFC002672F0B"\n}
|
|
55
|
+
assert_equal line, LogLine.new(line).send(:normalize_s3_format)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
should "normalize s3 format" do
|
|
59
|
+
line = %Q{044075be8905af794e579cfb92edacca4f1bfb4de13a87e51a163bd7ffd442bc rw-obs-1 [22/Apr/2010:06:00:23 +0000] 194.237.142.6 65a011a29cdf8ec533ec3d1ccaae921c CFCC97257CE8213A REST.GET.OBJECT log.gif "GET /log.gif?_item_id=1436&a=Mozilla%2F4.0%20(compatible%3B%20MSIE%207.0%3B%20Windows%20NT%206.0%3B%20SLCC1%3B%20.NET%20CLR%202.0.50727%3B%20.NET%20CLR%201.1.4322%3B%20InfoPath.1%3B%20.NET%20CLR%203.5.30729%3B%20.NET%20CLR%203.0.30618)&aid=jetshop&e=click&l=sv&n=microsoft%20internet%20explorer&o=pick_item&p=win32&r=http%3A%2F%2Fwww.tahorseware.se%2Fhund-c-155-1.aspx&s=1280x1024&sid=www.tahorseware.se&t=Biopet%20Omega-3%20%2C%20250%20ml&u=http%3A%2F%2Fwww.tahorseware.se%2Fbiopet-omega3--250-ml-p-1436-c-155.aspx&uid=1264576170959509276&x=31710&z=-120& HTTP/1.1" 200 - 35 35 4 3 "http://www.tahorseware.se/biopet-omega3--250-ml-p-1436-c-155.aspx" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 3.5.30729; .NET CLR 3.0.30618)" -\n}
|
|
60
|
+
normalized_line = %Q{194.237.142.6 - - [22/Apr/2010:06:00:23 +0000] "GET /log.gif?_item_id=1436&a=Mozilla%2F4.0%20(compatible%3B%20MSIE%207.0%3B%20Windows%20NT%206.0%3B%20SLCC1%3B%20.NET%20CLR%202.0.50727%3B%20.NET%20CLR%201.1.4322%3B%20InfoPath.1%3B%20.NET%20CLR%203.5.30729%3B%20.NET%20CLR%203.0.30618)&aid=jetshop&e=click&l=sv&n=microsoft%20internet%20explorer&o=pick_item&p=win32&r=http%3A%2F%2Fwww.tahorseware.se%2Fhund-c-155-1.aspx&s=1280x1024&sid=www.tahorseware.se&t=Biopet%20Omega-3%20%2C%20250%20ml&u=http%3A%2F%2Fwww.tahorseware.se%2Fbiopet-omega3--250-ml-p-1436-c-155.aspx&uid=1264576170959509276&x=31710&z=-120& HTTP/1.1" 200 35 "http://www.tahorseware.se/biopet-omega3--250-ml-p-1436-c-155.aspx" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 1.1.4322; InfoPath.1; .NET CLR 3.5.30729; .NET CLR 3.0.30618)" "-" "-"\n}
|
|
61
|
+
assert_equal normalized_line, LogLine.new(line).send(:normalize_s3_format)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
context "#normalize_apache_format" do
|
|
67
|
+
|
|
68
|
+
should "not change log lines of the right format" do
|
|
69
|
+
line = %Q{83.255.102.181 - - [22/Apr/2010:16:19:52 +0200] "GET /log.gif?a=Mozilla%2F4.0%20(compatible%3B%20MSIE%208.0%3B%20Windows%20NT%206.0%3B%20Trident%2F4.0%3B%20SLCC1%3B%20.NET%20CLR%202.0.50727%3B%20OfficeLiveConnector.1.3%3B%20OfficeLivePatch.0.0%3B%20.NET%20CLR%203.5.30729%3B%20.NET%20CLR%203.0.30618)&aid=jetshop&l=sv&n=microsoft%20internet%20explorer&o=visit_page&p=win32&s=1024x768&sid=www.24.se&t=Mobiltillbeh%C3%B6r%2C%20batteri%20laptop%20%2C%20datortillbeh%C3%B6r%20och%20mobilskal%20hos%2024.se&u=http%3A%2F%2Fwww.24.se%2FShowSearchResult.aspx%3F%26pagesize%3D50%26searchstring%3Dfl%C3%A4kt&uid=1271105199151801502&x=30270&z=-120& HTTP/1.1" 200 35 "http://www.24.se/ShowSearchResult.aspx?&pagesize=50&searchstring=fläkt" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; SLCC1; .NET CLR 2.0.50727; OfficeLiveConnector.1.3; OfficeLivePatch.0.0; .NET CLR 3.5.30729; .NET CLR 3.0.30618)" "-" "rwcdid=0A000104B386C34BD146BFC002672F0B"\n}
|
|
70
|
+
assert_equal line, LogLine.new(line).send(:normalize_apache_format)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
should "normalize apache format" do
|
|
74
|
+
line = %Q{124.191.88.9 - - [26/May/2009:23:59:50 +0000] "GET /log.gif?_current_price=99%20EUR&_image=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FDesignhouseStockholm%2Fart_placemats_stor.jpg&_item_id=6401&_thumbnail=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FDesignhouseStockholm%2Fart_placemats_liten.jpg&_title=%C2%A0ART%20PLACEMATS&_url=http%3A%2F%2Fdesignhousestockholm.crossroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D6401&a=Mozilla%2F5.0%20(Macintosh%3B%20U%3B%20Intel%20Mac%20OS%20X%2010_5_6%3B%20en-us)%20AppleWebKit%2F525.27.1%20(KHTML%2C%20like%20Gecko)%20Version%2F3.2.1%20Safari%2F525.27.1&aid=crossroads&l=en-us&n=netscape&o=view_item&p=macintel&r=http%3A%2F%2Fdesignhousestockholm.crossroads.se%2FPortal%2FProducts%2FProductGroup.asp%3FsectionId%3D2775&s=1440x900&sid=DHS_Shop&t=Product&u=http%3A%2F%2Fdesignhousestockholm.crossroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D6401&uid=1243382389648842685&x=9949&z=-600& HTTP/1.1" 200 35 "http://designhousestockholm.crossroads.se/Portal/Products/Product.asp?itemId=6401" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1"\n}
|
|
75
|
+
normalized_line = %Q{124.191.88.9 - - [26/May/2009:23:59:50 +0000] "GET /log.gif?_current_price=99%20EUR&_image=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FDesignhouseStockholm%2Fart_placemats_stor.jpg&_item_id=6401&_thumbnail=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FDesignhouseStockholm%2Fart_placemats_liten.jpg&_title=%C2%A0ART%20PLACEMATS&_url=http%3A%2F%2Fdesignhousestockholm.crossroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D6401&a=Mozilla%2F5.0%20(Macintosh%3B%20U%3B%20Intel%20Mac%20OS%20X%2010_5_6%3B%20en-us)%20AppleWebKit%2F525.27.1%20(KHTML%2C%20like%20Gecko)%20Version%2F3.2.1%20Safari%2F525.27.1&aid=crossroads&l=en-us&n=netscape&o=view_item&p=macintel&r=http%3A%2F%2Fdesignhousestockholm.crossroads.se%2FPortal%2FProducts%2FProductGroup.asp%3FsectionId%3D2775&s=1440x900&sid=DHS_Shop&t=Product&u=http%3A%2F%2Fdesignhousestockholm.crossroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D6401&uid=1243382389648842685&x=9949&z=-600& HTTP/1.1" 200 35 "http://designhousestockholm.crossroads.se/Portal/Products/Product.asp?itemId=6401" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1" "-" "-"\n}
|
|
76
|
+
assert_equal normalized_line, LogLine.new(line).send(:normalize_apache_format)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
should "normalize ugly lines too" do
|
|
80
|
+
normalized_line = @ugly_cdt.sub(/$/, ' "-" "-"')
|
|
81
|
+
assert_equal normalized_line, LogLine.new(@ugly_cdt).send(:normalize_apache_format)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
context "#timestamp" do
|
|
87
|
+
|
|
88
|
+
should "parse a normal timestamp" do
|
|
89
|
+
line = %Q{83.255.102.181 - - [22/Apr/2010:16:19:52 +0000] "GET /log.gif?a=Mozilla%2F4.0%20(compatible%3B%20MSIE%208.0%3B%20Windows%20NT%206.0%3B%20Trident%2F4.0%3B%20SLCC1%3B%20.NET%20CLR%202.0.50727%3B%20OfficeLiveConnector.1.3%3B%20OfficeLivePatch.0.0%3B%20.NET%20CLR%203.5.30729%3B%20.NET%20CLR%203.0.30618)&aid=jetshop&l=sv&n=microsoft%20internet%20explorer&o=visit_page&p=win32&s=1024x768&sid=www.24.se&t=Mobiltillbeh%C3%B6r%2C%20batteri%20laptop%20%2C%20datortillbeh%C3%B6r%20och%20mobilskal%20hos%2024.se&u=http%3A%2F%2Fwww.24.se%2FShowSearchResult.aspx%3F%26pagesize%3D50%26searchstring%3Dfl%C3%A4kt&uid=1271105199151801502&x=30270&z=-120& HTTP/1.1" 200 35 "http://www.24.se/ShowSearchResult.aspx?&pagesize=50&searchstring=fläkt" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; SLCC1; .NET CLR 2.0.50727; OfficeLiveConnector.1.3; OfficeLivePatch.0.0; .NET CLR 3.5.30729; .NET CLR 3.0.30618)" "-" "rwcdid=0A000104B386C34BD146BFC002672F0B"\n}
|
|
90
|
+
assert_equal "Thu Apr 22 16:19:52 UTC 2010", LogLine.new(line).timestamp.strftime('%a %b %d %H:%M:%S %Z %Y')
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
should "parse a timestamp with negative UTC timezone (will this ever happen?)" do
|
|
94
|
+
line = %Q{83.255.102.181 - - [22/Apr/2010:16:19:52 -0000] "GET /log.gif?a=Mozilla%2F4.0%20(compatible%3B%20MSIE%208.0%3B%20Windows%20NT%206.0%3B%20Trident%2F4.0%3B%20SLCC1%3B%20.NET%20CLR%202.0.50727%3B%20OfficeLiveConnector.1.3%3B%20OfficeLivePatch.0.0%3B%20.NET%20CLR%203.5.30729%3B%20.NET%20CLR%203.0.30618)&aid=jetshop&l=sv&n=microsoft%20internet%20explorer&o=visit_page&p=win32&s=1024x768&sid=www.24.se&t=Mobiltillbeh%C3%B6r%2C%20batteri%20laptop%20%2C%20datortillbeh%C3%B6r%20och%20mobilskal%20hos%2024.se&u=http%3A%2F%2Fwww.24.se%2FShowSearchResult.aspx%3F%26pagesize%3D50%26searchstring%3Dfl%C3%A4kt&uid=1271105199151801502&x=30270&z=-120& HTTP/1.1" 200 35 "http://www.24.se/ShowSearchResult.aspx?&pagesize=50&searchstring=fläkt" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; SLCC1; .NET CLR 2.0.50727; OfficeLiveConnector.1.3; OfficeLivePatch.0.0; .NET CLR 3.5.30729; .NET CLR 3.0.30618)" "-" "rwcdid=0A000104B386C34BD146BFC002672F0B"\n}
|
|
95
|
+
assert_equal "Thu Apr 22 16:19:52 UTC 2010", LogLine.new(line).timestamp.strftime('%a %b %d %H:%M:%S %Z %Y')
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
context "Job" do
|
|
103
|
+
setup do
|
|
104
|
+
@raw_prefix = "observer-raw-"
|
|
105
|
+
@path = "/tmp/test/observation_compiler"
|
|
106
|
+
FileUtils.rm_rf(@path)
|
|
107
|
+
@job = Job.new(:working_path => @path, :raw_logs_prefix => @raw_prefix)
|
|
108
|
+
@job.create_working_folders
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
context "#merge_raw_into_processed" do
|
|
112
|
+
|
|
113
|
+
should "merge all raw logs for a date, but split them to a file per observation date" do
|
|
114
|
+
mockup_log_file( File.join(@job.raw_working_path, @raw_prefix + "2010-01-02.merge_1"),
|
|
115
|
+
[ { :o => "view_item", :timestamp => Time.parse("2010-01-01 23:00:00") },
|
|
116
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 01:00:00") },
|
|
117
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 02:00:00") } ])
|
|
118
|
+
mockup_log_file( File.join(@job.raw_working_path, @raw_prefix + "2010-01-02.merge_2"),
|
|
119
|
+
[ { :o => "view_item", :timestamp => Time.parse("2010-01-01 23:30:00") },
|
|
120
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 01:30:00") },
|
|
121
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 02:30:00") } ])
|
|
122
|
+
|
|
123
|
+
@job.merge_raw_into_processed(Date.parse("2010-01-02"))
|
|
124
|
+
|
|
125
|
+
result_1 = File.read(File.join(@job.processed_working_path, @job.processed_log_name(Date.parse("2010-01-01"))))
|
|
126
|
+
expected_1 = mockup_log(
|
|
127
|
+
[ { :o => "view_item", :timestamp => Time.parse("2010-01-01 23:00:00") },
|
|
128
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-01 23:30:00") } ])
|
|
129
|
+
assert_equal expected_1, result_1
|
|
130
|
+
|
|
131
|
+
result_2 = File.read(File.join(@job.processed_working_path, @job.processed_log_name(Date.parse("2010-01-02"))))
|
|
132
|
+
expected_2 = mockup_log(
|
|
133
|
+
[ { :o => "view_item", :timestamp => Time.parse("2010-01-02 01:00:00") },
|
|
134
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 02:00:00") },
|
|
135
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 01:30:00") },
|
|
136
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 02:30:00") } ])
|
|
137
|
+
assert_equal expected_2, result_2
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
should "skip non-valid lines" do
|
|
141
|
+
mockup_log_file( File.join(@job.raw_working_path, @raw_prefix + "2010-01-02.merge_1"),
|
|
142
|
+
[ { :o => "view_item", :timestamp => Time.parse("2010-01-01 23:00:00") },
|
|
143
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 01:00:00") },
|
|
144
|
+
{ :o => "view_item", :timestamp => nil },
|
|
145
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 02:00:00") } ])
|
|
146
|
+
mockup_log_file( File.join(@job.raw_working_path, @raw_prefix + "2010-01-02.merge_2"),
|
|
147
|
+
[ { :o => "view_item", :timestamp => Time.parse("2010-01-01 23:30:00") },
|
|
148
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 01:30:00") },
|
|
149
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 02:30:00") },
|
|
150
|
+
{ :o => "view_item", :timestamp => nil }])
|
|
151
|
+
|
|
152
|
+
@job.merge_raw_into_processed(Date.parse("2010-01-02"))
|
|
153
|
+
|
|
154
|
+
result_1 = File.read(File.join(@job.processed_working_path, @job.processed_log_name(Date.parse("2010-01-01"))))
|
|
155
|
+
expected_1 = mockup_log(
|
|
156
|
+
[ { :o => "view_item", :timestamp => Time.parse("2010-01-01 23:00:00") },
|
|
157
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-01 23:30:00") } ])
|
|
158
|
+
assert_equal expected_1, result_1
|
|
159
|
+
|
|
160
|
+
result_2 = File.read(File.join(@job.processed_working_path, @job.processed_log_name(Date.parse("2010-01-02"))))
|
|
161
|
+
expected_2 = mockup_log(
|
|
162
|
+
[ { :o => "view_item", :timestamp => Time.parse("2010-01-02 01:00:00") },
|
|
163
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 02:00:00") },
|
|
164
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 01:30:00") },
|
|
165
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-02 02:30:00") } ])
|
|
166
|
+
assert_equal expected_2, result_2
|
|
167
|
+
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
context "#sort_processed" do
|
|
173
|
+
|
|
174
|
+
should "sort all processed files in range" do
|
|
175
|
+
file_name = File.join(@job.processed_working_path, @job.processed_log_name(Date.parse("2010-01-01")))
|
|
176
|
+
|
|
177
|
+
mockup_log_file(file_name,
|
|
178
|
+
[ { :o => "view_item", :timestamp => Time.parse("2010-01-01 02:01:01") },
|
|
179
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-01 01:02:01") },
|
|
180
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-01 01:01:02") },
|
|
181
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-01 01:01:01") } ])
|
|
182
|
+
|
|
183
|
+
@job.sort_processed((Date.parse "2010-01-01")..(Date.parse "2010-01-01"))
|
|
184
|
+
|
|
185
|
+
result = File.read(file_name)
|
|
186
|
+
expected = mockup_log(
|
|
187
|
+
[ { :o => "view_item", :timestamp => Time.parse("2010-01-01 01:01:01") },
|
|
188
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-01 01:01:02") },
|
|
189
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-01 01:02:01") },
|
|
190
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-01 02:01:01") } ])
|
|
191
|
+
assert_equal expected, result
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
should "keep unique log entries" do
|
|
195
|
+
file_name = File.join(@job.processed_working_path, @job.processed_log_name(Date.parse("2010-01-01")))
|
|
196
|
+
|
|
197
|
+
mockup_log_file(file_name,
|
|
198
|
+
[ { :o => "view_item", :timestamp => Time.parse("2010-01-01 01:01:01"), :ip => "11.11.11.11" },
|
|
199
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-01 01:01:01"), :ip => "11.11.11.11" },
|
|
200
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-01 01:01:01"), :ip => "22.22.22.22"},
|
|
201
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-01 01:01:01"), :ip => "22.22.22.22" } ])
|
|
202
|
+
|
|
203
|
+
@job.sort_processed((Date.parse "2010-01-01")..(Date.parse "2010-01-01"))
|
|
204
|
+
|
|
205
|
+
result = File.read(file_name)
|
|
206
|
+
expected = mockup_log(
|
|
207
|
+
[ { :o => "view_item", :timestamp => Time.parse("2010-01-01 01:01:01"), :ip => "11.11.11.11" },
|
|
208
|
+
{ :o => "view_item", :timestamp => Time.parse("2010-01-01 01:01:01"), :ip => "22.22.22.22" } ])
|
|
209
|
+
assert_equal expected, result
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
end
|