logmerge 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,162 @@
1
+ $TESTING = defined? $TESTING
2
+
3
+ require 'logmerge'
4
+
5
+ class LogMerge::IPResolver
6
+
7
+ ##
8
+ # Marshal format version number.
9
+
10
+ FORMAT = 0
11
+
12
+ ##
13
+ # Default number of threads.
14
+
15
+ THREADS = $TESTING ? 2 : 100
16
+
17
+ def self.create
18
+ resolver = nil
19
+
20
+ if File.exists? '.name_cache' then
21
+ data = File.read '.name_cache'
22
+ begin
23
+ resolver = Marshal.load data
24
+ rescue ArgumentError
25
+ end
26
+ end
27
+
28
+ resolver ||= self.new
29
+
30
+ yield resolver
31
+
32
+ ensure
33
+ begin
34
+ File.open '.name_cache', 'w' do |fp|
35
+ Marshal.dump resolver, fp
36
+ end
37
+ rescue Errno::EACCES
38
+ # ignore
39
+ end
40
+ end
41
+
42
+ def self.resolve(output = STDOUT, input = ARGF)
43
+ create do |resolver|
44
+ resolver.run output, input
45
+ end
46
+ end
47
+
48
+ def initialize(cache = {})
49
+ @cache = cache
50
+ @failures = []
51
+ @start_time = Time.now
52
+ @dns = Resolv::DNS.new
53
+
54
+ @output = nil
55
+ @done = false
56
+ @queue = SizedQueue.new THREADS
57
+ @threads = ThreadGroup.new
58
+ @buf = {}
59
+ @cur = 0
60
+ end
61
+
62
+ def expire_entries(start_time)
63
+ elapsed = (Time.now - start_time).to_i
64
+ @cache.each { |ip, (name, ttl)| @cache[ip][1] = ttl - elapsed }
65
+ @cache.delete_if { |ip, (name, ttl)| ttl <= 0 }
66
+ return nil
67
+ end
68
+
69
+ def flush
70
+ while @buf.has_key? @cur do
71
+ @output.puts @buf.delete(@cur)
72
+ @cur += 1
73
+ end
74
+ end
75
+
76
+ def marshal_dump
77
+ @failures.each { |ip| @cache.delete ip }
78
+
79
+ expire_entries @start_time # compact
80
+
81
+ return [FORMAT, @cache, @start_time]
82
+ end
83
+
84
+ def marshal_load(dumped)
85
+ format, cache, start_time = dumped
86
+
87
+ case format
88
+ when 0 then
89
+ send :initialize, cache
90
+ expire_entries start_time # expire
91
+ end
92
+ end
93
+
94
+ def resolve(ip)
95
+ name, = @cache[ip]
96
+
97
+ return name unless name.nil?
98
+
99
+ begin
100
+ dns_name = Resolv::IPv4.create(ip).to_name
101
+ rescue ArgumentError
102
+ return ip
103
+ end
104
+
105
+ @dns.each_resource dns_name, Resolv::DNS::Resource::IN::PTR do |res|
106
+ @cache[ip] = [res.name.to_s, res.ttl * 2]
107
+ return res.name.to_s
108
+ end
109
+
110
+ @cache[ip] = [ip, 0]
111
+ @failures << ip
112
+
113
+ return ip
114
+ end
115
+
116
+ def run(output, input)
117
+ start_threads
118
+ line_no = 0
119
+ @output = output
120
+
121
+ input.each_line do |line|
122
+ ip, rest = line.split ' ', 2
123
+ record = [line_no, ip, rest]
124
+ line_no += 1
125
+ @queue << record
126
+ flush if line_no % LogMerge::MAX_INFLIGHT == 0
127
+ end
128
+
129
+ @done = true
130
+
131
+ Thread.pass until @queue.empty?
132
+
133
+ @threads.enclose
134
+ until @threads.list.empty? do
135
+ @threads.list.first.join
136
+ end
137
+
138
+ flush
139
+ end
140
+
141
+ def start_threads
142
+ THREADS.times do
143
+ Thread.start do
144
+ @threads.add Thread.current
145
+ loop do
146
+ begin
147
+ line_no, ip, rest = @queue.pop true
148
+ rescue ThreadError
149
+ break if @done
150
+ Thread.pass
151
+ retry
152
+ end
153
+
154
+ name = resolve ip
155
+ @buf[line_no] = "#{name} #{rest}"
156
+ end
157
+ end
158
+ end
159
+ end
160
+
161
+ end
162
+
data/lib/logmerge.rb ADDED
@@ -0,0 +1,13 @@
1
+ require 'thread'
2
+ require 'time'
3
+
4
+ module LogMerge
5
+
6
+ MAX_INFLIGHT = 5000
7
+
8
+ end
9
+
10
+ require 'logmerge/merger'
11
+ require 'logmerge/resolver'
12
+ require 'logmerge/resolv'
13
+
data/resolv_test ADDED
@@ -0,0 +1,26 @@
1
+ #!/usr/local/bin/ruby -ws
2
+
3
+ # This checks Ruby's resolv.rb to see if #ttl is available. When #ttl is
4
+ # available, then lib/logmerge/resolv.rb can be removed.
5
+
6
+ $ruby ||= false
7
+ $mine ||= false
8
+
9
+ if $ruby then
10
+ require 'resolv'
11
+ elsif $mine then
12
+ require 'lib/logmerge/resolv'
13
+ else
14
+ raise ArgumentError, "#$0 -mine|-ruby"
15
+ end
16
+
17
+ dns = Resolv::DNS.new
18
+ ip = '138.217.248.136'
19
+ ipv4 = Resolv::IPv4.create ip
20
+ p ipv4.to_name
21
+
22
+ dns.each_resource ipv4.to_name, Resolv::DNS::Resource::IN::PTR do |r|
23
+ p r.name.to_s
24
+ p r.ttl
25
+ end
26
+
data/test/access.log ADDED
@@ -0,0 +1,10 @@
1
+ 138.217.248.136 - - [17/Jan/2006:00:00:00 -0800] "GET /javascripts/prototype.js HTTP/1.1" 200 7358 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
2
+ 68.142.250.78 - - [17/Jan/2006:00:00:00 -0800] "GET /things/view/123841 HTTP/1.0" 200 9146 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
3
+ 138.217.248.136 - - [17/Jan/2006:00:00:01 -0800] "GET /javascripts/enumerable.js HTTP/1.1" 200 1084 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
4
+ 202.160.180.40 - - [17/Jan/2006:00:00:01 -0800] "GET /teams/progress/1147887 HTTP/1.0" 302 128 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
5
+ 138.217.248.136 - - [17/Jan/2006:00:00:01 -0800] "GET /javascripts/effects.js HTTP/1.1" 200 5319 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
6
+ 68.189.40.2 - - [17/Jan/2006:00:00:01 -0800] "GET /entries/view/139942 HTTP/1.1" 200 2975 "-" "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/312.5.1 (KHTML, like Gecko) Safari/312.3.1"
7
+ 202.160.180.40 - - [17/Jan/2006:00:00:02 -0800] "GET /people/progress/princessauddie/1147887 HTTP/1.0" 200 7511 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
8
+ 138.217.248.136 - - [17/Jan/2006:00:00:02 -0800] "GET /javascripts/dragdrop.js HTTP/1.1" 200 5279 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
9
+ 192.168.1.254 - - [17/Jan/2006:00:00:02 -0800] "HEAD / HTTP/1.0" 200 0 "-" "-"
10
+ 216.109.121.70 - - [17/Jan/2006:00:00:02 -0800] "GET /rss/uber/author?username=twilightpumpkin HTTP/1.0" 200 15482 "-" "YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide; users 0; views 0)"
@@ -0,0 +1,86 @@
1
+ require 'stringio'
2
+ require 'test/unit'
3
+
4
+ $TESTING = true
5
+
6
+ require 'logmerge/merger'
7
+
8
+ class LogMerge::Merger
9
+
10
+ attr_accessor :streams
11
+ attr_accessor :buf
12
+
13
+ end
14
+
15
+ class TestMerger < Test::Unit::TestCase
16
+
17
+ def test_all_closed_eh_closed
18
+ merger = LogMerge::Merger.new []
19
+ assert_equal true, merger.all_closed?
20
+ end
21
+
22
+ def test_all_closed_eh_not_closed
23
+ merger = LogMerge::Merger.new [StringIO.new]
24
+ assert_equal true, merger.all_closed?
25
+ end
26
+
27
+ def test_buf_fill_fill
28
+ stream = StringIO.new <<-EOF
29
+ [17/Jan/2006:00:00:01 -0800]
30
+ [17/Jan/2006:00:00:02 -0800]
31
+ EOF
32
+ merger = LogMerge::Merger.new [stream]
33
+
34
+ assert_equal [1137484801, " [17/Jan/2006:00:00:01 -0800] \n"], merger.buf[0]
35
+
36
+ merger.buf_fill 0
37
+
38
+ assert_equal [1137484802, " [17/Jan/2006:00:00:02 -0800] \n"], merger.buf[0]
39
+ end
40
+
41
+ def test_buf_fill_empty
42
+ stream = StringIO.new <<-EOF
43
+ [17/Jan/2006:00:00:01 -0800]
44
+ EOF
45
+ merger = LogMerge::Merger.new [stream]
46
+
47
+ assert_equal [1137484801, " [17/Jan/2006:00:00:01 -0800] \n"], merger.buf[0]
48
+
49
+ merger.buf_fill 0
50
+
51
+ assert_equal true, merger.buf.empty?
52
+ assert_equal true, merger.streams.empty?
53
+ end
54
+
55
+ def test_initialize
56
+ stream1 = StringIO.new " [17/Jan/2006:00:00:01 -0800] \n"
57
+ stream2 = StringIO.new " [17/Jan/2006:00:00:02 -0800] \n"
58
+
59
+ merger = LogMerge::Merger.new [stream1, stream2]
60
+
61
+ assert_equal false, merger.all_closed?
62
+ assert_equal [1137484801, " [17/Jan/2006:00:00:01 -0800] \n"], merger.buf[0]
63
+ assert_equal [1137484802, " [17/Jan/2006:00:00:02 -0800] \n"], merger.buf[1]
64
+ end
65
+
66
+ def test_merge
67
+ stream1 = StringIO.new " [17/Jan/2006:00:00:02 -0800] \n"
68
+ stream2 = StringIO.new " [17/Jan/2006:00:00:01 -0800] \n"
69
+
70
+ merger = LogMerge::Merger.new [stream1, stream2]
71
+
72
+ output = StringIO.new
73
+
74
+ merger.merge output
75
+
76
+ expected = <<-EOF
77
+ [17/Jan/2006:00:00:01 -0800]
78
+ [17/Jan/2006:00:00:02 -0800]
79
+ EOF
80
+
81
+ assert_equal true, merger.all_closed?
82
+ assert_equal expected, output.string
83
+ end
84
+
85
+ end
86
+
@@ -0,0 +1,234 @@
1
+ require 'stringio'
2
+ require 'test/unit'
3
+
4
+ $TESTING = true
5
+
6
+ require 'logmerge/resolver'
7
+
8
+ class LogMerge::IPResolver
9
+
10
+ attr_accessor :cache
11
+ attr_accessor :failures
12
+ attr_accessor :buf
13
+ attr_accessor :output
14
+ attr_accessor :cur
15
+ attr_accessor :start_time
16
+ attr_accessor :queue
17
+
18
+ end
19
+
20
+ class TestResolver < Test::Unit::TestCase
21
+
22
+ def setup
23
+ @resolver = LogMerge::IPResolver.new
24
+ end
25
+
26
+ def test_class_create
27
+ File.unlink 'test/.name_cache' if File.exist? 'test/.name_cache'
28
+
29
+ assert_equal false, File.exist?('test/.name_cache')
30
+
31
+ Dir.chdir 'test' do
32
+ LogMerge::IPResolver.create do |res|
33
+ assert_equal({}, res.cache)
34
+ end
35
+ end
36
+
37
+ assert_equal true, File.exist?('test/.name_cache')
38
+
39
+ ensure
40
+ begin File.unlink 'test/.name_cache' rescue Errno::ENOENT; end
41
+ end
42
+
43
+ def test_class_create_with_cache
44
+ File.open 'test/.name_cache', 'w' do |fp|
45
+ fp.write util_dumped_resolver
46
+ end
47
+
48
+ Dir.chdir 'test' do
49
+ LogMerge::IPResolver.create do |res|
50
+ assert_equal({'65.61.137.68' => ['brain.43things.com', 500]}, res.cache)
51
+ end
52
+ end
53
+
54
+ assert_equal true, File.exist?('test/.name_cache')
55
+
56
+ ensure
57
+ begin File.unlink 'test/.name_cache' rescue Errno::ENOENT; end
58
+ end
59
+
60
+ def test_expire_entries_age
61
+ @resolver.cache = {
62
+ '65.61.137.68' => ['brain.43things.com', 500]
63
+ }
64
+
65
+ @resolver.expire_entries Time.now - 250
66
+
67
+ assert_equal ['65.61.137.68'], @resolver.cache.keys
68
+ assert_equal ['brain.43things.com', 250], @resolver.cache['65.61.137.68']
69
+ end
70
+
71
+ def test_expire_entries_delete
72
+ @resolver.cache = {
73
+ '65.61.137.68' => ['brain.43things.com', 500]
74
+ }
75
+
76
+ @resolver.expire_entries Time.now - 501
77
+
78
+ assert_equal true, @resolver.cache.empty?
79
+ end
80
+
81
+ def test_expire_entries_now
82
+ @resolver.cache = {
83
+ '65.61.137.68' => ['brain.43things.com', 500]
84
+ }
85
+
86
+ @resolver.expire_entries Time.now
87
+
88
+ assert_equal ['65.61.137.68'], @resolver.cache.keys
89
+ assert_equal ['brain.43things.com', 500], @resolver.cache['65.61.137.68']
90
+ end
91
+
92
+ def test_flush
93
+ @resolver.buf[0] = "one"
94
+ @resolver.buf[1] = "two"
95
+ @resolver.buf[2] = "three"
96
+ @resolver.buf[3] = "four"
97
+
98
+ @resolver.output = StringIO.new
99
+ @resolver.flush
100
+ assert_equal "one\ntwo\nthree\nfour\n", @resolver.output.string
101
+ end
102
+
103
+ def test_marshal_dump
104
+ dumped = @resolver.marshal_dump
105
+
106
+ assert_instance_of Array, dumped
107
+ assert_equal 0, dumped[0]
108
+ assert_equal({}, dumped[1])
109
+ assert_instance_of Time, dumped[2]
110
+ end
111
+
112
+ def test_marshal_dump_with_cache
113
+ @resolver.cache['65.61.137.68'] = ['brain.43things.com', 500]
114
+ dumped = @resolver.marshal_dump
115
+
116
+ assert_instance_of Array, dumped
117
+ assert_equal 0, dumped[0]
118
+ assert_equal({'65.61.137.68' => ['brain.43things.com', 500]}, dumped[1])
119
+ assert_instance_of Time, dumped[2]
120
+ end
121
+
122
+ def test_marshal_load
123
+ data = "\004\010U:\031LogMerge::IPResolver[\010i\000{\000u:\tTime\rw\202\032\200\032\027\253\037"
124
+ resolver = Marshal.load data
125
+
126
+ assert_instance_of LogMerge::IPResolver, resolver
127
+ assert_equal({}, resolver.cache)
128
+ assert_equal [], resolver.failures, "Make sure it was init'd properly"
129
+ end
130
+
131
+ def test_marshal_load_with_cache
132
+ resolver = Marshal.load util_dumped_resolver
133
+
134
+ assert_instance_of LogMerge::IPResolver, resolver
135
+ assert_equal({'65.61.137.68' => ["brain.43things.com", 500]},
136
+ resolver.cache)
137
+ end
138
+
139
+ def test_resolve
140
+ assert_equal 'www.43things.com', @resolver.resolve('65.61.137.67')
141
+ assert_equal '0.0.0.0', @resolver.resolve('0.0.0.0')
142
+
143
+ assert_equal ['0.0.0.0'], @resolver.failures
144
+
145
+ assert_equal ['0.0.0.0', '65.61.137.67'], @resolver.cache.keys
146
+ assert_equal 0, @resolver.cache['0.0.0.0'].last
147
+ assert_operator 0, :<, @resolver.cache['65.61.137.67'].last
148
+ end
149
+
150
+ def test_resolve_from_cache
151
+ @resolver.cache['65.61.137.68'] = ['brain.43things.com', 0]
152
+
153
+ assert_equal 'www.43things.com', @resolver.resolve('65.61.137.67')
154
+ assert_equal 'brain.43things.com', @resolver.resolve('65.61.137.68')
155
+ assert_equal '0.0.0.0', @resolver.resolve('0.0.0.0')
156
+
157
+ assert_equal ['0.0.0.0'], @resolver.failures
158
+
159
+ assert_equal ['0.0.0.0', '65.61.137.67', '65.61.137.68'],
160
+ @resolver.cache.keys
161
+ assert_equal 0, @resolver.cache['0.0.0.0'].last
162
+ assert_operator 0, :<, @resolver.cache['65.61.137.67'].last
163
+ assert_equal 0, @resolver.cache['65.61.137.68'].last
164
+ end
165
+
166
+ def test_resolve_bad_ip
167
+ assert_equal '0.0.0.256', @resolver.resolve('0.0.0.256')
168
+ end
169
+
170
+ def test_run
171
+ input = StringIO.new <<-EOF
172
+ 138.217.248.136 - - [17/Jan/2006:00:00:00 -0800] "GET /javascripts/prototype.js HTTP/1.1" 200 7358 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
173
+ 68.142.250.78 - - [17/Jan/2006:00:00:01 -0800] "GET /things/view/123841 HTTP/1.0" 200 9146 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
174
+ 138.217.248.136 - - [17/Jan/2006:00:00:02 -0800] "GET /javascripts/enumerable.js HTTP/1.1" 200 1084 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
175
+ 202.160.180.40 - - [17/Jan/2006:00:00:03 -0800] "GET /teams/progress/1147887 HTTP/1.0" 302 128 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
176
+ 138.217.248.136 - - [17/Jan/2006:00:00:04 -0800] "GET /javascripts/effects.js HTTP/1.1" 200 5319 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
177
+ 68.189.40.2 - - [17/Jan/2006:00:00:05 -0800] "GET /entries/view/139942 HTTP/1.1" 200 2975 "-" "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/312.5.1 (KHTML, like Gecko) Safari/312.3.1"
178
+ 202.160.180.40 - - [17/Jan/2006:00:00:06 -0800] "GET /people/progress/princessauddie/1147887 HTTP/1.0" 200 7511 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
179
+ 138.217.248.136 - - [17/Jan/2006:00:00:07 -0800] "GET /javascripts/dragdrop.js HTTP/1.1" 200 5279 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
180
+ 192.168.1.254 - - [17/Jan/2006:00:00:08 -0800] "HEAD / HTTP/1.0" 200 0 "-" "-"
181
+ 216.109.121.70 - - [17/Jan/2006:00:00:09 -0800] "GET /rss/uber/author?username=twilightpumpkin HTTP/1.0" 200 15482 "-" "YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide; users 0; views 0)"
182
+ EOF
183
+ output = StringIO.new
184
+
185
+ @resolver.cache = {
186
+ '216.109.121.70' => ['oc31.my.dcn.yahoo.net', 94],
187
+ '68.142.250.78' => ['lj2268.inktomisearch.com', 90],
188
+ '138.217.248.136' => ['CPE-138-217-248-136.wa.bigpond.net.au', 167760],
189
+ '68.189.40.2' => ['68-189-40-2.dhcp.rdng.ca.charter.com', 81360],
190
+ '202.160.180.40' => ['lj9027.inktomisearch.com', 90],
191
+ '192.168.1.254' => ['dhcp-254.coop.robotcoop.com', 7200]
192
+ }
193
+
194
+ @resolver.run output, input
195
+
196
+ assert_equal true, input.eof?, "Input empty."
197
+ assert_equal true, @resolver.queue.empty?, "Queue empty"
198
+ assert_equal 10, output.string.scan("\n").length
199
+
200
+ expected = <<-EOF
201
+ CPE-138-217-248-136.wa.bigpond.net.au - - [17/Jan/2006:00:00:00 -0800] "GET /javascripts/prototype.js HTTP/1.1" 200 7358 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
202
+ lj2268.inktomisearch.com - - [17/Jan/2006:00:00:01 -0800] "GET /things/view/123841 HTTP/1.0" 200 9146 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
203
+ CPE-138-217-248-136.wa.bigpond.net.au - - [17/Jan/2006:00:00:02 -0800] "GET /javascripts/enumerable.js HTTP/1.1" 200 1084 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
204
+ lj9027.inktomisearch.com - - [17/Jan/2006:00:00:03 -0800] "GET /teams/progress/1147887 HTTP/1.0" 302 128 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
205
+ CPE-138-217-248-136.wa.bigpond.net.au - - [17/Jan/2006:00:00:04 -0800] "GET /javascripts/effects.js HTTP/1.1" 200 5319 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
206
+ 68-189-40-2.dhcp.rdng.ca.charter.com - - [17/Jan/2006:00:00:05 -0800] "GET /entries/view/139942 HTTP/1.1" 200 2975 "-" "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/312.5.1 (KHTML, like Gecko) Safari/312.3.1"
207
+ lj9027.inktomisearch.com - - [17/Jan/2006:00:00:06 -0800] "GET /people/progress/princessauddie/1147887 HTTP/1.0" 200 7511 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
208
+ CPE-138-217-248-136.wa.bigpond.net.au - - [17/Jan/2006:00:00:07 -0800] "GET /javascripts/dragdrop.js HTTP/1.1" 200 5279 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
209
+ dhcp-254.coop.robotcoop.com - - [17/Jan/2006:00:00:08 -0800] "HEAD / HTTP/1.0" 200 0 "-" "-"
210
+ oc31.my.dcn.yahoo.net - - [17/Jan/2006:00:00:09 -0800] "GET /rss/uber/author?username=twilightpumpkin HTTP/1.0" 200 15482 "-" "YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide; users 0; views 0)"
211
+ EOF
212
+
213
+ assert_equal expected, output.string
214
+ end
215
+
216
+ def test_start_threads
217
+ threads = Thread.list.length
218
+
219
+ @resolver.start_threads
220
+
221
+ assert_operator threads, :<, Thread.list.length
222
+ end
223
+
224
+ ##
225
+ # HACK don't like this method since it is Time.now dependent.
226
+
227
+ def util_dumped_resolver
228
+ resolver = LogMerge::IPResolver.new
229
+ resolver.cache['65.61.137.68'] = ['brain.43things.com', 500]
230
+ resolver.start_time = Time.now
231
+ return Marshal.dump(resolver)
232
+ end
233
+ end
234
+
metadata ADDED
@@ -0,0 +1,59 @@
1
+ --- !ruby/object:Gem::Specification
2
+ rubygems_version: 0.8.11.6
3
+ specification_version: 1
4
+ name: logmerge
5
+ version: !ruby/object:Gem::Version
6
+ version: 1.0.0
7
+ date: 2006-01-20 00:00:00 -08:00
8
+ summary: Resolves IP addresses and merges Apache access logs.
9
+ require_paths:
10
+ - lib
11
+ email: eric@robotcoop.com
12
+ homepage:
13
+ rubyforge_project:
14
+ description:
15
+ autorequire:
16
+ default_executable:
17
+ bindir: bin
18
+ has_rdoc: false
19
+ required_ruby_version: !ruby/object:Gem::Version::Requirement
20
+ requirements:
21
+ - - ">"
22
+ - !ruby/object:Gem::Version
23
+ version: 0.0.0
24
+ version:
25
+ platform: ruby
26
+ signing_key:
27
+ cert_chain:
28
+ authors:
29
+ - Eric Hodel
30
+ files:
31
+ - LICENSE
32
+ - Manifest.txt
33
+ - README
34
+ - Rakefile
35
+ - bin/ip2name
36
+ - bin/logmerge
37
+ - lib/logmerge.rb
38
+ - lib/logmerge/merger.rb
39
+ - lib/logmerge/resolv.rb
40
+ - lib/logmerge/resolver.rb
41
+ - resolv_test
42
+ - test/access.log
43
+ - test/test_merger.rb
44
+ - test/test_resolver.rb
45
+ test_files: []
46
+
47
+ rdoc_options: []
48
+
49
+ extra_rdoc_files: []
50
+
51
+ executables:
52
+ - ip2name
53
+ - logmerge
54
+ extensions: []
55
+
56
+ requirements: []
57
+
58
+ dependencies: []
59
+