logmerge 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +28 -0
- data/Manifest.txt +14 -0
- data/README +35 -0
- data/Rakefile +63 -0
- data/bin/ip2name +8 -0
- data/bin/logmerge +6 -0
- data/lib/logmerge/merger.rb +74 -0
- data/lib/logmerge/resolv.rb +1892 -0
- data/lib/logmerge/resolver.rb +162 -0
- data/lib/logmerge.rb +13 -0
- data/resolv_test +26 -0
- data/test/access.log +10 -0
- data/test/test_merger.rb +86 -0
- data/test/test_resolver.rb +234 -0
- metadata +59 -0
@@ -0,0 +1,162 @@
|
|
1
|
+
$TESTING = defined? $TESTING
|
2
|
+
|
3
|
+
require 'logmerge'
|
4
|
+
|
5
|
+
class LogMerge::IPResolver
|
6
|
+
|
7
|
+
##
|
8
|
+
# Marshal format version number.
|
9
|
+
|
10
|
+
FORMAT = 0
|
11
|
+
|
12
|
+
##
|
13
|
+
# Default number of threads.
|
14
|
+
|
15
|
+
THREADS = $TESTING ? 2 : 100
|
16
|
+
|
17
|
+
def self.create
|
18
|
+
resolver = nil
|
19
|
+
|
20
|
+
if File.exists? '.name_cache' then
|
21
|
+
data = File.read '.name_cache'
|
22
|
+
begin
|
23
|
+
resolver = Marshal.load data
|
24
|
+
rescue ArgumentError
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
resolver ||= self.new
|
29
|
+
|
30
|
+
yield resolver
|
31
|
+
|
32
|
+
ensure
|
33
|
+
begin
|
34
|
+
File.open '.name_cache', 'w' do |fp|
|
35
|
+
Marshal.dump resolver, fp
|
36
|
+
end
|
37
|
+
rescue Errno::EACCES
|
38
|
+
# ignore
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.resolve(output = STDOUT, input = ARGF)
|
43
|
+
create do |resolver|
|
44
|
+
resolver.run output, input
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def initialize(cache = {})
|
49
|
+
@cache = cache
|
50
|
+
@failures = []
|
51
|
+
@start_time = Time.now
|
52
|
+
@dns = Resolv::DNS.new
|
53
|
+
|
54
|
+
@output = nil
|
55
|
+
@done = false
|
56
|
+
@queue = SizedQueue.new THREADS
|
57
|
+
@threads = ThreadGroup.new
|
58
|
+
@buf = {}
|
59
|
+
@cur = 0
|
60
|
+
end
|
61
|
+
|
62
|
+
def expire_entries(start_time)
|
63
|
+
elapsed = (Time.now - start_time).to_i
|
64
|
+
@cache.each { |ip, (name, ttl)| @cache[ip][1] = ttl - elapsed }
|
65
|
+
@cache.delete_if { |ip, (name, ttl)| ttl <= 0 }
|
66
|
+
return nil
|
67
|
+
end
|
68
|
+
|
69
|
+
def flush
|
70
|
+
while @buf.has_key? @cur do
|
71
|
+
@output.puts @buf.delete(@cur)
|
72
|
+
@cur += 1
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def marshal_dump
|
77
|
+
@failures.each { |ip| @cache.delete ip }
|
78
|
+
|
79
|
+
expire_entries @start_time # compact
|
80
|
+
|
81
|
+
return [FORMAT, @cache, @start_time]
|
82
|
+
end
|
83
|
+
|
84
|
+
def marshal_load(dumped)
|
85
|
+
format, cache, start_time = dumped
|
86
|
+
|
87
|
+
case format
|
88
|
+
when 0 then
|
89
|
+
send :initialize, cache
|
90
|
+
expire_entries start_time # expire
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def resolve(ip)
|
95
|
+
name, = @cache[ip]
|
96
|
+
|
97
|
+
return name unless name.nil?
|
98
|
+
|
99
|
+
begin
|
100
|
+
dns_name = Resolv::IPv4.create(ip).to_name
|
101
|
+
rescue ArgumentError
|
102
|
+
return ip
|
103
|
+
end
|
104
|
+
|
105
|
+
@dns.each_resource dns_name, Resolv::DNS::Resource::IN::PTR do |res|
|
106
|
+
@cache[ip] = [res.name.to_s, res.ttl * 2]
|
107
|
+
return res.name.to_s
|
108
|
+
end
|
109
|
+
|
110
|
+
@cache[ip] = [ip, 0]
|
111
|
+
@failures << ip
|
112
|
+
|
113
|
+
return ip
|
114
|
+
end
|
115
|
+
|
116
|
+
def run(output, input)
|
117
|
+
start_threads
|
118
|
+
line_no = 0
|
119
|
+
@output = output
|
120
|
+
|
121
|
+
input.each_line do |line|
|
122
|
+
ip, rest = line.split ' ', 2
|
123
|
+
record = [line_no, ip, rest]
|
124
|
+
line_no += 1
|
125
|
+
@queue << record
|
126
|
+
flush if line_no % LogMerge::MAX_INFLIGHT == 0
|
127
|
+
end
|
128
|
+
|
129
|
+
@done = true
|
130
|
+
|
131
|
+
Thread.pass until @queue.empty?
|
132
|
+
|
133
|
+
@threads.enclose
|
134
|
+
until @threads.list.empty? do
|
135
|
+
@threads.list.first.join
|
136
|
+
end
|
137
|
+
|
138
|
+
flush
|
139
|
+
end
|
140
|
+
|
141
|
+
def start_threads
|
142
|
+
THREADS.times do
|
143
|
+
Thread.start do
|
144
|
+
@threads.add Thread.current
|
145
|
+
loop do
|
146
|
+
begin
|
147
|
+
line_no, ip, rest = @queue.pop true
|
148
|
+
rescue ThreadError
|
149
|
+
break if @done
|
150
|
+
Thread.pass
|
151
|
+
retry
|
152
|
+
end
|
153
|
+
|
154
|
+
name = resolve ip
|
155
|
+
@buf[line_no] = "#{name} #{rest}"
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
end
|
162
|
+
|
data/lib/logmerge.rb
ADDED
data/resolv_test
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
#!/usr/local/bin/ruby -ws
|
2
|
+
|
3
|
+
# This checks Ruby's resolv.rb to see if #ttl is available. When #ttl is
|
4
|
+
# available, then lib/logmerge/resolv.rb can be removed.
|
5
|
+
|
6
|
+
$ruby ||= false
|
7
|
+
$mine ||= false
|
8
|
+
|
9
|
+
if $ruby then
|
10
|
+
require 'resolv'
|
11
|
+
elsif $mine then
|
12
|
+
require 'lib/logmerge/resolv'
|
13
|
+
else
|
14
|
+
raise ArgumentError, "#$0 -mine|-ruby"
|
15
|
+
end
|
16
|
+
|
17
|
+
dns = Resolv::DNS.new
|
18
|
+
ip = '138.217.248.136'
|
19
|
+
ipv4 = Resolv::IPv4.create ip
|
20
|
+
p ipv4.to_name
|
21
|
+
|
22
|
+
dns.each_resource ipv4.to_name, Resolv::DNS::Resource::IN::PTR do |r|
|
23
|
+
p r.name.to_s
|
24
|
+
p r.ttl
|
25
|
+
end
|
26
|
+
|
data/test/access.log
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
138.217.248.136 - - [17/Jan/2006:00:00:00 -0800] "GET /javascripts/prototype.js HTTP/1.1" 200 7358 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
2
|
+
68.142.250.78 - - [17/Jan/2006:00:00:00 -0800] "GET /things/view/123841 HTTP/1.0" 200 9146 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
|
3
|
+
138.217.248.136 - - [17/Jan/2006:00:00:01 -0800] "GET /javascripts/enumerable.js HTTP/1.1" 200 1084 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
4
|
+
202.160.180.40 - - [17/Jan/2006:00:00:01 -0800] "GET /teams/progress/1147887 HTTP/1.0" 302 128 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
5
|
+
138.217.248.136 - - [17/Jan/2006:00:00:01 -0800] "GET /javascripts/effects.js HTTP/1.1" 200 5319 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
6
|
+
68.189.40.2 - - [17/Jan/2006:00:00:01 -0800] "GET /entries/view/139942 HTTP/1.1" 200 2975 "-" "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/312.5.1 (KHTML, like Gecko) Safari/312.3.1"
|
7
|
+
202.160.180.40 - - [17/Jan/2006:00:00:02 -0800] "GET /people/progress/princessauddie/1147887 HTTP/1.0" 200 7511 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
8
|
+
138.217.248.136 - - [17/Jan/2006:00:00:02 -0800] "GET /javascripts/dragdrop.js HTTP/1.1" 200 5279 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
9
|
+
192.168.1.254 - - [17/Jan/2006:00:00:02 -0800] "HEAD / HTTP/1.0" 200 0 "-" "-"
|
10
|
+
216.109.121.70 - - [17/Jan/2006:00:00:02 -0800] "GET /rss/uber/author?username=twilightpumpkin HTTP/1.0" 200 15482 "-" "YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide; users 0; views 0)"
|
data/test/test_merger.rb
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'stringio'
|
2
|
+
require 'test/unit'
|
3
|
+
|
4
|
+
$TESTING = true
|
5
|
+
|
6
|
+
require 'logmerge/merger'
|
7
|
+
|
8
|
+
class LogMerge::Merger
|
9
|
+
|
10
|
+
attr_accessor :streams
|
11
|
+
attr_accessor :buf
|
12
|
+
|
13
|
+
end
|
14
|
+
|
15
|
+
class TestMerger < Test::Unit::TestCase
|
16
|
+
|
17
|
+
def test_all_closed_eh_closed
|
18
|
+
merger = LogMerge::Merger.new []
|
19
|
+
assert_equal true, merger.all_closed?
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_all_closed_eh_not_closed
|
23
|
+
merger = LogMerge::Merger.new [StringIO.new]
|
24
|
+
assert_equal true, merger.all_closed?
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_buf_fill_fill
|
28
|
+
stream = StringIO.new <<-EOF
|
29
|
+
[17/Jan/2006:00:00:01 -0800]
|
30
|
+
[17/Jan/2006:00:00:02 -0800]
|
31
|
+
EOF
|
32
|
+
merger = LogMerge::Merger.new [stream]
|
33
|
+
|
34
|
+
assert_equal [1137484801, " [17/Jan/2006:00:00:01 -0800] \n"], merger.buf[0]
|
35
|
+
|
36
|
+
merger.buf_fill 0
|
37
|
+
|
38
|
+
assert_equal [1137484802, " [17/Jan/2006:00:00:02 -0800] \n"], merger.buf[0]
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_buf_fill_empty
|
42
|
+
stream = StringIO.new <<-EOF
|
43
|
+
[17/Jan/2006:00:00:01 -0800]
|
44
|
+
EOF
|
45
|
+
merger = LogMerge::Merger.new [stream]
|
46
|
+
|
47
|
+
assert_equal [1137484801, " [17/Jan/2006:00:00:01 -0800] \n"], merger.buf[0]
|
48
|
+
|
49
|
+
merger.buf_fill 0
|
50
|
+
|
51
|
+
assert_equal true, merger.buf.empty?
|
52
|
+
assert_equal true, merger.streams.empty?
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_initialize
|
56
|
+
stream1 = StringIO.new " [17/Jan/2006:00:00:01 -0800] \n"
|
57
|
+
stream2 = StringIO.new " [17/Jan/2006:00:00:02 -0800] \n"
|
58
|
+
|
59
|
+
merger = LogMerge::Merger.new [stream1, stream2]
|
60
|
+
|
61
|
+
assert_equal false, merger.all_closed?
|
62
|
+
assert_equal [1137484801, " [17/Jan/2006:00:00:01 -0800] \n"], merger.buf[0]
|
63
|
+
assert_equal [1137484802, " [17/Jan/2006:00:00:02 -0800] \n"], merger.buf[1]
|
64
|
+
end
|
65
|
+
|
66
|
+
def test_merge
|
67
|
+
stream1 = StringIO.new " [17/Jan/2006:00:00:02 -0800] \n"
|
68
|
+
stream2 = StringIO.new " [17/Jan/2006:00:00:01 -0800] \n"
|
69
|
+
|
70
|
+
merger = LogMerge::Merger.new [stream1, stream2]
|
71
|
+
|
72
|
+
output = StringIO.new
|
73
|
+
|
74
|
+
merger.merge output
|
75
|
+
|
76
|
+
expected = <<-EOF
|
77
|
+
[17/Jan/2006:00:00:01 -0800]
|
78
|
+
[17/Jan/2006:00:00:02 -0800]
|
79
|
+
EOF
|
80
|
+
|
81
|
+
assert_equal true, merger.all_closed?
|
82
|
+
assert_equal expected, output.string
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
|
@@ -0,0 +1,234 @@
|
|
1
|
+
require 'stringio'
|
2
|
+
require 'test/unit'
|
3
|
+
|
4
|
+
$TESTING = true
|
5
|
+
|
6
|
+
require 'logmerge/resolver'
|
7
|
+
|
8
|
+
class LogMerge::IPResolver
|
9
|
+
|
10
|
+
attr_accessor :cache
|
11
|
+
attr_accessor :failures
|
12
|
+
attr_accessor :buf
|
13
|
+
attr_accessor :output
|
14
|
+
attr_accessor :cur
|
15
|
+
attr_accessor :start_time
|
16
|
+
attr_accessor :queue
|
17
|
+
|
18
|
+
end
|
19
|
+
|
20
|
+
class TestResolver < Test::Unit::TestCase
|
21
|
+
|
22
|
+
def setup
|
23
|
+
@resolver = LogMerge::IPResolver.new
|
24
|
+
end
|
25
|
+
|
26
|
+
def test_class_create
|
27
|
+
File.unlink 'test/.name_cache' if File.exist? 'test/.name_cache'
|
28
|
+
|
29
|
+
assert_equal false, File.exist?('test/.name_cache')
|
30
|
+
|
31
|
+
Dir.chdir 'test' do
|
32
|
+
LogMerge::IPResolver.create do |res|
|
33
|
+
assert_equal({}, res.cache)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
assert_equal true, File.exist?('test/.name_cache')
|
38
|
+
|
39
|
+
ensure
|
40
|
+
begin File.unlink 'test/.name_cache' rescue Errno::ENOENT; end
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_class_create_with_cache
|
44
|
+
File.open 'test/.name_cache', 'w' do |fp|
|
45
|
+
fp.write util_dumped_resolver
|
46
|
+
end
|
47
|
+
|
48
|
+
Dir.chdir 'test' do
|
49
|
+
LogMerge::IPResolver.create do |res|
|
50
|
+
assert_equal({'65.61.137.68' => ['brain.43things.com', 500]}, res.cache)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
assert_equal true, File.exist?('test/.name_cache')
|
55
|
+
|
56
|
+
ensure
|
57
|
+
begin File.unlink 'test/.name_cache' rescue Errno::ENOENT; end
|
58
|
+
end
|
59
|
+
|
60
|
+
def test_expire_entries_age
|
61
|
+
@resolver.cache = {
|
62
|
+
'65.61.137.68' => ['brain.43things.com', 500]
|
63
|
+
}
|
64
|
+
|
65
|
+
@resolver.expire_entries Time.now - 250
|
66
|
+
|
67
|
+
assert_equal ['65.61.137.68'], @resolver.cache.keys
|
68
|
+
assert_equal ['brain.43things.com', 250], @resolver.cache['65.61.137.68']
|
69
|
+
end
|
70
|
+
|
71
|
+
def test_expire_entries_delete
|
72
|
+
@resolver.cache = {
|
73
|
+
'65.61.137.68' => ['brain.43things.com', 500]
|
74
|
+
}
|
75
|
+
|
76
|
+
@resolver.expire_entries Time.now - 501
|
77
|
+
|
78
|
+
assert_equal true, @resolver.cache.empty?
|
79
|
+
end
|
80
|
+
|
81
|
+
def test_expire_entries_now
|
82
|
+
@resolver.cache = {
|
83
|
+
'65.61.137.68' => ['brain.43things.com', 500]
|
84
|
+
}
|
85
|
+
|
86
|
+
@resolver.expire_entries Time.now
|
87
|
+
|
88
|
+
assert_equal ['65.61.137.68'], @resolver.cache.keys
|
89
|
+
assert_equal ['brain.43things.com', 500], @resolver.cache['65.61.137.68']
|
90
|
+
end
|
91
|
+
|
92
|
+
def test_flush
|
93
|
+
@resolver.buf[0] = "one"
|
94
|
+
@resolver.buf[1] = "two"
|
95
|
+
@resolver.buf[2] = "three"
|
96
|
+
@resolver.buf[3] = "four"
|
97
|
+
|
98
|
+
@resolver.output = StringIO.new
|
99
|
+
@resolver.flush
|
100
|
+
assert_equal "one\ntwo\nthree\nfour\n", @resolver.output.string
|
101
|
+
end
|
102
|
+
|
103
|
+
def test_marshal_dump
|
104
|
+
dumped = @resolver.marshal_dump
|
105
|
+
|
106
|
+
assert_instance_of Array, dumped
|
107
|
+
assert_equal 0, dumped[0]
|
108
|
+
assert_equal({}, dumped[1])
|
109
|
+
assert_instance_of Time, dumped[2]
|
110
|
+
end
|
111
|
+
|
112
|
+
def test_marshal_dump_with_cache
|
113
|
+
@resolver.cache['65.61.137.68'] = ['brain.43things.com', 500]
|
114
|
+
dumped = @resolver.marshal_dump
|
115
|
+
|
116
|
+
assert_instance_of Array, dumped
|
117
|
+
assert_equal 0, dumped[0]
|
118
|
+
assert_equal({'65.61.137.68' => ['brain.43things.com', 500]}, dumped[1])
|
119
|
+
assert_instance_of Time, dumped[2]
|
120
|
+
end
|
121
|
+
|
122
|
+
def test_marshal_load
|
123
|
+
data = "\004\010U:\031LogMerge::IPResolver[\010i\000{\000u:\tTime\rw\202\032\200\032\027\253\037"
|
124
|
+
resolver = Marshal.load data
|
125
|
+
|
126
|
+
assert_instance_of LogMerge::IPResolver, resolver
|
127
|
+
assert_equal({}, resolver.cache)
|
128
|
+
assert_equal [], resolver.failures, "Make sure it was init'd properly"
|
129
|
+
end
|
130
|
+
|
131
|
+
def test_marshal_load_with_cache
|
132
|
+
resolver = Marshal.load util_dumped_resolver
|
133
|
+
|
134
|
+
assert_instance_of LogMerge::IPResolver, resolver
|
135
|
+
assert_equal({'65.61.137.68' => ["brain.43things.com", 500]},
|
136
|
+
resolver.cache)
|
137
|
+
end
|
138
|
+
|
139
|
+
def test_resolve
|
140
|
+
assert_equal 'www.43things.com', @resolver.resolve('65.61.137.67')
|
141
|
+
assert_equal '0.0.0.0', @resolver.resolve('0.0.0.0')
|
142
|
+
|
143
|
+
assert_equal ['0.0.0.0'], @resolver.failures
|
144
|
+
|
145
|
+
assert_equal ['0.0.0.0', '65.61.137.67'], @resolver.cache.keys
|
146
|
+
assert_equal 0, @resolver.cache['0.0.0.0'].last
|
147
|
+
assert_operator 0, :<, @resolver.cache['65.61.137.67'].last
|
148
|
+
end
|
149
|
+
|
150
|
+
def test_resolve_from_cache
|
151
|
+
@resolver.cache['65.61.137.68'] = ['brain.43things.com', 0]
|
152
|
+
|
153
|
+
assert_equal 'www.43things.com', @resolver.resolve('65.61.137.67')
|
154
|
+
assert_equal 'brain.43things.com', @resolver.resolve('65.61.137.68')
|
155
|
+
assert_equal '0.0.0.0', @resolver.resolve('0.0.0.0')
|
156
|
+
|
157
|
+
assert_equal ['0.0.0.0'], @resolver.failures
|
158
|
+
|
159
|
+
assert_equal ['0.0.0.0', '65.61.137.67', '65.61.137.68'],
|
160
|
+
@resolver.cache.keys
|
161
|
+
assert_equal 0, @resolver.cache['0.0.0.0'].last
|
162
|
+
assert_operator 0, :<, @resolver.cache['65.61.137.67'].last
|
163
|
+
assert_equal 0, @resolver.cache['65.61.137.68'].last
|
164
|
+
end
|
165
|
+
|
166
|
+
def test_resolve_bad_ip
|
167
|
+
assert_equal '0.0.0.256', @resolver.resolve('0.0.0.256')
|
168
|
+
end
|
169
|
+
|
170
|
+
def test_run
|
171
|
+
input = StringIO.new <<-EOF
|
172
|
+
138.217.248.136 - - [17/Jan/2006:00:00:00 -0800] "GET /javascripts/prototype.js HTTP/1.1" 200 7358 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
173
|
+
68.142.250.78 - - [17/Jan/2006:00:00:01 -0800] "GET /things/view/123841 HTTP/1.0" 200 9146 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
|
174
|
+
138.217.248.136 - - [17/Jan/2006:00:00:02 -0800] "GET /javascripts/enumerable.js HTTP/1.1" 200 1084 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
175
|
+
202.160.180.40 - - [17/Jan/2006:00:00:03 -0800] "GET /teams/progress/1147887 HTTP/1.0" 302 128 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
176
|
+
138.217.248.136 - - [17/Jan/2006:00:00:04 -0800] "GET /javascripts/effects.js HTTP/1.1" 200 5319 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
177
|
+
68.189.40.2 - - [17/Jan/2006:00:00:05 -0800] "GET /entries/view/139942 HTTP/1.1" 200 2975 "-" "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/312.5.1 (KHTML, like Gecko) Safari/312.3.1"
|
178
|
+
202.160.180.40 - - [17/Jan/2006:00:00:06 -0800] "GET /people/progress/princessauddie/1147887 HTTP/1.0" 200 7511 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
179
|
+
138.217.248.136 - - [17/Jan/2006:00:00:07 -0800] "GET /javascripts/dragdrop.js HTTP/1.1" 200 5279 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
180
|
+
192.168.1.254 - - [17/Jan/2006:00:00:08 -0800] "HEAD / HTTP/1.0" 200 0 "-" "-"
|
181
|
+
216.109.121.70 - - [17/Jan/2006:00:00:09 -0800] "GET /rss/uber/author?username=twilightpumpkin HTTP/1.0" 200 15482 "-" "YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide; users 0; views 0)"
|
182
|
+
EOF
|
183
|
+
output = StringIO.new
|
184
|
+
|
185
|
+
@resolver.cache = {
|
186
|
+
'216.109.121.70' => ['oc31.my.dcn.yahoo.net', 94],
|
187
|
+
'68.142.250.78' => ['lj2268.inktomisearch.com', 90],
|
188
|
+
'138.217.248.136' => ['CPE-138-217-248-136.wa.bigpond.net.au', 167760],
|
189
|
+
'68.189.40.2' => ['68-189-40-2.dhcp.rdng.ca.charter.com', 81360],
|
190
|
+
'202.160.180.40' => ['lj9027.inktomisearch.com', 90],
|
191
|
+
'192.168.1.254' => ['dhcp-254.coop.robotcoop.com', 7200]
|
192
|
+
}
|
193
|
+
|
194
|
+
@resolver.run output, input
|
195
|
+
|
196
|
+
assert_equal true, input.eof?, "Input empty."
|
197
|
+
assert_equal true, @resolver.queue.empty?, "Queue empty"
|
198
|
+
assert_equal 10, output.string.scan("\n").length
|
199
|
+
|
200
|
+
expected = <<-EOF
|
201
|
+
CPE-138-217-248-136.wa.bigpond.net.au - - [17/Jan/2006:00:00:00 -0800] "GET /javascripts/prototype.js HTTP/1.1" 200 7358 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
202
|
+
lj2268.inktomisearch.com - - [17/Jan/2006:00:00:01 -0800] "GET /things/view/123841 HTTP/1.0" 200 9146 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
|
203
|
+
CPE-138-217-248-136.wa.bigpond.net.au - - [17/Jan/2006:00:00:02 -0800] "GET /javascripts/enumerable.js HTTP/1.1" 200 1084 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
204
|
+
lj9027.inktomisearch.com - - [17/Jan/2006:00:00:03 -0800] "GET /teams/progress/1147887 HTTP/1.0" 302 128 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
205
|
+
CPE-138-217-248-136.wa.bigpond.net.au - - [17/Jan/2006:00:00:04 -0800] "GET /javascripts/effects.js HTTP/1.1" 200 5319 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
206
|
+
68-189-40-2.dhcp.rdng.ca.charter.com - - [17/Jan/2006:00:00:05 -0800] "GET /entries/view/139942 HTTP/1.1" 200 2975 "-" "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/312.5.1 (KHTML, like Gecko) Safari/312.3.1"
|
207
|
+
lj9027.inktomisearch.com - - [17/Jan/2006:00:00:06 -0800] "GET /people/progress/princessauddie/1147887 HTTP/1.0" 200 7511 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
208
|
+
CPE-138-217-248-136.wa.bigpond.net.au - - [17/Jan/2006:00:00:07 -0800] "GET /javascripts/dragdrop.js HTTP/1.1" 200 5279 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
209
|
+
dhcp-254.coop.robotcoop.com - - [17/Jan/2006:00:00:08 -0800] "HEAD / HTTP/1.0" 200 0 "-" "-"
|
210
|
+
oc31.my.dcn.yahoo.net - - [17/Jan/2006:00:00:09 -0800] "GET /rss/uber/author?username=twilightpumpkin HTTP/1.0" 200 15482 "-" "YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide; users 0; views 0)"
|
211
|
+
EOF
|
212
|
+
|
213
|
+
assert_equal expected, output.string
|
214
|
+
end
|
215
|
+
|
216
|
+
def test_start_threads
|
217
|
+
threads = Thread.list.length
|
218
|
+
|
219
|
+
@resolver.start_threads
|
220
|
+
|
221
|
+
assert_operator threads, :<, Thread.list.length
|
222
|
+
end
|
223
|
+
|
224
|
+
##
|
225
|
+
# HACK don't like this method since it is Time.now dependent.
|
226
|
+
|
227
|
+
def util_dumped_resolver
|
228
|
+
resolver = LogMerge::IPResolver.new
|
229
|
+
resolver.cache['65.61.137.68'] = ['brain.43things.com', 500]
|
230
|
+
resolver.start_time = Time.now
|
231
|
+
return Marshal.dump(resolver)
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
metadata
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
rubygems_version: 0.8.11.6
|
3
|
+
specification_version: 1
|
4
|
+
name: logmerge
|
5
|
+
version: !ruby/object:Gem::Version
|
6
|
+
version: 1.0.0
|
7
|
+
date: 2006-01-20 00:00:00 -08:00
|
8
|
+
summary: Resolves IP addresses and merges Apache access logs.
|
9
|
+
require_paths:
|
10
|
+
- lib
|
11
|
+
email: eric@robotcoop.com
|
12
|
+
homepage:
|
13
|
+
rubyforge_project:
|
14
|
+
description:
|
15
|
+
autorequire:
|
16
|
+
default_executable:
|
17
|
+
bindir: bin
|
18
|
+
has_rdoc: false
|
19
|
+
required_ruby_version: !ruby/object:Gem::Version::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">"
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 0.0.0
|
24
|
+
version:
|
25
|
+
platform: ruby
|
26
|
+
signing_key:
|
27
|
+
cert_chain:
|
28
|
+
authors:
|
29
|
+
- Eric Hodel
|
30
|
+
files:
|
31
|
+
- LICENSE
|
32
|
+
- Manifest.txt
|
33
|
+
- README
|
34
|
+
- Rakefile
|
35
|
+
- bin/ip2name
|
36
|
+
- bin/logmerge
|
37
|
+
- lib/logmerge.rb
|
38
|
+
- lib/logmerge/merger.rb
|
39
|
+
- lib/logmerge/resolv.rb
|
40
|
+
- lib/logmerge/resolver.rb
|
41
|
+
- resolv_test
|
42
|
+
- test/access.log
|
43
|
+
- test/test_merger.rb
|
44
|
+
- test/test_resolver.rb
|
45
|
+
test_files: []
|
46
|
+
|
47
|
+
rdoc_options: []
|
48
|
+
|
49
|
+
extra_rdoc_files: []
|
50
|
+
|
51
|
+
executables:
|
52
|
+
- ip2name
|
53
|
+
- logmerge
|
54
|
+
extensions: []
|
55
|
+
|
56
|
+
requirements: []
|
57
|
+
|
58
|
+
dependencies: []
|
59
|
+
|