logmerge 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +28 -0
- data/Manifest.txt +14 -0
- data/README +35 -0
- data/Rakefile +63 -0
- data/bin/ip2name +8 -0
- data/bin/logmerge +6 -0
- data/lib/logmerge/merger.rb +74 -0
- data/lib/logmerge/resolv.rb +1892 -0
- data/lib/logmerge/resolver.rb +162 -0
- data/lib/logmerge.rb +13 -0
- data/resolv_test +26 -0
- data/test/access.log +10 -0
- data/test/test_merger.rb +86 -0
- data/test/test_resolver.rb +234 -0
- metadata +59 -0
@@ -0,0 +1,162 @@
|
|
1
|
+
$TESTING = defined? $TESTING
|
2
|
+
|
3
|
+
require 'logmerge'
|
4
|
+
|
5
|
+
class LogMerge::IPResolver
|
6
|
+
|
7
|
+
##
|
8
|
+
# Marshal format version number.
|
9
|
+
|
10
|
+
FORMAT = 0
|
11
|
+
|
12
|
+
##
|
13
|
+
# Default number of threads.
|
14
|
+
|
15
|
+
THREADS = $TESTING ? 2 : 100
|
16
|
+
|
17
|
+
def self.create
|
18
|
+
resolver = nil
|
19
|
+
|
20
|
+
if File.exists? '.name_cache' then
|
21
|
+
data = File.read '.name_cache'
|
22
|
+
begin
|
23
|
+
resolver = Marshal.load data
|
24
|
+
rescue ArgumentError
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
resolver ||= self.new
|
29
|
+
|
30
|
+
yield resolver
|
31
|
+
|
32
|
+
ensure
|
33
|
+
begin
|
34
|
+
File.open '.name_cache', 'w' do |fp|
|
35
|
+
Marshal.dump resolver, fp
|
36
|
+
end
|
37
|
+
rescue Errno::EACCES
|
38
|
+
# ignore
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.resolve(output = STDOUT, input = ARGF)
|
43
|
+
create do |resolver|
|
44
|
+
resolver.run output, input
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def initialize(cache = {})
|
49
|
+
@cache = cache
|
50
|
+
@failures = []
|
51
|
+
@start_time = Time.now
|
52
|
+
@dns = Resolv::DNS.new
|
53
|
+
|
54
|
+
@output = nil
|
55
|
+
@done = false
|
56
|
+
@queue = SizedQueue.new THREADS
|
57
|
+
@threads = ThreadGroup.new
|
58
|
+
@buf = {}
|
59
|
+
@cur = 0
|
60
|
+
end
|
61
|
+
|
62
|
+
def expire_entries(start_time)
|
63
|
+
elapsed = (Time.now - start_time).to_i
|
64
|
+
@cache.each { |ip, (name, ttl)| @cache[ip][1] = ttl - elapsed }
|
65
|
+
@cache.delete_if { |ip, (name, ttl)| ttl <= 0 }
|
66
|
+
return nil
|
67
|
+
end
|
68
|
+
|
69
|
+
def flush
|
70
|
+
while @buf.has_key? @cur do
|
71
|
+
@output.puts @buf.delete(@cur)
|
72
|
+
@cur += 1
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def marshal_dump
|
77
|
+
@failures.each { |ip| @cache.delete ip }
|
78
|
+
|
79
|
+
expire_entries @start_time # compact
|
80
|
+
|
81
|
+
return [FORMAT, @cache, @start_time]
|
82
|
+
end
|
83
|
+
|
84
|
+
def marshal_load(dumped)
|
85
|
+
format, cache, start_time = dumped
|
86
|
+
|
87
|
+
case format
|
88
|
+
when 0 then
|
89
|
+
send :initialize, cache
|
90
|
+
expire_entries start_time # expire
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def resolve(ip)
|
95
|
+
name, = @cache[ip]
|
96
|
+
|
97
|
+
return name unless name.nil?
|
98
|
+
|
99
|
+
begin
|
100
|
+
dns_name = Resolv::IPv4.create(ip).to_name
|
101
|
+
rescue ArgumentError
|
102
|
+
return ip
|
103
|
+
end
|
104
|
+
|
105
|
+
@dns.each_resource dns_name, Resolv::DNS::Resource::IN::PTR do |res|
|
106
|
+
@cache[ip] = [res.name.to_s, res.ttl * 2]
|
107
|
+
return res.name.to_s
|
108
|
+
end
|
109
|
+
|
110
|
+
@cache[ip] = [ip, 0]
|
111
|
+
@failures << ip
|
112
|
+
|
113
|
+
return ip
|
114
|
+
end
|
115
|
+
|
116
|
+
def run(output, input)
|
117
|
+
start_threads
|
118
|
+
line_no = 0
|
119
|
+
@output = output
|
120
|
+
|
121
|
+
input.each_line do |line|
|
122
|
+
ip, rest = line.split ' ', 2
|
123
|
+
record = [line_no, ip, rest]
|
124
|
+
line_no += 1
|
125
|
+
@queue << record
|
126
|
+
flush if line_no % LogMerge::MAX_INFLIGHT == 0
|
127
|
+
end
|
128
|
+
|
129
|
+
@done = true
|
130
|
+
|
131
|
+
Thread.pass until @queue.empty?
|
132
|
+
|
133
|
+
@threads.enclose
|
134
|
+
until @threads.list.empty? do
|
135
|
+
@threads.list.first.join
|
136
|
+
end
|
137
|
+
|
138
|
+
flush
|
139
|
+
end
|
140
|
+
|
141
|
+
def start_threads
|
142
|
+
THREADS.times do
|
143
|
+
Thread.start do
|
144
|
+
@threads.add Thread.current
|
145
|
+
loop do
|
146
|
+
begin
|
147
|
+
line_no, ip, rest = @queue.pop true
|
148
|
+
rescue ThreadError
|
149
|
+
break if @done
|
150
|
+
Thread.pass
|
151
|
+
retry
|
152
|
+
end
|
153
|
+
|
154
|
+
name = resolve ip
|
155
|
+
@buf[line_no] = "#{name} #{rest}"
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
end
|
162
|
+
|
data/lib/logmerge.rb
ADDED
data/resolv_test
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
#!/usr/local/bin/ruby -ws
|
2
|
+
|
3
|
+
# This checks Ruby's resolv.rb to see if #ttl is available. When #ttl is
|
4
|
+
# available, then lib/logmerge/resolv.rb can be removed.
|
5
|
+
|
6
|
+
$ruby ||= false
|
7
|
+
$mine ||= false
|
8
|
+
|
9
|
+
if $ruby then
|
10
|
+
require 'resolv'
|
11
|
+
elsif $mine then
|
12
|
+
require 'lib/logmerge/resolv'
|
13
|
+
else
|
14
|
+
raise ArgumentError, "#$0 -mine|-ruby"
|
15
|
+
end
|
16
|
+
|
17
|
+
dns = Resolv::DNS.new
|
18
|
+
ip = '138.217.248.136'
|
19
|
+
ipv4 = Resolv::IPv4.create ip
|
20
|
+
p ipv4.to_name
|
21
|
+
|
22
|
+
dns.each_resource ipv4.to_name, Resolv::DNS::Resource::IN::PTR do |r|
|
23
|
+
p r.name.to_s
|
24
|
+
p r.ttl
|
25
|
+
end
|
26
|
+
|
data/test/access.log
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
138.217.248.136 - - [17/Jan/2006:00:00:00 -0800] "GET /javascripts/prototype.js HTTP/1.1" 200 7358 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
2
|
+
68.142.250.78 - - [17/Jan/2006:00:00:00 -0800] "GET /things/view/123841 HTTP/1.0" 200 9146 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
|
3
|
+
138.217.248.136 - - [17/Jan/2006:00:00:01 -0800] "GET /javascripts/enumerable.js HTTP/1.1" 200 1084 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
4
|
+
202.160.180.40 - - [17/Jan/2006:00:00:01 -0800] "GET /teams/progress/1147887 HTTP/1.0" 302 128 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
5
|
+
138.217.248.136 - - [17/Jan/2006:00:00:01 -0800] "GET /javascripts/effects.js HTTP/1.1" 200 5319 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
6
|
+
68.189.40.2 - - [17/Jan/2006:00:00:01 -0800] "GET /entries/view/139942 HTTP/1.1" 200 2975 "-" "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/312.5.1 (KHTML, like Gecko) Safari/312.3.1"
|
7
|
+
202.160.180.40 - - [17/Jan/2006:00:00:02 -0800] "GET /people/progress/princessauddie/1147887 HTTP/1.0" 200 7511 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
8
|
+
138.217.248.136 - - [17/Jan/2006:00:00:02 -0800] "GET /javascripts/dragdrop.js HTTP/1.1" 200 5279 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
9
|
+
192.168.1.254 - - [17/Jan/2006:00:00:02 -0800] "HEAD / HTTP/1.0" 200 0 "-" "-"
|
10
|
+
216.109.121.70 - - [17/Jan/2006:00:00:02 -0800] "GET /rss/uber/author?username=twilightpumpkin HTTP/1.0" 200 15482 "-" "YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide; users 0; views 0)"
|
data/test/test_merger.rb
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'stringio'
|
2
|
+
require 'test/unit'
|
3
|
+
|
4
|
+
$TESTING = true
|
5
|
+
|
6
|
+
require 'logmerge/merger'
|
7
|
+
|
8
|
+
class LogMerge::Merger
|
9
|
+
|
10
|
+
attr_accessor :streams
|
11
|
+
attr_accessor :buf
|
12
|
+
|
13
|
+
end
|
14
|
+
|
15
|
+
class TestMerger < Test::Unit::TestCase
|
16
|
+
|
17
|
+
def test_all_closed_eh_closed
|
18
|
+
merger = LogMerge::Merger.new []
|
19
|
+
assert_equal true, merger.all_closed?
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_all_closed_eh_not_closed
|
23
|
+
merger = LogMerge::Merger.new [StringIO.new]
|
24
|
+
assert_equal true, merger.all_closed?
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_buf_fill_fill
|
28
|
+
stream = StringIO.new <<-EOF
|
29
|
+
[17/Jan/2006:00:00:01 -0800]
|
30
|
+
[17/Jan/2006:00:00:02 -0800]
|
31
|
+
EOF
|
32
|
+
merger = LogMerge::Merger.new [stream]
|
33
|
+
|
34
|
+
assert_equal [1137484801, " [17/Jan/2006:00:00:01 -0800] \n"], merger.buf[0]
|
35
|
+
|
36
|
+
merger.buf_fill 0
|
37
|
+
|
38
|
+
assert_equal [1137484802, " [17/Jan/2006:00:00:02 -0800] \n"], merger.buf[0]
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_buf_fill_empty
|
42
|
+
stream = StringIO.new <<-EOF
|
43
|
+
[17/Jan/2006:00:00:01 -0800]
|
44
|
+
EOF
|
45
|
+
merger = LogMerge::Merger.new [stream]
|
46
|
+
|
47
|
+
assert_equal [1137484801, " [17/Jan/2006:00:00:01 -0800] \n"], merger.buf[0]
|
48
|
+
|
49
|
+
merger.buf_fill 0
|
50
|
+
|
51
|
+
assert_equal true, merger.buf.empty?
|
52
|
+
assert_equal true, merger.streams.empty?
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_initialize
|
56
|
+
stream1 = StringIO.new " [17/Jan/2006:00:00:01 -0800] \n"
|
57
|
+
stream2 = StringIO.new " [17/Jan/2006:00:00:02 -0800] \n"
|
58
|
+
|
59
|
+
merger = LogMerge::Merger.new [stream1, stream2]
|
60
|
+
|
61
|
+
assert_equal false, merger.all_closed?
|
62
|
+
assert_equal [1137484801, " [17/Jan/2006:00:00:01 -0800] \n"], merger.buf[0]
|
63
|
+
assert_equal [1137484802, " [17/Jan/2006:00:00:02 -0800] \n"], merger.buf[1]
|
64
|
+
end
|
65
|
+
|
66
|
+
def test_merge
|
67
|
+
stream1 = StringIO.new " [17/Jan/2006:00:00:02 -0800] \n"
|
68
|
+
stream2 = StringIO.new " [17/Jan/2006:00:00:01 -0800] \n"
|
69
|
+
|
70
|
+
merger = LogMerge::Merger.new [stream1, stream2]
|
71
|
+
|
72
|
+
output = StringIO.new
|
73
|
+
|
74
|
+
merger.merge output
|
75
|
+
|
76
|
+
expected = <<-EOF
|
77
|
+
[17/Jan/2006:00:00:01 -0800]
|
78
|
+
[17/Jan/2006:00:00:02 -0800]
|
79
|
+
EOF
|
80
|
+
|
81
|
+
assert_equal true, merger.all_closed?
|
82
|
+
assert_equal expected, output.string
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
|
@@ -0,0 +1,234 @@
|
|
1
|
+
require 'stringio'
|
2
|
+
require 'test/unit'
|
3
|
+
|
4
|
+
$TESTING = true
|
5
|
+
|
6
|
+
require 'logmerge/resolver'
|
7
|
+
|
8
|
+
class LogMerge::IPResolver
|
9
|
+
|
10
|
+
attr_accessor :cache
|
11
|
+
attr_accessor :failures
|
12
|
+
attr_accessor :buf
|
13
|
+
attr_accessor :output
|
14
|
+
attr_accessor :cur
|
15
|
+
attr_accessor :start_time
|
16
|
+
attr_accessor :queue
|
17
|
+
|
18
|
+
end
|
19
|
+
|
20
|
+
class TestResolver < Test::Unit::TestCase
|
21
|
+
|
22
|
+
def setup
|
23
|
+
@resolver = LogMerge::IPResolver.new
|
24
|
+
end
|
25
|
+
|
26
|
+
def test_class_create
|
27
|
+
File.unlink 'test/.name_cache' if File.exist? 'test/.name_cache'
|
28
|
+
|
29
|
+
assert_equal false, File.exist?('test/.name_cache')
|
30
|
+
|
31
|
+
Dir.chdir 'test' do
|
32
|
+
LogMerge::IPResolver.create do |res|
|
33
|
+
assert_equal({}, res.cache)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
assert_equal true, File.exist?('test/.name_cache')
|
38
|
+
|
39
|
+
ensure
|
40
|
+
begin File.unlink 'test/.name_cache' rescue Errno::ENOENT; end
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_class_create_with_cache
|
44
|
+
File.open 'test/.name_cache', 'w' do |fp|
|
45
|
+
fp.write util_dumped_resolver
|
46
|
+
end
|
47
|
+
|
48
|
+
Dir.chdir 'test' do
|
49
|
+
LogMerge::IPResolver.create do |res|
|
50
|
+
assert_equal({'65.61.137.68' => ['brain.43things.com', 500]}, res.cache)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
assert_equal true, File.exist?('test/.name_cache')
|
55
|
+
|
56
|
+
ensure
|
57
|
+
begin File.unlink 'test/.name_cache' rescue Errno::ENOENT; end
|
58
|
+
end
|
59
|
+
|
60
|
+
def test_expire_entries_age
|
61
|
+
@resolver.cache = {
|
62
|
+
'65.61.137.68' => ['brain.43things.com', 500]
|
63
|
+
}
|
64
|
+
|
65
|
+
@resolver.expire_entries Time.now - 250
|
66
|
+
|
67
|
+
assert_equal ['65.61.137.68'], @resolver.cache.keys
|
68
|
+
assert_equal ['brain.43things.com', 250], @resolver.cache['65.61.137.68']
|
69
|
+
end
|
70
|
+
|
71
|
+
def test_expire_entries_delete
|
72
|
+
@resolver.cache = {
|
73
|
+
'65.61.137.68' => ['brain.43things.com', 500]
|
74
|
+
}
|
75
|
+
|
76
|
+
@resolver.expire_entries Time.now - 501
|
77
|
+
|
78
|
+
assert_equal true, @resolver.cache.empty?
|
79
|
+
end
|
80
|
+
|
81
|
+
def test_expire_entries_now
|
82
|
+
@resolver.cache = {
|
83
|
+
'65.61.137.68' => ['brain.43things.com', 500]
|
84
|
+
}
|
85
|
+
|
86
|
+
@resolver.expire_entries Time.now
|
87
|
+
|
88
|
+
assert_equal ['65.61.137.68'], @resolver.cache.keys
|
89
|
+
assert_equal ['brain.43things.com', 500], @resolver.cache['65.61.137.68']
|
90
|
+
end
|
91
|
+
|
92
|
+
def test_flush
|
93
|
+
@resolver.buf[0] = "one"
|
94
|
+
@resolver.buf[1] = "two"
|
95
|
+
@resolver.buf[2] = "three"
|
96
|
+
@resolver.buf[3] = "four"
|
97
|
+
|
98
|
+
@resolver.output = StringIO.new
|
99
|
+
@resolver.flush
|
100
|
+
assert_equal "one\ntwo\nthree\nfour\n", @resolver.output.string
|
101
|
+
end
|
102
|
+
|
103
|
+
def test_marshal_dump
|
104
|
+
dumped = @resolver.marshal_dump
|
105
|
+
|
106
|
+
assert_instance_of Array, dumped
|
107
|
+
assert_equal 0, dumped[0]
|
108
|
+
assert_equal({}, dumped[1])
|
109
|
+
assert_instance_of Time, dumped[2]
|
110
|
+
end
|
111
|
+
|
112
|
+
def test_marshal_dump_with_cache
|
113
|
+
@resolver.cache['65.61.137.68'] = ['brain.43things.com', 500]
|
114
|
+
dumped = @resolver.marshal_dump
|
115
|
+
|
116
|
+
assert_instance_of Array, dumped
|
117
|
+
assert_equal 0, dumped[0]
|
118
|
+
assert_equal({'65.61.137.68' => ['brain.43things.com', 500]}, dumped[1])
|
119
|
+
assert_instance_of Time, dumped[2]
|
120
|
+
end
|
121
|
+
|
122
|
+
def test_marshal_load
|
123
|
+
data = "\004\010U:\031LogMerge::IPResolver[\010i\000{\000u:\tTime\rw\202\032\200\032\027\253\037"
|
124
|
+
resolver = Marshal.load data
|
125
|
+
|
126
|
+
assert_instance_of LogMerge::IPResolver, resolver
|
127
|
+
assert_equal({}, resolver.cache)
|
128
|
+
assert_equal [], resolver.failures, "Make sure it was init'd properly"
|
129
|
+
end
|
130
|
+
|
131
|
+
def test_marshal_load_with_cache
|
132
|
+
resolver = Marshal.load util_dumped_resolver
|
133
|
+
|
134
|
+
assert_instance_of LogMerge::IPResolver, resolver
|
135
|
+
assert_equal({'65.61.137.68' => ["brain.43things.com", 500]},
|
136
|
+
resolver.cache)
|
137
|
+
end
|
138
|
+
|
139
|
+
def test_resolve
|
140
|
+
assert_equal 'www.43things.com', @resolver.resolve('65.61.137.67')
|
141
|
+
assert_equal '0.0.0.0', @resolver.resolve('0.0.0.0')
|
142
|
+
|
143
|
+
assert_equal ['0.0.0.0'], @resolver.failures
|
144
|
+
|
145
|
+
assert_equal ['0.0.0.0', '65.61.137.67'], @resolver.cache.keys
|
146
|
+
assert_equal 0, @resolver.cache['0.0.0.0'].last
|
147
|
+
assert_operator 0, :<, @resolver.cache['65.61.137.67'].last
|
148
|
+
end
|
149
|
+
|
150
|
+
def test_resolve_from_cache
|
151
|
+
@resolver.cache['65.61.137.68'] = ['brain.43things.com', 0]
|
152
|
+
|
153
|
+
assert_equal 'www.43things.com', @resolver.resolve('65.61.137.67')
|
154
|
+
assert_equal 'brain.43things.com', @resolver.resolve('65.61.137.68')
|
155
|
+
assert_equal '0.0.0.0', @resolver.resolve('0.0.0.0')
|
156
|
+
|
157
|
+
assert_equal ['0.0.0.0'], @resolver.failures
|
158
|
+
|
159
|
+
assert_equal ['0.0.0.0', '65.61.137.67', '65.61.137.68'],
|
160
|
+
@resolver.cache.keys
|
161
|
+
assert_equal 0, @resolver.cache['0.0.0.0'].last
|
162
|
+
assert_operator 0, :<, @resolver.cache['65.61.137.67'].last
|
163
|
+
assert_equal 0, @resolver.cache['65.61.137.68'].last
|
164
|
+
end
|
165
|
+
|
166
|
+
def test_resolve_bad_ip
|
167
|
+
assert_equal '0.0.0.256', @resolver.resolve('0.0.0.256')
|
168
|
+
end
|
169
|
+
|
170
|
+
def test_run
|
171
|
+
input = StringIO.new <<-EOF
|
172
|
+
138.217.248.136 - - [17/Jan/2006:00:00:00 -0800] "GET /javascripts/prototype.js HTTP/1.1" 200 7358 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
173
|
+
68.142.250.78 - - [17/Jan/2006:00:00:01 -0800] "GET /things/view/123841 HTTP/1.0" 200 9146 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
|
174
|
+
138.217.248.136 - - [17/Jan/2006:00:00:02 -0800] "GET /javascripts/enumerable.js HTTP/1.1" 200 1084 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
175
|
+
202.160.180.40 - - [17/Jan/2006:00:00:03 -0800] "GET /teams/progress/1147887 HTTP/1.0" 302 128 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
176
|
+
138.217.248.136 - - [17/Jan/2006:00:00:04 -0800] "GET /javascripts/effects.js HTTP/1.1" 200 5319 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
177
|
+
68.189.40.2 - - [17/Jan/2006:00:00:05 -0800] "GET /entries/view/139942 HTTP/1.1" 200 2975 "-" "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/312.5.1 (KHTML, like Gecko) Safari/312.3.1"
|
178
|
+
202.160.180.40 - - [17/Jan/2006:00:00:06 -0800] "GET /people/progress/princessauddie/1147887 HTTP/1.0" 200 7511 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
179
|
+
138.217.248.136 - - [17/Jan/2006:00:00:07 -0800] "GET /javascripts/dragdrop.js HTTP/1.1" 200 5279 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
180
|
+
192.168.1.254 - - [17/Jan/2006:00:00:08 -0800] "HEAD / HTTP/1.0" 200 0 "-" "-"
|
181
|
+
216.109.121.70 - - [17/Jan/2006:00:00:09 -0800] "GET /rss/uber/author?username=twilightpumpkin HTTP/1.0" 200 15482 "-" "YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide; users 0; views 0)"
|
182
|
+
EOF
|
183
|
+
output = StringIO.new
|
184
|
+
|
185
|
+
@resolver.cache = {
|
186
|
+
'216.109.121.70' => ['oc31.my.dcn.yahoo.net', 94],
|
187
|
+
'68.142.250.78' => ['lj2268.inktomisearch.com', 90],
|
188
|
+
'138.217.248.136' => ['CPE-138-217-248-136.wa.bigpond.net.au', 167760],
|
189
|
+
'68.189.40.2' => ['68-189-40-2.dhcp.rdng.ca.charter.com', 81360],
|
190
|
+
'202.160.180.40' => ['lj9027.inktomisearch.com', 90],
|
191
|
+
'192.168.1.254' => ['dhcp-254.coop.robotcoop.com', 7200]
|
192
|
+
}
|
193
|
+
|
194
|
+
@resolver.run output, input
|
195
|
+
|
196
|
+
assert_equal true, input.eof?, "Input empty."
|
197
|
+
assert_equal true, @resolver.queue.empty?, "Queue empty"
|
198
|
+
assert_equal 10, output.string.scan("\n").length
|
199
|
+
|
200
|
+
expected = <<-EOF
|
201
|
+
CPE-138-217-248-136.wa.bigpond.net.au - - [17/Jan/2006:00:00:00 -0800] "GET /javascripts/prototype.js HTTP/1.1" 200 7358 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
202
|
+
lj2268.inktomisearch.com - - [17/Jan/2006:00:00:01 -0800] "GET /things/view/123841 HTTP/1.0" 200 9146 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
|
203
|
+
CPE-138-217-248-136.wa.bigpond.net.au - - [17/Jan/2006:00:00:02 -0800] "GET /javascripts/enumerable.js HTTP/1.1" 200 1084 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
204
|
+
lj9027.inktomisearch.com - - [17/Jan/2006:00:00:03 -0800] "GET /teams/progress/1147887 HTTP/1.0" 302 128 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
205
|
+
CPE-138-217-248-136.wa.bigpond.net.au - - [17/Jan/2006:00:00:04 -0800] "GET /javascripts/effects.js HTTP/1.1" 200 5319 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
206
|
+
68-189-40-2.dhcp.rdng.ca.charter.com - - [17/Jan/2006:00:00:05 -0800] "GET /entries/view/139942 HTTP/1.1" 200 2975 "-" "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/312.5.1 (KHTML, like Gecko) Safari/312.3.1"
|
207
|
+
lj9027.inktomisearch.com - - [17/Jan/2006:00:00:06 -0800] "GET /people/progress/princessauddie/1147887 HTTP/1.0" 200 7511 "-" "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
208
|
+
CPE-138-217-248-136.wa.bigpond.net.au - - [17/Jan/2006:00:00:07 -0800] "GET /javascripts/dragdrop.js HTTP/1.1" 200 5279 "http://www.43things.com/things/view/16508" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8) Gecko/20051111 Firefox/1.5"
|
209
|
+
dhcp-254.coop.robotcoop.com - - [17/Jan/2006:00:00:08 -0800] "HEAD / HTTP/1.0" 200 0 "-" "-"
|
210
|
+
oc31.my.dcn.yahoo.net - - [17/Jan/2006:00:00:09 -0800] "GET /rss/uber/author?username=twilightpumpkin HTTP/1.0" 200 15482 "-" "YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide; users 0; views 0)"
|
211
|
+
EOF
|
212
|
+
|
213
|
+
assert_equal expected, output.string
|
214
|
+
end
|
215
|
+
|
216
|
+
def test_start_threads
|
217
|
+
threads = Thread.list.length
|
218
|
+
|
219
|
+
@resolver.start_threads
|
220
|
+
|
221
|
+
assert_operator threads, :<, Thread.list.length
|
222
|
+
end
|
223
|
+
|
224
|
+
##
|
225
|
+
# HACK don't like this method since it is Time.now dependent.
|
226
|
+
|
227
|
+
def util_dumped_resolver
|
228
|
+
resolver = LogMerge::IPResolver.new
|
229
|
+
resolver.cache['65.61.137.68'] = ['brain.43things.com', 500]
|
230
|
+
resolver.start_time = Time.now
|
231
|
+
return Marshal.dump(resolver)
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
metadata
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
rubygems_version: 0.8.11.6
|
3
|
+
specification_version: 1
|
4
|
+
name: logmerge
|
5
|
+
version: !ruby/object:Gem::Version
|
6
|
+
version: 1.0.0
|
7
|
+
date: 2006-01-20 00:00:00 -08:00
|
8
|
+
summary: Resolves IP addresses and merges Apache access logs.
|
9
|
+
require_paths:
|
10
|
+
- lib
|
11
|
+
email: eric@robotcoop.com
|
12
|
+
homepage:
|
13
|
+
rubyforge_project:
|
14
|
+
description:
|
15
|
+
autorequire:
|
16
|
+
default_executable:
|
17
|
+
bindir: bin
|
18
|
+
has_rdoc: false
|
19
|
+
required_ruby_version: !ruby/object:Gem::Version::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">"
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 0.0.0
|
24
|
+
version:
|
25
|
+
platform: ruby
|
26
|
+
signing_key:
|
27
|
+
cert_chain:
|
28
|
+
authors:
|
29
|
+
- Eric Hodel
|
30
|
+
files:
|
31
|
+
- LICENSE
|
32
|
+
- Manifest.txt
|
33
|
+
- README
|
34
|
+
- Rakefile
|
35
|
+
- bin/ip2name
|
36
|
+
- bin/logmerge
|
37
|
+
- lib/logmerge.rb
|
38
|
+
- lib/logmerge/merger.rb
|
39
|
+
- lib/logmerge/resolv.rb
|
40
|
+
- lib/logmerge/resolver.rb
|
41
|
+
- resolv_test
|
42
|
+
- test/access.log
|
43
|
+
- test/test_merger.rb
|
44
|
+
- test/test_resolver.rb
|
45
|
+
test_files: []
|
46
|
+
|
47
|
+
rdoc_options: []
|
48
|
+
|
49
|
+
extra_rdoc_files: []
|
50
|
+
|
51
|
+
executables:
|
52
|
+
- ip2name
|
53
|
+
- logmerge
|
54
|
+
extensions: []
|
55
|
+
|
56
|
+
requirements: []
|
57
|
+
|
58
|
+
dependencies: []
|
59
|
+
|