blackhole 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/blackhole +86 -0
- data/lib/blackhole.rb +54 -0
- data/lib/hole.rb +147 -0
- data/lib/runner.rb +89 -0
- metadata +126 -0
data/bin/blackhole
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# encoding: utf-8
|
3
|
+
|
4
|
+
$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
|
5
|
+
|
6
|
+
require 'optparse'
|
7
|
+
require 'hole'
|
8
|
+
require 'runner'
|
9
|
+
require 'yaml'
|
10
|
+
|
11
|
+
|
12
|
+
options = {}
|
13
|
+
|
14
|
+
parser = OptionParser.new do |op|
|
15
|
+
op.banner = "Usage: Blackhole [options]"
|
16
|
+
|
17
|
+
op.separator "== Blackhole =="
|
18
|
+
op.separator ""
|
19
|
+
op.separator "Starting Blackhole"
|
20
|
+
op.separator "================================================================================="
|
21
|
+
op.separator "Options:"
|
22
|
+
|
23
|
+
##
|
24
|
+
## Daemonization / Logging options
|
25
|
+
##
|
26
|
+
op.on("--pidfile PATH", "DO YOU WANT ME TO WRITE A PIDFILE SOMEWHERE FOR U?") do |pid_file|
|
27
|
+
options[:pid_file] = pid_file
|
28
|
+
end
|
29
|
+
|
30
|
+
op.on("--logfile PATH", "I'LL POOP OUT LOGS HERE FOR U") do |log_file|
|
31
|
+
options[:log_file] = log_file
|
32
|
+
end
|
33
|
+
|
34
|
+
op.on("-v", "--verbosity LEVEL", "HOW MUCH POOP DO U WANT IN UR LOGS? [LEVEL=0:errors,1:some,2:lots of poop]") do |verbosity|
|
35
|
+
options[:verbosity] = verbosity.to_i
|
36
|
+
end
|
37
|
+
|
38
|
+
op.on("-K", "--kill", "SHUT DOWN Blackhole") do
|
39
|
+
options[:kill] = true
|
40
|
+
end
|
41
|
+
|
42
|
+
# Socket to Tug
|
43
|
+
op.on("-H", "--host HOST", "Blackhole will be place this Network Interface") do |host|
|
44
|
+
options[:host] = host
|
45
|
+
end
|
46
|
+
|
47
|
+
op.on("-P", "--port PORT", "Blackhole pull log from this hole") do |port|
|
48
|
+
options[:port] = port.to_i
|
49
|
+
end
|
50
|
+
|
51
|
+
##
|
52
|
+
## Mongo
|
53
|
+
##
|
54
|
+
op.on("--mongodb DATABASE", "STORE LOGS IN THIS DB") do |mongo_db|
|
55
|
+
options[:mongo_db] = mongo_db
|
56
|
+
end
|
57
|
+
|
58
|
+
# NOTE: this option can be given multiple times for a replica set
|
59
|
+
op.on("--mongohost HOSTPORT", "STORE LOGS IN THIS MONGO [eg, localhost or localhost:27017]") do |mongo_hostport|
|
60
|
+
options[:mongo_hostports] ||= []
|
61
|
+
options[:mongo_hostports] << mongo_hostport
|
62
|
+
end
|
63
|
+
|
64
|
+
op.on("-h", "--help", "WANNA LEARN MORE?") do
|
65
|
+
puts op
|
66
|
+
exit
|
67
|
+
end
|
68
|
+
|
69
|
+
op.separator ""
|
70
|
+
end
|
71
|
+
|
72
|
+
parser.parse!
|
73
|
+
|
74
|
+
cur_dir = Dir.pwd
|
75
|
+
|
76
|
+
options[:log_file] ||= "#{cur_dir}/blackhole.log"
|
77
|
+
options[:pid_file] ||= "#{cur_dir}/blackhole.pid"
|
78
|
+
options[:verbosity] ||= 2
|
79
|
+
options[:host] ||= "localhost"
|
80
|
+
options[:port] ||= 8889
|
81
|
+
options[:mongo_db_name] ||= "blackhole"
|
82
|
+
options[:mongo_hostport] ||= []
|
83
|
+
options[:mongo_hostport] << "localhost:27017"
|
84
|
+
|
85
|
+
p "blackhole options : #{options.inspect}"
|
86
|
+
Blackhole::Runner.new(options).run!
|
data/lib/blackhole.rb
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
module Blackhole
|
2
|
+
class << self
|
3
|
+
attr_accessor :logger
|
4
|
+
|
5
|
+
# Mongo connection
|
6
|
+
attr_accessor :mongo
|
7
|
+
|
8
|
+
# Info to connect to mongo
|
9
|
+
attr_accessor :mongo_db_name
|
10
|
+
attr_accessor :mongo_hostports
|
11
|
+
|
12
|
+
# Returns a mongo connection (NOT an em-mongo connection)
|
13
|
+
def mongo
|
14
|
+
@mongo ||= begin
|
15
|
+
require 'mongo' # require mongo here, as opposed to the top, because we don't want mongo included in the reactor (use em-mongo for that)
|
16
|
+
|
17
|
+
hostports = self.mongo_hostports || [["localhost", Mongo::Connection::DEFAULT_PORT]]
|
18
|
+
self.mongo_hostports = hostports.collect do |hp|
|
19
|
+
if hp.is_a?(String)
|
20
|
+
host, port = hp.split(":")
|
21
|
+
[host, port || Mongo::Connection::DEFAULT_PORT]
|
22
|
+
else
|
23
|
+
hp
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
if self.mongo_hostports.length == 1
|
28
|
+
hostport = self.mongo_hostports.first
|
29
|
+
Mongo::Connection.new(hostport[0], hostport[1], :pool_size => 5, :pool_timeout => 20).db(self.mongo_db_name || "blackhole")
|
30
|
+
else
|
31
|
+
raise "repls set con not impl"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def collection(collection_name)
|
37
|
+
@collections ||= {}
|
38
|
+
@collections[collection_name] ||= begin
|
39
|
+
c = mongo.collection(collection_name)
|
40
|
+
c.ensure_index([["t", 1]], {:background => true, :unique => true})
|
41
|
+
c.ensure_index([["host", 1]], {:background => true, :unique => true})
|
42
|
+
c
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def mongo_collection_name(namespace)
|
47
|
+
"blackhole_#{namespace}"
|
48
|
+
end
|
49
|
+
|
50
|
+
def collection_for(namespace)
|
51
|
+
collection(mongo_collection_name(namespace))
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
data/lib/hole.rb
ADDED
@@ -0,0 +1,147 @@
|
|
1
|
+
# Mouth의 sucker의 영감을 받아 고친 sucker
|
2
|
+
# MOuth : https://github.com/cypriss/mouth
|
3
|
+
require 'em-mongo'
|
4
|
+
require 'eventmachine'
|
5
|
+
|
6
|
+
module Blackhole
|
7
|
+
class HoleConnection < EM::Connection
|
8
|
+
attr_accessor :hole
|
9
|
+
|
10
|
+
def receive_data(data)
|
11
|
+
Blackhole.logger.debug "UDP packet: '#{data}'"
|
12
|
+
# store to memory
|
13
|
+
hole.store!(data)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
class Hole
|
18
|
+
# Host/Port to suck UDP packets on
|
19
|
+
attr_accessor :host
|
20
|
+
attr_accessor :port
|
21
|
+
|
22
|
+
# Actual EM::Mongo connection
|
23
|
+
attr_accessor :mongo
|
24
|
+
|
25
|
+
# Info to connect to mongo
|
26
|
+
attr_accessor :mongo_db_name
|
27
|
+
attr_accessor :mongo_hostports
|
28
|
+
|
29
|
+
# Stats
|
30
|
+
attr_accessor :udp_packets_received
|
31
|
+
attr_accessor :mongo_flushes
|
32
|
+
|
33
|
+
# Received Log
|
34
|
+
attr_accessor :logs
|
35
|
+
# sender info
|
36
|
+
attr_accessor :info
|
37
|
+
|
38
|
+
|
39
|
+
def initialize(options = {})
|
40
|
+
|
41
|
+
self.host = options[:host] || "localhost"
|
42
|
+
self.port = options[:port] || 8889
|
43
|
+
self.mongo_db_name = options[:mongo_db_name] || "stepper"
|
44
|
+
hostports = options[:mongo_hostports] || [["localhost", EM::Mongo::DEFAULT_PORT]]
|
45
|
+
self.mongo_hostports = hostports.collect do |hp|
|
46
|
+
if hp.is_a?(String)
|
47
|
+
host, port = hp.split(":")
|
48
|
+
[host, port || EM::Mongo::DEFAULT_PORT]
|
49
|
+
else
|
50
|
+
hp
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
self.udp_packets_received = 0
|
55
|
+
self.mongo_flushes = 0
|
56
|
+
self.logs = []
|
57
|
+
self.info = []
|
58
|
+
end
|
59
|
+
|
60
|
+
def tug!
|
61
|
+
EM.run do
|
62
|
+
# Connect to mongo now
|
63
|
+
self.mongo
|
64
|
+
|
65
|
+
EM.open_datagram_socket host, port, HoleConnection do |conn|
|
66
|
+
conn.hole = self
|
67
|
+
end
|
68
|
+
|
69
|
+
EM.add_periodic_timer(5) do
|
70
|
+
#Blackhole.logger.info "Stepping: #{self.stepping.inspect}"
|
71
|
+
#self.flush!
|
72
|
+
self.drain!
|
73
|
+
self.set_procline!
|
74
|
+
end
|
75
|
+
|
76
|
+
EM.next_tick do
|
77
|
+
Blackhole.logger.info "Blackhost started to tug logs..."
|
78
|
+
self.set_procline!
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def store!(data)
|
84
|
+
if /(?<seq>^<\d+>)(?<date>.{15})\s(?<hostname>[\w-]+)\s(?<filename>[^:]+):\s(?<log>.+)/ =~ data
|
85
|
+
unless log == ""
|
86
|
+
packet = {}
|
87
|
+
packet[:hostname] = hostname
|
88
|
+
packet[:filename] = filename
|
89
|
+
self.info << packet
|
90
|
+
packet[:date] = date
|
91
|
+
packet[:log] = log
|
92
|
+
packet[:time] = Time.now.to_i
|
93
|
+
self.logs << packet
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
self.udp_packets_received += 1
|
98
|
+
end
|
99
|
+
|
100
|
+
def drain!
|
101
|
+
|
102
|
+
temp_logs = self.logs.clone
|
103
|
+
self.logs = []
|
104
|
+
temp_info = self.info.clone
|
105
|
+
self.info = []
|
106
|
+
temp_info.uniq!
|
107
|
+
|
108
|
+
if temp_info.size > 0
|
109
|
+
collection_name = Blackhole.mongo_collection_name("info")
|
110
|
+
temp_info.each do |info|
|
111
|
+
info_tmp = {}
|
112
|
+
info_tmp[:host] = info[:hostname].to_s
|
113
|
+
info_tmp[:filename] = info[:filename].to_s
|
114
|
+
info_tmp[:port] = self.port
|
115
|
+
self.mongo.collection(collection_name).update(info_tmp, info_tmp, { upsert: true })
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
if temp_logs.size > 0
|
120
|
+
# sanitize collection name
|
121
|
+
collection_name = Blackhole.mongo_collection_name(self.port)
|
122
|
+
|
123
|
+
self.mongo.collection(collection_name).insert(temp_logs)
|
124
|
+
|
125
|
+
end
|
126
|
+
|
127
|
+
Blackhole.logger.info "Saved logs : #{temp_logs}"
|
128
|
+
|
129
|
+
self.mongo_flushes += 1
|
130
|
+
end
|
131
|
+
|
132
|
+
def mongo
|
133
|
+
@mongo ||= begin
|
134
|
+
if self.mongo_hostports.length == 1
|
135
|
+
EM::Mongo::Connection.new(*self.mongo_hostports.first).db(self.mongo_db_name)
|
136
|
+
else
|
137
|
+
raise "Ability to connect to a replica set not implemented."
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def set_procline!
|
143
|
+
$0 = "blackhole [#{self.port}] [UDP Recv: #{self.udp_packets_received}] [Mongo saves: #{self.mongo_flushes}]"
|
144
|
+
end
|
145
|
+
|
146
|
+
end # class Hole
|
147
|
+
end # module
|
data/lib/runner.rb
ADDED
@@ -0,0 +1,89 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'logger'
|
3
|
+
require 'hole'
|
4
|
+
require 'blackhole'
|
5
|
+
|
6
|
+
module Blackhole
|
7
|
+
class Runner
|
8
|
+
|
9
|
+
attr_accessor :log_file
|
10
|
+
attr_accessor :pid_file
|
11
|
+
attr_accessor :logger
|
12
|
+
attr_accessor :verbosity # 0: Only errors/warnings 1: informational 2: debug/all incomding UDP packets
|
13
|
+
attr_accessor :options
|
14
|
+
|
15
|
+
def initialize(opts={})
|
16
|
+
self.log_file = opts[:log_file]
|
17
|
+
self.pid_file = opts[:pid_file]
|
18
|
+
self.verbosity = opts[:verbosity]
|
19
|
+
self.options = opts
|
20
|
+
end
|
21
|
+
|
22
|
+
def run!
|
23
|
+
kill! if self.options[:kill]
|
24
|
+
|
25
|
+
puts "Starting Blackhole..."
|
26
|
+
|
27
|
+
daemonize!
|
28
|
+
save_pid!
|
29
|
+
setup_logging!
|
30
|
+
|
31
|
+
# Start the reactor!
|
32
|
+
hole = Blackhole::Hole.new(self.options)
|
33
|
+
hole.tug!
|
34
|
+
end
|
35
|
+
|
36
|
+
def kill!
|
37
|
+
if @pid_file
|
38
|
+
pid = File.read(@pid_file)
|
39
|
+
#logger.warn "Sending #{kill_command} to #{pid.to_i}"
|
40
|
+
Process.kill(:INT, pid.to_i)
|
41
|
+
else
|
42
|
+
#logger.warn "No pid_file specified"
|
43
|
+
end
|
44
|
+
ensure
|
45
|
+
exit(0)
|
46
|
+
end
|
47
|
+
|
48
|
+
def daemonize!
|
49
|
+
# Fork and continue in forked process
|
50
|
+
# Also calls setsid
|
51
|
+
# Also redirects all output to /dev/null
|
52
|
+
Process.daemon(true)
|
53
|
+
|
54
|
+
# Reset umask
|
55
|
+
File.umask(0000)
|
56
|
+
|
57
|
+
# Set the procline
|
58
|
+
$0 = "Blackhole [initializing]"
|
59
|
+
end
|
60
|
+
|
61
|
+
def save_pid!
|
62
|
+
if @pid_file
|
63
|
+
pid = Process.pid
|
64
|
+
FileUtils.mkdir_p(File.dirname(@pid_file))
|
65
|
+
File.open(@pid_file, 'w') { |f| f.write(pid) }
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def setup_logging!
|
70
|
+
if @log_file
|
71
|
+
STDERR.reopen(@log_file, 'a')
|
72
|
+
# Open a logger
|
73
|
+
self.logger = Logger.new(@log_file)
|
74
|
+
self.logger.level = case self.verbosity
|
75
|
+
when 0
|
76
|
+
Logger::WARN
|
77
|
+
when 1
|
78
|
+
Logger::INFO
|
79
|
+
else
|
80
|
+
Logger::DEBUG
|
81
|
+
end
|
82
|
+
Blackhole.logger = self.logger
|
83
|
+
|
84
|
+
self.logger.info "Blackhole Initialized..."
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
end # class Runner
|
89
|
+
end # module
|
metadata
ADDED
@@ -0,0 +1,126 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: blackhole
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
version: 0.1.0
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Kim, SeongSik
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2013-02-01 00:00:00 +09:00
|
18
|
+
default_executable:
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: em-mongo
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ~>
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
segments:
|
29
|
+
- 0
|
30
|
+
- 4
|
31
|
+
- 2
|
32
|
+
version: 0.4.2
|
33
|
+
type: :runtime
|
34
|
+
version_requirements: *id001
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: mongo
|
37
|
+
prerelease: false
|
38
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ~>
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
segments:
|
44
|
+
- 1
|
45
|
+
- 6
|
46
|
+
version: "1.6"
|
47
|
+
type: :runtime
|
48
|
+
version_requirements: *id002
|
49
|
+
- !ruby/object:Gem::Dependency
|
50
|
+
name: bson_ext
|
51
|
+
prerelease: false
|
52
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
53
|
+
none: false
|
54
|
+
requirements:
|
55
|
+
- - ~>
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
segments:
|
58
|
+
- 1
|
59
|
+
- 6
|
60
|
+
version: "1.6"
|
61
|
+
type: :runtime
|
62
|
+
version_requirements: *id003
|
63
|
+
- !ruby/object:Gem::Dependency
|
64
|
+
name: eventmachine
|
65
|
+
prerelease: false
|
66
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
67
|
+
none: false
|
68
|
+
requirements:
|
69
|
+
- - ~>
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
segments:
|
72
|
+
- 0
|
73
|
+
- 12
|
74
|
+
- 10
|
75
|
+
version: 0.12.10
|
76
|
+
type: :runtime
|
77
|
+
version_requirements: *id004
|
78
|
+
description: The blackhole is a UDP listening server. IT store udp packets into mongodb
|
79
|
+
email: kssminus@gmail.com
|
80
|
+
executables:
|
81
|
+
- blackhole
|
82
|
+
extensions: []
|
83
|
+
|
84
|
+
extra_rdoc_files: []
|
85
|
+
|
86
|
+
files:
|
87
|
+
- lib/blackhole.rb
|
88
|
+
- lib/hole.rb
|
89
|
+
- lib/runner.rb
|
90
|
+
- bin/blackhole
|
91
|
+
has_rdoc: true
|
92
|
+
homepage: http://github.com/kssminus/blackhole
|
93
|
+
licenses: []
|
94
|
+
|
95
|
+
post_install_message:
|
96
|
+
rdoc_options: []
|
97
|
+
|
98
|
+
require_paths:
|
99
|
+
- lib
|
100
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
101
|
+
none: false
|
102
|
+
requirements:
|
103
|
+
- - ">="
|
104
|
+
- !ruby/object:Gem::Version
|
105
|
+
segments:
|
106
|
+
- 1
|
107
|
+
- 9
|
108
|
+
- 0
|
109
|
+
version: 1.9.0
|
110
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
111
|
+
none: false
|
112
|
+
requirements:
|
113
|
+
- - ">="
|
114
|
+
- !ruby/object:Gem::Version
|
115
|
+
segments:
|
116
|
+
- 0
|
117
|
+
version: "0"
|
118
|
+
requirements: []
|
119
|
+
|
120
|
+
rubyforge_project:
|
121
|
+
rubygems_version: 1.3.7
|
122
|
+
signing_key:
|
123
|
+
specification_version: 3
|
124
|
+
summary: This will tug everything.
|
125
|
+
test_files: []
|
126
|
+
|