blackhole 0.1.0

data/bin/blackhole

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'optparse'
+require 'hole'
+require 'runner'
+require 'yaml'
+
+
+options = {}
+
+parser = OptionParser.new do |op|
+  op.banner = "Usage: Blackhole [options]"
+
+  op.separator "== Blackhole =="
+  op.separator ""
+  op.separator "Starting Blackhole"
+  op.separator "================================================================================="
+  op.separator "Options:"
+
+  ##
+  ## Daemonization / Logging options
+  ##
+  op.on("--pidfile PATH", "DO YOU WANT ME TO WRITE A PIDFILE SOMEWHERE FOR U?") do |pid_file|
+    options[:pid_file] = pid_file
+  end
+  
+  op.on("--logfile PATH", "I'LL POOP OUT LOGS HERE FOR U") do |log_file|
+    options[:log_file] = log_file
+  end
+  
+  op.on("-v", "--verbosity LEVEL", "HOW MUCH POOP DO U WANT IN UR LOGS? [LEVEL=0:errors,1:some,2:lots of poop]") do |verbosity|
+    options[:verbosity] = verbosity.to_i
+  end
+  
+  op.on("-K", "--kill", "SHUT DOWN Blackhole") do
+    options[:kill] = true
+  end
+ 
+  # Socket to Tug
+  op.on("-H", "--host HOST", "Blackhole will be place this Network Interface") do |host|
+    options[:host] = host
+  end
+  
+  op.on("-P", "--port PORT", "Blackhole pull log from this hole") do |port|
+    options[:port] = port.to_i
+  end
+  
+  ##
+  ## Mongo
+  ##
+  op.on("--mongodb DATABASE", "STORE LOGS IN THIS DB") do |mongo_db|
+    options[:mongo_db] = mongo_db
+  end
+  
+  # NOTE: this option can be given multiple times for a replica set
+  op.on("--mongohost HOSTPORT", "STORE LOGS IN THIS MONGO [eg, localhost or localhost:27017]") do |mongo_hostport|
+    options[:mongo_hostports] ||= []
+    options[:mongo_hostports] << mongo_hostport
+  end
+
+  op.on("-h", "--help", "WANNA LEARN MORE?") do
+    puts op
+    exit
+  end
+
+  op.separator ""
+end
+
+parser.parse!
+
+cur_dir  = Dir.pwd
+
+options[:log_file]        ||= "#{cur_dir}/blackhole.log"
+options[:pid_file]        ||= "#{cur_dir}/blackhole.pid"
+options[:verbosity]       ||= 2
+options[:host]            ||= "localhost"
+options[:port]            ||= 8889
+options[:mongo_db_name]   ||= "blackhole" 
+options[:mongo_hostport]  ||= []
+options[:mongo_hostport]  << "localhost:27017"
+
+p "blackhole options : #{options.inspect}"
+Blackhole::Runner.new(options).run!

data/lib/blackhole.rb

@@ -0,0 +1,54 @@
+module Blackhole
+  class << self
+    attr_accessor :logger
+    
+    # Mongo connection
+    attr_accessor :mongo
+    
+    # Info to connect to mongo
+    attr_accessor :mongo_db_name
+    attr_accessor :mongo_hostports
+
+    # Returns a mongo connection (NOT an em-mongo connection)
+    def mongo
+      @mongo ||= begin
+        require 'mongo'   # require mongo here, as opposed to the top, because we don't want mongo included in the reactor (use em-mongo for that)
+        
+        hostports = self.mongo_hostports || [["localhost", Mongo::Connection::DEFAULT_PORT]]
+        self.mongo_hostports = hostports.collect do |hp|
+          if hp.is_a?(String)
+            host, port = hp.split(":")
+            [host, port || Mongo::Connection::DEFAULT_PORT]
+          else
+            hp
+          end
+        end
+        
+        if self.mongo_hostports.length == 1
+          hostport = self.mongo_hostports.first
+          Mongo::Connection.new(hostport[0], hostport[1], :pool_size => 5, :pool_timeout => 20).db(self.mongo_db_name || "blackhole")
+        else
+          raise "repls set con not impl"
+        end
+      end
+    end
+    
+    def collection(collection_name)
+      @collections ||= {}
+      @collections[collection_name] ||= begin
+        c = mongo.collection(collection_name)
+        c.ensure_index([["t", 1]], {:background => true, :unique => true})
+        c.ensure_index([["host", 1]], {:background => true, :unique => true})
+        c
+      end
+    end
+    
+    def mongo_collection_name(namespace)
+      "blackhole_#{namespace}"
+    end
+    
+    def collection_for(namespace)
+      collection(mongo_collection_name(namespace))
+    end
+  end
+end

data/lib/hole.rb

@@ -0,0 +1,147 @@
+# Mouth의 sucker의 영감을 받아 고친 sucker
+# MOuth : https://github.com/cypriss/mouth
+require 'em-mongo'
+require 'eventmachine'
+
+module Blackhole
+  class HoleConnection < EM::Connection
+    attr_accessor :hole
+  
+    def receive_data(data)
+      Blackhole.logger.debug "UDP packet: '#{data}'"
+      # store to memory
+      hole.store!(data)
+    end
+  end
+  
+  class Hole
+    # Host/Port to suck UDP packets on
+    attr_accessor :host
+    attr_accessor :port
+
+    # Actual EM::Mongo connection
+    attr_accessor :mongo
+    
+    # Info to connect to mongo
+    attr_accessor :mongo_db_name
+    attr_accessor :mongo_hostports
+    
+    # Stats
+    attr_accessor :udp_packets_received
+    attr_accessor :mongo_flushes
+    
+    # Received Log
+    attr_accessor :logs
+    # sender info
+    attr_accessor :info
+
+
+    def initialize(options = {})
+    
+      self.host = options[:host] || "localhost"
+      self.port = options[:port] || 8889
+      self.mongo_db_name = options[:mongo_db_name] || "stepper"
+      hostports = options[:mongo_hostports] || [["localhost", EM::Mongo::DEFAULT_PORT]]
+      self.mongo_hostports = hostports.collect do |hp|
+        if hp.is_a?(String)
+          host, port = hp.split(":")
+          [host, port || EM::Mongo::DEFAULT_PORT]
+        else
+          hp
+        end
+      end
+      
+      self.udp_packets_received = 0
+      self.mongo_flushes = 0
+      self.logs = []
+      self.info = []
+    end
+   
+    def tug!
+      EM.run do
+        # Connect to mongo now
+        self.mongo
+
+        EM.open_datagram_socket host, port, HoleConnection do |conn|
+          conn.hole = self
+        end
+        
+        EM.add_periodic_timer(5) do
+          #Blackhole.logger.info "Stepping: #{self.stepping.inspect}"
+          #self.flush!
+          self.drain!
+          self.set_procline!
+        end
+
+        EM.next_tick do
+          Blackhole.logger.info "Blackhost started to tug logs..."
+          self.set_procline!
+        end
+      end
+    end
+    
+    def store!(data)
+      if /(?<seq>^<\d+>)(?<date>.{15})\s(?<hostname>[\w-]+)\s(?<filename>[^:]+):\s(?<log>.+)/ =~ data
+        unless log == ""
+          packet = {}
+          packet[:hostname] = hostname
+          packet[:filename] = filename
+          self.info << packet
+          packet[:date] = date
+          packet[:log] = log
+          packet[:time] = Time.now.to_i
+          self.logs << packet
+        end
+      end
+
+      self.udp_packets_received += 1
+    end
+    
+    def drain!
+
+      temp_logs = self.logs.clone
+      self.logs = []
+      temp_info = self.info.clone
+      self.info = []
+      temp_info.uniq!
+      
+      if temp_info.size > 0
+        collection_name = Blackhole.mongo_collection_name("info") 
+        temp_info.each do |info|
+          info_tmp = {}
+          info_tmp[:host] = info[:hostname].to_s
+          info_tmp[:filename] = info[:filename].to_s
+          info_tmp[:port] = self.port
+          self.mongo.collection(collection_name).update(info_tmp, info_tmp, { upsert: true })
+        end
+      end
+
+      if temp_logs.size > 0
+        # sanitize collection name
+        collection_name = Blackhole.mongo_collection_name(self.port) 
+
+        self.mongo.collection(collection_name).insert(temp_logs)
+        
+      end
+
+      Blackhole.logger.info "Saved logs : #{temp_logs}" 
+
+      self.mongo_flushes += 1
+    end
+    
+    def mongo
+      @mongo ||= begin
+        if self.mongo_hostports.length == 1
+          EM::Mongo::Connection.new(*self.mongo_hostports.first).db(self.mongo_db_name)
+        else
+          raise "Ability to connect to a replica set not implemented."
+        end
+      end
+    end
+    
+    def set_procline!
+      $0 = "blackhole [#{self.port}] [UDP Recv: #{self.udp_packets_received}] [Mongo saves: #{self.mongo_flushes}]"
+    end
+    
+  end # class Hole
+end # module

data/lib/runner.rb

@@ -0,0 +1,89 @@
+require 'fileutils'
+require 'logger'
+require 'hole'
+require 'blackhole'
+
+module Blackhole
+  class Runner
+    
+    attr_accessor :log_file
+    attr_accessor :pid_file
+    attr_accessor :logger
+    attr_accessor :verbosity    # 0: Only errors/warnings 1: informational 2: debug/all incomding UDP packets
+    attr_accessor :options
+    
+    def initialize(opts={})
+      self.log_file = opts[:log_file]
+      self.pid_file = opts[:pid_file]
+      self.verbosity = opts[:verbosity]
+      self.options = opts
+    end
+    
+    def run!
+      kill! if self.options[:kill]
+      
+      puts "Starting Blackhole..."
+      
+      daemonize!
+      save_pid!
+      setup_logging!      
+
+      # Start the reactor!
+      hole = Blackhole::Hole.new(self.options)
+      hole.tug!
+    end
+    
+    def kill!
+      if @pid_file
+        pid = File.read(@pid_file)
+        #logger.warn "Sending #{kill_command} to #{pid.to_i}"
+        Process.kill(:INT, pid.to_i)
+      else
+        #logger.warn "No pid_file specified"
+      end
+    ensure
+      exit(0)
+    end
+    
+    def daemonize!
+      # Fork and continue in forked process
+      # Also calls setsid
+      # Also redirects all output to /dev/null
+      Process.daemon(true)
+       
+      # Reset umask
+      File.umask(0000)
+      
+      # Set the procline
+      $0 = "Blackhole [initializing]"
+    end
+    
+    def save_pid!
+      if @pid_file
+        pid = Process.pid
+        FileUtils.mkdir_p(File.dirname(@pid_file))
+        File.open(@pid_file, 'w') { |f| f.write(pid) }
+      end
+    end
+    
+    def setup_logging!
+      if @log_file
+        STDERR.reopen(@log_file, 'a')
+        # Open a logger
+        self.logger = Logger.new(@log_file)
+        self.logger.level = case self.verbosity
+        when 0
+          Logger::WARN
+        when 1
+          Logger::INFO
+        else
+          Logger::DEBUG
+        end
+        Blackhole.logger = self.logger
+        
+        self.logger.info "Blackhole Initialized..."
+      end
+    end
+    
+  end # class Runner
+end # module

metadata

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification 
+name: blackhole
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Kim, SeongSik
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-02-01 00:00:00 +09:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: em-mongo
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 4
+        - 2
+        version: 0.4.2
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: mongo
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 6
+        version: "1.6"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: bson_ext
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 6
+        version: "1.6"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 12
+        - 10
+        version: 0.12.10
+  type: :runtime
+  version_requirements: *id004
+description: The blackhole is a UDP listening server. IT store udp packets into mongodb
+email: kssminus@gmail.com
+executables: 
+- blackhole
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/blackhole.rb
+- lib/hole.rb
+- lib/runner.rb
+- bin/blackhole
+has_rdoc: true
+homepage: http://github.com/kssminus/blackhole
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 9
+      - 0
+      version: 1.9.0
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: This will tug everything.
+test_files: []
+