typhon 0.1.0

data/bin/typhon

@@ -0,0 +1,63 @@
+#!/usr/bin/ruby
+
+require 'typhon'
+require 'optparse'
+
+configdir = "/etc/typhon"
+daemon = false
+pidfile = nil
+
+opt = OptionParser.new
+
+opt.on("--config [DIR]", "Directory for configuration file and heads") do |v|
+    configdir = v
+end
+
+opt.on("--daemonize", "-d", "Daemonize the process") do |v|
+    daemon = true
+end
+
+opt.on("--pid [PIDFILE]", "-p", "Write a pidfile") do |v|
+    pidfile = v
+end
+
+opt.parse!
+
+raise "The directory #{configdir} does not exist" unless File.directory?(configdir)
+
+def stop_and_exit(daemon, pidfile, signal)
+    Typhon::Log.info("Exiting after signal #{signal}")
+
+    if daemon && pidfile
+        File.unlink(pidfile)
+    end
+
+    exit
+end
+
+Signal.trap('INT') { stop_and_exit(daemon, pidfile, :int) }
+Signal.trap('TERM') { stop_and_exit(daemon, pidfile, :term) }
+
+typhon = Typhon.new(configdir)
+
+Typhon.files.each do |f|
+    Typhon::Log.info("Tailing log #{f}")
+end
+
+if daemon
+    raise "Pidfile #{pidfile} exist" if pidfile && File.exist?(pidfile)
+
+    Typhon.daemonize do
+        if pidfile
+            begin
+                File.open(pidfile, 'w') {|f| f.write(Process.pid) }
+            rescue
+            end
+        end
+
+        Typhon::Log.info("Running in the background as pid #{Process.pid}")
+        typhon.tail
+    end
+else
+    typhon.tail
+end

data/lib/typhon.rb

@@ -0,0 +1,82 @@
+class Typhon
+    require 'rubygems'
+    require 'yaml'
+    require 'eventmachine'
+    require 'eventmachine-tail'
+    require 'typhon/heads'
+    require 'typhon/log'
+    require 'typhon/config'
+    require 'typhon/stompclient'
+
+    class << self
+        def heads
+            Heads.heads
+        end
+
+        def files
+            Heads.heads.keys
+        end
+
+        def grow(options, &blk)
+            raise "Heads need a name" unless options[:name]
+            raise "Heads need files" unless options[:files]
+
+            Heads.register_head(options[:name], options[:files], blk)
+        end
+
+        def stomp=(stomp)
+            @stomp = stomp
+        end
+
+        def stomp
+            @stomp
+        end
+
+        def daemonize
+            fork do
+                Process.setsid
+                exit if fork
+                Dir.chdir('/tmp')
+                STDIN.reopen('/dev/null')
+                STDOUT.reopen('/dev/null', 'a')
+                STDERR.reopen('/dev/null', 'a')
+
+                yield
+            end
+        end
+    end
+
+    attr_reader :heads
+
+    def initialize(path="/etc/typhon")
+        @configdir = path
+
+        Config[:configdir] = path
+        Config.loadconfig
+
+        @heads = Heads.new
+        @stomp = nil
+    end
+
+    def tail
+        EM.run do
+            @heads.loadheads
+
+            if Config[:stomp]
+                Log.debug("Connecting to Stomp Server %s:%d" % [ Config[:stomp][:server], Config[:stomp][:port] ])
+                @stomp = EM.connect Config[:stomp][:server], Config[:stomp][:port], Typhon::StompClient, {:auto_reconnect => true, :timeout => 2}
+                Typhon.stomp = @stomp
+            end
+
+            EM.add_periodic_timer(10) do
+                @heads.loadheads
+            end
+
+            if Config[:stat_log_frequency] > 0
+                EM.add_periodic_timer(Config[:stat_log_frequency]) do
+                    @heads.log_stats
+                end
+            end
+        end
+    end
+end

data/lib/typhon/config.rb

@@ -0,0 +1,48 @@
+class Typhon
+    class Config
+        include Enumerable
+
+        @settings = {:loglevel => :info, :stomp => false, :stat_log_frequency => 3600}
+
+        class << self
+            attr_reader :settings
+
+            def []=(key,val)
+                @settings[key] = val
+            end
+
+            def [](key)
+                @settings[key]
+            end
+
+            def include?(key)
+                @settings.include?(key)
+            end
+
+            def each
+                @settings.each_pair do |k, v|
+                    yield({k => v})
+                end
+            end
+
+            def loadconfig
+                raise "Set configdir" unless @settings.include?(:configdir)
+
+                file = File.join([@settings[:configdir], "typhon.yaml"])
+
+                raise "Cannot find file #{file}" unless File.exist?(file)
+                @settings.merge!(YAML.load_file(file))
+            end
+
+            def method_missing(k, *args, &block)
+                return @settings[k] if @settings.include?(k)
+
+                k = k.to_s.gsub("_", ".")
+                return @settings[k] if @settings.include?(k)
+
+                super
+            end
+        end
+    end
+end
+

data/lib/typhon/heads.rb

@@ -0,0 +1,160 @@
+class Typhon
+    class Heads
+        # these methods are here to help the DSL have somewhere to
+        # store instances of the heads and some utilities to manage
+        # them.  This is effectively a global named scope that just
+        # holds blocks of codes
+        class << self
+            def register_head(name, files, head)
+                @heads ||= {}
+
+                [files].flatten.each do |file|
+                    @heads[file] ||= {}
+
+                    raise "Already have a head called #{name} for file #{file}" if @heads[file].include?(name)
+
+                    @heads[file][name] = head
+
+                    Log.debug("Registered a new head: #{name} for file #{file}")
+                end
+            end
+
+            def clear!
+                Log.debug("Clearing previously loaded heads")
+                @heads = {}
+            end
+
+            def heads
+                @heads || {}
+            end
+
+            def files
+                @heads.keys
+            end
+        end
+
+        def initialize
+            @dir = File.join(Config.configdir, "heads")
+            @tails = {}
+            @linecount = 0
+            @starttime = Time.now
+        end
+
+        def log_stats
+            uptime = seconds_to_human((Time.now - @starttime).to_i)
+
+            Log.info("Up for #{uptime} read #{@linecount} lines")
+        end
+
+        # Handles a line of text from a log file by finding the
+        # heads associated with that file and calling them all
+        def feed(file, pos, text)
+            return unless Heads.heads.include?(file)
+
+            Heads.heads[file].each_pair do |name, head|
+                begin
+                    head.call(file, pos, text)
+                    @linecount += 1
+                rescue Exception => e
+                    Log.error("Failed to handle line from #{file}##{pos} with head #{name}: #{e.class}: #{e}")
+                end
+            end
+        end
+
+        # Loads/Reload all the heads from disk, a trigger file is used that
+        # the user can touch the trigger and it will initiate a complete reload
+        def loadheads
+            if File.exist?(triggerfile)
+                triggerage = File::Stat.new(triggerfile).mtime.to_f
+            else
+                triggerage = 0
+            end
+
+            @loaded ||= 0
+
+            if (@loaded < triggerage) || @loaded == 0
+                Heads.clear!
+                headfiles.each do |head|
+                    loadhead(head)
+                end
+            end
+
+            starttails
+
+            @loaded = Time.now.to_f
+        end
+
+        # Start EM tailers for each known file.  If a file has become orphaned
+        # by all its heads being removed then close the tail
+        def starttails
+            # for all the files that have interested heads start tailers
+            Typhon.files.each do |file|
+                unless @tails.include?(file)
+                    Log.debug("Starting a new tailer for #{file}")
+                    @tails[file] = EventMachine::file_tail(file) do |ft, line|
+                        self.feed(ft.path, ft.position, line)
+                    end
+                end
+            end
+
+            # for all the tailers make sure there are files, else close the tailer
+            @tails.keys.each do |file|
+                unless Typhon.files.include?(file)
+                    Log.debug("Closing tailer for #{file} there are no heads attached")
+
+                    begin
+                        @tails[file].close
+                    rescue
+                    end
+                end
+            end
+        end
+
+        def loadhead(head)
+            Log.debug("Loading head #{head}")
+            load head
+        rescue Exception => e
+            puts "Failed to load #{head}: #{e.class}: #{e}"
+            p e.backtrace
+        end
+
+        def headfiles
+            if File.directory?(@dir)
+                Dir.entries(@dir).grep(/head.rb$/).map do |f|
+                     File.join([@dir, f])
+                end
+            else
+                raise "#{@dir} is not a directory"
+            end
+        end
+
+        def triggerfile
+            File.join([@dir, "reload.txt"])
+        end
+
+        # borrowed from ohai, thanks Adam.
+        def seconds_to_human(seconds)
+            days = seconds.to_i / 86400
+            seconds -= 86400 * days
+
+            hours = seconds.to_i / 3600
+            seconds -= 3600 * hours
+
+            minutes = seconds.to_i / 60
+            seconds -= 60 * minutes
+
+            if days > 1
+                return sprintf("%d days %02d hours %02d minutes %02d seconds", days, hours, minutes, seconds)
+            elsif days == 1
+                return sprintf("%d day %02d hours %02d minutes %02d seconds", days, hours, minutes, seconds)
+            elsif hours > 0
+                return sprintf("%d hours %02d minutes %02d seconds", hours, minutes, seconds)
+            elsif minutes > 0
+                return sprintf("%d minutes %02d seconds", minutes, seconds)
+            else
+                return sprintf("%02d seconds", seconds)
+            end
+        end
+
+    end
+end

data/lib/typhon/log.rb

@@ -0,0 +1,55 @@
+class Typhon
+    class Log
+        require 'syslog'
+
+        include Syslog::Constants
+
+        @configured = false
+
+        @known_levels = [:debug, :info, :warn, :error, :fatal]
+
+        class << self
+            def log(msg, severity=:debug)
+                configure unless @configured
+
+                if @known_levels.index(severity) >= @known_levels.index(@active_level)
+                    Syslog.send(valid_levels[severity.to_sym], "#{from} #{msg}")
+                end
+            rescue Exception => e
+                STDERR.puts("Failed to log: #{e.class}: #{e}: original log message: #{severity}: #{msg}")
+                STDERR.puts(e.backtrace.join("\n\t"))
+            end
+
+            def configure
+                Syslog.close if Syslog.opened?
+                Syslog.open(File.basename($0))
+
+                @active_level = Config[:loglevel]
+
+                raise "Unknown log level #{@active_level} specified" unless valid_levels.include?(@active_level)
+
+                @configured = true
+            end
+
+            # figures out the filename that called us
+            def from
+                from = File.basename(caller[4])
+            end
+
+            def valid_levels
+                {:info  => :info,
+                 :warn  => :warning,
+                 :debug => :debug,
+                 :fatal => :crit,
+                 :error => :err}
+            end
+
+            def method_missing(level, *args, &block)
+                super unless [:info, :warn, :debug, :fatal, :error].include?(level)
+
+                log(args[0], level)
+            end
+        end
+    end
+end
+

data/lib/typhon/stompclient.rb

@@ -0,0 +1,49 @@
+class Typhon
+    class StompClient < EM::Connection
+        include EM::Protocols::Stomp
+
+        def initialize(params={})
+            @connected = false
+            @options = {:auto_reconnect => true, :timeout => 2, :max_queue_size => 500}
+            @queue = EM::Queue.new
+        end
+
+        def connection_completed
+            connect :login => Config[:stomp][:user], :passcode => Config[:stomp][:pass]
+
+            Log.debug("Authenticated to %s:%d" % [ Config[:stomp][:server], Config[:stomp][:port] ])
+            @connected = true
+        end
+
+        def unbind
+            Log.error("Connection to %s:%d failed" % [ Config[:stomp][:server], Config[:stomp][:port] ])
+            @connected = false
+
+            EM.add_timer(@options[:timeout]) do
+                Log.debug("Connecting to Stomp Server %s:%d" % [ Config[:stomp][:server], Config[:stomp][:port] ])
+                reconnect Config[:stomp][:server], Config[:stomp][:port]
+            end
+        end
+
+        def connected?
+            (@connected && !error?)
+        end
+
+        def publish(topic, message, param={})
+            if connected?
+                send(topic, message, param)
+
+                until @queue.empty? do
+                    @queue.pop do |msg|
+                        send(msg[:topic], msg[:message], msg[:param])
+                    end
+                end
+            else
+                if @queue.size < @options[:max_queue_size]
+                    @queue.push({:topic => topic, :message => message, :param => param})
+                end
+            end
+        end
+    end
+end
+

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification 
+name: typhon
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- R.I.Pienaar
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-09-29 00:00:00 +01:00
+default_executable: typhon
+dependencies: []
+
+description: Single daemon that tails many files and route lines through your own logic
+email: rip@devco.net
+executables: 
+- typhon
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/typhon
+- lib/typhon/config.rb
+- lib/typhon/stompclient.rb
+- lib/typhon/heads.rb
+- lib/typhon/log.rb
+- lib/typhon.rb
+has_rdoc: true
+homepage: https://github.com/ripienaar/typhon/
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Wrapper around eventmachine-tail to make writing custom logtailers easy
+test_files: []
+