gltail 0.0.7

data/lib/gl_tail/parsers/pix-fwsm.rb

@@ -0,0 +1,50 @@
+# gl_tail.rb - OpenGL visualization of your server traffic
+# Copyright 2007 Erlend Simonsen <mr@fudgie.org>
+#
+# Licensed under the GNU General Public License v2 (see LICENSE)
+#
+
+# Parser which handles logs from Cisco PIX or FWSM firewalls
+# should also handle ASA logs, with minimal change...
+# Leif Sawyer (leif@denali.net)
+#
+
+class PixParser < Parser
+  def parse( line )
+    if line.include?(': Built')
+        _, firewall, type, direction, srcif, src, srcport, dstif, dst, dstport =
+               /^.* \d+ \d+:\d+:\d+ \[?([a-zA-Z0-9\-]+)\/?\]?.* %(FWSM|PIX)-\d+-\d+: Built (\w+)bound \w+ connection \d+ for (\w+):([a-zA-Z0-9.]+)\/([a-zA-Z0-9.]+) \(.*\) to (\w+):([a-zA-Z0-9.]+)\/([a-zA-Z0-9.]+)/.match(line).to_a
+
+        if firewall
+          add_activity(:block => 'firewall', :name => firewall)
+          if direction == 'out'
+            add_activity(:block => 'hosts', :name => src)
+            add_activity(:block => 'sites', :name => dst)
+          else
+            add_activity(:block => 'hosts', :name => dst)
+            add_activity(:block => 'sites', :name => src)
+          end
+          printf("%sbound from %s firewall '%s', srcif=%s, src=%s, srcport=%s, dstif=%s, dst=%s, dstport=%s...\n", direction, type, firewall, srcif, src, srcport, dstif, dst, dstport ) if $VRB > 0
+        end
+
+    elsif line.include?('Accessed URL')
+          _, firewall, type, client, server, url = /^.* \d+ \d+:\d+:\d+ \[?([a-zA-Z0-9\-]+)\/?\]?.* %(FWSM|PIX)-\d+-\d+: ([a-zA-Z0-9.]+) Accessed URL ([a-zA-Z0-9.]+):(.*)[\?]?/.match(line).to_a
+        if firewall
+          add_activity(:block => 'firewall', :name => firewall)
+          add_activity(:block => 'hosts', :name => client)
+          add_activity(:block => 'sites', :name => server)
+          add_activity(:block => 'urls', :name => url)
+          printf("%s firewall '%s': client %s accessed url %s on host %s\n", type, firewall, client, url, server) if $VRB > 0
+        end
+
+#    elsif line.include?(': Deny')
+        # Deny udp src outside:_SRC_IP_/_SRC_PORT_ dst inside:_DST_IP_/_DST_PORT_ by access-group "_ACL_NAME"
+#       printf("ACL denied access ...\n") if $VRB > 0
+
+#    elsif line.include?('static translation')
+        # Teardown static translation from inside:_SRC_IP_ to dmz-anc-csa:_DST_IP_ duration 0:01:00
+#       printf("static translation ...\n") if $VRB > 0
+
+    end
+ end
+end

data/lib/gl_tail/parsers/postfix.rb

@@ -0,0 +1,119 @@
+# gl_tail.rb - OpenGL visualization of your server traffic
+# Copyright 2007 Erlend Simonsen <mr@fudgie.org>
+#
+# Licensed under the GNU General Public License v2 (see LICENSE)
+
+# Parser which handles Postfix logs
+class PostfixParser < Parser
+  def parse( line )
+    if line.include?(': connect from')
+      _, host, ip = /: connect from ([^\[]+)\[(\d+.\d+.\d+.\d+)\]/.match(line).to_a
+      if host
+        host = ip if host == 'unknown'
+        add_activity(:block => 'smtp', :name => host, :size => 0.03)
+      end
+    elsif line.include?(' sasl_username=')
+      _, username = /, sasl_username=(.*)/.match(line).to_a
+      add_activity(:block => 'logins', :name => "#{username}/sasl", :size => 0.1)
+    elsif line.include?('NOQUEUE: reject:')
+      #
+      # Parse rejection messages, including RBL rejections.
+      #  The rejection status could be displayed with the actual error codes if desired.
+      #   Change :name => 'rejected'
+      #   To     :name => status
+      #   Or     :name => extstatus
+      #
+      _, host, ip, status, extstatus, rejectreason, from, to, proto, helo = /: reject: RCPT from ([^\[]+)\[(\d+.\d+.\d+.\d+)\]: (\d+) (\d.\d.\d) (.*) from=<([^>]+)> to=<([^>]+)> proto=(.*) helo=<([^>]+)>/.match(line).to_a
+      add_activity(:block => 'status', :name => 'rejected', :size => 0.03)
+      host = ip if host == 'unknown'
+      if not rejectreason.nil?
+        if rejectreason.include?(' blocked using ')
+          rbltype = 'rbl'
+          if rejectreason.include?('Sender address')
+            # RHSBL-stle rejection message
+            _, ip, rbl, rbltext = /Sender address \[([^>]+)\] blocked using (.*)\; (.*)\;/.match(rejectreason).to_a
+            rbltype = 'rhsbl'
+          else
+            # Plain RBL rejection message
+            _, ip, rbl, rbltext = /Client host \[(\d+.\d+.\d+.\d+)\] blocked using (.*)\; (.*)\;/.match(rejectreason).to_a
+          end
+          if not rbl.nil?
+            #            add_activity(:block => 'rejections', :name => host, :size => 0.03)
+            add_activity(:block => 'rejections', :name => rbltype + ' ' + rbl, :size => 0.03)
+          end
+        else
+          # Generic rejection message, print the whole thing.
+          _, address, reason = /<([^>]+)>\: (.*);/.match(rejectreason).to_a
+          #            add_activity(:block => 'rejections', :name => host, :size => 0.03)
+          add_activity(:block => 'rejections', :name => reason, :size => 0.03)
+        end
+      end
+    elsif line.include?(' from=<')
+      _, from, size = /: from=<([^>]+)>, size=(\d+)/.match(line).to_a
+      if from
+        add_activity(:block => 'mail from', :name => from, :size => size.to_f/100000.0)
+      end
+    elsif line.include?(' to=<')
+      if line.include?('relay=local')
+        # Incoming
+        _, to, delay, status = /: to=<([^>]+)>, .*delay=([\d.]+).*status=([^ ]+)/.match(line).to_a
+        add_activity(:block => 'mail to', :name => to, :size => delay.to_f/10.0, :type => 5, :color => [1.0, 0.0, 1.0, 1.0])
+        add_activity(:block => 'status', :name => 'received', :size => delay.to_f/10.0, :type => 3)
+      else
+        # Outgoing
+        _, to, relay_host, delay, status = /: to=<([^>]+)>.*relay=([^\[,]+).*delay=([\d.]+).*status=([^ ]+)/.match(line).to_a
+        add_activity(:block => 'mail from', :name => to, :size => delay.to_f/10.0)
+        add_activity(:block => 'smtp', :name => relay_host, :size => delay.to_f/10.0)
+        add_activity(:block => 'status', :name => status, :size => delay.to_f/10.0, :type => 3)
+      end
+    elsif line.include?('spamd:') and (line.include?('clean message') or line.include?('identified spam'))
+      #
+      # Parse spamd log entries for the result summary, and add the clean/spam status to the status block.
+      #
+      # NOTE/TODO: Much more could be done with this block, including averaging scores and processing times
+      #
+      _, status, score, mailaddr, proctime, size = /: spamd: (\w+ \w+) \((\d+\.\d+)\/.*\) for (.*):.* in (\d+\.\d+) seconds, (\d+) bytes/.match(line).to_a
+      if not status.nil?
+        status = status.include?('clean') ? 'clean' : 'spam'
+        add_activity(:block => 'status', :name => status, :size => proctime.to_f/10.0)
+      end
+    elsif line.include?('clamd[')
+      #
+      # Parse clamd log entries. Print the name of the detected virus.
+      #
+      _, virusname = /clamd\[\d+\]: .*: (.*) FOUND/.match(line).to_a
+      add_activity(:block => 'status', :name => 'virus', :size => 0.03)
+      add_activity(:block => 'viruses', :name => virusname, :size => 0.03)
+    elsif line.include?(': warning:')
+      #
+      # Parse warning messages. If it is a known warning, shorten the message, otherwise print the full text
+      #
+      _, warningtext = /: warning: (.*)/.match(line).to_a
+      if warningtext.include?('malformed domain name')
+        warningtext = 'Malformed Domain Name'
+      elsif warningtext.include?('non-SMTP command')
+        warningtext = 'Non-SMTP Command'
+      elsif warningtext.include?('Non-recoverable failure in name resolution')
+        warningtext = 'DNS Failure'
+      elsif warningtext.include?('hostname nor servname provided')
+        warningtext = 'Host Verification Failure'
+      elsif warningtext.include?('address not listed for hostname')
+        warningtext = 'Hostname Without Address'
+      elsif warningtext.include?('Connection rate limit exceeded')
+        warningtext = 'Per-Host Connection Rate Exceeded'
+      elsif warningtext.include?('Connection concurrency limit exceeded')
+        warningtext = 'Per-Host Connection concurrency Exceeded'
+      elsif warningtext.include?('numeric domain name in resource data')
+        warningtext = 'Numeric Domain Name'
+      elsif warningtext.include?('numeric hostname')
+        warningtext = 'Numeric Host Name'
+      elsif warningtext.include?('valid_hostname: empty hostname')
+        warningtext = 'Empty Hostname'
+      elsif warningtext.include?('Illegal address syntax')
+        warningtext = 'Illegal Address Syntax'
+      end
+      add_activity(:block => 'status', :name => 'warning', :size => 0.03)
+      add_activity(:block => 'warnings', :name => warningtext, :size => 0.03)
+    end
+  end
+end

data/lib/gl_tail/parsers/postgresql.rb

@@ -0,0 +1,42 @@
+# gl_tail.rb - OpenGL visualization of your server traffic
+# Copyright 2007 Erlend Simonsen <mr@fudgie.org>
+#
+# Licensed under the GNU General Public License v2 (see LICENSE)
+#
+
+# Parser which handles PostgreSQL logs
+class PostgreSQLParser < Parser
+  def parse( line )
+    # here's an example parser for postgres log files; adjust accordingly for different logfile setups.
+    #
+    # postgresql.conf:
+    #    log_line_prefix = '[%d, %t] '
+    #    log_connections = on
+    #    log_disconnections = on
+    #    log_duration = on
+    #    log_statement = 'all'
+
+    _, database, datetime, activity, description = /^\[(.*), (.* .* .*)\] LOG:  ([a-zA-Z0-9\s]*): (.*)/.match(line).to_a
+
+    if database
+      add_activity(:block => 'database', :name => database, :size => 0.2)
+    else
+      _, datetime, activity, description = /^(.* .* .*) LOG:  ([a-zA-Z0-9\s]*): (.*)/.match(line).to_a
+    end
+
+    if activity
+      activity = 'vacuum' if(description.include?('vacuum') || activity == 'autovacuum')
+      case activity
+      when 'duration'
+        add_activity(:block => 'database', :name => 'duration', :size => description.to_f / 100.0)
+      when 'statement'
+        add_activity(:block => 'database', :name => 'activity', :size => 0.2)
+      when 'connection authorized', 'disconnection'
+        add_activity(:block => 'database', :name => 'login/logout', :size => 0.2)
+      when 'vacuum'
+        add_event(:block => 'database', :name => 'vacuum', :message => description, :update_stats => true, :color => [1.0, 1.0, 0.0, 1.0])
+      end
+    end
+
+  end
+end

data/lib/gl_tail/parsers/pureftpd.rb

@@ -0,0 +1,30 @@
+# gl_tail.rb - OpenGL visualization of your server traffic
+# Copyright 2007 Erlend Simonsen <mr@fudgie.org>
+#
+# Licensed under the GNU General Public License v2 (see LICENSE)
+#
+
+# Parser which handles logs from PureFTPD
+class PureftpdParser < Parser
+  def parse( line )
+    _, host, domain, user, date, url, status, size = /^([\d\S.]+) (\S+) (\S+) \[([^\]]+)\] \"(.+?)\" (\d+) ([\S]+)/.match(line).to_a
+
+    if host
+      user = host if user == 'ftp'
+
+      method, url = url.split(" ")
+      url = method if url.nil?
+
+      if method == "PUT"
+        add_activity(:block => 'urls', :name => url, :size => size.to_i, :type => 5)
+      else
+        add_activity(:block => 'urls', :name => url, :size => size.to_i)
+      end
+      add_activity(:block => 'sites', :name => server.name, :size => size.to_i) # Size of activity based on size of request
+      add_activity(:block => 'users', :name => user, :size => size.to_i)
+
+      add_activity(:block => 'content', :name => 'file')
+      add_activity(:block => 'status', :name => status, :type => 3) # don't show a blob
+    end
+  end
+end

data/lib/gl_tail/parsers/qmail.rb

@@ -0,0 +1,30 @@
+# gl_tail.rb - OpenGL visualization of your server traffic
+# Copyright 2007 Erlend Simonsen <mr@fudgie.org>
+#
+# Licensed under the GNU General Public License v2 (see LICENSE)
+#
+
+# Parser which handles qmail logs
+class QmailParser < Parser
+  def parse( line )
+    if line.include?(' logged in from ')
+      _, user, host, ip = /: User \'([^']+)\' of \'([^']+)\' logged in from (\d+.\d+.\d+.\d+)/.match(line).to_a
+      if host
+        add_activity(:block => 'logins', :name => user+'@'+host, :size => 0.05)
+        add_activity(:block => 'sites', :name => server.name, :size => 0.05)
+      end
+    elsif line.include?(' to local ')
+      _, prefix, host = / to local ([^@]+)@(.*)/.match(line).to_a
+      if host
+        add_activity(:block => 'mail to', :name => host, :size => 0.05)
+        add_activity(:block => 'sites', :name => server.name, :size => 0.05)
+      end
+    elsif line.include?(' to remote ')
+      _, prefix, host = / to remote ([^@]+)@(.*)/.match(line).to_a
+      if host
+        add_activity(:block => 'mail from', :name => host, :size => 0.05)
+        add_activity(:block => 'sites', :name => server.name, :size => 0.05)
+      end
+    end
+  end
+end

data/lib/gl_tail/parsers/rails.rb

@@ -0,0 +1,41 @@
+# gl_tail.rb - OpenGL visualization of your server traffic
+# Copyright 2007 Erlend Simonsen <mr@fudgie.org>
+#
+# Licensed under the GNU General Public License v2 (see LICENSE)
+#
+
+# Parser which handles Rails access logs
+class RailsParser < Parser
+  def parse( line )
+    #Completed in 0.02100 (47 reqs/sec) | Rendering: 0.01374 (65%) | DB: 0.00570 (27%) | 200 OK [http://example.com/whatever/whatever]
+    _, ms, url = /^Completed in ([\d.]+) .* \[([^\]]+)\]/.match(line).to_a
+
+    if url
+      _, host, url = /^http[s]?:\/\/([^\/]+)(.*)/.match(url).to_a
+
+      add_activity(:block => 'sites', :name => host, :size => ms.to_f) # Size of activity based on request time.
+      add_activity(:block => 'urls', :name => HttpHelper.generalize_url(url), :size => ms.to_f)
+      add_activity(:block => 'slow requests', :name => url, :size => ms.to_f)
+      add_activity(:block => 'content', :name => 'page')
+
+      # Events to pop up
+      add_event(:block => 'info', :name => "Logins", :message => "Login...", :update_stats => true, :color => [0.5, 1.0, 0.5, 1.0]) if url.include?('/login')
+      add_event(:block => 'info', :name => "Sales", :message => "$", :update_stats => true, :color => [1.5, 0.0, 0.0, 1.0]) if url.include?('/checkout')
+      add_event(:block => 'info', :name => "Signups", :message => "New User...", :update_stats => true, :color => [1.0, 1.0, 1.0, 1.0]) if(url.include?('/signup') || url.include?('/users/create'))
+    elsif line.include?('Processing ')
+      #Processing TasksController#update_sheet_info (for 123.123.123.123 at 2007-10-05 22:34:33) [POST]
+      _, host = /^Processing .* \(for (\d+.\d+.\d+.\d+) at .*\).*$/.match(line).to_a
+      if host
+        add_activity(:block => 'users', :name => host)
+      end
+    elsif line.include?('Error (')
+      _, error, msg = /^([^ ]+Error) \((.*)\):/.match(line).to_a
+      if error
+        add_event(:block => 'info', :name => "Exceptions", :message => error, :update_stats => true, :color => [1.0, 0.0, 0.0, 1.0])
+        add_event(:block => 'info', :name => "Exceptions", :message => msg, :update_stats => false, :color => [1.0, 0.0, 0.0, 1.0])
+        add_activity(:block => 'warnings', :name => msg)
+
+      end
+    end
+  end
+end

data/lib/gl_tail/parsers/squid.rb

@@ -0,0 +1,25 @@
+# gl_tail.rb - OpenGL visualization of your server traffic
+# Copyright 2007 Erlend Simonsen <mr@fudgie.org>
+#
+# Licensed under the GNU General Public License v2 (see LICENSE)
+#
+
+# Parser which handles squid logs
+class SquidParser < Parser
+  def parse( line )
+    _, delay, host, size, method, uri, user = /\d+.\d+ +(\d+) +(\d+.\d+.\d+.\d+.\d+).+ (\d+) (.+) (.+) (.+) .+ .+/.match(line).to_a
+    if host
+      if method != 'ICP_QUERY'
+        size = size.to_f / 100000.0
+        #Uncomment if you authenticate to use the proxy
+        #add_activity(:block => 'users', :name => user, :size => size)
+        add_activity(:block => 'hosts', :name => host, :size => size)
+        add_activity(:block => 'types', :name => method, :size => size) if method
+        _, site = /http:\/\/(.+?)\/.+/.match(uri).to_a
+        if site:
+            add_activity(:block => 'sites', :name => site, :size => size)
+        end
+      end
+    end
+  end
+end

data/lib/gl_tail/parsers/tshark.rb

@@ -0,0 +1,25 @@
+# gl_tail.rb - OpenGL visualization of your server traffic
+# Copyright 2007 Erlend Simonsen <mr@fudgie.org>
+#
+# Licensed under the GNU General Public License v2 (see LICENSE)
+#
+
+# Parser which handles tshark logs
+class TSharkParser < Parser
+
+  def parse( line )
+    if(line.include?('->'))
+      time, srcip, arrow, destip, type, = line.split(" ")
+      add_activity(:block => 'users', :name => srcip)
+      add_activity(:block => 'types', :name => type)
+    end
+
+    if(line.include?('DNS Standard query A'))
+      foo, name = line.split(" A ")
+      if(name != nil)
+        add_event(:block => 'status', :name => "DNS Queries", :message => "DNS Request: " + name, :update_stats => true, :color => [1.5, 1.0, 0.5, 1.0])
+      end
+    end 
+  end 
+    
+end

data/lib/gl_tail/resolver.rb

@@ -0,0 +1,69 @@
+require 'resolv-replace'
+
+class Resolver
+  include GlTail::Configurable
+
+  config_attribute :reverse_ip_lookups, "Lookup Hostnames"
+  config_attribute :reverse_timeout,    "Wait how long for DNS reply [s]"
+
+  def self.instance
+    @@instance ||= Resolver.new
+  end
+
+  def initialize
+    @cache = { }
+    @thread = nil
+    @reverse_ip_lookups = true
+    @reverse_timeout = 1.5
+    @queue = Queue.new
+  end
+
+  attr_reader :queue, :cache
+
+  def start
+    @thread = Thread.new {
+      while @reverse_ip_lookups
+        ip, element = @queue.pop
+        if @cache.include? ip
+          element.name = @cache[ip]
+        else
+          begin
+            timeout(@reverse_timeout.to_f) {
+              puts "[Resolver] Looking for #{ip}" if $DBG > 0
+              hostname = Resolv.getname(ip)
+              puts "[Resolver] Got #{hostname}[#{ip}]" if $DBG > 0
+              @cache[ip] = hostname
+              element.name = hostname
+            }
+          rescue Timeout::Error
+            puts "[Resolver] Timeout!" if $DBG > 0
+          rescue Resolv::ResolvError
+            # No result, don't bother retrying
+            @cache[ip] = ip
+          end
+        end
+      end
+    }
+  end
+
+  def lookup(ip, element)
+    return ip if not @reverse_ip_lookups
+
+    if name = cache[ip]
+      return name
+    else
+      puts "[Resolver] Pushing #{ip} for lookup" if $DBG > 0
+      queue.push([ip, element])
+    end
+
+    return ip
+
+  end
+
+  def self.resolv(ip, element)
+    instance.lookup(ip, element)
+  end
+
+end
+
+Resolver.instance.start

data/lib/gl_tail/server.rb

@@ -0,0 +1,46 @@
+# gl_tail.rb - OpenGL visualization of your server traffic
+# Copyright 2007 Erlend Simonsen <mr@fudgie.org>
+#
+# Licensed under the GNU General Public License v2 (see LICENSE)
+#
+
+class Server
+  attr_reader :name, :host, :color, :parser
+
+
+  def initialize(options)
+    @name = options[:name] || options[:host]
+    @host = options[:host]
+    @color = options[:color] || [1.0, 1.0, 1.0, 1.0]
+    @parser = Parser.registry[ options[:parser] ] || Parser.registry[ :apache ]
+    @blocks = options[:blocks]
+    @max_size = 1.0
+
+    # instantiate the parser
+    @parser = @parser.new( self )
+
+  end
+
+  #block, message, size
+  def add_activity(options = { })
+    size = $CONFIG.min_blob_size
+    if options[:size]
+      size = options[:size].to_f
+      @max_size = size if size > @max_size
+      size = $CONFIG.min_blob_size + ((size / @max_size) * ($CONFIG.max_blob_size - $CONFIG.min_blob_size))
+      options[:size] = size
+    end
+
+    block = @blocks[options[:block]].add_activity( { :name => @name, :color => @color, :size => $CONFIG.min_blob_size }.update(options) ) if (options[:block] && @blocks[options[:block]])
+  end
+
+  #block, message
+  def add_event(options = { })
+    block = @blocks[options[:block]].add_event( { :name => @name, :color => @color, :size => $CONFIG.min_blob_size}.update(options) ) if (options[:block] && @blocks[options[:block]])
+  end
+
+  def update
+    @max_size = @max_size * 0.99 if(@max_size * 0.99 > 1.0)
+  end
+
+end