varnishops 0.0.1

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,24 @@
+# varnishops
+
+varnishops is a tool for analyzing and categorizing traffic on your varnish servers in realtime.
+
+It gathers the output of varnishncsa and create statistics such as request rate, output bandwidth or hitratio.
+
+URLs from varnishncsa are categorized using custom regular expressions, based on your assets' paths and classification (example provided in **ext/**)
+
+varnishops is written in Ruby (tested with MRI 1.9.3) and depends only on varnishncsa.
+
+## Setup
+
+ * git clone https://github.com/Fotolia/varnishops
+OR
+ * gem install varnishops
+
+
+By default, all URLs will be categorized as "other", and statistics will be computed based on the global varnish traffic
+
+Categories can be added by defining filters (ruby regexps). See example file in **ext/**.
+
+## Credits
+
+varnishops is heavily inspired by [mctop](http://github.com/etsy/mctop) from Etsy's folks.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/varnishops

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(__FILE__),'..','lib')
+
+require 'cmdline'
+require 'varnish_pipe'
+require 'ui'
+
+@config = CmdLine.parse(ARGV)
+
+@regexs = {}
+# load regex config file
+begin
+  load @config[:config_file]
+rescue Exception => e
+  puts "Unable to load config.rb (#{e.message})"
+  exit 1
+end
+
+pipe = VarnishPipe.new(@config, @regexs)
+ui = UI.new(@config)
+
+done = false
+
+# trap most of the typical signals
+%w[ INT QUIT HUP KILL ].each do |sig|
+  Signal.trap(sig) do
+    puts "** Caught signal #{sig} - exiting"
+    done = true
+  end
+end
+
+# kick the pipe thread off
+pipe_thr = Thread.new { pipe.start }
+
+# main loop
+until done do
+  ui.header
+  ui.footer
+  ui.render_stats(pipe)
+  refresh
+
+  key = ui.input_handler
+  case key
+    when /[Qq]/
+      done = true
+    when /[Kk]/
+      ui.sort_mode = :key
+    when /[Cc]/
+      ui.sort_mode = :calls
+    when /[Rr]/
+      ui.sort_mode = :reqps
+    when /[Bb]/
+      ui.sort_mode = :bps
+    when /[Hh]/
+      ui.sort_mode = :hitratio
+    when /[Pp]/
+      ui.show_percent = !ui.show_percent
+    when /[Tt]/
+      ui.sort_order =  ui.sort_order == :desc ? :asc : :desc
+  end
+end
+
+# stop threads
+ui.stop
+pipe.stop

data/config.rb

@@ -0,0 +1,2 @@
+@regexs = {
+}

data/ext/config.rb

@@ -0,0 +1,17 @@
+# Example regex file, expressions order matters !
+@regexs = {
+  /^(images)\/.*_(\d+)\.(png)$/ => [1,3,2]
+  /^\/(\w+)\/.*\.(\w+)$/ => [1, 2],
+}
+
+# will match URLs and display them such as:
+#
+# /images/11/22/logo_400.png   => images:png:400
+#
+# first, then:
+#
+# /assets/your/own/path/app.js => assets:js
+#
+# /images/00/11/banner.jpg     => images:jpg
+#
+# validate these expressions using the fine http://rubular.com/

data/lib/cmdline.rb

@@ -0,0 +1,47 @@
+require 'optparse'
+
+class CmdLine
+  def self.parse(args)
+    @config = {}
+
+    opts = OptionParser.new do |opt|
+      @config[:config_file] = File.join(File.dirname(File.dirname(__FILE__)), 'config.rb')
+      opt.on '-c', '--config-file=FILE', String, 'Config file containing regular expressions to categorize traffic' do |config_file|
+        @config[:config_file] = File.expand_path(config_file)
+      end
+
+      @config[:refresh_rate] = 500
+      opt.on '-r', '--refresh=MS', Integer, 'Refresh the stats display every MS milliseconds' do |refresh_rate|
+        @config[:refresh_rate] = refresh_rate
+      end
+
+      @config[:avg_period] = 60
+      opt.on '-a', '--avg-period=SEC', Integer, 'Compute averages over SEC seconds' do |avg_period|
+        @config[:avg_period] = avg_period
+      end
+
+      opt.on_tail '-h', '--help', 'Show usage info' do
+        puts opts
+        exit
+      end
+    end
+
+    opts.parse!
+
+    # limit data structures size
+    @config[:avg_period] = 600 if @config[:avg_period] > 600
+    @config[:avg_period] = 10 if @config[:avg_period] < 10
+
+    unless File.exists?(@config[:config_file])
+      puts "#{@config[:config_file]}: no such file"
+      exit 1
+    end
+
+    unless system("which varnishncsa")
+      puts "varnishncsa not found. install it or set a correct PATH"
+      exit 1
+    end
+
+    @config
+  end
+end

data/lib/ui.rb

@@ -0,0 +1,193 @@
+require 'curses'
+
+include Curses
+
+class UI
+  attr_accessor :sort_order, :sort_mode, :show_percent
+
+  def initialize(config)
+    @config = config
+
+    @stat_cols    = %w[ calls req/s kB/s hitratio ]
+    @stat_col_width = 15
+    @key_col_width  = 30
+    @avg_period = @config[:avg_period]
+    @sort_mode = :reqps
+    @sort_order = :desc
+    @show_percent = false
+
+    @commands = {
+      'B' => "sort by bandwidth",
+      'C' => "sort by call number",
+      'H' => "sort by hitratio",
+      'K' => "sort by key",
+      'P' => "display percentages",
+      'Q' => "quit",
+      'R' => "sort by request rate",
+      'T' => "toggle sort order (asc|desc)"
+    }
+
+    init_screen
+    cbreak
+    curs_set(0)
+
+    # set keyboard input timeout - sneaky way to manage refresh rate
+    Curses.timeout = @config[:refresh_rate]
+
+    if can_change_color?
+      start_color
+      init_pair(0, COLOR_WHITE, COLOR_BLACK)
+      init_pair(1, COLOR_WHITE, COLOR_BLUE)
+      init_pair(2, COLOR_WHITE, COLOR_RED)
+    end
+  end
+
+  def header
+    # pad stat columns to @stat_col_width
+    @stat_cols = @stat_cols.map { |c| sprintf("%#{@stat_col_width}s", c) }
+
+    @url_col_width = cols - @key_col_width - (@stat_cols.length * @stat_col_width)
+
+    attrset(color_pair(1))
+    setpos(0,0)
+    addstr(sprintf "%-#{@key_col_width}s%-#{@url_col_width}s%s", "request pattern", "last url", @stat_cols.join)
+  end
+
+  def footer
+    footer_text = @commands.map { |k,v| "#{k}:#{v}" }.join(' | ')
+    setpos(lines - 1, 0)
+    attrset(color_pair(2))
+    addstr(sprintf "%-#{cols}s", footer_text)
+  end
+
+  def render_stats(pipe)
+    render_start_t = Time.now.to_f * 1000
+
+    # subtract header + footer lines
+    maxlines = lines - 3
+    offset = 1
+
+    # construct and render footer stats line
+    setpos(lines - 2, 0)
+    attrset(color_pair(2))
+    header_summary = sprintf "%-24s %-14s %-14s %-14s %-14s",
+      "sort mode: #{@sort_mode.to_s} (#{@sort_order.to_s})",
+      "| requests: #{pipe.stats[:total_reqs]}",
+      "| bytes out: #{pipe.stats[:total_bytes]}",
+      "| duration: #{Time.now.to_i - pipe.start_time.to_i}s",
+      "| samples: #{@avg_period}"
+    addstr(sprintf "%-#{cols}s", header_summary)
+
+    # reset colours for main key display
+    attrset(color_pair(0))
+
+    top = []
+    totals = {}
+
+    pipe.semaphore.synchronize do
+      # we may have seen no packets received on the pipe thread
+      return if pipe.start_time.nil?
+
+      time_since_start = Time.now.to_f - pipe.start_time
+      elapsed = time_since_start < @avg_period ? time_since_start : @avg_period
+
+      # calculate hits+misses, request rate, bandwidth and hitratio
+      pipe.stats[:requests].each do |key,values|
+        total_hits = values[:hit].inject(:+)
+        total_calls = total_hits + values[:miss].inject(:+)
+        total_bytes = values[:bytes].inject(:+)
+
+        pipe.stats[:calls][key] = total_calls
+        pipe.stats[:reqps][key] = total_calls.to_f / elapsed
+        pipe.stats[:bps][key] = total_bytes.to_f / elapsed
+        pipe.stats[:hitratio][key] = total_hits.to_f * 100 / total_calls
+      end
+
+      if @sort_mode == :key
+        top = pipe.stats[:requests].sort { |a,b| a[0] <=> b[0] }
+      else
+        top = pipe.stats[@sort_mode].sort { |a,b| a[1] <=> b[1] }
+      end
+
+      totals = {
+        :calls => pipe.stats[:calls].values.inject(:+),
+        :reqps => pipe.stats[:reqps].values.inject(:+),
+        :bps => pipe.stats[:bps].values.inject(:+)
+      }
+    end
+
+    unless @sort_order == :asc
+      top.reverse!
+    end
+
+    for i in 0..maxlines-1
+      if i < top.length
+        k = top[i][0]
+
+        # if the key is too wide for the column truncate it and add an ellipsis
+        display_key = k.length > @key_col_width ? "#{k[0..@key_col_width-4]}..." : k
+        display_url = pipe.stats[:requests][k][:last]
+        display_url = display_url.length > @url_col_width ? "#{display_url[0..@url_col_width-4]}..." : display_url
+
+        # render each key
+        if @show_percent
+          line = sprintf "%-#{@key_col_width}s%-#{@url_col_width}s %14.2f %14.2f %14.2f %14.2f",
+                   display_key, display_url,
+                   pipe.stats[:calls][k].to_f / totals[:calls] * 100,
+                   pipe.stats[:reqps][k] / totals[:reqps] * 100,
+                   pipe.stats[:bps][k] / totals[:bps] * 100,
+                   pipe.stats[:hitratio][k]
+        else
+          line = sprintf "%-#{@key_col_width}s%-#{@url_col_width}s %14.d %14.2f %14.2f %14.2f",
+                   display_key, display_url,
+                   pipe.stats[:calls][k],
+                   pipe.stats[:reqps][k],
+                   pipe.stats[:bps][k] / 1000,
+                   pipe.stats[:hitratio][k]
+        end
+      else
+        # clear remaining lines
+        line = " "*cols
+      end
+
+      setpos(1 + i, 0)
+      addstr(line)
+    end
+
+    # Display column totals
+    unless @show_percent
+      setpos(top.length + 2, 0)
+      line = sprintf "%-#{@key_col_width + @url_col_width}s %14.d %14.2f %14.2f",
+               "TOTAL", totals[:calls], totals[:reqps], totals[:bps] / 1000
+      addstr(line)
+    end
+
+    # print render time in status bar
+    runtime = (Time.now.to_f * 1000) - render_start_t
+    attrset(color_pair(2))
+    setpos(lines - 2, cols - 24)
+    addstr(sprintf "render time: %4.3f (ms)", runtime)
+  end
+
+  def input_handler
+    # Curses.getch has a bug in 1.8.x causing non-blocking
+    # calls to block reimplemented using IO.select
+    if RUBY_VERSION =~ /^1.8/
+      refresh_secs = @config[:refresh_rate].to_f / 1000
+
+      if IO.select([STDIN], nil, nil, refresh_secs)
+        c = getch
+        c.chr
+      else
+        nil
+      end
+    else
+      getch
+    end
+  end
+
+  def stop
+    nocbreak
+    close_screen
+  end
+end

data/lib/varnish_pipe.rb

@@ -0,0 +1,75 @@
+class VarnishPipe
+  attr_accessor :stats, :semaphore, :start_time
+
+  def initialize(config, regexs)
+    @stats = {
+      :total_reqs => 0,
+      :total_bytes => 0,
+      :requests => {},
+      :calls => {},
+      :hitratio => {},
+      :reqps => {},
+      :bps => {},
+    }
+
+    @semaphore = Mutex.new
+    @avg_period = config[:avg_period]
+    @default_key = "other"
+
+    @regexs = regexs
+  end
+
+  def start
+    @stop = false
+    @start_time = Time.new.to_f
+    @start_ts = @start_time.to_i
+
+    IO.popen("varnishncsa -F '%U %{Varnish:hitmiss}x %b'").each_line do |line|
+      if line =~ /^(\S+) (\w+) (\d+)$/
+        url, status, bytes = $1, $2, $3
+        key = nil
+
+        @regexs.each do |k,v|
+          if k.match(url)
+            key = v.map{|x| "#{$~[x]}" }.join(":")
+            break
+          end
+        end
+
+        key = @default_key unless key
+
+        @semaphore.synchronize do
+          duration = (Time.now.to_i - @start_ts)
+          it = duration / @avg_period
+          idx = duration % @avg_period
+
+          @stats[:requests][key] ||= {
+            :hit => Array.new(@avg_period, 0),
+            :miss => Array.new(@avg_period, 0),
+            :bytes => Array.new(@avg_period, 0),
+            :it => Array.new(@avg_period, 0),
+            :last => ""
+          }
+
+          cur_req = @stats[:requests][key]
+          if cur_req[:it][idx] < it
+            cur_req[status.to_sym][idx] = 0
+            cur_req[:bytes][idx] = 0
+          end
+          cur_req[status.to_sym][idx] += 1
+          cur_req[:bytes][idx] += bytes.to_i
+          cur_req[:it][idx] = it
+          cur_req[:last] = url
+          @stats[:total_reqs] += 1
+          @stats[:total_bytes] += bytes.to_i
+        end
+      end
+
+      break if @stop
+    end
+  end
+
+  def stop
+    @stop = true
+  end
+end

data/varnishops.gemspec

@@ -0,0 +1,17 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |gem|
+  gem.name          = "varnishops"
+  gem.version       = "0.0.1"
+  gem.authors       = ["Jonathan Amiez"]
+  gem.email         = ["jonathan.amiez@fotolia.com"]
+  gem.description   = %q{varnishops - a realtime varnish log analyzer}
+  gem.summary       = %q{varnishops - an interactive terminal app for analyzing varnish activity with hitratio, bandwidth and request rate per type of files}
+  gem.homepage      = "https://github.com/Fotolia/varnishops"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,57 @@
+--- !ruby/object:Gem::Specification
+name: varnishops
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jonathan Amiez
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-10 00:00:00.000000000 Z
+dependencies: []
+description: varnishops - a realtime varnish log analyzer
+email:
+- jonathan.amiez@fotolia.com
+executables:
+- varnishops
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- bin/varnishops
+- config.rb
+- ext/config.rb
+- lib/cmdline.rb
+- lib/ui.rb
+- lib/varnish_pipe.rb
+- varnishops.gemspec
+homepage: https://github.com/Fotolia/varnishops
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: varnishops - an interactive terminal app for analyzing varnish activity with
+  hitratio, bandwidth and request rate per type of files
+test_files: []