varnish-rb 0.2.0

data/Gemfile

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+gem 'ffi'
+
+group :development do
+  gem "bundler"
+  gem "jeweler"
+  gem "rake", "~> 0.9.2"
+end
+
+group :examples do
+  gem 'eventmachine'
+end

data/Gemfile.lock

@@ -0,0 +1,29 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    eventmachine (1.0.3)
+    eventmachine (1.0.3-java)
+    ffi (1.0.9)
+    ffi (1.0.9-java)
+    git (1.2.5)
+    jeweler (1.8.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+      rdoc
+    json (1.7.7)
+    json (1.7.7-java)
+    rake (0.9.6)
+    rdoc (4.0.1)
+      json (~> 1.4)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  bundler
+  eventmachine
+  ffi
+  jeweler
+  rake (~> 0.9.2)

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 ZephirWorks
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+varnish-rb
+==========
+
+varnish-rb provides a bridge between Ruby and [Varnish 3](http://varnish-cache.org/).
+
+
+Low-level access to Varnishlog
+------------------------------
+
+Low-level access to the varnishlog SHM is provided via Varnish::VSL and the underlying class Varnish::VSM.
+Ruby methods are mapped to the corresponding C APIs through FFI. This guarantees extremely high performance
+(around 200k entries per second) at the cost.
+
+This API is currently undocumented, because the underlying varnishlog API is not officially supported and
+subject to changes. See examples/log.rb for an example.
+
+
+Varnishlog EventMachine connection
+----------------------------------
+
+EM::VarnishLog::Connection is a subclass of EM::Connection that provides a stream of log entries. It processes
+log entries in a background thread outside the reactor and pushes them onto an EM::Channel. Channel subscribers
+within the reactor thread can receive these entries and process them further.
+
+Due to the complexity of using threads with EventMachine, and the need for very high performance, this API is
+currently tested only on JRuby (1.6.2 as of this writing).
+
+Pushing one log entry at a time will incur a very high overhead that limits the number of entries that can be
+processed per second. This gem provides an EM::BufferedChannel that implements double buffering. Using this channel
+it is possible to process about 150k entries per second, corresponding to 3-4k HTTP requests per second. This
+makes it possible to process moderate traffic in a development or preproduction environment, but you would have to
+be out of your mind to use it in production.
+
+See examples/log\_tail.rb and especially examples/request\_tail.rb for a taste of how to use this API.
+
+Contributing to varnish-rb
+==========================
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+Copyright
+=========
+
+Copyright (c) 2011 ZephirWorks. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,52 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "varnish-rb"
+  gem.homepage = "http://github.com/andreacampi/varnish-rb"
+  gem.license = "MIT"
+  gem.summary = %Q{A bridge between Ruby and Varnish 3}
+  gem.description = %Q{varnish-rb provides a bridge between Ruby and [Varnish 3](http://varnish-cache.org/)}
+  gem.email = "andrea.campi@zephirworks.com"
+  gem.authors = ["Andrea Campi"]
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+# require 'rake/testtask'
+# Rake::TestTask.new(:test) do |test|
+#   test.libs << 'lib' << 'test'
+#   test.pattern = 'test/**/test_*.rb'
+#   test.verbose = true
+# end
+# 
+# require 'rcov/rcovtask'
+# Rcov::RcovTask.new do |test|
+#   test.libs << 'test'
+#   test.pattern = 'test/**/test_*.rb'
+#   test.verbose = true
+#   test.rcov_opts << '--exclude "gems/*"'
+# end
+# 
+# task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "varnish-rb #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/examples/log.rb

@@ -0,0 +1,30 @@
+require 'varnish'
+
+class Log
+  def initialize
+    @vd = Varnish::VSM.VSM_New
+    Varnish::VSL.VSL_Setup(@vd)
+    Varnish::VSL.VSL_Open(@vd, 1)
+
+    @count = 0
+    @t = nil
+  end
+
+  def run
+    Varnish::VSL.VSL_Dispatch(@vd, self.method(:callback).to_proc, FFI::MemoryPointer.new(:pointer))
+  end
+
+private
+  def callback(*args)
+    @t = Time.now if @count == 0
+    @count += 1
+
+    if (@count % 100000) == 0
+      puts "received #{@count} messages in #{(Time.now - @t).to_s} seconds"
+      @t = Time.now
+    end
+    return 0
+  end
+end
+
+Log.new.run

data/examples/log_tail.rb

@@ -0,0 +1,21 @@
+require 'eventmachine'
+require 'em/varnish_log/connection'
+require 'em/buffered_channel'
+
+EM.run do
+  count = 0
+  t = nil
+
+  channel = EM::BufferedChannel.new
+  stream = EM::VarnishLog::Connection.new(channel)
+  stream.run
+
+  channel.subscribe do |msg|
+    t = Time.now if count == 0
+    count += 1
+    if (count % 100000) == 0
+      puts "received #{count} messages in #{(Time.now - t).to_s} seconds"
+      t = Time.now
+    end
+  end
+end

data/examples/request_stream.rb

@@ -0,0 +1,27 @@
+require 'em/varnish_log/connection'
+require 'em/buffered_channel'
+
+class RequestStream
+  def initialize
+    @channel = EM::BufferedChannel.new
+    @callbacks = []
+
+    listen
+  end
+
+  def onrequest(&block)
+    @callbacks << block
+  end
+
+private
+
+  def listen
+    EM::VarnishLog::Connection.new(@channel).run
+
+    @channel.subscribe do |msg|
+      if msg[:tag] == :reqend
+        @callbacks.each { |c| c.call(msg) }
+      end
+    end
+  end
+end

data/examples/request_tail.rb

@@ -0,0 +1,17 @@
+require 'eventmachine'
+require File.expand_path('request_stream', File.dirname(__FILE__))
+
+EM.run do
+  count = 0
+  t = nil
+
+  stream = RequestStream.new
+  stream.onrequest do |data|
+    t = Time.now if count == 0
+    count += 1
+    if (count % 1000) == 0
+      puts "received #{count} requests in #{(Time.now - t).to_s} seconds"
+      t = Time.now
+    end
+  end
+end

data/lib/em/buffered_channel.rb

@@ -0,0 +1,68 @@
+require 'eventmachine'
+
+module EM
+  #
+  # A subclass of EM::Channel that implements double buffering (using a
+  # circular array of arrays) to achieve higher packet rate.
+  #
+  # The buffer size and the number of buffers are critical to achieving good
+  # performance, you may need to tune them.
+  #
+  class BufferedChannel < EM::Channel
+    def initialize
+      super
+      @buffer = BufferSet.new(2000, 200)
+    end
+
+    def push(*items)
+      @buffer.push(*items) do |buf|
+        EM.schedule do
+          @subs.values.each do |s|
+            begin
+              buf.each do |i|
+                s.call i
+              end
+            ensure
+              buf.clear
+            end
+          end
+        end
+      end
+    end
+
+    class BufferSet
+      def initialize(size, nbufs)
+        @size = size
+        @nbufs = nbufs
+
+        @buffers = []
+
+        setup
+      end
+
+      def push(data)
+        raise "WTF #{@buffers.length}" unless @buffers.length == @nbufs
+
+        if @buffers[0].length > @size
+          if @buffers[1].length > 0
+            raise "panic! the next buffer is still full!"
+          end
+
+          buf = @buffers.shift
+          @buffers << buf
+
+          yield buf
+        end
+
+        @buffers.first << data
+      end
+
+    private
+      def setup
+        (0..@nbufs-1).each do |n|
+          @buffers[n] = []
+        end
+      end
+    end
+  end
+end

data/lib/em/varnish_log/connection.rb

@@ -0,0 +1,56 @@
+require 'eventmachine'
+require 'varnish'
+
+module EM
+  module VarnishLog
+    #
+    # A subclass of EM::Connection that reads from the Varnish log SHM and
+    # pushes single log entries on the channel passed to its constructor.
+    #
+    # Note that simply using an EM::Channel pretty much guarantees awful
+    # performance, suitable only for extremely low workloads in a development
+    # environment. The provided EM::BufferedChannel improves things quite a
+    # bit: it has proved able to push about 150k entries per second
+    # (corresponding to about 4k HTTP req/s), using a full CPU on a modern
+    # MBP.
+    #
+    class Connection
+      class << self
+        attr_reader :channel
+      end
+
+      def initialize(channel)
+        @channel = channel
+      end
+
+      def run
+        vd = Varnish::VSM.VSM_New
+        Varnish::VSL.VSL_Setup(vd)
+        Varnish::VSL.VSL_Open(vd, 1)
+
+        callback = Proc.new { |*args| cb(*args) }
+
+        Thread.new do
+          begin
+            Varnish::VSL.VSL_Dispatch(vd, callback, FFI::MemoryPointer.new(:pointer))
+          rescue => e
+            puts "exception in thread: #{e.inspect}"
+          ensure
+            EM.stop
+          end
+        end
+      rescue => e
+        puts "exception in post_init: #{e.inspect}"
+      end
+
+    private
+      def cb(priv, tag, fd, len, spec, ptr, bitmap)
+        str = ptr.read_string(len)
+        @channel.push(:tag => tag, :fd => fd, :data => str, :spec => spec, :bitmap => bitmap)
+        0 # ok
+      rescue => e
+        puts "exception in cb: #{e.inspect}"
+      end
+    end
+  end
+end

data/lib/varnish.rb

@@ -0,0 +1,9 @@
+module Varnish
+  LIBVARNISHAPI = [
+    'libvarnishapi.1',    # Mac OS X
+    'libvarnishapi.so.1'  # Debian / Ubuntu
+  ]
+end
+
+require 'varnish/vsm'
+require 'varnish/vsl'

data/lib/varnish/utils.rb

@@ -0,0 +1,2 @@
+require 'varnish/utils/buffer_set'
+require 'varnish/utils/timer'

data/lib/varnish/utils/timer.rb

@@ -0,0 +1,20 @@
+module Varnish
+  module Utils
+    module Timer
+      def timer_init
+        @count = 0
+        @time = 0
+        @interval = 40000
+      end
+
+      def timer_count
+        @time = Time.now if @count == 0
+        @count += 1
+        if (@count % @interval) == 0
+          puts "Got #{@count} calls in #{(Time.now - @time).to_f}s"
+          @time = Time.now
+        end
+      end
+    end
+  end
+end

data/lib/varnish/vsl.rb

@@ -0,0 +1,17 @@
+require 'varnish'
+require 'ffi'
+require 'varnish/vsl/enum'
+
+module Varnish
+  module VSL
+    extend FFI::Library
+    ffi_lib Varnish::LIBVARNISHAPI
+    include Enum
+
+    callback :VSL_handler_f, [:pointer, VslTag, :int, :int, VslSpec, :pointer, :int64], :int
+
+    attach_function 'VSL_Setup',    [ :pointer ], :void
+    attach_function 'VSL_Open',     [ :pointer, :int ], :int
+    attach_function 'VSL_Dispatch', [ :pointer, :VSL_handler_f, :pointer ], :int
+  end
+end

data/lib/varnish/vsl/enum.rb

@@ -0,0 +1,81 @@
+require 'varnish'
+require 'ffi'
+
+module Varnish
+  module VSL
+    module Enum
+      extend FFI::Library
+
+      VslTag = enum(
+        :debug,
+        :error,
+        :cli,
+        :statsess,
+        :reqend,
+        :sessionopen,
+        :sessionclose,
+        :backendopen,
+        :backendxid,
+        :backendreuse,
+        :backendclose,
+        :httpgarbage,
+        :backend,
+        :length,
+
+        :fetcherror,
+
+        :rxrequest,
+        :rxresponse,
+        :rxstatus,
+        :rxurl,
+        :rxprotocol,
+        :rxheader,
+
+        :txrequest,
+        :txresponse,
+        :txstatus,
+        :txurl,
+        :txprotocol,
+        :txheader,
+
+        :objrequest,
+        :objresponse,
+        :objstatus,
+        :objurl,
+        :objprotocol,
+        :objheader,
+
+        :lostheader,
+
+        :ttl,
+        :fetch_body,
+        :vcl_acl,
+        :vcl_call,
+        :vcl_trace,
+        :vcl_return,
+        :vcl_error,
+        :reqstart,
+        :hit,
+        :hitpass,
+        :expban,
+        :expkill,
+        :workthread,
+
+        :esi_xmlerror,
+
+        :hash,
+
+        :backend_health,
+        :vcl_log,
+
+        :gzip
+      )
+
+      VslSpec = enum(
+        :spec_unknown,
+        :spec_client,
+        :spec_backend
+      )
+    end
+  end
+end

data/lib/varnish/vsm.rb

@@ -0,0 +1,11 @@
+require 'varnish'
+require 'ffi'
+
+module Varnish
+  module VSM
+    extend FFI::Library
+    ffi_lib Varnish::LIBVARNISHAPI
+
+    attach_function 'VSM_New', [], :pointer
+  end
+end

data/varnish-rb.gemspec

@@ -0,0 +1,67 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "varnish-rb"
+  s.version = "0.2.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Andrea Campi"]
+  s.date = "2013-03-29"
+  s.description = "varnish-rb provides a bridge between Ruby and [Varnish 3](http://varnish-cache.org/)"
+  s.email = "andrea.campi@zephirworks.com"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "examples/log.rb",
+    "examples/log_tail.rb",
+    "examples/request_stream.rb",
+    "examples/request_tail.rb",
+    "lib/em/buffered_channel.rb",
+    "lib/em/varnish_log/connection.rb",
+    "lib/varnish.rb",
+    "lib/varnish/utils.rb",
+    "lib/varnish/utils/timer.rb",
+    "lib/varnish/vsl.rb",
+    "lib/varnish/vsl/enum.rb",
+    "lib/varnish/vsm.rb",
+    "varnish-rb.gemspec"
+  ]
+  s.homepage = "http://github.com/andreacampi/varnish-rb"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.25"
+  s.summary = "A bridge between Ruby and Varnish 3"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<ffi>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+      s.add_development_dependency(%q<rake>, ["~> 0.9.2"])
+    else
+      s.add_dependency(%q<ffi>, [">= 0"])
+      s.add_dependency(%q<bundler>, [">= 0"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+      s.add_dependency(%q<rake>, ["~> 0.9.2"])
+    end
+  else
+    s.add_dependency(%q<ffi>, [">= 0"])
+    s.add_dependency(%q<bundler>, [">= 0"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+    s.add_dependency(%q<rake>, ["~> 0.9.2"])
+  end
+end
+

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification
+name: varnish-rb
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+  prerelease: 
+platform: ruby
+authors:
+- Andrea Campi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: varnish-rb provides a bridge between Ruby and [Varnish 3](http://varnish-cache.org/)
+email: andrea.campi@zephirworks.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- VERSION
+- examples/log.rb
+- examples/log_tail.rb
+- examples/request_stream.rb
+- examples/request_tail.rb
+- lib/em/buffered_channel.rb
+- lib/em/varnish_log/connection.rb
+- lib/varnish.rb
+- lib/varnish/utils.rb
+- lib/varnish/utils/timer.rb
+- lib/varnish/vsl.rb
+- lib/varnish/vsl/enum.rb
+- lib/varnish/vsm.rb
+- varnish-rb.gemspec
+homepage: http://github.com/andreacampi/varnish-rb
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2339583890201378041
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: A bridge between Ruby and Varnish 3
+test_files: []