thinner 0.1.0

data/documentation/css/dawn.css

@@ -0,0 +1,121 @@
+pre.dawn .MetaSeparator {
+   font-weight: bold;
+   background-color: #DCDCDC;
+   color: #19356D;
+}
+pre.dawn .SupportVariable {
+   color: #234A97;
+}
+pre.dawn .Constant {
+   font-weight: bold;
+   color: #811F24;
+}
+pre.dawn .EmbeddedSource {
+   background-color: #829AC2;
+}
+pre.dawn .StringRegexpConstantCharacterEscape {
+   font-weight: bold;
+   color: #811F24;
+}
+pre.dawn .Support {
+   color: #691C97;
+}
+pre.dawn .MarkupList {
+   color: #693A17;
+}
+pre.dawn .Storage {
+   color: #A71D5D;
+   font-style: italic;
+}
+pre.dawn .line-numbers {
+   background-color: #7496CF;
+   color: #000000;
+}
+pre.dawn .StringConstant {
+   font-weight: bold;
+   color: #696969;
+}
+pre.dawn .MarkupUnderline {
+   text-decoration: underline;
+   color: #080808;
+}
+pre.dawn .MarkupHeading {
+   font-weight: bold;
+   color: #19356D;
+}
+pre.dawn .SupportConstant {
+   color: #B4371F;
+}
+pre.dawn .MarkupQuote {
+   background-color: #C5C5C5;
+   color: #0B6125;
+   font-style: italic;
+}
+pre.dawn .StringRegexpSpecial {
+   font-weight: bold;
+   color: #CF5628;
+}
+pre.dawn .InvalidIllegal {
+   background-color: #B52A1D;
+   color: #F8F8F8;
+   font-style: italic;
+}
+pre.dawn .MarkupDeleted {
+   color: #B52A1D;
+}
+pre.dawn .MarkupRaw {
+   background-color: #C5C5C5;
+   color: #234A97;
+}
+pre.dawn .SupportFunction {
+   color: #693A17;
+}
+pre.dawn .PunctuationSeparator {
+   color: #794938;
+}
+pre.dawn .StringRegexp {
+   color: #CF5628;
+}
+pre.dawn .StringEmbeddedSource {
+   background-color: #829AC2;
+   color: #080808;
+}
+pre.dawn .MarkupLink {
+   color: #234A97;
+   font-style: italic;
+}
+pre.dawn .MarkupBold {
+   font-weight: bold;
+   color: #080808;
+}
+pre.dawn .StringVariable {
+   color: #234A97;
+}
+pre.dawn .String {
+   color: #0B6125;
+}
+pre.dawn .Keyword {
+   color: #794938;
+}
+pre.dawn {
+   background-color: #F5F5F5;
+   color: #080808;
+}
+pre.dawn .MarkupItalic {
+   color: #080808;
+   font-style: italic;
+}
+pre.dawn .InvalidDeprecated {
+   font-weight: bold;
+   color: #B52A1D;
+}
+pre.dawn .Variable {
+   color: #234A97;
+}
+pre.dawn .Entity {
+   color: #BF4F24;
+}
+pre.dawn .Comment {
+   color: #5A525F;
+   font-style: italic;
+}

data/documentation/css/styles.css

@@ -0,0 +1,53 @@
+body {
+  font-family: Garamond, Baskerville, "Baskerville Old Face", "Hoefler Text", "Times New Roman", serif;
+  font-size: 16px;
+  line-height:20px;
+  width: 600px;
+  margin-left:auto;
+  margin-right:auto;
+  background: #f4f4f4;
+}
+a.propublica{
+  position:absolute;
+  background: transparent url(../images/proplogo.png) no-repeat -40px -20px;
+  top: 0;
+  left: 0;
+  width: 160px;
+  height: 141px;
+}
+
+pre {
+  font-family: Monaco, Courier, monospace;
+  font-size: 12px;
+  line-height: 16px;
+  padding:0.5em 1em;
+  overflow: auto;
+  border-left: 4px solid #143D8D;
+  margin-left: 1em;
+}
+a {
+  color: #143D8D;
+  text-decoration: none;
+  font-weight: bold;
+}
+ul {
+  margin:0 1em;
+  padding:0;
+  list-style: none;
+}
+li {
+  margin:0;
+  padding:0;
+}
+strong {
+  font-family: Monaco, Courier, monospace;
+  font-weight: normal;
+  background: #dadee5;
+  border: 1px solid #aaa;
+  padding: 1px 2px;
+  font-size: 12px;
+}
+p{ margin: 0 0 1em 0 }
+h3{
+  margin-bottom: 0px;
+}

data/documentation/examples/configure.rb

@@ -0,0 +1,25 @@
+Thinner.configure do |config|
+  # Number of urls to purge at one time. These purge requests are fired in quick
+  # succession. Thinner is perfectly capable of killing a Varnish server, by
+  # overloading the worker thread, so be really conservative with this option.
+  config.batch_length = 10
+
+  # The amount of time to sleep between purges in seconds.
+  config.sleep_time   = 1
+
+  # The server address and management port. See:
+  #   http://www.varnish-cache.org/trac/wiki/ManagementPort
+  # for details.
+  config.server       = "127.0.0.1:6082"
+
+  # By default, every time you call Thinner.purge! thinner spins off a new
+  # instance of Thinner::Client and terminates any old instances that are
+  # running. If you want to have overlapping instances set this to true.
+  # It's not recommended to have multiple Thinner::Client's running at the
+  # same time.
+  config.no_kill      = false
+
+  # The log file (either a string or file object) to log the current batch to.
+  # Defaults to STDOUT
+  config.log_file     = STDOUT
+end

data/documentation/examples/purge.rb

@@ -0,0 +1,5 @@
+# The urls in this array are purged in order, so you'll want to structure it
+# according to usage.
+arr << ["/some_route", "/"]
+
+Thinner.purge! arr

data/documentation/index.html.erb

@@ -0,0 +1,64 @@
+<%
+$:.unshift File.expand_path(File.dirname(__FILE__), "/../lib/thinner")
+DOCS = "documentation/examples/"
+require 'uv'
+def code_for(file)
+  return '' unless File.exists?("#{DOCS}#{file}.rb")
+  file = File.open("#{DOCS}#{file}.rb").read
+  Uv.parse(file, "xhtml", "ruby", false, "dawn", false)
+end
+%>
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="content-type" content="text/html;charset=UTF-8" />
+    <title>Thinner -- Version <%= File.read('VERSION') %></title>
+    <link rel="stylesheet" type="text/css" href="documentation/css/styles.css" />
+    <link rel="stylesheet" type="text/css" href="documentation/css/dawn.css" />
+  </head>
+  <body>
+    <a href="http://www.propublica.org" class="propublica">&nbsp;</a>
+    <h1>Thinner &ndash; Version <%= File.read('VERSION') %></h1>
+    <p><a href="https://github.com/propublica/Thinner">Thinner</a> is a utility
+      for purging urls from a Varnish server.</p>
+    <p>When you are deploying code changes to a server under load, the uncached
+       load can quickly bring down your backend server even with Varnish's grace
+       mode. Often, allowing stale caches to stick around for a while saves
+       both server performance and sanity.</p>
+    <p>Thinner gives you fine-grained control over wildcard purging, and rolls
+      purges out slowly. Your users will see stale pages from the previous deploy
+      until Thinner has finished invalidating the stale cache at a rate that you set.
+      If you have a bunch of pages you need to invalidate en masse, but don't
+      want to risk overloading your server, Thinner is for you.</p>
+    <p>All that being said, Thinner isn't really a solution for observing
+      model changes and purging associated urls. If you have a highly dynamic
+      application, it's worlds better to handle purging via a
+      <a href="http://github.com/russ/lacquer/blob/master/lib/lacquer/delayed_job_job.rb">job server</a>
+      outside of the request-response flow.</p>
+    <p><a href="doc/index.html">API docs</a> | <a href="http://github.com/propublica/thinner/issues">Issue Tracker</a></p>
+    <h2>Installation</h2>
+    <p>Thinner is available via rubygems:
+    <pre>gem install thinner</pre>
+    <h2>Usage</h2>
+    <p>Thinner has both a library and command-line interface. To use it as a gem
+      you'll first have to configure how it works by calling <strong>Thinner.configure</strong>.
+      Here's a quick rundown of all of the options available:</p>
+      <%= code_for "configure" %>
+    <p>Once you have the configuration in place call <strong>purge!</strong> with
+      an array of urls:</p>
+      <%= code_for "purge" %>
+    <p>Thinner will then fork a background process and purge the urls. You can
+      check the progress of the purge by tailing the log file or with:</p>
+      <pre>varnishlog | grep purge</pre>
+    <p>If ruby's not your cup of tea, Thinner also has a command line interface.
+      Once you've installed the gem run <strong>thinner -h</strong> to see the
+      available options.</p>
+    <p>The command line interface accepts a newline separated list of urls via
+      stdin by setting the <strong>-e</strong> flag. So you'll be able to use
+      the command like so:</p>
+      <pre>cat urls_to_purge.txt | bin/thinner -e</pre>
+    <h2>Change Log</h2>
+    <h3>0.1.0</h3>
+    <p>Initial release.</p>
+    </body>
+</html>

data/index.html

@@ -0,0 +1,87 @@
+
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="content-type" content="text/html;charset=UTF-8" />
+    <title>Thinner -- Version 0.1.0
+</title>
+    <link rel="stylesheet" type="text/css" href="documentation/css/styles.css" />
+    <link rel="stylesheet" type="text/css" href="documentation/css/dawn.css" />
+  </head>
+  <body>
+    <a href="http://www.propublica.org" class="propublica">&nbsp;</a>
+    <h1>Thinner &ndash; Version 0.1.0
+</h1>
+    <p><a href="https://github.com/propublica/Thinner">Thinner</a> is a utility
+      for purging urls from a Varnish server.</p>
+    <p>When you are deploying code changes to a server under load, the uncached
+       load can quickly bring down your backend server even with Varnish's grace
+       mode. Often, allowing stale caches to stick around for a while saves
+       both server performance and sanity.</p>
+    <p>Thinner gives you fine-grained control over wildcard purging, and rolls
+      purges out slowly. Your users will see stale pages from the previous deploy
+      until Thinner has finished invalidating the stale cache at a rate that you set.
+      If you have a bunch of pages you need to invalidate en masse, but don't
+      want to risk overloading your server, Thinner is for you.</p>
+    <p>All that being said, Thinner isn't really a solution for observing
+      model changes and purging associated urls. If you have a highly dynamic
+      application, it's worlds better to handle purging via a
+      <a href="http://github.com/russ/lacquer/blob/master/lib/lacquer/delayed_job_job.rb">job server</a>
+      outside of the request-response flow.</p>
+    <p><a href="doc/index.html">API docs</a> | <a href="http://github.com/propublica/thinner/issues">Issue Tracker</a></p>
+    <h2>Installation</h2>
+    <p>Thinner is available via rubygems:
+    <pre>gem install thinner</pre>
+    <h2>Usage</h2>
+    <p>Thinner has both a library and command-line interface. To use it as a gem
+      you'll first have to configure how it works by calling <strong>Thinner.configure</strong>.
+      Here's a quick rundown of all of the options available:</p>
+      <pre class="dawn"><span class="Support">Thinner</span><span class="PunctuationSeparator">.</span><span class="Entity">configure</span> <span class="Keyword">do </span><span class="PunctuationSeparator">|</span><span class="Variable">config</span><span class="PunctuationSeparator">|</span>
+<span class="Comment">  <span class="Comment">#</span> Number of urls to purge at one time. These purge requests are fired in quick</span>
+<span class="Comment">  <span class="Comment">#</span> succession. Thinner is perfectly capable of killing a Varnish server, by</span>
+<span class="Comment">  <span class="Comment">#</span> overloading the worker thread, so be really conservative with this option.</span>
+  config<span class="PunctuationSeparator">.</span><span class="Entity">batch_length</span> <span class="Keyword">=</span> <span class="Constant">10</span>
+
+<span class="Comment">  <span class="Comment">#</span> The amount of time to sleep between purges in seconds.</span>
+  config<span class="PunctuationSeparator">.</span><span class="Entity">sleep_time</span>   <span class="Keyword">=</span> <span class="Constant">1</span>
+
+<span class="Comment">  <span class="Comment">#</span> The server address and management port. See:</span>
+<span class="Comment">  <span class="Comment">#</span>   http://www.varnish-cache.org/trac/wiki/ManagementPort</span>
+<span class="Comment">  <span class="Comment">#</span> for details.</span>
+  config<span class="PunctuationSeparator">.</span><span class="Entity">server</span>       <span class="Keyword">=</span> &quot;127.0.0.1:6082&quot;
+
+<span class="Comment">  <span class="Comment">#</span> By default, every time you call Thinner.purge! thinner spins off a new</span>
+<span class="Comment">  <span class="Comment">#</span> instance of Thinner::Client and terminates any old instances that are</span>
+<span class="Comment">  <span class="Comment">#</span> running. If you want to have overlapping instances set this to true.</span>
+<span class="Comment">  <span class="Comment">#</span> It's not recommended to have multiple Thinner::Client's running at the</span>
+<span class="Comment">  <span class="Comment">#</span> same time.</span>
+  config<span class="PunctuationSeparator">.</span><span class="Entity">no_kill</span>      <span class="Keyword">=</span> <span class="Constant">false</span>
+
+<span class="Comment">  <span class="Comment">#</span> The log file (either a string or file object) to log the current batch to.</span>
+<span class="Comment">  <span class="Comment">#</span> Defaults to STDOUT</span>
+  config<span class="PunctuationSeparator">.</span><span class="Entity">log_file</span>     <span class="Keyword">=</span> <span class="Variable">STDOUT</span>
+<span class="Keyword">end</span>
+</pre>
+    <p>Once you have the configuration in place call <strong>purge!</strong> with
+      an array of urls:</p>
+      <pre class="dawn"><span class="Comment"><span class="Comment">#</span> The urls in this array are purged in order, so you'll want to structure it</span>
+<span class="Comment"><span class="Comment">#</span> according to usage.</span>
+arr <span class="Keyword">&lt;&lt;</span> [&quot;/some_route&quot;<span class="PunctuationSeparator">,</span> &quot;/&quot;]
+
+<span class="Support">Thinner</span><span class="PunctuationSeparator">.</span><span class="Entity">purge!</span> arr
+</pre>
+    <p>Thinner will then fork a background process and purge the urls. You can
+      check the progress of the purge by tailing the log file or with:</p>
+      <pre>varnishlog | grep purge</pre>
+    <p>If ruby's not your cup of tea, Thinner also has a command line interface.
+      Once you've installed the gem run <strong>thinner -h</strong> to see the
+      available options.</p>
+    <p>The command line interface accepts a newline separated list of urls via
+      stdin by setting the <strong>-e</strong> flag. So you'll be able to use
+      the command like so:</p>
+      <pre>cat urls_to_purge.txt | bin/thinner -e</pre>
+    <h2>Change Log</h2>
+    <h3>0.1.0</h3>
+    <p>Initial release.</p>
+    </body>
+</html>

data/lib/thinner/client.rb

@@ -0,0 +1,88 @@
+require 'logger'
+
+module Thinner
+
+  # A Thinner::Client runs as a background process and purges a list of urls
+  # in batches.
+  class Client
+
+    # A list of successfully purged urls.
+    attr_reader :purged_urls
+
+    # The list of Errors we want to catch.
+    ERRORS = [Varnish::Error, Varnish::BrokenConnection, Varnish::CommandFailed, Timeout::Error, Errno::ECONNREFUSED]
+
+    # Before purging, each Thinner::Client grabs various configuration settings
+    # and makes a copy of the passed in urls.
+    def initialize(urls)
+      @batch        = Thinner.configuration.batch_length
+      @timeout      = Thinner.configuration.sleep_time
+      @varnish      = Varnish::Client.new Thinner.configuration.server
+      @log_file     = Thinner.configuration.log_file
+      @purged_urls  = []
+      @urls         = Array.new urls
+      @length       = @urls.length
+      logger
+      handle_errors
+    end
+
+    # Kickstart the purging process and loop through the array until there aren't
+    # any urls left to purge. Each time the loop runs it will update the process
+    # label with the first url in the list.
+    def run!
+      while @urls.length > 0
+        @current_job = @urls.slice! 0, @batch
+        $0 = "#{PROCESS_IDENTIFIER}: purging #{@current_job.first}"
+        purge_urls
+        sleep @timeout
+      end
+      close_log
+    end
+
+    private
+
+    # Once a batch is ready the Client fires off purge requests on the list of
+    # urls.
+    def purge_urls
+      @current_job.each do |url|
+        begin
+          @varnish.start if @varnish.stopped?
+          while(!@varnish.running?) do sleep 0.1 end
+          if @varnish.purge :url, url
+            @logger.info "Purged url: #{url}"
+            @purged_urls << url
+          else
+            @logger.warn "Could not purge: #{url}"
+          end
+        rescue *ERRORS => e
+          @logger.warn "Error on url: #{url}, message: #{e}"
+          sleep @timeout
+        end
+      end
+    end
+
+    # Trap certain signals so the Client can report back the progress of the
+    # job and close the log.
+    def handle_errors
+      trap('TERM') { close_log }
+      trap('KILL') { close_log }
+      trap('INT')  { close_log }
+    end
+
+    # The logger redirects all STDOUT writes to a logger instance.
+    def logger
+      if !@log_file.respond_to?(:write)
+        STDOUT.reopen(File.open(@log_file, (File::WRONLY | File::APPEND | File::CREAT)))
+      end
+      @logger = Logger.new(STDOUT)
+    end
+
+    # Log the purged urls and exit the process.
+    def close_log
+      @logger.info "Purged #{@purged_urls.length} of #{@length} urls."
+      @logger.info "Exiting..."
+    end
+
+  end
+
+end

data/lib/thinner/command_line.rb

@@ -0,0 +1,80 @@
+require 'optparse'
+require File.expand_path("#{File.dirname __FILE__}/../thinner.rb")
+
+module Thinner
+
+  class CommandLine
+
+    # Usage and summary
+    BANNER = <<-EOF
+Thinner purges varnish caches as slowly as you need it to.
+
+Documentation: http://propublica.github.com/thinner/
+
+Usage: thinner OPTIONS URL
+
+Options:
+EOF
+    # Create a Thinner::CommandLine, parse any associated options, grab a list
+    # of urls and start the process
+    def initialize
+      @urls = []
+      options!
+      @urls ||= ARGV
+      run!
+    end
+
+    # Build a Thinner::Configuration instance from the passed in options and go
+    # to the races.
+    def run!
+      Thinner.configure do |config|
+        @options.each_pair do |key, value|
+          config.send("#{key}=".to_sym, value)
+        end
+      end
+      Thinner.purge! @urls
+    end
+
+    private
+
+    # Parse the command line options using OptionParser.
+    def options!
+      @options = {}
+      @option_parser = OptionParser.new(BANNER) do |opts|
+        opts.on("-b", "--batch_length BATCH", "Number of urls to purge at once") do |b|
+          @options[:batch_length] = b.to_i
+        end
+        opts.on("-t", "--sleep_time SLEEP", "Time to wait in between batches") do |t|
+          @options[:sleep_time] = t.to_i
+        end
+        opts.on("-e", "--stdin", "Use stdin for urls") do
+          ARGF.each_line do |url|
+            @urls << url.chomp
+          end
+        end
+        opts.on("-s", "--server SERVER", "Varnish url, e.g. 127.0.0.1:6082") do |s|
+          @options[:server] = s
+        end
+        opts.on("-o", "--log_file LOG_PATH", "Log file to output to (default: Standard Out") do |o|
+          @options[:log_file] = o
+        end
+        opts.on("-n", "--no-kill", "Don't kill the running purgers if they exist") do |n|
+          @options[:no_kill] = n
+        end
+        opts.on_tail("-h", "--help", "Display this help message") do
+          puts opts.help
+          exit
+        end
+      end
+
+      begin
+        @option_parser.parse!(ARGV)
+      rescue OptionParser::InvalidOption => e
+        puts e.message
+        exit(1)
+      end
+    end
+
+  end
+
+end

data/lib/thinner/configuration.rb

@@ -0,0 +1,37 @@
+module Thinner
+
+  # Thinner::Configuration holds the various settings for Thinner
+  class Configuration
+
+    attr_accessor :batch_length, :sleep_time, :server, :log_file, :no_kill
+
+    # Create a Thinner::Configuration instance with sane defaults.
+    def initialize
+      # Number of urls to purge at one time. These purge requests are fired in quick
+      # succession. Thinner is perfectly capable of killing a Varnish server, by
+      # overloading the worker thread, so be really conservative with this option.
+      @batch_length = 10
+
+      # The amount of time to sleep between purges in seconds.
+      @sleep_time   = 1
+
+      # The server address and management port. See:
+      #   http://www.varnish-cache.org/trac/wiki/ManagementPort
+      # for details.
+      @server       = "127.0.0.1:6082"
+
+      # By default, every time you call Thinner.purge! thinner spins off a new
+      # instance of Thinner::Client and terminates any old instances that are
+      # running. If you want to have overlapping instances set this to true.
+      # It's not recommended to have multiple Thinner::Client's running at the
+      # same time.
+      @no_kill      = false
+
+      # The log file (either a string or file object) to log the current batch to.
+      # Defaults to STDOUT
+      @log_file     = STDOUT
+    end
+
+  end
+
+end

data/lib/thinner/purger.rb

@@ -0,0 +1,46 @@
+module Thinner
+
+  # A Thinner::Purger dispatches a client and ensures only one instance of a
+  # Thinner::Client is running at a given time.
+  class Purger
+
+    # Each Purger accepts a list of urls to pass on to the client to purge.
+    def initialize(urls)
+      @urls = urls
+    end
+
+    # After the configuration is in place and the Purger has a list of urls,
+    # it can fork a client process to run in the background. By default the
+    # Purger will kill any old Thinner::Client processes still running so as
+    # to not double up on purge requests.
+    def purge!
+      self.class.stop! unless Thinner.configuration.no_kill
+      puts "==== Starting purge see: #{Thinner.configuration.log_file} for finished urls."
+      client_id = fork {
+        Client.new(@urls).run!
+      }
+      Process.detach(client_id)
+    end
+
+    # A list of Thinner::Client process ids -- adapted from resque.
+    def self.job_ids
+      lines = `ps -A -o pid,command | grep #{PROCESS_IDENTIFIER}`.split("\n").map do |line|
+        line.split(' ')[0].to_i
+      end
+    end
+
+    # Before we spin up a new client each running process is killed by pid. Each
+    # killed process's id is logged in the Thinner log file.
+    def self.stop!
+      job_ids.each do |pid|
+        begin
+          Process.kill("KILL", pid.to_i)
+          puts "==== Killing process: #{pid}"
+        rescue Errno::ESRCH
+        end
+      end
+    end
+
+  end
+
+end

data/lib/thinner.rb

@@ -0,0 +1,37 @@
+module Thinner
+
+  # The base location of the Thinner gem.
+  ROOT               = File.expand_path "#{File.dirname __FILE__}/.."
+
+  # The Thinner version.
+  VERSION            = File.read("#{ROOT}/VERSION").chomp
+
+  # The process label to run each Thinner::Client under.
+  PROCESS_IDENTIFIER = "Thinner"
+
+  # Set up the configuration instance as a class level accessor.
+  class << self; attr_accessor :configuration; end
+
+  # Set any thinner settings by passing in a block.
+  def self.configure
+    self.configuration ||= Configuration.new
+    yield configuration
+  end
+
+  # Halt any running instances of Thinner::Client
+  def self.stop!
+    Purger.stop!
+  end
+
+  # Begin purging urls.
+  def self.purge! urls
+    Purger.new(urls).purge!
+  end
+
+end
+
+require "klarlack"
+require "logger"
+require "#{Thinner::ROOT}/lib/thinner/configuration"
+require "#{Thinner::ROOT}/lib/thinner/client"
+require "#{Thinner::ROOT}/lib/thinner/purger"

data/test/helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'thinner'
+Thinner.configure {}
+URLS = ["/"]
+