varnisher 1.0.beta.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 89bc9abf14d10e978e9d63af4652deebd7912205
+  data.tar.gz: 3074562679934cc26c5b413a281fe9a4aef31f23
+SHA512:
+  metadata.gz: ec489165fc9a6b70f8824cfef3b962657467ad319dc584a4478497d76eedf8c0adf74d6af8b1b4201935038a57affc156d1e33ec41927f1eb39d5cac6147a112
+  data.tar.gz: b41896764dfab0db49743b7d30236de8f6ea30810b72f6e135d8b252ac8bd9b58c91c65fd9c10a61c7e2cb16bc7f7a869a3999d2f2e2dacdfd57abe59852bca6

data/LICENSE

@@ -0,0 +1,8 @@
+Copyright (c) 2011 Rob Miller
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,86 @@
+# Varnish Toolkit
+
+Administering Varnish is generally a breeze, but sometimes you want to do one of the few things that aren't painless out of the box. Hopefully, that's where this toolbox comes in.
+
+Varnish Toolkit relies on the wonderful libraries [hpricot](http://hpricot.com/), by why the lucky stiff, and [Parallel](https://github.com/grosser/parallel), by Michael Grosser. If you don't have them, install them with:
+
+    $ sudo gem install hpricot parallel
+
+## Usage
+
+    Usage: varnish.rb [options] action target
+        -h, --help                       Display this help
+        -v, --verbose                    Output more information
+        -H, --hostname HOSTNAME          Hostname/IP address of your Varnish server. Default is localhost
+        -p, --port PORT                  Port your Varnish server is listening on. Default is 80
+        -n, --num-pages NUM              Number of pages to crawl when in spider mode. -1 will crawl all pages
+        -#, --hashes                     If true, /foo.html#foo and /foo.html#bar will be seen as different in spider mode
+        -q, --ignore-query-string        If true, /foo?foo=bar and /foo?foo=baz will be seen as the same in spider mode
+
+If you find yourself typing certain parameters every time you use the script, you can specify them in an RC file called `.varnishrc` in your home directory. The file format is YAML and the default options are, if you want to paste and override them:
+
+    verbose: false
+    hostname: localhost
+    port: 80
+    num_pages: 100
+    ignore_hash: true
+    ignore_query_string: false
+
+## Examples
+
+### Purging a page and all the resources on it
+
+Quite often, it's necessary redevelop a page on a website in a way that involves changes not only to the page but also to CSS files, images, JavaScript files, etc. Purging pages in this instance can be a painful process, or at least one that requires a few `ban` commands in `varnishadm`. No longer!
+
+Just enter:
+
+	$ varnish.rb purge http://www.example.com/path/to/page
+
+...and `/path/to/page`, along with all its images, CSS files, JavaScript files, and other external accoutrements, will be purged from Varnish's cache. 
+
+As a bonus, this action is multithreaded, meaning even resource-heavy pages should purge quickly and evenly.
+
+This action requires your VCL to have something like the following, which is fairly standard:
+
+	if (req.request == "PURGE") {
+        if ( client.ip ~ auth ) {
+            ban("obj.http.x-url == " + req.url + " && obj.http.x-host == " + req.http.host);
+            error 200 "Purged.";
+        }
+    }
+
+(For an explanation of just what `obj.http.x-url` means, and why you should use it rather than `req.url`, see [this page](http://kristianlyng.wordpress.com/2010/07/28/smart-bans-with-varnish/).)
+
+### Purging an entire domain
+
+Provided your VCL has something akin to the following in it:
+
+	if ( req.request == "DOMAINPURGE" ) {
+            if ( client.ip ~ auth ) {
+                    ban("obj.http.x-host == " + req.http.host);
+                    error 200 "Purged.";
+            }
+    }
+
+...then you should be able to quickly purge an entire domain's worth of pages and resources by simply issuing the command:
+
+	$ varnish.rb purge www.example.com
+
+### Repopulating the cache
+
+If you've purged a whole domain, and particularly if your backend is slow, you might want to quickly repopulate the cache so that users never see your slow misses. Well, you can! Use the `spider` action:
+
+	$ varnish.rb spider www.example.com
+
+`spider` accepts either a hostname or a URL as its starting point, and will only fetch pages on the same domain as its origin. You can limit the number of pages it will process using the `-n` parameter:
+
+	$ varnish.rb -n 500 spider www.example.com
+
+If you'd like to combine purging and spidering, you can use the `reindex` action:
+
+	$ varnish.rb reindex www.example.com
+
+…which is functionally equivalent to:
+
+	$ varnish.rb purge www.example.com
+	$ varnish.rb spider www.example.com

data/bin/varnisher

@@ -0,0 +1,96 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'yaml'
+
+require 'varnisher'
+
+$options = {
+  :verbose             => false,
+  :hostname            => 'localhost',
+  :port                => 80,
+  :num_pages           => 100,
+  :ignore_hash         => true,
+  :spider_threads      => 16,
+  :ignore_query_string => false
+}
+
+rcfile = File.expand_path("~/.varnishrc")
+if FileTest.readable? rcfile
+  rc = YAML::load(File.open(rcfile))
+  # Convert to symbols
+  rc = rc.inject({}){ |memo,(k,v)| memo[k.to_sym] = v; memo }
+  $options.merge!(rc)
+end
+
+optparse = OptionParser.new do |opts|
+  opts.banner = 'Usage: varnish.rb [options] action target'
+
+  opts.on('-h', '--help', 'Display this help') do
+    puts opts
+  end
+
+  opts.on('-v', '--verbose', 'Output more information') do
+    $options[:verbose] = true
+  end
+
+  opts.on('-H', '--hostname HOSTNAME', 'Hostname/IP address of your Varnish server. Default is localhost') do |hostname|
+    $options[:hostname] = hostname
+  end
+
+  opts.on('-p', '--port PORT', 'Port your Varnish server is listening on. Default is 80') do |port|
+    $options[:port] = port
+  end
+
+  opts.on('-n', '--num-pages NUM', 'Number of pages to crawl when in spider mode. -1 will crawl all pages') do |num|
+    $options[:num_pages] = num.to_i
+  end
+
+  opts.on('-t', '--spider-threads NUM', 'Number of threads to use when spidering. Default is 16') do |num|
+    $options[:spider_threads] = num.to_i
+  end
+
+  opts.on('-#', '--hashes', 'If true, /foo.html#foo and /foo.html#bar will be seen as different in spider mode') do
+    $options[:ignore_hash] = false 
+  end
+
+  opts.on('-q', '--ignore-query-string', 'If true, /foo?foo=bar and /foo?foo=baz will be seen as the same in spider mode') do
+    $options[:ignore_query_string] = true
+  end
+end
+
+optparse.parse!
+
+# All our libs use these constants.
+PROXY_HOSTNAME = $options[:hostname]
+PROXY_PORT = $options[:port].to_i
+
+if ( ARGV.length < 2 )
+  puts "You must specify both an action and a target."
+end
+
+action = ARGV[0]
+target = ARGV[1]
+
+case action
+  when "purge"
+    # If target is a valid URL, then assume we're purging a page and its contents.
+    if target =~ /^[a-z]+:\/\//
+      Varnisher::PagePurger.new target
+    end
+
+    # If target is a hostname, assume we want to purge an entire domain.
+    if target =~ /^(([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])$/
+      Varnisher::DomainPurger.new target
+    end
+  
+  when "spider"
+    Varnisher::Spider.new target
+
+  when "reindex"
+    Varnisher::DomainPurger.new target
+    Varnisher::Spider.new target
+
+  else
+    puts "Invalid action."
+end

data/lib/varnisher.rb

@@ -0,0 +1,4 @@
+require_relative 'varnisher/spider'
+require_relative 'varnisher/domainpurger'
+require_relative 'varnisher/pagepurger'
+

data/lib/varnisher/domainpurger.rb

@@ -0,0 +1,27 @@
+require 'net/http'
+
+# This requires a special bit of VCL:
+#
+# if ( req.request == "DOMAINPURGE" ) {
+#   if ( client.ip ~ auth ) {
+#     ban("obj.http.x-host == " + req.http.host);
+#     error 200 "Purged.";
+#   }
+# }
+
+module Varnisher
+  class DomainPurger
+    def initialize(domain)
+      s = TCPSocket.open(PROXY_HOSTNAME, PROXY_PORT)
+      s.print("DOMAINPURGE / HTTP/1.1\r\nHost: #{domain}\r\n\r\n")
+
+      if s.read =~ /HTTP\/1\.1 200 Purged\./
+        puts "Purged  #{domain}"
+      else
+        puts "Failed to purge #{domain}"
+      end
+
+      s.close
+    end
+  end
+end

data/lib/varnisher/pagepurger.rb

@@ -0,0 +1,181 @@
+require 'rubygems'
+require 'hpricot'
+require 'net/http'
+require 'parallel'
+
+module Varnisher
+  class PagePurger
+    
+    def initialize(url)
+      @url = url
+      @uri = URI.parse(url)
+      
+      @urls = []
+      
+      # First, purge the URL itself; that means we'll get up-to-date references within that page.
+      puts "Purging #{@url}...\n\n"
+      purge(@url)
+      
+      # Then, do a fresh GET of the page and queue any resources we find on it.
+      puts "Looking for external resources on #{@url}..."
+
+      if $options[:verbose]
+        puts "\n\n"
+      end
+
+      fetch_page(@url)
+
+      if $options[:verbose]
+        puts "\n"
+      end
+
+      puts "#{@urls.length} total resources found.\n\n"
+
+      if @urls.length == 0
+        puts "No resources found. Abort!"
+        return
+      end
+      
+      # Let's figure out which of these resources we can actually purge — whether they're on our server, etc.
+      puts "Tidying resources...\n"
+      tidy_resources
+      puts "#{@urls.length} purgeable resources found.\n\n"
+      
+      # Now, purge all of the resources we just queued.
+      puts "Purging resources..."
+
+      if $options[:verbose]
+        puts "\n\n"
+      end
+
+      purge_queue
+
+      if $options[:verbose]
+        puts "\n"
+      end
+      
+      puts "Nothing more to do!\n\n"
+    end
+    
+    # Sends a PURGE request to the Varnish server, asking it to purge the given URL from its cache.
+    def purge(url)
+      begin
+        uri = URI.parse(URI.encode(url.to_s.strip))
+      rescue
+        puts "Couldn't parse URL for purging: #{$!}"
+        return
+      end
+
+      s = TCPSocket.open(PROXY_HOSTNAME, PROXY_PORT)
+      s.print("PURGE #{uri.path} HTTP/1.1\r\nHost: #{uri.host}\r\n\r\n")
+
+      if $options[:verbose]
+        if s.read =~ /HTTP\/1\.1 200 Purged\./
+          puts "Purged  #{url}"
+        else
+          puts "Failed to purge #{url}"
+        end
+      end
+
+      s.close
+    end
+    
+    # Fetches a page and parses out any external resources (e.g. JavaScript files, images, CSS files) it finds on it.
+    def fetch_page(url)
+      begin
+        uri = URI.parse(URI.encode(url.to_s.strip))
+      rescue
+        puts "Couldn't parse URL for resource-searching: #{url}"
+        return
+      end
+      
+      headers = {
+        "User-Agent"     => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.106 Safari/535.2",
+        "Accept-Charset" => "utf-8", 
+        "Accept"         => "text/html"
+      }
+      
+      begin
+        doc = Hpricot(Net::HTTP.get_response(uri).body)
+      rescue
+        puts "Hmm, I couldn't seem to fetch that URL. Sure it's right?\n"
+        return
+      end
+
+      find_resources(doc) do |resource|
+        if $options[:verbose]
+            puts "Found #{resource}"
+          end
+        queue_resource(resource)
+      end
+    end
+
+    def find_resources(doc)
+      return unless doc.respond_to? 'search'
+
+      # A bash at an abstract representation of resources. All you need is an XPath, and what attribute to select from the matched elements.
+      resource = Struct.new :name, :xpath, :attribute
+      resources = [ 
+        resource.new('stylesheet', 'link[@rel*=stylesheet]', 'href'),
+        resource.new('JavaScript file', 'script[@src]', 'src'),
+        resource.new('image file', 'img[@src]', 'src')
+      ]
+
+      resources.each { |resource|
+        doc.search(resource.xpath).each { |e|
+          att = e.get_attribute(resource.attribute)
+          yield att
+        }
+      }
+    end
+    
+    # Adds a URL to the processing queue.
+    def queue_resource(url)
+      @urls << url.to_s
+    end
+    
+    def tidy_resources
+      valid_urls = []
+      
+      @urls.each { |url|
+        # If we're dealing with a host-relative URL (e.g. <img src="/foo/bar.jpg">), absolutify it.
+        if url.to_s =~ /^\//  
+          url = @uri.scheme + "://" + @uri.host + url.to_s
+        end
+
+        # If we're dealing with a path-relative URL, make it relative to the current directory.
+        unless url.to_s =~ /[a-z]+:\/\//
+          # Take everything up to the final / in the path to be the current directory.
+          /^(.*)\//.match(@uri.path)
+          url = @uri.scheme + "://" + @uri.host + $1 + "/" + url.to_s
+        end
+        
+        begin
+          uri = URI.parse(url)
+        rescue
+          next 
+        end
+        
+        # Skip URLs that aren't HTTP, or that are on different domains.
+        next if uri.scheme != "http"
+        next if uri.host != @uri.host
+
+        valid_urls << url
+      }
+
+      @urls = valid_urls.dup
+    end
+    
+    # Processes the queue of URLs, sending a purge request for each of them.
+    def purge_queue()
+      Parallel.map(@urls) { |url|
+        if $options[:verbose]
+          puts "Purging #{url}..."
+        end
+
+        purge(url)
+      }
+    end
+
+  end
+end

data/lib/varnisher/spider.rb

@@ -0,0 +1,164 @@
+require 'rubygems'
+require 'hpricot'
+require 'net/http'
+require 'parallel'
+
+module Varnisher
+  class Spider
+
+    def initialize(url)
+      if url =~ /^(([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])$/
+        url = 'http://' + url
+      end
+
+      @uri = URI.parse(url)
+
+      @pages_hit = 0
+
+      @visited = []
+      @to_visit = []
+
+      puts "Beginning spider of #{url}"
+      crawl_page(url)
+      spider
+      puts "Done; #{@pages_hit} pages hit."
+    end
+
+    def queue_link(url)
+      @to_visit << url
+    end
+
+    def crawl_page(url, limit = 10)
+      # Don't crawl a page twice
+      return if @visited.include? url
+
+      # Let's not hit this again
+      @visited << url
+
+      begin
+        uri = URI.parse(URI.encode(url.to_s.strip))
+      rescue
+        return
+      end
+
+      headers = {
+        "User-Agent"     => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31",
+        "Accept-Charset" => "ISO-8859-1,utf-8;q=0.7,*;q=0.3",
+        "Accept"         => "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+      }
+
+      begin
+        req = Net::HTTP::Get.new(uri.path, headers)
+        response = Net::HTTP.start(uri.host, uri.port) { |http| http.request(req) }
+
+        case response
+        when Net::HTTPRedirection
+          return crawl_page(response['location'], limit - 1)
+        when Net::HTTPSuccess
+          doc = Hpricot(response.body)
+        end
+      rescue
+        return
+      end
+
+      @pages_hit += 1
+
+      if $options[:verbose]
+        puts "Fetched #{url}..."
+      end
+
+      find_links(doc, url) do |link|
+        next if @visited.include? link
+        next if @to_visit.include? link
+
+        @to_visit << link
+      end
+    end
+
+    def find_links(doc, url)
+      return unless doc.respond_to? 'search'
+
+      begin
+        uri = URI.parse(URI.encode(url.to_s.strip))
+      rescue
+        return
+      end
+
+      hrefs = []
+
+      # Looks like a valid document! Let's parse it for links
+      doc.search("//a[@href]").each do |e|
+        hrefs << e.get_attribute("href")
+      end
+
+      # Let's also look for commented-out URIs
+      doc.search("//comment()").each do |e|
+        e.to_html.scan(/https?:\/\/[^\s\"]*/) { |url| hrefs << url; }
+      end
+
+      hrefs.each do |href|
+          # Skip mailto links
+          next if href =~ /^mailto:/
+
+          # If we're dealing with a host-relative URL (e.g. <img src="/foo/bar.jpg">), absolutify it.
+          if href.to_s =~ /^\//
+            href = uri.scheme + "://" + uri.host + href.to_s
+          end
+
+          # If we're dealing with a path-relative URL, make it relative to the current directory.
+          unless href.to_s =~ /[a-z]+:\/\//
+            # Take everything up to the final / in the path to be the current directory.
+            if uri.path =~ /\//
+              /^(.*)\//.match(uri.path)
+              path = $1
+            # If we're on the homepage, then we don't need a path.
+            else
+              path = ""
+            end
+
+            href = uri.scheme + "://" + uri.host + path + "/" + href.to_s
+          end
+
+          # At this point, we should have an absolute URL regardless of
+          # its original format.
+
+          # Strip hash links
+          if ( $options[:ignore_hash] )
+            href.gsub!(/(#.*?)$/, '')
+          end
+
+          # Strip query strings
+          if ( $options[:ignore_query_string] )
+            href.gsub!(/(\?.*?)$/, '')
+          end
+
+          begin
+            href_uri = URI.parse(href)
+          rescue
+            # No harm in this — if we can't parse it as a URI, it probably isn't one (`javascript:` links, etc.) and we can safely ignore it.
+            next
+          end
+
+          next if href_uri.host != uri.host
+          next unless href_uri.scheme =~ /^https?$/
+
+          yield href
+      end
+    end
+
+    def spider
+      Parallel.in_threads($options[:spider_threads]) { |thread_number|
+          # We've crawled too many pages
+          next if @pages_hit > $options[:num_pages] && $options[:num_pages] >= 0
+
+          while @to_visit.length > 0 do
+            begin
+              url = @to_visit.pop
+            end while ( @visited.include? url )
+
+            crawl_page(url)
+          end
+        }
+    end
+  end
+end

data/lib/varnisher/version.rb

@@ -0,0 +1,3 @@
+module Varnisher
+  VERSION = "1.0.beta.1"
+end

metadata

@@ -0,0 +1,81 @@
+--- !ruby/object:Gem::Specification
+name: varnisher
+version: !ruby/object:Gem::Version
+  version: 1.0.beta.1
+platform: ruby
+authors:
+- Rob Miller
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: hpricot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.6
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+description: Some tools that make working with the Varnish HTTP cache easier, including
+  things like doing mass purges of entire domains.
+email: rob@bigfish.co.uk
+executables:
+- varnisher
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/varnisher
+- lib/varnisher/domainpurger.rb
+- lib/varnisher/pagepurger.rb
+- lib/varnisher/spider.rb
+- lib/varnisher/version.rb
+- lib/varnisher.rb
+- LICENSE
+- README.md
+homepage: http://github.com/robmiller/varnisher
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Helpful tools for working with Varnish caches
+test_files: []