varnisher 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f28d1ab92622c71c86bf45e3762de993496f2e7
-  data.tar.gz: f59a59018cef6491a2684c3646f293a2aae80141
+  metadata.gz: 53a788e751893e32d605e1221b9cc42012a3bda3
+  data.tar.gz: 79b9cdeb072f45e6042bfdad7f68f4c60ec5d574
 SHA512:
-  metadata.gz: 1f05dac54745831b650f7b809408778866f23a5bdd5176b0c46e10ce329d258d4567f3f45ebce80bb01a1e7a99359b56616de3f05d5095c0c6872f9e6f4429b6
-  data.tar.gz: d87c226e4a485a47c29864c7d67972bb64916d1e33e2da93d546fcd4aae2627d468d885f98dc4aacd866d9e94a3039eedeb81083ccc90196f843074a2783183c
+  metadata.gz: 974f483c163d0ead6adb56c18ee672ed8ac586618412e9614366069a8a54d763a1f2d6b1bf65ae0ab08a2d3090dbf9dc3f4fe39ebf3bdee97e94ef471713c851
+  data.tar.gz: de2a2698bf9e9aaa992884a0512a40df21c668763d0aa2241be6cb24fb4b70acda32821b2a73db7db818640484c640783c39d02aebd8d1c0c8254cb1af302586

data/README.md

@@ -1,6 +1,6 @@
 # varnisher
 
-[![Build Status](https://travis-ci.org/robmiller/varnisher.png?branch=master)](https://travis-ci.org/robmiller/varnisher)
+[![Build Status](https://travis-ci.org/robmiller/varnisher.png?branch=master)](https://travis-ci.org/robmiller/varnisher) [![Dependency Status](https://gemnasium.com/robmiller/varnisher.png)](https://gemnasium.com/robmiller/varnisher)
 
 Administering Varnish is generally a breeze, but sometimes you want to
 do one of the few things that aren't painless out of the box. Hopefully,

data/lib/varnisher.rb

@@ -1,5 +1,6 @@
 require 'net/http'
 
+require_relative 'varnisher/urls'
 require_relative 'varnisher/spider'
 require_relative 'varnisher/purger'
 require_relative 'varnisher/domainpurger'

data/lib/varnisher/pagepurger.rb

@@ -29,8 +29,6 @@ module Varnisher
     def initialize(url)
       @url = url
       @uri = URI.parse(url)
-
-      @urls = []
     end
 
     # Sends a PURGE request to the Varnish server, asking it to purge
@@ -69,6 +67,8 @@ module Varnisher
     # Purges all the resources on the given page.
     def purge_resources
       fetch_page
+      parse_page
+      @urls = find_resources
 
       return if @urls.empty?
 
@@ -84,16 +84,18 @@ module Varnisher
       Varnisher.log.info "Looking for external resources on #{@url}..."
 
       begin
-        @doc = Nokogiri::HTML(Net::HTTP.get_response(@uri).body)
+        @html = Net::HTTP.get_response(@uri).body
       rescue
         Varnisher.log.info "Hmm, I couldn't fetch that URL. Sure it's right?\n"
         return
       end
+    end
 
-      @urls = find_resources
-
-      Varnisher.log.debug ''
-      Varnisher.log.info "#{@urls.length} total resources found.\n"
+    # Parses the raw HTML we've fetched into a Nokogiri document.
+    #
+    # @api private
+    def parse_page
+      @doc = Nokogiri::HTML(@html)
     end
 
     # Returns an array of resources contained within the current page.
@@ -107,18 +109,31 @@ module Varnisher
     # @return [Array] An array of strings, each representing a URL
     #
     # @api private
-    def find_resources
-      found = []
+    def find_resources(&block)
+      found = Varnisher::Urls.new
+
+      self.class.resources.each do |resource|
+        found += find_resource(resource, &block)
+      end
+
+      Varnisher.log.debug ''
+      Varnisher.log.info "#{found.length} total resources found.\n"
+
+      found
+    end
+
+    # Given a resource, will return instances of that resource found in
+    # the current document.
+    def find_resource(resource, &block)
+      found = Varnisher::Urls.new
 
-      self.class.resources.each do |res|
-        @doc.css(res.selector).each do |e|
-          attribute = e[res.attribute]
+      @doc.css(resource.selector).each do |e|
+        attribute = e[resource.attribute]
 
-          Varnisher.log.debug("Found resource: #{attribute}")
+        Varnisher.log.debug("Found resource: #{attribute}")
 
-          yield attribute if block_given?
-          found << attribute
-        end
+        yield attribute if block_given?
+        found << attribute
       end
 
       found
@@ -133,8 +148,7 @@ module Varnisher
     def tidy_resources
       Varnisher.log.info 'Tidying resources...'
 
-      @urls = @urls.map { |url| URI.join(@uri, url) }
-        .select { |uri| uri.scheme == 'http' && uri.host == @uri.host }
+      @urls = @urls.make_absolute(@uri).with_hostname(@uri.host)
 
       Varnisher.log.info "#{@urls.length} purgeable resources found.\n"
     end

data/lib/varnisher/purger.rb

@@ -81,7 +81,9 @@ module Varnisher
 
       begin
         http = Net::HTTP.new(hostname, port)
-        response = http.request(@request_method.new(@path))
+        request = @request_method.new(@path)
+        request.add_field("Host", @host)
+        response = http.request(request)
       rescue Timeout::Error
         return false
       end

data/lib/varnisher/spider.rb

@@ -43,6 +43,9 @@ module Varnisher
 
       @visited = []
       @to_visit = []
+
+      @threads = Varnisher.options['threads']
+      @num_pages = Varnisher.options['num-pages']
     end
 
     # Adds a link to the queue of pages to be visited.
@@ -103,16 +106,14 @@ module Varnisher
     #
     # @api private
     def find_links(doc, uri)
-      hrefs = []
+      urls = Varnisher::Urls.new(get_anchors(doc) + get_commented_urls(doc))
 
-      hrefs  = get_anchors(doc)
-      hrefs += get_commented_urls(doc)
+      urls = urls.make_absolute(uri).with_hostname(uri.host)
 
-      hrefs = valid_urls(hrefs, uri)
-      hrefs = remove_hashes(hrefs)
-      hrefs = remove_query_strings(hrefs)
+      urls = urls.without_hashes if Varnisher.options['ignore-hashes']
+      urls = urls.without_query_strings if Varnisher.options['ignore-query-strings']
 
-      hrefs
+      urls
     end
 
     # Given an HTML document, will return all the URLs that exist as
@@ -131,67 +132,6 @@ module Varnisher
       doc.xpath('//comment()').flat_map { |e| URI.extract(e.to_html, 'http') }
     end
 
-    # Given a set of URLs, will return only the ones that are valid for
-    # spidering.
-    #
-    # That means URLs that have the same hostname as the hostname we
-    # started from, and that are on the HTTP scheme rather than HTTPS
-    # (since Varnish doesn't support HTTPS).
-    #
-    # Additionally, some normalisation will be performed, so that the
-    # URLs are absolute (using the page that they were fetched from as
-    # the base, just like a browser would).
-    #
-    # @return [Array] An array of URIs
-    def valid_urls(hrefs, uri)
-      hrefs.map { |u| URI.join(uri, URI.escape(u)) }
-        .select { |u| u.scheme == 'http' && u.host == @uri.host }
-    end
-
-    # Given a set of URLs, will normalise them according to their URL
-    # minus the hash; that is, normalise them so that:
-    #
-    # foo#bar
-    #
-    # and:
-    #
-    # foo#baz
-    #
-    # Are considered the same.
-    #
-    # @return [Array] An array of URIs
-    def remove_hashes(hrefs)
-      return hrefs unless Varnisher.options['ignore-hashes']
-
-      hrefs = hrefs.group_by do |h|
-        URI.parse(h.scheme + '://' + h.host + h.path.to_s + h.query.to_s)
-      end
-
-      hrefs.keys
-    end
-
-    # Given a set of URLs, will normalise them according to their URL
-    # minus the query string; that is, normalise them so that:
-    #
-    # foo?foo=bar
-    #
-    # and:
-    #
-    # foo?foo=baz
-    #
-    # Are considered the same.
-    #
-    # @return [Array] An array of URIs
-    def remove_query_strings(hrefs)
-      return hrefs unless Varnisher.options['ignore-query-strings']
-
-      hrefs = hrefs.group_by do |h|
-        URI.parse(h.scheme + '://' + h.host + h.path.to_s)
-      end
-
-      hrefs.keys
-    end
-
     # Pops a URL from the queue of yet-to-be-visited URLs, ensuring that
     # it's not one that we've visited before.
     #
@@ -207,6 +147,16 @@ module Varnisher
       url
     end
 
+    # Returns true if the spider has visited the maximum number of pages
+    # it's allowed to.
+    def limit_reached?
+      @visited.length > @num_pages && @num_pages >= 0
+    end
+
+    def pages_remaining?
+      @to_visit.length > 0
+    end
+
     # Kicks off the spidering process.
     #
     # Fires up Parallel in as many threads as have been configured, and
@@ -221,13 +171,8 @@ module Varnisher
 
       crawl_page(@uri)
 
-      threads = Varnisher.options['threads']
-      num_pages = Varnisher.options['num-pages']
-
-      Parallel.in_threads(threads) do |thread_number|
-        next if @visited.length > num_pages && num_pages >= 0
-
-        crawl_page(pop_url) while @to_visit.length > 0
+      Parallel.in_threads(@threads) do |_|
+        crawl_page(pop_url) while pages_remaining? and !limit_reached?
       end
 
       Varnisher.log.info "Done; #{@visited.length} pages hit."

data/lib/varnisher/urls.rb

@@ -0,0 +1,92 @@
+module Varnisher
+  # A collection for URLs, that exposes some useful behaviour (like
+  # selecting only URLs that have a given hostname, or converting
+  # relative URLs to absolute).
+  class Urls
+    include Enumerable
+    extend Forwardable
+
+    # Given an array of URLs (either strings or URI objects), store them
+    # in the collection.
+    def initialize(urls = [])
+      @urls = Array(urls)
+      @urls = make_uris
+    end
+
+    # Coerces the values of the current collection into being URI
+    # objects, which allows strings to be passed initially.
+    def make_uris
+      coerced = urls.map do |url|
+        begin
+          URI(url)
+        rescue
+          nil
+        end
+      end
+
+      coerced.compact
+    end
+
+    # Given a relative URL and a base to work from, will return the
+    # absolute form of that URL.
+    #
+    # For example:
+    #
+    #   absolute_url('http://www.example.com', '/foo')
+    #   # => "http://www.example.com/foo"
+    #
+    #   absolute_url('http://www.example.com/foo', 'bar')
+    #   # => "http://www.example.com/bar"
+    #
+    #   absolute_url('http://www.example.com/foo/bar', 'baz')
+    #   # => "http://www.example.com/foo/baz"
+    def absolute_url(base, url)
+      URI.join(base, URI.escape(url.to_s))
+    end
+
+    # Returns a new collection containing absolute versions of all the
+    # URLs in the current collection.
+    def make_absolute(base)
+      Urls.new(urls.map { |uri| absolute_url(base, uri) })
+    end
+
+    # Returns a new collection containing only the URLs in this
+    # collection that match the given hostname.
+    def with_hostname(hostname)
+      Urls.new(urls.select { |uri| uri.scheme == 'http' && uri.host == hostname })
+    end
+
+    # Returns a new collection containing the URLs in the current
+    # collection, normalised according to their hash.
+    def without_hashes
+      normalised = urls.group_by do |url|
+        url.fragment = nil
+        url
+      end
+
+      Urls.new(normalised.keys)
+    end
+
+    # Returns a new collection containing the URLs in the current
+    # collection without their query string values.
+    def without_query_strings
+      normalised = urls.group_by do |h|
+        url.query = nil
+        url
+      end
+
+      Urls.new(normalised.keys)
+    end
+
+    # Allows the addition of two collections by accessing the underlying
+    # array.
+    def +(other)
+      Urls.new(urls + other.urls)
+    end
+
+    def_delegators :urls, :each, :<<, :length, :empty?, :include?
+
+    protected
+    attr_reader :urls
+  end
+end

data/lib/varnisher/version.rb

@@ -1,3 +1,3 @@
 module Varnisher
-  VERSION = '1.0.1'
+  VERSION = '1.0.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: varnisher
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Rob Miller
@@ -14,112 +14,112 @@ dependencies:
   name: main
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 5.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 5.2.0
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.0
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.8.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 10.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 10.1.0
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 5.0.6
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 5.0.6
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.13.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.13.0
 - !ruby/object:Gem::Dependency
   name: letters
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.4.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.4.1
 - !ruby/object:Gem::Dependency
   name: rubygems-tasks
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.2.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.2.4
 description: Some tools that make working with the Varnish HTTP cache easier, including
@@ -130,16 +130,17 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- Gemfile
+- LICENSE
+- README.md
 - bin/varnisher
+- lib/varnisher.rb
 - lib/varnisher/domainpurger.rb
 - lib/varnisher/pagepurger.rb
 - lib/varnisher/purger.rb
 - lib/varnisher/spider.rb
+- lib/varnisher/urls.rb
 - lib/varnisher/version.rb
-- lib/varnisher.rb
-- LICENSE
-- README.md
-- Gemfile
 homepage: http://github.com/robmiller/varnisher
 licenses:
 - MIT
@@ -150,17 +151,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Helpful tools for working with Varnish caches