sitemap_check 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d49c855487981b357e247bd51099795f9e8c1444
-  data.tar.gz: 8787dcdc878e484471cb5a998a4dce27cf1f9eec
+  metadata.gz: 2fcf90cc5916a816ef6a0c7f191c7739066f5303
+  data.tar.gz: 632022e800ace98dbf9cbd9e3a3185f042a56756
 SHA512:
-  metadata.gz: 1c50fa5f6518a2b6329a4a3e481cf4aa167af4fd1c772d4b3aec6fdf7f0493689c4c131f4eb0c424c3eec735689611b4be0e30d7ca252cd606566cdc394542c4
-  data.tar.gz: 3325d58fb39689c119f54234e4000489121642046e0f440b1ae8ab2a48ae0200c1bb17e434b3e1252119c0ad61868c3571554121fd359bd3636b747457e6518b
+  metadata.gz: de78fed953cfac1403d376f69841d736711fe8c3fe335f971ca712be882ab69930aca8392066e8251075dd610f6f893d9cfe7b7ed854b2df4770c4cd94e024ad
+  data.tar.gz: ffb5a6e6781ebe7d9eea77b42f5766d8e4f9e7cc6348cb482c687c3f2467b79cbcf2a5bd7f74d818fad395a1a6bb67aca742cf26ace822999f3cc6fd1b26bdd1

data/bin/sitemap_check

@@ -1,4 +1,4 @@
 #!/usr/bin/env ruby
 
 require "sitemap_check"
-SitemapCheck.check ARGV[0]
+SitemapCheck.check ENV.fetch("CHECK_URL", ARGV[0])

data/lib/sitemap_check.rb

@@ -9,12 +9,13 @@ class SitemapCheck
     new(url).check
   end
 
-  def initialize(url = nil, http = HTTPClient.new)
+  def initialize(check_url)
     self.start_time = Time.now
     self.exit_code = 0
-    check_url = url || ENV.fetch("CHECK_URL")
+    check_url = check_url
     puts "Expanding Sitemaps from #{check_url}"
-    self.sitemaps = Sitemap.new(check_url, http).sitemaps
+    self.sitemaps = Sitemap.new(check_url).sitemaps
+    Typhoeus::Config.user_agent = "SitemapCheckbot/#{VERSION} (+https://github.com/reevoo/sitemap_check)"
   end
 
   def check
@@ -65,6 +66,7 @@ class SitemapCheck
 
   def check_pages_in(sitemap)
     puts "Checking #{sitemap.url}"
+    sitemap.check_pages
     if sitemap.missing_pages.any?
       missing_pages(sitemap)
     else

data/lib/sitemap_check/page.rb

@@ -1,35 +1,38 @@
-require "httpclient"
+require "typhoeus"
+require "sitemap_check/logger"
+require "colorize"
 
 class SitemapCheck
   class Page
-    def initialize(url, http = HTTPClient.new, holdoff = 1)
+    def initialize(url, logger = Logger.new)
       self.url = url
-      self.http = http
-      self.tries = 0
-      self.holdoff = holdoff
+      self.request = Typhoeus::Request.new(self.url, method: :head, followlocation: true)
+      self.logger = logger
+      setup_callbacks
     end
 
-    attr_reader :url, :error
-
-    def exists?
-      @_exists ||= http.head(url, follow_redirect: true).ok?
-    rescue SocketError, HTTPClient::ConnectTimeoutError, Errno::ETIMEDOUT => e
-      self.tries += 1
-      if tries < 5
-        sleep holdoff
-        retry
-      else
-        self.error = e
-        @_exists = true
-      end
-    rescue HTTPClient::BadResponseError => e
-      self.error = e
-      @_exists = true
-    end
+    attr_reader :url, :request, :exists, :error
 
     protected
 
-    attr_accessor :http, :tries, :holdoff
-    attr_writer :url, :error
+    attr_writer :url, :request
+    attr_accessor :logger
+
+    def setup_callbacks # rubocop:disable Metrics/AbcSize
+      request.on_complete do |response|
+        if response.success?
+          @exists = true
+        elsif response.timed_out?
+          @exists = true
+          logger.log "  warning: request to #{url} timed out".magenta
+        elsif response.code == 404
+          @exists = false
+          logger.log "  missing: #{url}".magenta
+        else
+          @error = true
+          logger.log "  error: (#{response.code}) while connecting to #{url}".magenta
+        end
+      end
+    end
   end
 end

data/lib/sitemap_check/sitemap.rb

@@ -1,32 +1,36 @@
-require "httpclient"
+require "typhoeus"
 require "sitemap_check/page"
 require "sitemap_check/logger"
 require "nokogiri"
-require "colorize"
 
 class SitemapCheck
   class Sitemap
-    def initialize(url, http = HTTPClient.new, logger = Logger.new)
+    def initialize(url, logger = Logger.new)
       self.logger = logger
       self.url = url
       self.checked = 0
-      self.http = http
-      self.queue = Queue.new
+      self.hydra = Typhoeus::Hydra.new(max_concurrency: concurency)
       setup_doc
     end
 
-    attr_reader :url, :checked
+    attr_reader :url, :checked, :pages
+
+    def check_pages
+      queue_pages
+      hydra.run
+      self.checked = pages.count
+    end
 
     def sitemaps
       expanded_sitemaps = maps.map do |sitemap|
-        map = Sitemap.new(sitemap.loc.text, http)
+        map = Sitemap.new(sitemap.loc.text)
         [map] + map.sitemaps
       end.flatten
       (expanded_sitemaps + [self]).uniq(&:url)
     end
 
     def missing_pages
-      @_misssing ||= find_missing_pages
+      pages.reject(&:exists)
     end
 
     def errored_pages
@@ -39,7 +43,7 @@ class SitemapCheck
 
     protected
 
-    attr_accessor :http, :doc, :logger, :queue
+    attr_accessor :hydra, :doc, :logger
     attr_writer :url, :checked
 
     private
@@ -48,46 +52,19 @@ class SitemapCheck
       ENV.fetch("CONCURRENCY", "10").to_i
     end
 
-    def find_missing_pages
-      queue_pages
-      check_pages
-      pages.reject(&:exists?)
-    end
-
-    def check_pages
-      concurency.times.map do
-        Thread.new do
-          begin
-            nil while check_page(queue.pop(true))
-          rescue ThreadError
-            nil
-          end
-        end
-      end.each(&:join)
-      self.checked = pages.count
-    end
-
-    def check_page(page)
-      return unless page
-      logger.log "  missing: #{page.url}".red unless page.exists?
-      logger.log "  warning: error connecting to #{page.url}".magenta if page.error
-    end
-
     def queue_pages
-      pages.each { |page| queue.push page }
+      pages.each { |page| hydra.queue page.request }
     end
 
     def setup_doc
-      response = http.get(url, follow_redirect: true)
-      return unless (@ok = response.ok?)
+      response = Typhoeus.get(url, followlocation: true)
+      return unless (@ok = response.success?)
       self.doc = Nokogiri::Slop(response.body)
       doc.remove_namespaces!
-    rescue HTTPClient::BadResponseError
-      @ok = false
     end
 
     def pages
-      doc.urlset.url.map { |url| Page.new(url.loc.text, http) }
+      @pages ||= doc.urlset.url.map { |url| Page.new(url.loc.text, logger) }
     rescue NoMethodError
       []
     end

data/lib/sitemap_check/version.rb

@@ -1,3 +1,3 @@
 class SitemapCheck
-  VERSION = "0.1.6"
+  VERSION = "0.1.7"
 end

data/sitemap_check.gemspec

@@ -18,12 +18,12 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(/^bin\//) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "nokogiri", "~> 1.5"
-  spec.add_dependency "httpclient", "~> 2.6"
-  spec.add_dependency "colorize", "~> 0.7"
-  spec.add_development_dependency "bundler", "~> 1.9"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.1"
+  spec.add_dependency "nokogiri", "~> 1.7"
+  spec.add_dependency "typhoeus", "~> 1.1"
+  spec.add_dependency "colorize", "~> 0.8"
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "rspec", "~> 3.5"
   spec.add_development_dependency "reevoocop"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "codeclimate-test-reporter"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sitemap_check
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Ed Robinson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-14 00:00:00.000000000 Z
+date: 2017-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -16,84 +16,84 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
-  name: httpclient
+  name: typhoeus
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.14'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '3.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '3.5'
 - !ruby/object:Gem::Dependency
   name: reevoocop
   requirement: !ruby/object:Gem::Requirement