sitemap_check 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 51d3f2f99c25a2034868ae1434c94e9adafdb50a
+  data.tar.gz: adf5452975f758257bfabf1d87e471bd2c2fb98a
+SHA512:
+  metadata.gz: b9736873b07433315af0134f0cd5df00a39d9dc7b316fc2066bf3d00423dc38007e32cbc587320058e1360180d14549f7c9e48585744d263fdcf128461b055d6
+  data.tar.gz: 9d6411787e55f4767cf8a10224b04ed4696b3381bfc425a2010c960a2c81512658bde3299b0abf56df580b028a531af6cc09249be44ff547926f63672e15f742

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.ruby-version

@@ -0,0 +1 @@
+2.2.2

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.2.2

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,3 @@
+# Contributor Code of Conduct
+
+Matz is Nice And So We Are Nice

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sitemap_check.gemspec
+gemspec

data/LICENCE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Reevoo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,11 @@
+# Sitemap Check
+
+## Install
+
+`gem install sitemap_check`
+
+## Usage
+
+```bash
+$ CHECK_URL=http://reevoo.com/sitemap_index.xml sitemap_check
+```

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+require 'reevoocop/rake_task'
+require 'rspec/core/rake_task'
+
+ReevooCop::RakeTask.new(:reevoocop)
+RSpec::Core::RakeTask.new(:spec)
+
+task default: [:spec, :reevoocop]
+task release: [:spec, :reevoocop]
+task build:   [:spec, :reevoocop]

data/bin/sitemap_check

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'sitemap_check'
+SitemapCheck.check

data/lib/sitemap_check/page.rb

@@ -0,0 +1,33 @@
+require 'httpclient'
+
+class SitemapCheck
+  class Page
+    def initialize(url, http = HTTPClient.new, holdoff = 1)
+      self.url = url
+      self.http = http
+      self.tries = 0
+      self.holdoff = holdoff
+    end
+
+    attr_reader :url
+
+    def exists?
+      @_exists ||= http.head(url, follow_redirect: true).ok?
+    rescue SocketError, HTTPClient::ConnectTimeoutError
+      self.tries += 1
+      if tries < 5
+        sleep holdoff
+        retry
+      else
+        @_exists = false
+      end
+    rescue HTTPClient::BadResponseError
+      @_exists = false
+    end
+
+    protected
+
+    attr_accessor :http, :tries, :holdoff
+    attr_writer :url
+  end
+end

data/lib/sitemap_check/sitemap.rb

@@ -0,0 +1,86 @@
+require 'httpclient'
+require 'sitemap_check/page'
+require 'nokogiri'
+require 'colorize'
+
+class SitemapCheck
+  class Sitemap
+    def initialize(url, http = HTTPClient.new)
+      self.url = url
+      self.checked = 0
+      self.http = http
+      setup_doc
+    end
+
+    attr_reader :url, :checked
+
+    def sitemaps
+      expanded_sitemaps = maps.map do |sitemap|
+        map = Sitemap.new(sitemap.loc.text, http)
+        [map] + map.sitemaps
+      end.flatten
+      (expanded_sitemaps + [self]).uniq(&:url)
+    end
+
+    def missing_pages
+      @_misssing ||= find_missing_pages
+    end
+
+    def exists? # rubocop:disable Style/TrivialAccessors
+      @ok
+    end
+
+    protected
+
+    attr_accessor :http, :doc
+    attr_writer :url, :checked
+
+    private
+
+    def concurency
+      ENV.fetch('CONCURENCY', 10)
+    end
+
+    def find_missing_pages # rubocop:disable Metrics/AbcSize
+      q = Queue.new
+      mutex = Mutex.new
+      pages.each { |page| q.push page }
+      concurency.times.map do
+        Thread.new do
+          begin
+            while (page = q.pop(true))
+              unless page.exists?
+                puts "  missing: #{page.url}".red
+                page
+              end
+              mutex.synchronize { self.checked += 1 }
+            end
+          rescue ThreadError # rubocop:disable Lint/HandleExceptions
+          end
+        end
+      end.each(&:join)
+      pages.reject(&:exists?)
+    end
+
+    def setup_doc
+      response = http.get(url, follow_redirect: true)
+      return unless (@ok = response.ok?)
+      self.doc = Nokogiri::Slop(response.body)
+      doc.remove_namespaces!
+    rescue HTTPClient::BadResponseError
+      @ok = false
+    end
+
+    def pages
+      doc.urlset.url.map { |url| Page.new(url.loc.text, http) }
+    rescue NoMethodError
+      []
+    end
+
+    def maps
+      doc.sitemapindex.sitemap
+    rescue NoMethodError
+      []
+    end
+  end
+end

data/lib/sitemap_check/version.rb

@@ -0,0 +1,3 @@
+class SitemapCheck
+  VERSION = '0.1.0'
+end

data/lib/sitemap_check.rb

@@ -0,0 +1,71 @@
+require 'colorize'
+require 'sitemap_check/sitemap'
+
+class SitemapCheck
+
+  def self.check
+    $stdout.sync = true
+    new.check
+  end
+
+  def initialize(http = HTTPClient.new)
+    self.exit_code = 0
+    puts "Expanding Sitemaps from #{ENV['CHECK_URL']}"
+    self.sitemaps = Sitemap.new(ENV['CHECK_URL'], http).sitemaps
+  end
+
+  def check
+    check_indexes
+    check_pages
+    exit exit_code
+  end
+
+  protected
+
+  attr_accessor :sitemaps, :exit_code
+
+  private
+
+  def check_indexes
+    sitemaps.reject(&:exists?).each do |sitemap|
+      puts "#{sitemap.url} does not exist".red.bold
+      self.exit_code = 1
+    end
+    puts ''
+  end
+
+  def good_sitemaps
+    sitemaps.select(&:exists?)
+  end
+
+  def check_pages
+    good_sitemaps.each { |sitemap| check_pages_in(sitemap) }
+  end
+
+  def check_pages_in(sitemap)
+    puts "Checking #{sitemap.url}"
+    if sitemap.missing_pages.any?
+      missing_pages(sitemap)
+    else
+      if sitemap.checked > 0
+        a_ok(sitemap)
+      else
+        nothing_doing
+      end
+    end
+    puts ''
+  end
+
+  def missing_pages(sitemap)
+    self.exit_code = 1
+    puts "checked #{sitemap.checked} pages and #{sitemap.missing_pages.count} were missing".red.bold
+  end
+
+  def a_ok(sitemap)
+    puts "checked #{sitemap.checked} pages and everything was ok".green.bold
+  end
+
+  def nothing_doing
+    puts 'this sitemap did not contain any pages'.green
+  end
+end

data/sitemap_check.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sitemap_check/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'sitemap_check'
+  spec.version       = SitemapCheck::VERSION
+  spec.authors       = ['Ed Robinson']
+  spec.email         = ['ed@reevoo.com']
+
+  spec.summary       = 'Check for broken links in your sitemap'
+  spec.homepage      = 'https://github.com/reevoo/sitemap_check'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(/^spec\//) }
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(/^bin\//) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'nokogiri', '~> 1.5'
+  spec.add_dependency 'httpclient', '~> 2.6'
+  spec.add_dependency 'colorize', '~> 0.7'
+  spec.add_development_dependency 'bundler', '~> 1.9'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.1'
+  spec.add_development_dependency 'reevoocop'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'codeclimate-test-reporter'
+end

metadata

@@ -0,0 +1,186 @@
+--- !ruby/object:Gem::Specification
+name: sitemap_check
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ed Robinson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: reevoocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- ed@reevoo.com
+executables:
+- sitemap_check
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENCE
+- README.md
+- Rakefile
+- bin/sitemap_check
+- lib/sitemap_check.rb
+- lib/sitemap_check/page.rb
+- lib/sitemap_check/sitemap.rb
+- lib/sitemap_check/version.rb
+- sitemap_check.gemspec
+homepage: https://github.com/reevoo/sitemap_check
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Check for broken links in your sitemap
+test_files: []