blinkr 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e61e7418f9b5b9eed589dd944f72f2b21a0d6336
+  data.tar.gz: 873968583abfeeb3494065a9650f9cb363598e1a
+SHA512:
+  metadata.gz: 5f05594eea95d76a50dcbb5dd634bceecde12b33e87e562314b9f8cd0aa7c71d64ba95ba7a023e6461895da21645b147d728ffd8b2d38d255084d632a41d6b37
+  data.tar.gz: 60e9ab67b9c54c82b6ba249a58a8fee766cf5d37505a3a06a0dc1b0f13d9094ce9ef2e0be039fd676a8942a4df5a964849bb8c42361fe0a28fdefac1c330bdc4

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.swp

data/.ruby-gemset

@@ -0,0 +1 @@
+blink

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.1.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in blinkr.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2014 Pete Muir
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,56 @@
+# Blinkr
+
+A broken link checker for websites
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'blinkr'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install blinkr
+
+## Usage
+
+To run blinkr against your site, assuming you have a `sitemap.xml` in the root of your site:
+
+````
+blinkr -u http://www.jboss.org
+````
+
+If you want to customize blinkr, create a config file `blinkr.yaml`. For example:
+
+````
+# Links not to check
+skips:
+  - !ruby/regexp /^\/video\/((?!91710755).)*\/$/
+  - !ruby/regexp /^\/quickstarts\/((?!eap\/kitchensink).)*\/*$/
+  - !ruby/regexp /^\/boms\/((?!eap\/jboss-javaee-6_0).)*\/*$/
+  - !ruby/regexp /^\/archetypes\/((?!eap\/jboss-javaee6-webapp-archetype).)*\/*$/
+# The output file to write the report to
+report: _tmp/blinkr.html
+# The URL to check
+base_url: http://www.jboss.org
+sitemap: http://www.jboss.org/my_sitemap.xml
+````
+
+Specify a custom config file on the command link:
+
+````
+blinkr -c my_blinkr.yaml
+````
+
+## Contributing
+
+1. Fork it ( http://github.com/pmuir/blinkr/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+

data/Rakefile

@@ -0,0 +1,21 @@
+require "bundler/gem_tasks"
+
+GEMFILE = "blinkr-#{Blinkr::VERSION}.gem"
+
+desc "Run all tests and build the gem"
+task :build do
+  system "gem build blinkr.gemspec"
+end
+
+namespace :release do
+  desc "Release the gem to rubygems"
+  task :push => [ :build, :tag ] do
+    system "gem push #{GEMFILE}"
+  end
+
+  desc "Create tag #{Blinkr::VERSION} in git"
+  task :tag do
+    system "git tag #{Blinkr::VERSION}"
+  end
+end
+

data/bin/blinkr

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require 'blinkr'
+require 'optparse'
+
+options = {}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: blinkr [options]"
+
+  opts.on("-c", "--config FILE", "specify the config.yaml file") do |opt|
+    options[:config] = opt
+  end
+  opts.on("-u", "--base-url URL", "specify the URL of the site root") do |opt|
+    options[:base_url] = opt
+  end
+end.parse!
+
+Blinkr.run options[:base_url], options[:config]
+

data/blinkr.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'blinkr/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "blinkr"
+  spec.version       = Blinkr::VERSION
+  spec.authors       = ["Pete Muir"]
+  spec.email         = ["pmuir@bleepbleep.org.uk"]
+  spec.summary       = %q{A simple broken link checker}
+  spec.homepage      = ""
+  spec.license       = "Apache-2.0"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '~> 2.0'
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency 'rake', '~> 10.3'
+  spec.add_dependency 'nokogiri', '~> 1.5'
+  spec.add_dependency 'typhoeus', '~> 0.6'
+  spec.add_dependency 'slim', '~> 2.0'
+end

data/lib/blinkr.rb

@@ -0,0 +1,16 @@
+require 'blinkr/version'
+require 'blinkr/check'
+require 'blinkr/report'
+require 'yaml'
+
+module Blinkr
+  def self.run(base_url, config = 'blinkr.yaml')
+    if !config.nil? && File.exists?(config)
+      config = YAML.load_file(config) 
+    else
+      config = {}
+    end
+    blinkr = Blinkr::Check.new(base_url || config['base_url'], sitemap: config['sitemap'], skips: config['skips'], max_retrys: config['max_retrys'], max_page_retrys: config['max_page_retrys'])
+    Blinkr::Report.render(blinkr.check, config['report'])
+  end
+end

data/lib/blinkr/cache.rb

@@ -0,0 +1,16 @@
+module Blinkr
+  class Cache
+    def initialize
+      @memory = {}
+    end
+
+    def get(request)
+      @memory[request]
+    end
+
+    def set(request, response)
+      @memory[request] = response unless response.timed_out?
+    end
+  end
+end
+

data/lib/blinkr/check.rb

@@ -0,0 +1,99 @@
+require 'nokogiri'
+require 'uri'
+require 'typhoeus'
+require 'blinkr/cache'
+require 'ostruct'
+
+module Blinkr 
+  class Check
+
+    def initialize base_url, sitemap: sitemap, skips: skips, max_retrys: max_retrys, max_page_retrys: max_page_retrys
+      raise "Must specify base_url" if base_url.nil?
+      unless sitemap.nil?
+        @sitemap = sitemap
+      else
+        @sitemap = URI.join(base_url, 'sitemap.xml').to_s
+      end
+      @skips = skips || []
+      @base_url = base_url
+      @max_retrys = max_retrys || 3
+      @max_page_retrys = max_page_retrys || @max_retrys
+      Typhoeus::Config.cache = Blinkr::Cache.new
+      @hydra = Typhoeus::Hydra.new(max_concurrency: 200)
+    end
+
+    def check
+      @errors = {}
+      puts "Loading sitemap from #{@sitemap}"
+      if @sitemap =~ URI::regexp
+        sitemap = Nokogiri::XML(Typhoeus.get(@sitemap, followlocation: true).body)
+      else
+        sitemap = Nokogiri::XML(File.open(@sitemap))
+      end
+      sitemap.css('loc').each do |loc|
+        request = Typhoeus::Request.new(
+          loc.content,
+          method: :get,
+          followlocation: true
+        )
+        perform(request, @max_page_retrys) do |resp|
+          page = Nokogiri::HTML(resp.body)
+          page.css('a[href]').each do |a|
+            check_attr(a.attribute('href'), loc.content)
+          end
+          page.css('img[src]').each do |img|
+            check_attr(img.attribute('src'), loc.content)
+          end
+        end
+      end
+      @hydra.run
+      @errors
+    end
+
+    private
+
+    def check_attr attr, src
+      url = attr.value
+      unless url.nil?
+        begin
+          uri = URI(url)
+          uri = URI.join(src, url) if uri.path.nil? || uri.path.empty?
+          uri = URI.join(@base_url, uri) if uri.scheme.nil?
+          url = uri.to_s
+        rescue Exception => e
+        end
+        if uri.nil? || uri.is_a?(URI::HTTP)
+          request = Typhoeus::Request.new(
+            url,
+            method: :head,
+            followlocation: true
+          )
+          perform request do |resp|
+            unless resp.success?
+              @errors[request] ||= OpenStruct.new({ :url => url, :code => resp.code.to_i, :message => resp.return_message, :refs => [] })
+              @errors[request].refs << OpenStruct.new({:src => src, :line_no => attr.line, :snippet => attr.parent.to_s})
+            end
+          end
+        end
+      end
+    end
+
+    def perform req, limit = @max_retrys
+      if @skips.none? { |regex| regex.match(req.base_url) }
+        req.on_complete do |resp|
+          if resp.timed_out?
+            if limit > 0
+              perform(Typhoeus::Request.new(req.base_url, req.options), limit - 1, &Proc.new)
+            else
+              yield OpenStruct.new({:success => false, :code => '0', :return_message => "Server timed out"})
+            end
+          else
+            yield resp
+          end
+        end
+        @hydra.queue req
+      end
+    end
+  end
+end
+

data/lib/blinkr/report.html.slim

@@ -0,0 +1,27 @@
+doctype html 
+head
+  link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap.min.css"
+  meta name="viewport" content="width=device-width, initial-scale=1"
+  title Blinkr
+body
+
+  .container
+    h1 Blinkr
+    p.lead 
+      | Detected broken links
+    - errors.each do |req, err|
+      .panel.panel-danger
+        .panel-heading
+          a href="#{req.base_url}" target='_blank'
+            | #{req.base_url} 
+            i.glyphicon.glyphicon-new-window
+          .pull-right.badge
+            | #{err.code if err.code > 0} #{err.message}
+        ul.list-group
+          - err.refs.each do |ref|
+            li.list-group-item
+              a href="view-source:#{ref.src}" target='_blank'
+                | #{ref.src} (line #{ref.line_no}) 
+                i.glyphicon.glyphicon-new-window
+              pre #{ref.snippet}
+

data/lib/blinkr/report.rb

@@ -0,0 +1,14 @@
+require 'slim'
+require 'ostruct'
+
+module Blinkr
+  class Report
+
+    TMPL = File.expand_path('report.html.slim', File.dirname(__FILE__))
+
+    def self.render(errors, out_file)
+      out_file ||= 'blinkr.html'
+      File.open(out_file, 'w') { |file| file.write(Slim::Template.new(TMPL).render(OpenStruct.new({:errors => errors}))) }
+    end
+  end
+end

data/lib/blinkr/version.rb

@@ -0,0 +1,4 @@
+module Blinkr
+  VERSION='0.0.1'
+end
+

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: blinkr
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Pete Muir
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: slim
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: 
+email:
+- pmuir@bleepbleep.org.uk
+executables:
+- blinkr
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/blinkr
+- blinkr.gemspec
+- lib/blinkr.rb
+- lib/blinkr/cache.rb
+- lib/blinkr/check.rb
+- lib/blinkr/report.html.slim
+- lib/blinkr/report.rb
+- lib/blinkr/version.rb
+homepage: ''
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A simple broken link checker
+test_files: []