broken_link_finder 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 87ed607a6fa9669142546f0af4d62c11fb8767ae25c54f4c27373cc823706368
+  data.tar.gz: f90f38d18482eed566995288b669cfc71c0d66fe177e44850e8732bd7a6532b5
+SHA512:
+  metadata.gz: 7450e7671c5559de801e3c8f8eae7c01ae0c37bfb7c1a476f95639ea987139d7638fc14655eb4b3948f87b3021c1716e3842a902bc09c46413e1a16529cdf870
+  data.tar.gz: 3b89341b5c35de1ed61b1ab39d324d5f7c33d5d69597f87286710921f4aff7b20ce3368b5889d446895ea8cc7b1ad2ecc9630ced62310fef56ef743457a2dcb1

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+**/spike.rb
+main.rb
+.DS_Store
+.byebug_history

data/.ruby-version

@@ -0,0 +1 @@
+2.5.3

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.2.2
+before_install: gem install bundler -v 1.13.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in broken_link_finder.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,52 @@
+PATH
+  remote: .
+  specs:
+    broken_link_finder (0.2.0)
+      wgit
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.6.0)
+      public_suffix (>= 2.0.2, < 4.0)
+    bson (4.5.0)
+    byebug (11.0.1)
+    coderay (1.1.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    hashdiff (0.4.0)
+    method_source (0.9.2)
+    mini_portile2 (2.4.0)
+    minitest (5.11.3)
+    mongo (2.8.0)
+      bson (>= 4.4.2, < 5.0.0)
+    nokogiri (1.10.3)
+      mini_portile2 (~> 2.4.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    public_suffix (3.1.0)
+    rake (10.5.0)
+    safe_yaml (1.0.5)
+    webmock (3.5.1)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+    wgit (0.0.8)
+      mongo (~> 2.6)
+      nokogiri (~> 1.10)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  broken_link_finder!
+  bundler (~> 2.0)
+  byebug (~> 11.0)
+  minitest (~> 5.0)
+  pry (~> 0.12)
+  rake (~> 10.0)
+  webmock (~> 3.5)
+
+BUNDLED WITH
+   2.0.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Michael Telford
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,81 @@
+# Broken Link Finder
+
+Does what it says on the tin. Finds a website's broken links. 
+
+Simply point it at a website and it will crawl all of its webpages searching for and identifing any broken links. You will then be presented with a nice concise summary of the broken links found.
+
+## Made Possible By
+
+This repository utilises the awesome `wgit` Ruby gem. See its [repository](https://github.com/michaeltelford/wgit) for more details.
+
+The only gotcha is that `wgit` doesn't currently follow redirects meaning they will appear as broken links in the results.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'broken_link_finder'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install broken_link_finder
+
+## Usage
+
+Below is a sample script which crawls a website and outputs its broken links to a file.
+
+> main.rb
+
+```ruby
+require 'broken_link_finder'
+
+finder = BrokenLinkFinder::Finder.new
+finder.crawl_site "http://txti.es" # Also, see Finder#crawl_url for a single webpage.
+finder.pretty_print_broken_links
+```
+
+Then execute the script with:
+
+    $ ruby main.rb
+
+The output should look something like:
+
+```text
+Below is a breakdown of the different pages and their broken links...
+
+The following broken links exist in http://txti.es/about:
+http://twitter.com/thebarrytone
+http://twitter.com/nwbld
+http://twitter.com/txties
+https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=84L4BDS86FBUU
+
+The following broken links exist in http://txti.es/how:
+http://en.wikipedia.org/wiki/Markdown
+http://imgur.com
+```
+
+## TODO
+
+- Create a `broken_link_finder` executable.
+- Add logger functionality (especially useful in the console during development).
+- Update the `wgit` gem as soon as redirects are implemented.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub [here](https://github.com/michaeltelford/broken-link-finder).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "pry"
+require "broken_link_finder"
+require 'wgit/core_ext'
+
+# Call reload to load all recent code changes.
+def reload
+  original_verbose = $VERBOSE
+  $VERBOSE = nil # Temporarily suppress ruby warnings.
+  load 'load.rb'
+  include BrokenLinkFinder
+  $VERBOSE = original_verbose # Restore ruby warnings.
+  true
+end
+
+# You can add fixtures and/or initialization code here...
+reload
+url = "http://txti.es/"
+finder = Finder.new
+
+binding.pry

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/broken_link_finder.gemspec

@@ -0,0 +1,46 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'broken_link_finder/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "broken_link_finder"
+  spec.version       = BrokenLinkFinder::VERSION
+  spec.author        = "Michael Telford"
+  spec.email         = "michael.telford@live.com"
+
+  spec.summary       = "Finds a website's broken links and reports back to you with a summary."
+  spec.description   = "Finds a website's broken links using the 'wgit' gem and reports back to you with a summary."
+  spec.homepage      = "https://github.com/michaeltelford/broken-link-finder"
+  spec.license       = "MIT"
+  spec.metadata      = { 
+    "source_code_uri" => "https://github.com/michaeltelford/broken-link-finder",
+  }
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "bin"
+  # spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '~> 2.5' # Only works with ruby 2.5.x
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "pry", "~> 0.12"
+  spec.add_development_dependency "byebug", "~> 11.0"
+  spec.add_development_dependency "webmock", "~> 3.5"
+
+  spec.add_runtime_dependency "wgit"
+end

data/lib/broken_link_finder.rb

@@ -0,0 +1,2 @@
+require_relative "./broken_link_finder/version"
+require_relative "./broken_link_finder/finder"

data/lib/broken_link_finder/finder.rb

@@ -0,0 +1,106 @@
+require 'wgit'
+
+module BrokenLinkFinder
+  class Finder
+    attr_reader :broken_links
+
+    # Create a new Finder instance.
+    def initialize
+      @broken_links = {}
+      @crawler = Wgit::Crawler.new
+    end
+
+    # Clear/empty the @broken_links Hash.
+    def clear_broken_links
+      @broken_links = {}
+    end
+
+    # Finds broken links within an entire site and appends them to the
+    # @broken_links array.
+    def crawl_site(url)
+      clear_broken_links
+      url = Wgit::Url.new(url)
+      crawled_pages = []
+
+      @crawler.crawl_site(url) do |doc|
+        # Ensure the given website url is valid.
+        raise "Invalid URL: #{url}" if doc.url == url and doc.empty?
+
+        # Ensure we only process each page once.
+        next if crawled_pages.include?(doc.url)
+        crawled_pages << doc.url
+
+        # Get all page links and determine which are broken.
+        next unless doc
+        links = doc.internal_full_links + doc.external_links
+        find_broken_links(doc.url, links)
+      end
+
+      !@broken_links.empty?
+    end
+
+    # Finds broken links within a single page and appends them to the
+    # @broken_links array.
+    def crawl_url(url)
+      clear_broken_links
+      url = Wgit::Url.new(url)
+
+      # Ensure the given page url is valid.
+      doc = @crawler.crawl_url(url)
+      raise "Invalid URL: #{url}" unless doc
+
+      # Get all page links and determine which are broken.
+      links = doc.internal_full_links + doc.external_links
+      find_broken_links(url, links)
+
+      !@broken_links.empty?
+    end
+
+    # Pretty prints the contents of broken_links into a stream e.g. Kernel
+    # (STDOUT) or a file. 
+    # Returns true if there were broken links and vice versa. 
+    def pretty_print_broken_links(stream = Kernel)
+      raise "stream must respond_to? :puts" unless stream.respond_to? :puts
+      
+      if (@broken_links.empty?)
+        stream.puts("Good news, there are no broken links!")
+        false
+      else
+        stream.puts("Below is a breakdown of the different pages and their \
+broken links...")
+        stream.puts("")
+
+        @broken_links.each do |page, links|
+          stream.puts("The following broken links exist in #{page}:")
+          links.each do |link|
+            stream.puts(link)
+          end
+          stream.puts("")
+        end
+        true
+      end
+    end
+
+    private
+
+    # Finds which links are broken and append the details to @broken_links.
+    def find_broken_links(url, links)
+      links.each do |link|
+        ok = @crawler.crawl_url(link)
+        if not ok # a.k.a. if the link is broken...
+          append_broken_link(url, link)
+        end
+      end
+    end
+
+    # Append url => [link] to @broken_links.
+    def append_broken_link(url, link)
+      unless @broken_links[url]
+        @broken_links[url] = []
+      end
+      @broken_links[url] << link
+    end
+
+    alias_method :crawl_page, :crawl_url
+  end
+end

data/lib/broken_link_finder/version.rb

@@ -0,0 +1,3 @@
+module BrokenLinkFinder
+  VERSION = "0.2.0"
+end

data/load.rb

@@ -0,0 +1,2 @@
+load "./lib/broken_link_finder/version.rb"
+load "./lib/broken_link_finder/finder.rb"

metadata

@@ -0,0 +1,159 @@
+--- !ruby/object:Gem::Specification
+name: broken_link_finder
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Michael Telford
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-05-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: wgit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Finds a website's broken links using the 'wgit' gem and reports back
+  to you with a summary.
+email: michael.telford@live.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-version"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- broken_link_finder.gemspec
+- lib/broken_link_finder.rb
+- lib/broken_link_finder/finder.rb
+- lib/broken_link_finder/version.rb
+- load.rb
+homepage: https://github.com/michaeltelford/broken-link-finder
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/michaeltelford/broken-link-finder
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.5'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.8
+signing_key: 
+specification_version: 4
+summary: Finds a website's broken links and reports back to you with a summary.
+test_files: []