checklinks 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 77b94b65c8046e3e8c4aadb5a21d1ea0e511011462497aaa304cb44519af05c8
+  data.tar.gz: 6b46cf2b5108b9ddc2028b58bf6253b95b3696bef60a75748aa997e30ad27b3b
+SHA512:
+  metadata.gz: be687604bef24ea7ece0d9322d68589de3e8b3ec84055138b2369c4eea9c3b6128b619bfd47fb6d75b04b584ca3a9109eb1b242733207d0bd2f5fd0fefae7aae
+  data.tar.gz: 9a383deb39970cd0eea213e12fe1de467ec9d6c9c9ce8e47db27bc85d5a0a07c0ec41f68d927b5123c09edeabc94689f4b671618aaff111bfa019015a5b802ee

data/.gitignore

@@ -0,0 +1 @@
+*.gem

data/.ruby-version

@@ -0,0 +1 @@
+2.5.8

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,17 @@
+PATH
+  remote: .
+  specs:
+    checklinks (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  checklinks!
+
+BUNDLED WITH
+   1.17.2

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Chris Seaton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,48 @@
+# Checklinks
+
+Checklinks finds broken or redirected links in a static webpage.
+
+```
+% gem install checklinks
+% checklinks **/*.md
+```
+
+It uses code from [awesome_bot](https://github.com/dkhamsing/awesome_bot), but the output is much more terse so it's easier to find which links are broken, and it also prints link locations which makes updating them easier.
+
+## Configuration
+
+```
+% checklinks --config .checklinks.yaml **/*.md
+```
+
+```yaml
+ignore:
+  exact:
+  - ...
+  prefixes:
+  - ...
+  suffixes:
+  - ...
+  forbidden:
+    exact:
+    - ...
+    prefixes:
+    - ...
+    suffixes:
+    - ...
+  redirect:
+    exact:
+    - ...
+    prefixes:
+    - ...
+    suffixes:
+    - ...
+```
+
+## Author
+
+[Chris Seaton](https://chrisseaton.com/)
+
+## License
+
+Copyright © 2020 Chris Seaton. Available under the MIT license.

data/bin/checklinks

@@ -0,0 +1,163 @@
+#!/usr/bin/env ruby
+
+require 'set'
+require 'yaml'
+
+require 'checklinks'
+
+# Process command line arguments
+
+files = []
+config = {}
+
+args = ARGV.dup
+until args.empty?
+  arg = args.shift
+  if arg.start_with?('-')
+    case arg
+    when '--config'
+      config_file = args.shift
+      raise unless config_file
+      config = YAML.load(File.read(config_file))
+    else
+      raise
+    end
+  else
+    files.push arg
+  end
+end
+
+# Create worklists - ahead of processing anything as they refer to each other cylically
+
+file_worklist = Checklinks::Worklist.new(*files)
+url_worklist = Checklinks::Worklist.new
+link_collector = Checklinks::Worklist.collector
+status_worklist = Checklinks::Worklist.new
+error_collector = Checklinks::Worklist.collector
+
+# Process files to produce URLs and links
+
+file_worklist.process(Set.new) do |filename, found_urls|
+  File.open(filename, encoding: Encoding::UTF_8) do |file|
+    line_number = 1
+    file.each_line do |line|
+      urls = URI.extract(line, /http()s?/)
+      urls = Checklinks::Awesome.links_filter(urls)
+      urls.each do |url|
+        next unless found_urls.add?(url)
+        url_worklist.push url
+        link_collector.push [url, filename, line_number]
+      end
+      line_number += 1
+    end
+  end
+end
+
+# Process URLs to produce status updates and errors
+
+url_worklist.process_concurrently(16) do |url|
+  status_worklist.push [:try, url]
+  if config.dig('ignore', 'exact')&.any? { |exact| url.start_with?(exact) } ||
+      config.dig('ignore', 'prefixes')&.any? { |prefix| url.start_with?(prefix) } ||
+      config.dig('ignore', 'suffixes')&.any? { |suffix| url.end_with?(suffix) }
+    status_worklist.push [:ignored]
+    next
+  end
+  begin
+    code, headers = Checklinks::Awesome.net_status(url, true)
+  rescue Errno::ECONNREFUSED
+    code = :refused
+  rescue Errno::ETIMEDOUT, Net::OpenTimeout, Net::ReadTimeout
+    code = :timeout
+  rescue OpenSSL::SSL::SSLError
+    code = :https
+  rescue => e
+    code = e.class.name
+  end
+  if code == 200
+    status_worklist.push [:ok]
+  else
+    if (301..302).include?(code) && (config.dig('ignore', 'redirect', 'exact')&.any? { |exact| url.start_with?(exact) } ||
+        config.dig('ignore', 'redirect', 'prefixes')&.any? { |prefix| url.start_with?(prefix) } ||
+        config.dig('ignore', 'redirect', 'suffixes')&.any? { |suffix| url.end_with?(suffix) })
+      status_worklist.push [:ignored]
+      next
+    end
+    if code == 403 && (config.dig('ignore', 'forbidden', 'exact')&.any? { |exact| url.start_with?(exact) } ||
+        config.dig('ignore', 'forbidden', 'prefixes')&.any? { |prefix| url.start_with?(prefix) } ||
+        config.dig('ignore', 'forbidden', 'suffixes')&.any? { |suffix| url.end_with?(suffix) })
+      status_worklist.push [:ignored]
+      next
+    end
+    status_worklist.push [:error]
+    error_collector.push [url, code, headers]
+  end
+end
+
+# Process status updates to print progress on the screen
+
+status_worklist.process({ok: 0, error: 0, ignored: 0, clear: 0}) do |message, state|
+  print "\r#{' '*state[:clear]}\r"
+  case message.first
+  when :try
+    puts message.last
+  when :ok
+    state[:ok] += 1
+  when :ignored
+    state[:ignored] += 1
+  when :error
+    state[:error] += 1
+    error_collector.push 
+  end
+  line = "[#{state[:ok].to_s.rjust(5)} ok, #{state[:ignored].to_s.rjust(5)} ignored, #{state[:error].to_s.rjust(5)} errors, #{file_worklist.size.to_s.rjust(5)} files left, #{url_worklist.size.to_s.rjust(5)} URLs left]"
+  print line
+  state[:clear] = line.length
+end
+
+# Run everything to conclusion
+
+Checklinks::Worklist.close file_worklist, url_worklist
+status_state = status_worklist.close
+links = link_collector.close
+errors = error_collector.close
+
+# Print a report
+
+puts "\r#{' '*status_state[:clear]}\r"
+
+if errors.empty?
+  puts 'Ok!'
+else
+  puts 'Errors:'
+
+  # Create a map of errored URLs back to their locations
+
+  urls = errors.map(&:first).to_set
+  url_links = {}
+  links.each do |url, filename, line_number|
+    next unless urls.include?(url)
+    references = url_links[url]
+    unless references
+      references = []
+      url_links[url] = references
+    end
+    references.push [filename, line_number]
+  end
+
+  # Print each error
+
+  errors.each do |url, code, headers|
+    message = code
+    if (301..302).include?(code)
+      message = "#{code} -> #{headers['location']}"
+    else
+      message = code
+    end
+    puts "  #{url} (#{message})"
+    url_links[url].each do |filename, line_number|
+      puts "    #{filename}:#{line_number}"
+    end
+  end
+end
+
+exit errors.empty?

data/checklinks.gemspec

@@ -0,0 +1,21 @@
+require_relative 'lib/checklinks/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'checklinks'
+  spec.version       = Checklinks::VERSION
+  spec.authors       = ['Chris Seaton']
+  spec.email         = ['chris@chrisseaton.com']
+
+  spec.summary       = 'Find broken links in your static website'
+  spec.homepage      = 'https://github.com/chrisseaton/checklinks/'
+  spec.license       = 'MIT'
+
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0")
+  end
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = '>= 2.5.8'
+end

data/lib/checklinks.rb

@@ -0,0 +1,3 @@
+require 'checklinks/version'
+require 'checklinks/worklist'
+require 'checklinks/awesome'

data/lib/checklinks/awesome.rb

@@ -0,0 +1,105 @@
+module Checklinks
+  module Awesome
+    # This module is code copied from awesome_bot by dkhamsing
+
+    # The MIT License (MIT)
+    #
+    # Copyright (c) 2015 
+    #
+    # Permission is hereby granted, free of charge, to any person obtaining a copy
+    # of this software and associated documentation files (the "Software"), to deal
+    # in the Software without restriction, including without limitation the rights
+    # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    # copies of the Software, and to permit persons to whom the Software is
+    # furnished to do so, subject to the following conditions:
+    #
+    # The above copyright notice and this permission notice shall be included in all
+    # copies or substantial portions of the Software.
+    #
+    # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    # SOFTWARE.
+
+    def self.links_filter(list)
+      list.reject { |x| x.length < 9 }
+        .map do |x|
+          x.gsub(',','%2c').gsub(/'.*/, '').gsub(/,.*/, '')
+        end
+        .map do |x|
+          if x.include? ')]'
+            x.gsub /\)\].*/, ''
+          elsif (x.scan(')').count == 1) && (x.scan('(').count == 1)
+            x
+          elsif (x.scan(')').count == 2) && (x.scan('(').count == 1)
+            x.gsub(/\)\).*/, ')')
+          elsif (x.scan(')').count > 0)
+            if (x.include? 'wikipedia')
+              if (x.scan(')').count >= 1) && (x.scan('(').count == 0)
+                x.gsub(/\).*/, '')
+              else
+                x
+              end
+            else
+              x.gsub(/\).*/, '')
+            end
+          elsif x.include? '[' # adoc
+            x.gsub(/\[.*/, '')
+          elsif x[-1]=='.' || x[-1]==':'
+            x[0..-2]
+          elsif x[-1]=='.'
+            x[0..-2]
+          elsif x[-3..-1]=='%2c'
+            x[0..-4]
+          else
+            x
+          end
+        end
+    end
+
+    def self.net_status(url, timeout=30, head)
+      require 'net/http'
+      require 'openssl'
+      require 'uri'
+
+      uri = URI.parse url
+      Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == 'https', :open_timeout => timeout) do |http|
+        ua = {'User-Agent' => 'checklinks'}
+        if head
+          request = Net::HTTP::Head.new(uri,ua)
+        else
+          request = Net::HTTP::Get.new(uri,ua)
+        end
+
+        if uri.userinfo
+          auth_user, auth_pass = uri.userinfo.split(/:/)
+          request.basic_auth auth_user, auth_pass
+        end
+
+        response = http.request request
+
+        code = response.code==nil ? 200 : response.code.to_i
+
+        headers = {}
+        response.each do |k, v|
+          headers[k] = v.force_encoding("utf-8")
+        end
+
+        # handle incomplete redirect
+        loc = headers['location']
+        unless loc.nil?
+          loc_uri = URI.parse loc
+          if loc_uri.scheme.nil?
+            new_loc = uri.scheme + '://' + uri.host + loc
+            headers['location'] = new_loc
+          end
+        end
+
+        return [code, headers]
+      end
+    end
+  end
+end

data/lib/checklinks/version.rb

@@ -0,0 +1,6 @@
+module Checklinks
+  MAJOR_VERSION = 0
+  MINOR_VERSION = 1
+  PATCH_VERSION = 0
+  VERSION = "#{MAJOR_VERSION}.#{MINOR_VERSION}.#{PATCH_VERSION}"
+end

data/lib/checklinks/worklist.rb

@@ -0,0 +1,79 @@
+module Checklinks
+  # A worklist is a list of jobs to do and possibly a pool of threads
+  # working on it
+  class Worklist
+    # Create a worklist, possibly with a list of jobs to start with
+    def initialize(*jobs)
+      @state = nil
+      @queue = Queue.new
+      @active = Queue.new
+      @processing = false
+      push *jobs
+    end
+
+    # Push more jobs onto the worklist
+    def push(*jobs)
+      jobs.each do |job|
+        raise unless job
+        @queue.push job
+      end
+    end
+
+    # Process the jobs in the worklist in a background thread. You can pass a
+    # state object which is then passed into the block to process each job.
+    def process(state=nil)
+      @state = state
+      process_concurrently(1) do |job|
+        yield job, @state
+      end
+    end
+
+    # Process the jobs in the worklist in multiple concurrent background threads
+    def process_concurrently(n)
+      raise if @threads
+      @threads = n.times.map {
+        Thread.new {
+          until @queue.closed? && @queue.empty?
+            job = @queue.pop
+            next unless job
+            @active.push :active
+            yield job
+            @active.pop
+          end
+        }
+      }
+    end
+
+    # How many jobs are left to process
+    def size
+      # This is a a bit racy - we don't update these two queues atomically
+      @queue.size + @active.size
+    end
+
+    # Declare that you're not going to push any more jobs, and wait for the
+    # current jobs to be processed if you've started that
+    def close
+      @threads.size.times do
+        @queue.push nil
+      end
+      @queue.close
+      @threads.each &:join
+      @state
+    end
+
+    # Close a set of worklists
+    def self.close(*worklists)
+      worklists.map(&:close)
+    end
+
+    # Create a worklist that collects values into an array and returns
+    # it when closed
+    def self.collector
+      worklist = Worklist.new
+      worklist.process([]) do |value, collected|
+        collected.push value
+      end
+      worklist
+    end
+  end
+end

metadata

@@ -0,0 +1,57 @@
+--- !ruby/object:Gem::Specification
+name: checklinks
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Chris Seaton
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-09-13 00:00:00.000000000 Z
+dependencies: []
+description:
+email:
+- chris@chrisseaton.com
+executables:
+- checklinks
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- bin/checklinks
+- checklinks.gemspec
+- lib/checklinks.rb
+- lib/checklinks/awesome.rb
+- lib/checklinks/version.rb
+- lib/checklinks/worklist.rb
+homepage: https://github.com/chrisseaton/checklinks/
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.8
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.7.6.2
+signing_key:
+specification_version: 4
+summary: Find broken links in your static website
+test_files: []