sla 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20e5f6a05020d9aecc2ad6754a7a22cee70e18f45e3bc5f679daabb2042c749a
-  data.tar.gz: 2d5b0290dd9dd25169bbb89ff20df4aea66db616ea65585a56a982866579f568
+  metadata.gz: 408ec43b2728bd5d2a1461b747b487506976e020894684d8270154a5e9b807a1
+  data.tar.gz: 8620e841cde6cc02ba87e2c35a32d278c197f65363e85fad6fd4adec3adcbf1e
 SHA512:
-  metadata.gz: cc586125c98e82786f0c747fcdf9d21f45d933564e4aba416036d21a39a70c72bdb974665c8e6a6f5c5b17a4ad323a52bb896205f4e5bb7e49002dd4370b1d84
-  data.tar.gz: 10fadfccf9da8a7b427f6512bfb6365f890dc70cc01646280a5de51aeebee38026942dc78b30e1c5dac9196015e3e9565a804bf13e8c9c3c7c2bed00c9a5ca49
+  metadata.gz: d7b55730a4502b09df9c59ca8a3f0b49aa2986a2b6b548f8f26e8a8e5650075497acaef75f020e93354eb2f33d0538c6176ad3d3d070e11b0600e8ca6eddab34
+  data.tar.gz: 94eeaa6c32a22a8f894bf6c65b0b74084120a071897028d4888952ac7d7dd28d10dcfe289a5230bd053c06b2a307ee32ddcf96e50f8bfc900c94eaa03b60a4ca

data/bin/sla

@@ -1,10 +1,18 @@
 #!/usr/bin/env ruby
-
 require 'sla'
+include Colsole
+
+router = MisterBin::Runner.new handler: SLA::Command
 
 begin
-  SLA::CommandLine.execute ARGV
+  exit router.run ARGV
+
 rescue SLA::BrokenLinks => e
   puts "(exit code 1)"
   exit 1
+
+rescue Interrupt
+  puts "\nGoodbye"
+  exit 1
+
 end

data/lib/sla.rb

@@ -2,14 +2,20 @@ require 'webcache'
 require 'nokogiri'
 require 'colsole'
 require 'singleton'
-require 'super_docopt'
+require 'mister_bin'
 require 'uri'
 
 require 'sla/version'
 require 'sla/exceptions'
-require 'sla/link'
+require 'sla/page'
 require 'sla/checker'
-require 'sla/command_line'
+
+require 'sla/formatters/base'
+require 'sla/formatters/verbose'
+require 'sla/formatters/simple'
+require 'sla/formatters/tty'
+
+require 'sla/command'
 
 if ENV['BYEBUG']
   # :nocov:

data/lib/sla/checker.rb

@@ -1,42 +1,60 @@
 module SLA
   class Checker
-    include Colsole
+    attr_reader :max_depth, :ignore, :check_external
 
-    attr_accessor :max_depth, :checked_links, :check_external, :ignore
+    def initialize(max_depth: 5, ignore: nil, check_external: false)
+      @max_depth = max_depth
+      @ignore = ignore
+      @check_external = check_external
+    end
 
-    def initialize
-      @max_depth = 10
-      @checked_links = []
-      @check_external = false
-      @ignore = []
+    def deeply_checked
+      @deeply_checked ||= {}
     end
 
-    def count
-      checked_links.count
+    def checked
+      @checked ||= {}
     end
 
-    def check(link, depth=1, &block)
-      link = Link.new link, depth: depth if link.is_a? String
+    def check(page, &block)
+      return if ignore? page
+      return if page.depth >= max_depth
+      return unless page.valid?
+
+      yield [:source, page] if block_given?
 
-      return if link.external? && !check_external
-      ignore.each do |ignored|
-        return if link.ident.start_with? ignored
+      pages = page.pages
+      pages.reject! { |page| page.external? } if !check_external
+
+      pages.each do |page|
+        if checked.has_key? page.url or ignore? page
+          yield [:skip, page] if block_given?
+        else
+          checked[page.url] = true
+          yield [:check, page] if block_given?
+        end
       end
 
-      link.validate
-      yield link if block_given?
+      pages.each do |page|
+        next if deeply_checked.has_key? page.url
+        deeply_checked[page.url] = true
+        next if page.external?
+        check page, &block
+      end
+    end
 
-      return if checked_links.include? link.url
+  private
 
-      checked_links.push link.url
+    def ignore?(page)
+      return false unless ignore
 
-      return if link.external?
-      return unless link.valid?
-      return if depth >= max_depth
-      
-      link.sublinks.each do |sublink|
-        check sublink, depth+1, &block
+      ignore.each do |text|
+        return true if page.url.include? text
       end
+
+      false
     end
+
   end
-end
+end
+

data/lib/sla/command.rb

@@ -0,0 +1,70 @@
+module SLA
+  class Command < MisterBin::Command
+    include Colsole
+
+    help "Site Link Analyzer"
+
+    version VERSION
+
+    usage "sla URL [options]"
+    usage "sla --help | -h | --version"
+
+    param "URL", "URL to scan"
+
+    option "--verbose, -v", "Show detailed output"
+    option "--simple, -s", "Show simple output of errors only"
+    option "--depth, -d DEPTH", "Set crawling depth [default: 5]"
+    option "--external, -x", "Also check external links"
+    option "--ignore, -i URLS", "Specify a list of space delimited patterns to skip\nURLs that contain any of the strings in this list will be skipped"
+    option "--cache, -c LIFE", "Set cache life [default: 1d]. LIFE can be in any of the following formats:\n  10  = 10 seconds\n  20s = 20 seconds\n  10m = 10 minutes\n  10h = 10 hours\n  10d = 10 days"
+    option "--cache-dir DIR", "Set the cache directory"
+
+    example "sla example.com"
+    example "sla example.com -c10m -d10"
+    example "sla example.com --cache-dir my_cache"
+    example "sla example.com --depth 10"
+    example "sla example.com --cache 30d --external"
+    example "sla example.com --simple > out.log"
+    example "sla example.com --ignore \"/admin /customer/login\""
+
+    environment "SLA_SLEEP", "Set number of seconds to sleep between calls (for debugging purposes)"
+
+    def run
+      WebCache.life = args['--cache']
+      WebCache.dir  = args['--cache-dir'] if args['--cache-dir']
+
+      max_depth = args['--depth'].to_i
+      url = args['URL']
+      ignore = args['--ignore']
+      ignore = ignore.split " " if ignore
+      check_external = args['--external']
+
+      checker = Checker.new max_depth: max_depth,
+        ignore: ignore, check_external: check_external
+
+      formatter = if args['--verbose'] 
+        Formatters::Verbose.new
+      elsif args['--simple']
+        Formatters::Simple.new
+      else
+        Formatters::TTY.new
+      end
+
+      execute url, checker, formatter
+    end
+
+    def execute(url, checker, formatter)
+      page = Page.new url
+      checker.check page do |action, page|
+        success = formatter.handle action, page
+        sleep ENV['SLA_SLEEP'].to_f if ENV['SLA_SLEEP']
+      end
+
+      formatter.footer
+
+      unless formatter.success? or ENV['SLA_ALLOW_FAILS']
+        raise BrokenLinks
+      end
+    end
+  end
+end

data/lib/sla/formatters/base.rb

@@ -0,0 +1,34 @@
+module SLA
+  module Formatters
+    class Base
+      include Colsole
+
+      attr_accessor :count, :failed
+
+      def initialize
+        @count = 0
+        @failed = 0
+      end
+
+      def success?
+        failed == 0
+      end
+
+      def handle(action, page)
+        # :nocov:
+        raise NotImplementedError
+        # :nocov:
+      end
+
+      def footer_prefix
+        "\n"
+      end
+
+      def footer
+        color = success? ? '!txtgrn!' : '!txtred!'
+        say "#{footer_prefix}#{color}Checked #{count} pages with #{failed} failures"
+      end
+
+    end
+  end
+end

data/lib/sla/formatters/simple.rb

@@ -0,0 +1,26 @@
+module SLA
+  module Formatters
+    class Simple < Base
+      attr_reader :last_source
+
+      def handle(action, page)
+        @last_source = page.url if action == :source
+
+        return unless action == :check
+        @count += 1
+
+        return if page.valid?
+
+        @failed += 1
+
+        if last_source
+          say "!txtpur!SOURCE  #{last_source}"
+          @last_source = nil
+        end
+
+        say "  !txtred!FAIL!txtrst!  #{page.depth}  #{page.url}"
+      end
+
+    end
+  end
+end

data/lib/sla/formatters/tty.rb

@@ -0,0 +1,43 @@
+module SLA
+  module Formatters
+    class TTY < Base
+      attr_reader :last_source
+
+      def handle(action, page)
+        screen_width = terminal_width
+
+        @last_source = page.url if action == :source
+
+        return unless action == :check
+        @count += 1
+
+        if page.valid?
+          status = "PASS"
+          color = "!txtgrn!"
+        else
+          @failed += 1
+          status = "FAIL"
+          color = "!txtred!"
+
+          if last_source
+            resay "!txtpur!SOURCE  #{last_source}"
+            @last_source = nil
+          end
+
+          resay "  !txtred!FAIL!txtrst!  #{page.depth}  #{page.url}"
+        end
+
+        message = "[#{failed}/#{count} @ #{page.depth}] #{status}"
+        remaining_width = screen_width - message.size - 4
+        url = page.url[0..remaining_width]
+        resay "[#{failed}/#{count} @ #{page.depth}] #{color}#{status}!txtrst! #{url} "
+      end
+
+      def footer_prefix
+        terminal? ? "\033[2K\n" : "\n"
+      end
+
+
+    end
+  end
+end

data/lib/sla/formatters/verbose.rb

@@ -0,0 +1,30 @@
+module SLA
+  module Formatters
+    class Verbose < Base
+      def handle(action, page)
+        case action
+        when :source
+          say "\n!txtpur!SOURCE  #{page.url}"
+        
+        when :check
+          @count += 1
+
+          if page.valid?
+            status = "PASS"
+            color = "!txtgrn!"
+          else
+            @failed += 1
+            status = "FAIL"
+            color = "!txtred!"
+          end
+
+          say "  #{color}#{status}!txtrst!  #{page.depth}  #{page.url}"
+
+        when :skip
+          say "  !txtblu!SKIP!txtrst!  #{page.depth}  #{page.url}"
+
+        end
+      end
+    end
+  end
+end

data/lib/sla/page.rb

@@ -0,0 +1,83 @@
+class Page
+  attr_reader :uri, :parent, :depth
+
+  def initialize(uri, parent: nil, depth: 0)
+    if uri.is_a? String
+      uri = "http://#{uri}" unless uri.start_with? 'http'
+      uri = URI.parse uri
+      uri.fragment = false
+    end
+
+    @uri, @parent, @depth = uri, parent, depth
+  end
+
+  def error
+    response.error
+  end
+
+  def external?
+    byebug unless uri.respond_to? :host
+    uri.host != parent.uri.host
+  end
+
+  def inspect
+    "#<Page url: #{url}, depth: #{depth}>"
+  end
+
+  def pages
+    @pages ||= pages!
+  end
+
+  def url
+    uri.to_s
+  end
+
+  def valid?
+    !response.error
+  end
+
+private
+
+  def anchors
+    @anchors ||= dom.css('a[href]')
+  end
+
+  def content
+    @content ||= response.content
+  end
+
+  def dom
+    @dom ||= Nokogiri::HTML content
+  end
+
+  def normalize_url(new_url)
+    new_url = URI.parse new_url
+    new_url.fragment = false
+
+    result = new_url.absolute? ? new_url : URI.join(url, new_url)
+
+    result.scheme =~ /^http/ ? result.to_s : nil
+  end
+
+  def pages!
+    result = {}
+    anchors.each do |a|
+      url = normalize_url a['href']
+      next unless url
+      page = Page.new url, parent: self, depth: depth+1
+      result[url] = page
+    end
+    result.values
+  end
+
+  def response
+    @response ||= response!
+  end
+
+  def response!
+    response = WebCache.get url
+    @uri = response.base_uri
+    response
+  end
+
+end

data/lib/sla/version.rb

@@ -1,3 +1,3 @@
 module SLA
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sla
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Danny Ben Shitrit
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-26 00:00:00.000000000 Z
+date: 2019-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colsole
@@ -17,6 +17,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -24,48 +27,51 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.1
 - !ruby/object:Gem::Dependency
-  name: super_docopt
+  name: mister_bin
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.7'
 - !ruby/object:Gem::Dependency
   name: webcache
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.10'
 description: Check for broken links on a website
 email: db@dannyben.com
 executables:
@@ -77,10 +83,13 @@ files:
 - bin/sla
 - lib/sla.rb
 - lib/sla/checker.rb
-- lib/sla/command_line.rb
-- lib/sla/docopt.txt
+- lib/sla/command.rb
 - lib/sla/exceptions.rb
-- lib/sla/link.rb
+- lib/sla/formatters/base.rb
+- lib/sla/formatters/simple.rb
+- lib/sla/formatters/tty.rb
+- lib/sla/formatters/verbose.rb
+- lib/sla/page.rb
 - lib/sla/version.rb
 homepage: https://github.com/DannyBen/sla
 licenses:

data/lib/sla/command_line.rb

@@ -1,83 +0,0 @@
-module SLA
-  class CommandLine < SuperDocopt::Base
-    include Colsole
-
-    version VERSION
-    docopt File.expand_path 'docopt.txt', __dir__
-    subcommands ['check']
-
-    def before_execute
-      WebCache.life = args['--cache']
-      WebCache.dir  = args['--cache-dir'] if args['--cache-dir']
-    end
-
-    def check
-      checker = Checker.new
-      checker.max_depth = args['--depth'].to_i
-      logfile = args['--log']
-      start_url = args['DOMAIN']
-      ignore = args['--ignore']
-      ignore = ignore.split " " if ignore
-      screen_width = terminal_width
-      
-      checker.check_external = args['--external']
-      checker.ignore = ignore if ignore
-
-      start_url = "http://#{start_url}" unless start_url[0..3] == 'http'
-
-      File.unlink logfile if logfile and File.exist? logfile
-
-      count = 1
-      failed = 0
-
-      log = []
-
-      checker.check start_url do |link|
-        status = link.status
-        colored_status = color_status status
-        if status != '200'
-          failed +=1 
-          resay "#{colored_status} #{link.ident}"
-          log.push "#{status}  #{link.ident}" if logfile
-        end
-
-        message = "[#{failed}/#{count} @ #{link.depth}] #{status}"
-        remaining_width = screen_width - message.size - 4
-        trimmed_link = link.ident[0..remaining_width]
-        
-        resay "[#{failed}/#{count} @ #{link.depth}] #{colored_status} #{trimmed_link} "
-        count += 1
-
-        sleep ENV['SLA_SLEEP'].to_f if ENV['SLA_SLEEP']
-      end
-
-      color = failed > 0 ? '!txtred!' : '!txtgrn!'
-      resay "#{color}Done checking #{count} links with #{failed} failures"
-
-      if logfile      
-        logstring = log.join("\n") + "\n"
-        File.write logfile, logstring
-      end
-
-      if failed > 0 and !ENV['SLA_ALLOW_FAILS']
-        raise BrokenLinks 
-      end
-    end
-
-  private
-
-    def color_status(status)
-      case status
-      when '200'
-        '!txtgrn!200!txtrst!'
-      when '404'
-        '!txtred!404!txtrst!'
-      else
-        # :nocov:
-        status
-        # :nocov:
-      end
-    end
-
-  end
-end

data/lib/sla/docopt.txt

@@ -1,43 +0,0 @@
-SLA
-
-Usage:
-  sla check DOMAIN [options]
-  sla (-h|--help|--version)
-
-Commands:
-  check
-    Start checking for broken links on a given domain.
-  
-Options:
-  --depth, -d DEPTH
-    Set crawling depth [default: 5].
-
-  --cache, -c LIFE
-    Set cache life [default: 1d]. LIFE can be in any of the 
-    following formats:
-      10  = 10 seconds
-      20s = 20 seconds
-      10m = 10 minutes
-      10h = 10 hours
-      10d = 10 days
-
-  --cache-dir DIR
-    Set the cache directory.
-
-  --external, -x
-    Also check external links.
-
-  --log, -l LOGFILE
-    Save errors to log file.
-
-  --ignore, -i URLS
-    Specify a list of space delimited URLs to skip.
-    URLs that start with the strings in this list will be skipped.
-
-Examples:
-  sla check example.com
-  sla check example.com -c10m -d10
-  sla check example.com --cache-dir my_cache
-  sla check example.com --depth 10 --log my_log.log
-  sla check example.com --cache 30d
-  sla check example.com --ignore "/admin /customer/login"

data/lib/sla/link.rb

@@ -1,84 +0,0 @@
-module SLA
-  class Link
-    attr_accessor :text, :href, :status, :depth, :real_uri
-    attr_reader :parent
-
-    def initialize(href, opts={})
-      @href   = href
-      @text   = opts[:text]
-      @depth  = opts[:depth] || 1
-      self.parent = opts[:parent] || @href
-    end
-
-    def valid?
-      validate
-      status == '200'
-    end
-
-    def validate
-      content
-    end
-
-    def content
-      @content ||= content!
-    end
-
-    def content!
-      response = WebCache.get url
-      @status = response.error ? '404' : '200'
-      @real_uri = response.base_uri
-      response.content
-    end
-
-    def ident
-      external? ? full_uri.to_s : full_uri.request_uri
-    end
-
-    def url
-      full_uri.to_s
-    end
-
-    def doc
-      @doc ||= Nokogiri::HTML content
-    end
-
-    def sublinks
-      @sublinks ||= sublinks!
-    end
-
-    def sublinks!
-      anchors = doc.css('a[href]')
-      result = []
-      anchors.each do |a|
-        link = Link.new a['href'], text: a.text, parent: real_uri, depth: depth+1
-        result.push link if link.relevant?
-      end
-      result
-    end
-
-    def uri
-      @uri ||= URI.parse href
-    end
-
-    def parent=(url)
-      @parent = url.is_a?(String) ? URI.parse(url) : url
-    end
-
-    def path
-      uri.request_uri
-    end
-
-    def full_uri
-      return uri if uri.absolute? || !parent.absolute?
-      URI.join parent, href
-    end
-
-    def external?
-      parent.host != full_uri.host
-    end
-
-    def relevant?
-      full_uri.scheme =~ /^http/
-    end
-  end
-end