site_health 0.1.0 → 0.2.0

data/Rakefile

@@ -1,6 +1,8 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/bin/console

@@ -1,14 +1,8 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
-require "bundler/setup"
-require "site_health"
+require 'bundler/setup'
+require 'site_health'
 
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
+require 'irb'
 IRB.start(__FILE__)

data/bin/setup

@@ -4,5 +4,3 @@ IFS=$'\n\t'
 set -vx
 
 bundle install
-
-# Do any other automated setup that you need to do here

data/exe/site_health

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'optparse'
+
+# for dev purposes
+require 'bundler/setup' if ENV['SITE_HEALTH_GEM_DEV']
+require 'site_health'
+
+def site_report(url, fields, progress)
+  puts 'severity,title,url' if progress
+  nurse = SiteHealth.check(url) do |n|
+    n.clerk do |clerk|
+      clerk.every_issue do |issue|
+        puts [issue.severity, issue.title, issue.url].join(',') if progress
+      end
+    end
+  end
+  SiteHealth::IssuesReport.new(nurse.issues) do |r|
+    r.fields = fields if fields
+  end
+end
+
+options = {}
+OptionParser.new do |parser|
+  parser.banner = 'Usage: site_health --help'
+  parser.default_argv = ARGV
+
+  parser.on('--url=val0', String, '') do |url|
+    options[:url] = url
+  end
+
+  parser.on('--fields=priority,title,url', Array, 'Issue fields to include - by default all fields are included') do |fields| # rubocop:disable Metrics/LineLength
+    options[:fields] = fields
+  end
+
+  parser.on('--output=result.csv', String, 'Output format, .csv or .json') do |output|
+    options[:output] = output
+  end
+
+  parser.on('--[no-]progress', '') do |progress|
+    options[:progress] = progress
+  end
+
+  parser.on('-h', '--help', 'How to use') do
+    puts parser
+    exit
+  end
+end.parse!
+
+def die(error_klass, message)
+  fail(error_klass, "#{message} - run `site_health --help`")
+end
+
+def perform(output, url, fields, progress, format_method)
+  File.write(
+    output,
+    site_report(url, fields, progress).public_send(format_method)
+  )
+end
+
+url = options.fetch(:url) { die(ArgumentError, '--url is required') }
+output = options.fetch(:output) { die(ArgumentError, '--output is required') }
+progress = options.fetch(:progress, true)
+fields = options.fetch(:fields, nil)
+
+extension = File.extname(output)
+case extension
+when '.csv' then perform(output, url, fields, progress, :to_csv)
+when '.json' then perform(output, url, fields, progress, :to_json)
+when ''
+  die(ArgumentError, "missing file extension from '#{output}'")
+else
+  die(ArgumentError, "unknown file extension '#{extension}'")
+end

data/lib/site_health.rb

@@ -1,136 +1,112 @@
-require "spidr"
-require 'w3c_validators'
-require "site_health/version"
+# frozen_string_literal: true
 
-require "site_health/key_struct"
+require 'logger'
+require 'spidr'
 
-require 'site_health/journals/css_journal'
-require 'site_health/journals/html_journal'
-require 'site_health/journals/xml_journal'
-require 'site_health/journals/w3c_journal'
+require 'site_health/version'
+require 'site_health/configuration/configuration'
 
-require "site_health/checkers/css_page"
-require "site_health/checkers/html_page"
-require "site_health/checkers/xml_page"
+require 'site_health/key_struct'
+require 'site_health/url_map'
+require 'site_health/link'
 
+require 'site_health/checkers/checker'
+require 'site_health/nurse'
+require 'site_health/issues_report'
+
+# Top-level module/namespace
 module SiteHealth
-  def self.check(site)
-    Check.call(site: site)
+  def self.require_optional_dependency(path, gem_name: nil)
+    gem_name ||= path
+    require path
+  rescue LoadError => e
+    message_parts = [
+      e.message,
+      "unable to require file from '#{gem_name}' gem",
+      'please install it',
+    ]
+    raise(LoadError, message_parts.join(' -- '))
   end
 
-  class Check
-    def self.call(**args)
-      new(**args).call
-    end
+  # @param [Checker] klass that inherits from Checker
+  # @return [see SiteHealth#registered_checkers]
+  def self.register_checker(klass)
+    registered_checkers[klass.name.to_sym] = klass
+    registered_checkers
+  end
 
-    BrokenLinkJournal = KeyStruct.new(:url, :exists_on)
+  # @return [Hash] all registered checkers
+  def self.registered_checkers
+    @checkers ||= {}
+  end
 
-    HTTPCodeJournal = KeyStruct.new(:url, :code)
-    class HTTPCodeJournal
-      def error?
-        code >= 400
-      end
+  # @param [Symbol, String] name of the checker to be loaded
+  # @return [Checker] loaded class that should inherits from Checker
+  def self.load_checker(name)
+    name_key = name.to_sym
+    registered_checkers.fetch(name_key) do
+      require "site_health/checkers/#{name}"
+      registered_checkers[name_key]
     end
+  end
 
-    ChecksJournal = KeyStruct.new(
-      :missing_html_title,
-      :broken_urls,
-      :http_error_urls,
-      :html_error_urls,
-      :html_warning_urls,
-      :xml_error_urls,
-      :css_error_urls,
-      :css_warning_urls
-    )
-
-    attr_reader :site
-
-    def initialize(site:)
-      @site = site
+  # @param [String] site to be checked
+  # @param config [SiteHealth::Configuration] the configuration to use
+  # @yieldparam [SiteHealth::Nurse] nurse (a.k.a agent)
+  # @return [Hash] journal data
+  # @see Nurse#journal
+  def self.check(site, config: SiteHealth.config)
+    nurse = Nurse.new(config: config)
+    yield(nurse) if block_given?
+
+    Spidr.site(site) do |spider|
+      spider.every_failed_url { |url| nurse.check_failed_url(url) }
+      spider.every_page { |page| nurse.check_page(page) }
     end
 
-    def call
-      url_map = Hash.new { |hash, key| hash[key] = [] }
-
-      missing_html_title = []
-      http_error_urls = []
-      html_error_urls = []
-      html_warning_urls = []
-      xml_error_urls = []
-      css_error_urls = []
-      css_warning_urls = []
-
-      spider = Spidr.site(site) do |spider|
-        spider.every_link do |origin, destination|
-          url_map[destination] << origin
-        end
-
-        spider.every_page do |page|
-          code_journal = HTTPCodeJournal.new(url: page.url, code: page.code)
-          http_error_urls << code_journal if code_journal.error?
-
-          if page.css?
-            result = Checkers::CSSPage.check(page)
-            xml_error_urls << result if result.errors?
-          end
-
-          if page.xml?
-            result = Checkers::XMLPage.check(page)
-            xml_error_urls << result if result.errors?
-          end
-
-          if page.html?
-            result = Checkers::HTMLPage.check(page)
-            missing_html_title << result if result.missing_title?
-            html_error_urls << result if result.errors?
-          end
-        end
-      end
+    nurse.punch_out!
+  end
 
-      http_error_urls = map_http_error_urls(http_error_urls, url_map)
-      broken_urls = broken_links(spider, url_map) + http_error_urls
-
-      ChecksJournal.new(
-        missing_html_title: missing_html_title,
-        broken_urls: broken_urls,
-        http_error_urls: http_error_urls,
-        html_error_urls: html_error_urls,
-        html_warning_urls: html_warning_urls,
-        xml_error_urls: xml_error_urls,
-        css_error_urls: css_error_urls,
-        css_warning_urls: css_warning_urls
-      )
-    end
+  # @param [Array<String>, String] urls to be checked
+  # @param config [SiteHealth::Configuration] the configuration to use
+  # @yieldparam [SiteHealth::Nurse] nurse (a.k.a agent)
+  # @return [Hash] journal data
+  # @see Nurse#journal
+  def self.check_urls(urls, config: SiteHealth.config)
+    nurse = Nurse.new(config: config)
+    yield(nurse) if block_given?
 
-    def validate_css_page(page, errors)
-      css_checker = Checkers::CSSPage.new(page)
-      result = css_checker.check
-      return unless result.errors?
+    agent = Spidr::Agent.new
 
-      result
-    end
+    Array(urls).each do |url|
+      page = agent.get_page(url)
 
-    def map_http_error_urls(urls, url_map)
-      urls.map do |failed_url|
-        BrokenLinkJournal.new(url: failed_url, exists_on: url_map[failed_url])
+      if page.nil?
+        nurse.check_failed_url(url)
+        next
       end
-    end
 
-    # Finds all pages which have broken links:
-    def broken_links(spider, url_map)
-      # FIXME: spider#failures only returns timeout errors etc and not HTTP error status codes..
-      #        so we need to have 2 types of "failed" URLs
-      spider.failures.map do |failed_url|
-        BrokenLinkJournal.new(url: failed_url, exists_on: url_map[failed_url])
-      end
+      nurse.check_page(page)
     end
 
-    # @return [W3CValidators::Results]
-    # @raise [W3CValidators::ValidatorUnavailable] the service is offline or returns 400 Bad Request
-    # @see https://github.com/w3c-validators/w3c_validators/issues/39 we really want to use #validate_text instead of #validate_uri but due to the linked issue thats not possible
-    def validate_html(html_url)
-      validator = W3CValidators::NuValidator.new
-      validator.validate_uri(html_url)
-    end
+    nurse.punch_out!
+  end
+
+  # @see Configuration#logger
+  def self.logger
+    config.logger
+  end
+
+  # @return [Configuration] the current configuration
+  # @yieldparam [Configuration] the current configuration
+  def self.configure
+    @configuration ||= Configuration.new
+    yield(@configuration) if block_given?
+    @configuration
+  end
+
+  # @return [Configuration] the current configuration
+  def self.config
+    configure
   end
 end

data/lib/site_health/check_data.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module SiteHealth
+  class CheckData
+    include Enumerable
+
+    def initialize
+      @data = {}
+    end
+
+    def [](key)
+      @data[key]
+    end
+
+    # Adds data
+    # @param [Hash] the hash to be added
+    # @return [Hash] the current data
+    def add(hash)
+      @data.merge!(hash)
+    end
+
+    # @return [TrueClass, FalseClass] true if there is no data
+    def empty?
+      @data.empty?
+    end
+
+    def each(&block)
+      @data.each(&block)
+    end
+
+    def to_h
+      @data.to_h
+    end
+  end
+end

data/lib/site_health/checkers/checker.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require 'site_health/check_data'
+require 'site_health/issues'
+require 'site_health/issue'
+
+module SiteHealth
+  # Parent class for all checkers (all checkers must inheirit from this class)
+  class Checker
+    # All possible page types that can be checked
+    CHECKABLE_TYPES = %i[
+      plain_text
+      directory
+      xsl
+      rss
+      atom
+      ms_word
+      pdf
+      zip
+      javascript
+      json
+      css
+      xml
+      html
+    ].freeze
+
+    def self.name(name = '__get_value__')
+      if name == '__get_value__'
+        return @name if @name
+
+        @name = (super() || SecureRandom.hex).downcase.gsub(/sitehealth::/, '')
+        return @name
+      end
+
+      @name = name.to_s
+    end
+
+    def self.types(types = '__get_value__')
+      if types == '__get_value__'
+        @types ||= CHECKABLE_TYPES
+        return @types
+      end
+
+      @types = Array(types).map(&:to_sym)
+    end
+
+    # @param [Hash] types
+    # the issues data - optional, if not present it will return the current data
+    # @return [Hash] the issues types data
+    def self.issue_types(types = :__get_value__)
+      if types == :__get_value__
+        return @issue_types ||= {}
+      end
+
+      default = types.fetch(:_default, {})
+      @issue_types = types.map do |key, data|
+        issue_data = { code: key }.merge!(default).merge!(data)
+        [key, issue_data]
+      end.to_h
+    end
+
+    attr_reader :page, :config, :logger, :issues, :data
+
+    # @param [Spidr::Page] page the crawled page
+    # @param config [SiteHealth::Configuration]
+    def initialize(page, config: SiteHealth.config)
+      @page = page
+      @config = config
+      @logger = config.logger
+      @issues = Issues.new(name)
+      @data = CheckData.new
+    end
+
+    # Run the checker
+    # @yieldparam [Checker] yields self
+    # @return [CheckerResult] returns self
+    def call
+      timer = Timer.measure { check }
+      add_data(
+        started_at: timer.started_at,
+        finished_at: timer.finished_at,
+        runtime_in_seconds: timer.diff.to_f
+      )
+      yield(self) if block_given?
+      self
+    end
+
+    # @return [String] the page URL
+    def url
+      page.url
+    end
+
+    # @return [String] the name of the checker
+    def name
+      self.class.name
+    end
+
+    # @return [Array<Symbol>] list of page types the checker will run on
+    def types
+      self.class.types
+    end
+
+    # @return [Hash] issue types data
+    def issue_types
+      self.class.issue_types
+    end
+
+    # @return [Boolean] determines whether the checker should run
+    def should_check?
+      types.any? { |type| page.public_send("#{type}?") }
+    end
+
+    # Adds an issue
+    # @return [Array<Issue>] the current list of issues
+    # @see Issue#initialize for supported arguments
+    def add_issue(**args)
+      issues << Issue.new({ name: name, url: page.url }.merge!(**args))
+    end
+
+    def add_issue_type(type, **args)
+      data = issue_types.fetch(type) do
+        raise(ArgumentError, "unknown issue type #{type}, known types are: #{issue_types.keys.join(', ')}") # rubocop:disable Metrics/LineLength
+      end
+
+      add_issue(data.merge(**args))
+    end
+
+    # Adds data
+    # @param [Hash] the hash to be added
+    # @return [Hash] the current data
+    def add_data(hash)
+      data.add(hash)
+    end
+
+    # @return [Hash] hash representation of the object
+    def to_h
+      {
+        name: name.to_sym,
+        data: data.to_h,
+        issues: issues.map(&:to_h),
+      }
+    end
+
+    protected
+
+    # Abstract method that subclasses must implement
+    # @raise [NotImplementedError] subclasses must implement
+    def check
+      raise(NotImplementedError, 'please implement!')
+    end
+  end
+end