yuno-proofreader 0.1.0

data/lib/yuno/proofreader/http/client.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "openssl"
+require "timeout"
+require "uri"
+require "zlib"
+
+require_relative "net_http_transport"
+
+module Yuno
+  module Proofreader
+    module Http
+      PRODUCT_TOKEN = "yuno-proofreader"
+      DEFAULT_USER_AGENT = "#{PRODUCT_TOKEN}/#{VERSION} (+https://gitlab.com/alsen-ca/yuno-proofreader)".freeze
+
+      Settings = Data.define(:user_agent, :delay, :max_redirects, :timeout, :max_bytes) do
+        def initialize(user_agent: DEFAULT_USER_AGENT, delay: 0.5, max_redirects: 3,
+                       timeout: 30, max_bytes: 5_000_000)
+          super
+        end
+      end
+
+      Response = Data.define(:url, :status, :body, :content_type) do
+        def success? = status.between?(200, 299)
+      end
+
+      # Fetches web pages politely: it says who it is, waits between requests, follows a few redirects and
+      # never reads more than max_bytes. It returns the answer for any HTTP status, error statuses included,
+      # and raises FetchError only when no answer could be obtained.
+      class Client
+        REDIRECTS = [301, 302, 303, 307, 308].freeze
+        ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5"
+        NETWORK_ERRORS = [
+          SocketError, SystemCallError, IOError, Timeout::Error, OpenSSL::SSL::SSLError,
+          Net::ProtocolError, Net::HTTPBadResponse, Zlib::Error
+        ].freeze
+        REAL_SLEEP = ->(seconds) { sleep(seconds) }
+        REAL_CLOCK = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+
+        def initialize(settings: Settings.new, transport: NetHttpTransport.new,
+                       sleeper: REAL_SLEEP, clock: REAL_CLOCK)
+          @settings = settings
+          @transport = transport
+          @sleeper = sleeper
+          @clock = clock
+          @last_request_at = nil
+        end
+
+        def get(url)
+          uri = parse(url)
+          (@settings.max_redirects + 1).times do
+            pace
+            raw = request(uri)
+            target = redirect_target(uri, raw)
+            return response_from(uri, raw) unless target
+
+            uri = target
+          end
+          raise FetchError, "Too many redirects for #{url}"
+        end
+
+        private
+
+        def parse(url)
+          uri = URI.parse(url.to_s.strip)
+          return uri if %w[http https].include?(uri.scheme) && !uri.host.to_s.empty?
+
+          raise FetchError, "Not a web address: #{url.inspect}"
+        rescue URI::InvalidURIError
+          raise FetchError, "Not a valid web address: #{url.inspect}"
+        end
+
+        # Waits so that at least settings.delay seconds pass between two requests.
+        def pace
+          now = @clock.call
+          if @last_request_at
+            wait = @settings.delay - (now - @last_request_at)
+            if wait.positive?
+              @sleeper.call(wait)
+              now += wait
+            end
+          end
+          @last_request_at = now
+        end
+
+        def request(uri)
+          raw = @transport.call(uri, headers: request_headers, **limits)
+          raw.with(headers: raw.headers.transform_keys { |name| name.to_s.downcase })
+        rescue *NETWORK_ERRORS => e
+          raise FetchError, "Could not fetch #{uri}: #{e.message}"
+        end
+
+        def request_headers
+          { "User-Agent" => @settings.user_agent, "Accept" => ACCEPT }
+        end
+
+        def limits
+          { timeout: @settings.timeout, max_bytes: @settings.max_bytes }
+        end
+
+        def redirect_target(uri, raw)
+          return unless REDIRECTS.include?(raw.status)
+
+          location = raw.headers["location"].to_s.strip
+          raise FetchError, "#{uri} redirects without saying where to" if location.empty?
+
+          parse(URI.join(uri.to_s, location).to_s)
+        rescue URI::Error
+          raise FetchError, "#{uri} redirects to an invalid address"
+        end
+
+        def response_from(uri, raw)
+          type = raw.headers["content-type"].to_s
+          Response.new(url: uri.to_s, status: raw.status, body: decode(raw.body, type), content_type: type)
+        end
+
+        # The body as clean UTF-8: the charset from the Content-Type header if there is one, otherwise UTF-8.
+        def decode(body, content_type)
+          charset = content_type[/charset=["']?([\w-]+)/i, 1]
+          text = body.dup
+          return text.force_encoding(Encoding::UTF_8).scrub if charset.nil? || charset.casecmp?("utf-8")
+
+          text.force_encoding(charset).encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+        rescue ArgumentError, EncodingError
+          body.dup.force_encoding(Encoding::UTF_8).scrub
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/http/net_http_transport.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+
+module Yuno
+  module Proofreader
+    module Http
+      # What a transport returns: the raw status, the headers (lowercase names) and the body.
+      RawResponse = Data.define(:status, :headers, :body)
+
+      # Sends one GET request with Net::HTTP and reads at most max_bytes of the answer.
+      # It follows no redirects and repairs no encodings. The client does both.
+      class NetHttpTransport
+        def call(uri, headers:, timeout:, max_bytes:)
+          raw = nil
+          http(uri, timeout).request(Net::HTTP::Get.new(uri, headers)) do |response|
+            raw = RawResponse.new(status: response.code.to_i, headers: headers_of(response),
+                                  body: read_limited(response, max_bytes))
+          end
+          raw
+        end
+
+        private
+
+        def http(uri, timeout)
+          Net::HTTP.new(uri.host, uri.port).tap do |http|
+            http.use_ssl = uri.scheme == "https"
+            http.open_timeout = timeout
+            http.read_timeout = timeout
+          end
+        end
+
+        def headers_of(response)
+          response.to_hash.transform_values { |values| values.join(", ") }
+        end
+
+        def read_limited(response, max_bytes)
+          raise too_large(max_bytes) if response["content-length"].to_i > max_bytes
+
+          buffer = String.new(encoding: Encoding::BINARY)
+          response.read_body do |chunk|
+            buffer << chunk
+            raise too_large(max_bytes) if buffer.bytesize > max_bytes
+          end
+          buffer
+        end
+
+        def too_large(max_bytes)
+          FetchError.new("The answer is larger than #{max_bytes} bytes")
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/http/robots.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Yuno
+  module Proofreader
+    module Http
+      # The rules of a robots.txt for one crawler name. Rules are matched the way search engines do:
+      # the longest matching pattern wins and Allow wins a tie. Percent-encoding is not normalized.
+      class Robots
+        Rule = Data.define(:allow, :pattern, :regexp)
+
+        def self.allow_all = new([])
+
+        # Uses the group that names this crawler, or else the group for all crawlers (*).
+        def self.parse(text, agent:)
+          groups = groups_in(text.to_s)
+          name = agent.to_s.downcase
+          chosen = groups.select { |group| group[:agents].include?(name) }
+          chosen = groups.select { |group| group[:agents].include?("*") } if chosen.empty?
+          new(chosen.flat_map { |group| group[:rules] }.filter_map { |allow, value| rule(allow, value) })
+        end
+
+        def self.groups_in(text)
+          groups = []
+          after_agent = false
+          fields_in(text).each do |field, value|
+            if field == "user-agent"
+              groups << { agents: [], rules: [] } unless after_agent
+              groups.last[:agents] << value.downcase
+              after_agent = true
+            elsif %w[allow disallow].include?(field)
+              groups.last[:rules] << [field == "allow", value] unless groups.empty?
+              after_agent = false
+            end
+          end
+          groups
+        end
+
+        # The "field: value" pairs of the file, without comments and blank or malformed lines.
+        def self.fields_in(text)
+          text.each_line.filter_map do |line|
+            field, value = line.sub(/#.*/, "").strip.split(/\s*:\s*/, 2)
+            [field.downcase, value] if field && value
+          end
+        end
+
+        def self.rule(allow, value)
+          return if value.empty?
+
+          source = value.chomp("$").split("*", -1).map { |part| Regexp.escape(part) }.join(".*")
+          anchor = value.end_with?("$") ? "\\z" : ""
+          Rule.new(allow:, pattern: value, regexp: Regexp.new("\\A#{source}#{anchor}"))
+        end
+        private_class_method :groups_in, :fields_in, :rule
+
+        def initialize(rules)
+          @rules = rules
+        end
+
+        def allowed?(url)
+          uri = URI.parse(url.to_s)
+          target = uri.path.to_s.empty? ? "/" : uri.path
+          target += "?#{uri.query}" if uri.query
+          best = @rules.select { |rule| rule.regexp.match?(target) }
+                       .max_by { |rule| [rule.pattern.length, rule.allow ? 1 : 0] }
+          best.nil? || best.allow
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/init/ask.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Init
+      # Asks questions on a terminal and reads the answers. Without a terminal it must not be used to ask:
+      # interactive? says which it is.
+      class Ask
+        def initialize(input:, out:)
+          @input = input
+          @out = out
+        end
+
+        def interactive? = @input.respond_to?(:tty?) && @input.tty?
+
+        # One line of text. An empty answer gives the default, which may be nil.
+        def line(question, default: nil)
+          @out.print("#{question}#{" [#{default}]" if default}: ")
+          answer = read
+          answer.empty? ? default : answer
+        end
+
+        # Asks until something is typed.
+        def required(question, default: nil)
+          loop do
+            answer = line(question, default:)
+            return answer unless answer.nil? || answer.empty?
+
+            @out.puts "An answer is needed here."
+          end
+        end
+
+        def yes?(question, default: true)
+          loop do
+            answer = line("#{question} #{default ? '[Y/n]' : '[y/N]'}").to_s.downcase
+            return default if answer.empty?
+            return answer.start_with?("y") if %w[y yes n no].include?(answer)
+
+            @out.puts "Please answer y or n."
+          end
+        end
+
+        # Lines until an empty one. An empty list means the question was skipped.
+        def list(question)
+          @out.puts "#{question} One per line, an empty line to finish."
+          lines = []
+          loop do
+            @out.print("  > ")
+            answer = read
+            return lines if answer.empty?
+
+            lines << answer
+          end
+        end
+
+        private
+
+        def read
+          text = @input.gets
+          raise Error, "The input ended before all the questions were answered." if text.nil?
+
+          text.strip
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/init/detector.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module Yuno
+  module Proofreader
+    module Init
+      # What a homepage says about the languages of its site. languages is a list of { "code", "prefix" } with the
+      # default language first. origin is where the page was found (scheme and host), description its meta
+      # description when it is short enough to suggest, and problems say what could not be used.
+      Detection = Data.define(:languages, :origin, :description, :problems)
+
+      # Reads the language versions a homepage announces with its hreflang tags. The tool tells languages apart by
+      # the start of the address (/es/...), so a version on another host, or deeper than one path segment, is
+      # reported and left out.
+      class Detector
+        CODE = /\A[a-z]{2,3}(-[a-z0-9]{2,8})*\z/
+        ALTERNATES = 'link[rel~="alternate"][hreflang]'
+        MAX_DESCRIPTION = 300
+
+        def self.call(html, url:, website:) = new(html, url, website).call
+
+        def initialize(html, url, website)
+          @doc = Nokogiri::HTML(html.to_s)
+          @url = url
+          @website = website
+          @problems = []
+        end
+
+        def call
+          languages = ordered(distinct(versions))
+          Detection.new(languages:, origin: Parsing.origin(@url), description:, problems: @problems)
+        end
+
+        private
+
+        def alternates
+          @doc.css(ALTERNATES).filter_map do |link|
+            code = link["hreflang"].to_s.strip.downcase
+            href = Analysis::Links.absolute(@url, link["href"])
+            [code, href] if !code.empty? && href
+          end
+        end
+
+        def versions
+          found = alternates.reject { |pair| pair.first == "x-default" }.uniq(&:first)
+          found.filter_map { |code, href| version(code, href) }
+        end
+
+        # { "code", "prefix" } for a version, or nil with the reason noted.
+        def version(code, href)
+          return note("#{code}: not a language code the tool can use") unless code.match?(CODE)
+          return note("#{code}: its address #{href} is on another host") unless Analysis::Links.same_site?(href,
+                                                                                                           @website)
+
+          parts = Analysis::Links.segments(href)
+          return note("#{code}: its address #{href} is deeper than a language prefix") if parts.size > 1
+
+          { "code" => code, "prefix" => parts.empty? ? "" : "/#{parts.first}" }
+        end
+
+        def note(message)
+          @problems << message
+          nil
+        end
+
+        # Two versions cannot share a prefix: the first one is kept.
+        def distinct(list)
+          kept = {}
+          list.each do |entry|
+            first = kept[entry["prefix"]]
+            if first
+              note("#{entry['code']}: shares its address prefix with #{first['code']}")
+            else
+              kept[entry["prefix"]] =
+                entry
+            end
+          end
+          kept.values
+        end
+
+        # The default language first: the one without a prefix, else the one x-default points to, else the first.
+        def ordered(list)
+          default = list.find { |entry| entry["prefix"].empty? } || x_default(list) || list.first
+          default ? [default, *(list - [default])] : []
+        end
+
+        def x_default(list)
+          target = alternates.find { |code, _href| code == "x-default" }&.last
+          target && list.find { |entry| Analysis::Links.key(target) == Analysis::Links.key("#{@website}#{entry['prefix']}") }
+        end
+
+        def description
+          text = Findings::Text.collapse(@doc.at_css('meta[name="description"]')&.[]("content"))
+          text unless text.empty? || text.length > MAX_DESCRIPTION
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/init/interview.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Init
+      # Works out the settings of a new site: from the command's options, from what the homepage says about itself
+      # and, on a terminal, by asking. Only the address and the languages are needed to write a file, the business
+      # description is needed before an audit, and everything else is optional.
+      class Interview
+        LISTS = {
+          "skip_paths" => ["skip-path",
+                           "Pages you do not want audited, written as in the address (for example /thank-you)."],
+          "rules" => ["rule", "Writing rules, such as how prices or dates are written."],
+          "ignore_terms" => ["ignore-term", "Words that are correct as written and must never be flagged: brand " \
+                                            "names, place names, local words."],
+          "intentional_differences" => ["difference", "Differences between language versions that are intended, " \
+                                                      "such as a section that exists in one language only."]
+        }.freeze
+        LANGUAGE_QUESTION = "Languages, the default first: a code, or code:/prefix (for example de, es:/es, en:/en)"
+
+        # flags: the options (Flags::Parsed). web: looks things up. offline: make no request. yes: accept what was
+        # detected, and ask nothing.
+        def initialize(flags:, ask:, web:, out:, offline: false, yes: false)
+          @flags = flags
+          @ask = ask
+          @web = web
+          @out = out
+          @offline = offline
+          @yes = yes
+        end
+
+        # The settings as a hash with string keys, in the shape of the settings file.
+        def call
+          intro
+          origin = website
+          detection = read_homepage(origin)
+          origin = better_origin(origin, detection)
+          settings = { "website" => origin, "languages" => languages(detection), **about(detection), **lists }
+          settings["sitemap_url"] = @flags.values["sitemap"] if @flags.values["sitemap"]
+          check_sitemap(settings)
+          settings
+        end
+
+        private
+
+        def asking? = @ask.interactive? && !@yes
+
+        def intro
+          return unless asking?
+
+          @out.puts "A few questions to write your settings file. An empty answer skips a question that is optional."
+        end
+
+        def accept?(question) = asking? ? @ask.yes?(question) : true
+
+        # ---- the address ----
+
+        def website
+          given = @flags.values["website"]
+          if given
+            return Parsing.origin(given) || raise(ConfigError,
+                                                  "--website is not a web address (got #{given.inspect})")
+          end
+          raise ConfigError, "Say which site to set up: --website ADDRESS" unless @ask.interactive?
+
+          loop do
+            origin = Parsing.origin(@ask.required("Address of the site (for example example.com)"))
+            return origin if origin
+
+            @out.puts "That is not a web address."
+          end
+        end
+
+        def read_homepage(origin)
+          return if @offline
+
+          html, url = @web.homepage(origin)
+          Detector.call(html, url:, website: origin)
+        rescue Error => e
+          @out.puts "The homepage could not be read (#{e.message}), so nothing could be detected from it."
+          nil
+        end
+
+        def better_origin(origin, detection)
+          found = detection&.origin
+          return origin if found.nil? || found == origin
+
+          accept?("#{origin} leads to #{found}. Use #{found} as the address of the site?") ? found : origin
+        end
+
+        # ---- the languages ----
+
+        def languages(detection)
+          given = @flags.lists["language"]
+          return with_notes(checked(given.map { |item| Parsing.language(item) })) unless given.empty?
+
+          detected = Array(detection&.languages)
+          return with_notes(detected) if !detected.empty? && announce(detection) && accept?("Use these languages?")
+
+          with_notes(typed_languages)
+        end
+
+        def checked(list)
+          problem = Parsing.language_problem(list)
+          raise ConfigError, "--language: #{problem}" if problem
+
+          list
+        end
+
+        def announce(detection)
+          @out.puts "The homepage announces these languages (the first is the default):"
+          detection.languages.each do |entry|
+            @out.puts "  #{entry['code']}  #{entry['prefix'].empty? ? '(no prefix)' : entry['prefix']}"
+          end
+          detection.problems.each { |problem| @out.puts "  Left out: #{problem}" }
+        end
+
+        def typed_languages
+          unless @ask.interactive?
+            raise ConfigError, "Say which languages the site has: --language CODE[:PREFIX], once for each, the " \
+                               "default first"
+          end
+
+          loop do
+            list = Parsing.languages(@ask.required(LANGUAGE_QUESTION))
+            problem = Parsing.language_problem(list)
+            return list unless problem
+
+            @out.puts "Not accepted: #{problem}"
+          end
+        end
+
+        def with_notes(list)
+          given = @flags.lists["note"].to_h do |item|
+            item.split("=", 2).then do |code, text|
+              [code.to_s.strip.downcase, text.to_s.strip]
+            end
+          end
+          list.map do |language|
+            note = given.fetch(language["code"]) { ask_note(language["code"]) }
+            note.to_s.empty? ? language : language.merge("notes" => note)
+          end
+        end
+
+        def ask_note(code)
+          @ask.line("Notes for #{code}: its variant and form of address (Enter to skip)") if asking?
+        end
+
+        # ---- the site ----
+
+        def about(detection)
+          { "business" => text("business", "What the site is and what it offers, in a sentence or two",
+                               required: true, suggestion: detection&.description),
+            "audience" => text("audience", "Who reads the site (Enter to skip)"),
+            "tone" => text("tone", "How the text should sound, such as formal or warm (Enter to skip)") }
+        end
+
+        def text(name, question, required: false, suggestion: nil)
+          given = @flags.values[name]
+          return given.strip if given
+          return (suggestion if @yes) unless asking?
+
+          required ? @ask.required(question, default: suggestion) : @ask.line(question)
+        end
+
+        def lists
+          LISTS.to_h { |setting, (flag, question)| [setting, list(flag, question)] }
+        end
+
+        def list(flag, question)
+          given = @flags.lists[flag]
+          return given.map(&:strip).reject(&:empty?) unless given.empty?
+
+          asking? ? @ask.list(question) : []
+        end
+
+        # ---- the sitemap ----
+
+        # Looks for the sitemap. When none is found and there is someone to ask, asks for its address once.
+        def check_sitemap(settings)
+          return if @offline
+
+          2.times do
+            return if sitemap_found?(settings) || !asking?
+
+            address = @ask.line("Address of your sitemap (Enter to skip)")
+            return if address.nil? || address.empty?
+
+            settings["sitemap_url"] = address
+          end
+        end
+
+        def sitemap_found?(settings)
+          count, paired = @web.sitemap(settings)
+          @out.puts sitemap_line(count, paired)
+          true
+        rescue Error => e
+          @out.puts "No usable sitemap was found (#{e.message})."
+          false
+        end
+
+        def sitemap_line(count, paired)
+          return "Found a sitemap: #{count} pages, paired by language." if paired
+
+          "Found a sitemap, but its pages could not be paired by language. Run: yuno-proofreader sitemap"
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/init/parsing.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Yuno
+  module Proofreader
+    module Init
+      # Turns what a person typed, or gave as an option, into settings.
+      module Parsing
+        SCHEME = %r{\A[a-z][a-z0-9+.-]*://}i
+
+        module_function
+
+        # "example.com" or "https://example.com/en" give "https://example.com". Anything that is not a web address
+        # gives nil.
+        def origin(text)
+          value = text.to_s.strip
+          return if value.empty?
+
+          value = "https://#{value}" unless value.match?(SCHEME)
+          uri = URI.parse(value)
+          return unless %w[http https].include?(uri.scheme) && !uri.host.to_s.empty?
+
+          "#{uri.scheme}://#{uri.host}#{":#{uri.port}" unless uri.port == uri.default_port}"
+        rescue URI::Error
+          nil
+        end
+
+        # "es:/es" gives code es with the prefix /es, and "de" gives code de with no prefix.
+        def language(item)
+          code, prefix = item.to_s.split(":", 2)
+          { "code" => code.to_s.strip.downcase, "prefix" => prefix.to_s.strip }
+        end
+
+        def languages(text) = text.to_s.split(",").map(&:strip).reject(&:empty?).map { |item| language(item) }
+
+        # What is wrong with a list of languages, in the words of the settings check, or nil when it is fine.
+        def language_problem(list)
+          Site.from_hash("website" => "https://example.com", "languages" => list)
+          nil
+        rescue ConfigError => e
+          e.message
+        end
+      end
+    end
+  end
+end