yuno-proofreader 0.1.0

data/lib/yuno/proofreader/errors.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    class Error < StandardError; end
+
+    class ConfigError < Error; end
+
+    class SitemapError < Error; end
+
+    class FetchError < Error; end
+
+    class StoreError < Error; end
+
+    class LlmError < Error; end
+  end
+end

data/lib/yuno/proofreader/findings/finding.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Findings
+      # One suggested change, as the model reports it: where it is, what it quotes and what it suggests.
+      class Finding < Data.define(:language, :category, :effort, :original, :suggested, :reason)
+        CATEGORIES = %w[accent typo grammar repetition html alt_text parity seo stale style other].freeze
+        MAX_QUOTE = 2_000
+        MAX_SUGGESTION = 2_000
+        MAX_REASON = 500
+
+        # findings: the usable ones. discarded: a line for each item that was not usable, saying why.
+        Parsed = Data.define(:findings, :discarded)
+
+        # items: the list of findings from a model answer. An item that cannot be used is discarded with a
+        # reason, so one bad item never costs the others. categories: the ones allowed for this list.
+        # fallback: the one used for anything else.
+        def self.parse_all(items, languages:, categories: CATEGORIES, fallback: "other")
+          return Parsed.new(findings: [], discarded: []) if items.nil?
+          return Parsed.new(findings: [], discarded: ["the findings are not a list"]) unless items.is_a?(Array)
+
+          findings = []
+          discarded = []
+          items.each_with_index do |item, index|
+            finding, problem = build(item, languages:, categories:, fallback:)
+            if finding
+              findings << finding
+            else
+              discarded << "finding #{index + 1} #{problem}"
+            end
+          end
+          Parsed.new(findings:, discarded:)
+        end
+
+        # Returns [finding, nil], or [nil, what is wrong with the item].
+        def self.build(item, languages:, categories:, fallback:)
+          return [nil, "is not an object"] unless item.is_a?(Hash)
+
+          raw = item.transform_keys(&:to_s)
+          language = language_for(raw["language"], languages)
+          problem = problem_with(raw, language)
+          return [nil, problem] if problem
+
+          category = category_for(raw["category"], categories, fallback)
+          [new(language:, category:, effort: effort_for(raw["effort"], category), original: clean(raw["original"]),
+               suggested: cut(clean(raw["suggested"]), MAX_SUGGESTION), reason: cut(clean(raw["reason"]), MAX_REASON)),
+           nil]
+        end
+
+        def self.problem_with(raw, language)
+          original = clean(raw["original"])
+          suggested = clean(raw["suggested"])
+          return "has an unknown language #{raw['language'].inspect}" unless language
+          return "quotes no text" if original.empty?
+          return "quotes more than #{MAX_QUOTE} characters" if original.length > MAX_QUOTE
+          return "suggests nothing" if suggested.empty?
+
+          "suggests the text it quotes" if suggested == original
+        end
+
+        # An exact code wins. Otherwise a regional code (de-AT) counts as its language (de).
+        def self.language_for(value, languages)
+          code = clean(value).downcase
+          languages.find { |language| language == code } ||
+            languages.sort_by { |language| -language.length }.find { |language| code.start_with?("#{language}-") }
+        end
+
+        def self.category_for(value, categories, fallback)
+          category = clean(value).downcase
+          categories.include?(category) ? category : fallback
+        end
+
+        # Quick means a fix that needs no thinking. Anything unclear is bigger, and alt text is always bigger.
+        def self.effort_for(value, category)
+          return "bigger" if category == "alt_text"
+
+          clean(value).downcase == "quick" ? "quick" : "bigger"
+        end
+
+        # Text and numbers become trimmed text. Anything else (a nested structure, nothing) becomes empty.
+        def self.clean(value)
+          case value
+          when String, Numeric then value.to_s.strip
+          else ""
+          end
+        end
+
+        def self.cut(text, max) = text[0, max]
+
+        private_class_method :build, :problem_with, :language_for, :category_for, :effort_for, :clean, :cut
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/findings/matching.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Findings
+      # When two findings, or a finding and a suggestion in the sheet, are about the same place.
+      module Matching
+        MIN_OVERLAP = 6
+        FAMILIES = { "accent" => "spelling", "typo" => "spelling", "grammar" => "wording", "style" => "wording" }.freeze
+
+        module_function
+
+        # Spelling and wording issues match across their categories, because the model files one issue under either.
+        def family(category) = FAMILIES.fetch(category.to_s, category.to_s)
+
+        # Same language, and one quoted text contains the other (ignoring spaces, capitals and entities).
+        # A short quote has to match exactly, so a word like "the" does not match half the page.
+        def same_spot?(first, second)
+          return false unless first.language == second.language
+
+          one = key(first.original)
+          other = key(second.original)
+          return false if one.empty? || other.empty?
+          return true if one == other
+
+          shorter, longer = [one, other].sort_by(&:length)
+          shorter.length >= MIN_OVERLAP && longer.include?(shorter)
+        end
+
+        def key(text) = Text.squash(text).downcase
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/findings/merger.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Findings
+      # Combines the findings of the audit and of the independent second review. A finding both found counts as
+      # confirmed. Where both found it, the audit's wording is kept, because its categories are the more reliable.
+      class Merger
+        Merged = Data.define(:finding, :found_by)
+
+        BOTH = "both"
+        AUDIT = "audit"
+        REVIEW = "review"
+        UNREVIEWED = "audit (unreviewed)"
+
+        # REVIEW: is nil when the second review failed. An empty list means it ran and found nothing.
+        def self.call(audit, review) = new(audit, review).call
+
+        def initialize(audit, review)
+          @audit = audit
+          @review = review
+        end
+
+        def call
+          return @audit.map { |finding| Merged.new(finding:, found_by: UNREVIEWED) } if @review.nil?
+
+          confirmed = Array.new(@review.size, false)
+          merged = @audit.map { |finding| compare(finding, confirmed) }
+          merged + @review.each_index.reject { |index| confirmed[index] }
+                          .map { |index| Merged.new(finding: @review[index], found_by: REVIEW) }
+        end
+
+        private
+
+        def compare(finding, confirmed)
+          hits = @review.each_index.select { |index| Matching.same_spot?(finding, @review[index]) }
+          hits.each { |index| confirmed[index] = true }
+          Merged.new(finding:, found_by: hits.empty? ? AUDIT : BOTH)
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/findings/outline.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Findings
+      # The structure of a page's HTML: its header, navigation and footer areas, and its headings.
+      # It answers one question: which headings is a position in the page under?
+      # It reads the markup with regular expressions, so it is made for the cleaned HTML the tool works with.
+      class Outline
+        Region = Data.define(:name, :start, :finish)
+        Heading = Data.define(:level, :start, :inner, :region)
+
+        REGION = %r{<(head|header|nav|footer)\b.*?</\1\s*>}mi
+        HEADING = %r{<h([1-6])\b[^>]*>(.*?)</h\1\s*>}mi
+        MAX_LABEL = 60
+
+        def initialize(html)
+          @regions = matches(REGION, html).map do |match|
+            Region.new(name: match[1].downcase, start: match.begin(0), finish: match.end(0))
+          end
+          @headings = matches(HEADING, html).map { |match| build_heading(match) }
+        end
+
+        # The headings as [level, text, region name or nil], for looking at the structure.
+        def headings = @headings.map { |heading| [heading.level, label(heading), heading.region&.name] }
+
+        # "header", "footer" or "head" for those areas. Anywhere else the headings above the position, from the
+        # first h2 down, such as "What we offer > Tours". The h1 is left out, because everything is under it.
+        def path_at(position)
+          region = region_at(position)
+          return "head" if region&.name == "head"
+
+          parts = chain_at(position, region).select { |heading| heading.level > 1 }.map { |heading| label(heading) }
+          path = [region&.name, *parts].compact.join(" > ")
+          path.empty? ? "(before first h2)" : path
+        end
+
+        private
+
+        def matches(regexp, html)
+          html.to_enum(:scan, regexp).map { Regexp.last_match }
+        end
+
+        def build_heading(match)
+          start = match.begin(0)
+          Heading.new(level: match[1].to_i, start:, inner: match[2], region: region_at(start))
+        end
+
+        def region_at(position)
+          @regions.find { |region| position >= region.start && position < region.finish }
+        end
+
+        # The headings that are open at the position: a new heading closes every heading of the same or a lower
+        # level. Headings in other areas (a footer, say) do not count.
+        def chain_at(position, region)
+          @headings.each_with_object([]) do |heading, chain|
+            next if heading.start > position || heading.region != region
+
+            chain.pop while chain.any? && chain.last.level >= heading.level
+            chain << heading
+          end
+        end
+
+        def label(heading)
+          text = Text.collapse(Text.decode(Text.strip_tags(heading.inner)))
+          return "(empty h#{heading.level})" if text.empty?
+
+          text.length > MAX_LABEL ? "#{text[0, MAX_LABEL - 1]}…" : text
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/findings/plan.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Findings
+      # One thing the reconciliation decided. kind is :added, :reappeared, :seen_again, :suppressed, :duplicate,
+      # :ignored, :closed or :close_skipped.
+      Decision = Data.define(:kind, :summary, :reason)
+
+      # What to write: the rows to add, the changes to rows that exist (row number => fields), and why.
+      Plan = Data.define(:additions, :updates, :decisions) do
+        def apply(store)
+          store.update_suggestions(updates)
+          store.add_suggestions(additions)
+          self
+        end
+
+        def counts = decisions.map(&:kind).tally
+
+        def empty? = additions.empty? && updates.empty?
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/findings/reconciler.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Findings
+      # Decides what one page's findings mean for the suggestions already in the sheet. It changes nothing: it
+      # returns a Plan with every change and the reason for it.
+      class Reconciler
+        ACTIVE = ["open", "on staging"].freeze
+        PRIORITY = { active: 0, held: 1, done: 2 }.freeze
+        RANK = { "code" => 0, "both" => 1 }.freeze
+        REAPPEARED = "[Reappeared after being marked done] "
+        HTTP_ADVICE = "Check why this URL does not load"
+        MAX_REASON = 500
+        MAX_SUMMARY = 50
+
+        # A suggestion as matching and closing see it. row is nil for one added in this run.
+        Entry = Data.define(:row, :language, :category, :original, :status, :verified, :reason, :seen)
+        Addition = Data.define(:record, :rank)
+        Check = Data.define(:presence, :place) do
+          def verified = { exact: "yes", loose: "loose", none: "no" }.fetch(presence)
+        end
+
+        # page: the fetched page. findings: Merger::Merged items. existing: the page's rows in the sheet.
+        def self.call(**inputs) = new(**inputs).call
+
+        def initialize(page:, findings:, existing: [], ignore_terms: [], now: Time.now.utc)
+          @page = page
+          @findings = findings
+          @now = now
+          @ignored = ignore_terms.map { |term| Text.norm(term).downcase }.reject(&:empty?)
+          @snapshots = page.snapshots.to_h { |snapshot| [snapshot.language, snapshot] }
+          @sources = page.snapshots.select(&:ok?).to_h do |snapshot|
+            [snapshot.language, Source.from_snapshot(snapshot)]
+          end
+          @entries = existing.map { |record| entry_of(record) }
+          @claimed = {}
+          @skipped = {}
+          @additions = []
+          @updates = {}
+          @decisions = []
+        end
+
+        def call
+          (code_findings + @findings).each { |merged| consider(merged) }
+          close_resolved
+          Plan.new(additions: ranked, updates: @updates, decisions: @decisions)
+        end
+
+        private
+
+        # ---- findings ----
+
+        def consider(merged)
+          finding = merged.finding
+          return note(:ignored, finding, "its quoted text is on the ignore list") if ignored?(finding)
+
+          match = best_match(finding)
+          return add(merged) unless match
+
+          case kind_of(match)
+          when :active then repeat(match, finding)
+          when :done then add(merged, reappeared: true)
+          else note(:suppressed, finding, "an earlier suggestion for it is marked #{match.status.inspect}")
+          end
+        end
+
+        def repeat(match, finding)
+          if match.row.nil? || @claimed[match.row]
+            return note(:duplicate, finding, "it repeats another finding of this run")
+          end
+          if check(finding).presence == :none
+            return note(:ignored, finding, "its quoted text is not on the page, so it does not count as seen again")
+          end
+
+          @claimed[match.row] = true
+          @updates[match.row] = { last_seen: @now, amount_seen: match.seen + 1 }
+          note(:seen_again, finding, "row #{match.row} was seen again")
+        end
+
+        def add(merged, reappeared: false)
+          finding = merged.finding
+          found = check(finding)
+          reason = reappeared ? cut("#{REAPPEARED}#{finding.reason}") : finding.reason
+          @entries << new_entry(finding, found, reason)
+          @additions << Addition.new(record: record_for(merged, found, reason), rank: rank(merged))
+          note(reappeared ? :reappeared : :added, finding,
+               reappeared ? "its text came back after the suggestion was marked done" : "a new finding")
+        end
+
+        def record_for(merged, found, reason)
+          finding = merged.finding
+          Store::SuggestionRecord.build(
+            timestamp: @now, page: @page.name, language: finding.language, category: finding.category,
+            effort: finding.effort, approximate_place: found.place, original: finding.original,
+            suggested: finding.suggested, reason:, verified: found.verified, status: "open",
+            found_by: merged.found_by, last_seen: @now, amount_seen: 1
+          )
+        end
+
+        def new_entry(finding, found, reason)
+          Entry.new(row: nil, language: finding.language, category: finding.category, original: finding.original,
+                    status: "open", verified: found.verified, reason:, seen: 1)
+        end
+
+        # Findings made by code: a version that answered with an HTTP error. A block by robots.txt or a network
+        # error has no status and is not a finding about the site.
+        def code_findings
+          @snapshots.values.filter_map do |snapshot|
+            next unless snapshot.status && snapshot.status != 200
+
+            finding = Finding.new(language: snapshot.language, category: "http", effort: "bigger",
+                                  original: snapshot.url, suggested: HTTP_ADVICE, reason: snapshot.problem.to_s)
+            Merger::Merged.new(finding:, found_by: "code")
+          end
+        end
+
+        def ignored?(finding) = @ignored.include?(Text.norm(finding.original).downcase)
+
+        # ---- matching ----
+
+        def best_match(finding)
+          hits = @entries.each_with_index.select { |entry, _index| matches?(entry, finding) }
+          hits.min_by { |entry, index| [PRIORITY.fetch(kind_of(entry)), index] }&.first
+        end
+
+        def matches?(entry, finding)
+          Matching.family(entry.category) == Matching.family(finding.category) && Matching.same_spot?(entry, finding)
+        end
+
+        # :active (open or on staging), :done, or :held (rejected, or a status the tool does not know).
+        def kind_of(entry)
+          status = entry.status.empty? ? "open" : entry.status
+          return :active if ACTIVE.include?(status)
+
+          status == "done" ? :done : :held
+        end
+
+        def entry_of(record)
+          language = record.language.to_s.strip.downcase
+          status = record.status.to_s.strip.downcase
+          verified = record.verified.to_s.strip.downcase
+          Entry.new(row: record.row, language:, category: record.category.to_s.strip, original: record.original.to_s,
+                    status:, verified:, reason: record.reason.to_s, seen: record.amount_seen || 1)
+        end
+
+        # ---- what the page says ----
+
+        def check(finding)
+          return Check.new(presence: :exact, place: "n/a") if finding.category == "http"
+
+          verification = @sources[finding.language]&.verify(finding.original)
+          return Check.new(presence: :none, place: "") unless verification
+
+          Check.new(presence: verification.presence, place: verification.place.to_s)
+        end
+
+        # ---- closing ----
+
+        def close_resolved
+          @entries.each do |entry|
+            next unless entry.row && kind_of(entry) == :active && !@claimed[entry.row]
+
+            why = resolved_because(entry)
+            close(entry, why) if why
+          end
+        end
+
+        def resolved_because(entry)
+          snapshot = @snapshots[entry.language]
+          return unless snapshot&.ok?
+          return "the page loads again" if entry.category == "http"
+          return unless entry.verified == "yes"
+          return skip_close(entry) unless snapshot.complete?
+
+          "its quoted text is no longer on the page" unless @sources[entry.language].presence(entry.original) == :exact
+        end
+
+        def skip_close(entry)
+          return if @skipped[entry.language]
+
+          @skipped[entry.language] = true
+          @decisions << Decision.new(kind: :close_skipped, summary: "#{entry.language} version",
+                                     reason: "the page looks incomplete, so open suggestions in this language " \
+                                             "were not closed")
+          nil
+        end
+
+        def close(entry, why)
+          reason = cut("[Auto-closed #{@now.strftime('%Y-%m-%d')}: #{why}] #{entry.reason}")
+          @updates[entry.row] = { status: "done", reason:, last_seen: @now }
+          @decisions << Decision.new(kind: :closed, summary: describe(entry), reason: why)
+        end
+
+        # ---- the plan ----
+
+        def rank(merged) = [RANK.fetch(merged.found_by, 2), merged.finding.effort == "quick" ? 0 : 1]
+
+        def ranked
+          ordered = @additions.each_with_index.sort_by { |addition, index| [addition.rank, index] }
+          ordered.map { |addition, _index| addition.record }
+        end
+
+        def note(kind, item, reason)
+          @decisions << Decision.new(kind:, summary: describe(item), reason:)
+          nil
+        end
+
+        def describe(item)
+          text = item.original.to_s
+          text = "#{text[0, MAX_SUMMARY - 1]}…" if text.length > MAX_SUMMARY
+          "#{item.category} (#{item.language}): #{text}"
+        end
+
+        def cut(text) = text[0, MAX_REASON]
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/findings/source.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Findings
+      # One language version of a page, prepared for searching. It answers two questions about a quoted text:
+      # is it really on the page, and under which headings does it sit?
+      class Source
+        # presence is :exact, :loose or :none. place is the heading path, or nil when it is not known.
+        Verification = Data.define(:presence, :place) do
+          def exact? = presence == :exact
+        end
+
+        attr_reader :html
+
+        def self.from_snapshot(snapshot) = new(snapshot.html)
+
+        def initialize(html)
+          @html = Text.collapse(html)
+        end
+
+        def outline = @outline ||= Outline.new(@html)
+
+        # :exact when the text is on the page as it stands, apart from runs of whitespace and the way entities
+        # are written. :loose when it only matches with every space ignored, for example after a spacing-only
+        # change. :none when it is not there. Capital letters count.
+        def presence(quote)
+          text = quote.to_s.strip
+          return :none if text.empty?
+          return :exact if exact?(text)
+
+          loose?(text) ? :loose : :none
+        end
+
+        # The headings the first occurrence of the text is under, or nil. The page is searched as sent, so a
+        # quote that only matches once entities such as é are decoded has no place.
+        def place(quote)
+          index = position(quote)
+          index && outline.path_at(index)
+        end
+
+        def verify(quote)
+          found = presence(quote)
+          Verification.new(presence: found, place: found == :none ? nil : place(quote))
+        end
+
+        private
+
+        def position(quote)
+          text = Text.collapse(quote)
+          return if text.empty?
+
+          [text, Text.escape(text)].uniq.filter_map { |candidate| @html.index(candidate) }.min
+        end
+
+        def exact?(text)
+          plain = Text.collapse(text)
+          decoded = Text.norm(text)
+          @html.include?(plain) || decoded_html.include?(decoded) || visible.include?(decoded)
+        end
+
+        def loose?(text)
+          squashed = Text.norm(text).delete(" ")
+          return false if squashed.empty?
+
+          loose_html.include?(squashed) || loose_visible.include?(squashed)
+        end
+
+        def decoded_html = @decoded_html ||= Text.collapse(Text.decode(@html))
+
+        def visible = @visible ||= Text.collapse(Text.decode(Text.strip_tags(@html)))
+
+        def loose_html = @loose_html ||= decoded_html.delete(" ")
+
+        def loose_visible = @loose_visible ||= visible.delete(" ")
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/findings/text.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module Yuno
+  module Proofreader
+    module Findings
+      # Text helpers for comparing a quoted text with a page: whitespace, HTML entities and tags.
+      module Text
+        SPACE = /[[:space:]]+/
+        ESCAPES = { "&" => "&amp;", "<" => "&lt;", ">" => "&gt;", '"' => "&quot;", "'" => "&#39;" }.freeze
+
+        module_function
+
+        # Runs of whitespace (non-breaking spaces included) become one space, and the ends are trimmed.
+        def collapse(string) = string.to_s.gsub(SPACE, " ").strip
+
+        # Turns HTML entities into the characters they stand for. Tags are left exactly as they are.
+        def decode(string)
+          escaped = string.to_s.gsub("<", "&lt;").gsub(">", "&gt;")
+          Nokogiri::HTML4.fragment("<div>#{escaped}</div>").text
+        end
+
+        # Entities decoded and whitespace collapsed: the form in which two texts are compared.
+        def norm(string) = collapse(decode(string))
+
+        # Entities decoded and all whitespace removed: the loosest form of comparing.
+        def squash(string) = decode(string).gsub(SPACE, "")
+
+        def strip_tags(html) = html.to_s.gsub(/<[^>]*>/m, "")
+
+        # The way a server writes a text into HTML, for finding a quote in the markup.
+        def escape(string) = string.to_s.gsub(/[&<>"']/, ESCAPES)
+      end
+    end
+  end
+end