yuno-proofreader 0.1.0

data/lib/yuno/proofreader/sitemap/importer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Sitemap
+      # Adds the pages of a sitemap pairing to the page queue. Pages that are already there are left alone,
+      # so it is safe to run again. Pages that left the sitemap are counted but never removed.
+      class Importer
+        Result = Data.define(:added, :existing, :gone)
+
+        def self.call(pairing, store, site) = new(pairing, store, site).call
+
+        def initialize(pairing, store, site)
+          @pairing = pairing
+          @store = store
+          @language = site.default_language
+        end
+
+        def call
+          raise SitemapError, "The pages could not be paired, so nothing was imported." unless @pairing.resolved?
+
+          known = @store.pages.map { |record| record.key(@language) }
+          fresh = @pairing.pages.reject { |page| known.include?(page.key(@language)) }
+          @store.add_pages(fresh)
+          listed = @pairing.pages.map { |page| page.key(@language) }
+          Result.new(added: fresh.size, existing: @pairing.pages.size - fresh.size, gone: (known - listed).size)
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/sitemap/loader.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Yuno
+  module Proofreader
+    module Sitemap
+      # Finds a site's sitemap, reads it (following a sitemap index) and pairs its pages by language.
+      # Discovery order: sitemap_url from the settings, then the Sitemap lines of robots.txt, then /sitemap.xml.
+      class Loader
+        MAX_DEPTH = 2
+        MAX_SITEMAPS = 25
+        MAX_ENTRIES = 50_000
+
+        Result = Data.define(:pairing, :entries, :sources, :warnings)
+
+        def self.call(site, client) = new(site, client).call
+
+        def initialize(site, client)
+          @site = site
+          @client = client
+          @entries = []
+          @sources = []
+          @warnings = []
+        end
+
+        def call
+          discover.each { |url| read(url, 0) }
+          raise SitemapError, unreadable_message if @sources.empty?
+
+          entries = unique(@entries)
+          pairing = Pairer.call(entries, @site)
+          Result.new(pairing:, entries:, sources: @sources, warnings: @warnings + pairing.warnings)
+        end
+
+        private
+
+        def unreadable_message
+          reasons = @warnings.select { |warning| warning.start_with?("Could not read the sitemap") }
+          "#{reasons.join('. ')}. If it lives somewhere else, set sitemap_url in your site settings."
+        end
+
+        def discover
+          return [@site.sitemap_url] if @site.sitemap_url
+
+          declared = robots_sitemaps
+          declared.empty? ? ["#{@site.website}/sitemap.xml"] : declared
+        end
+
+        def robots_sitemaps
+          response = @client.get("#{@site.website}/robots.txt")
+          response.success? ? declared_in(response.body) : []
+        rescue FetchError => e
+          @warnings << "Could not read robots.txt (#{e.message}). Using the default sitemap address."
+          []
+        end
+
+        def declared_in(text)
+          on_site_only(text.lines.filter_map { |line| line.sub(/#.*/, "")[/\A\s*sitemap:\s*(\S+)/i, 1] })
+        end
+
+        def read(url, depth)
+          return if @sources.include?(url)
+
+          if @sources.size >= MAX_SITEMAPS
+            @warnings << "More than #{MAX_SITEMAPS} sitemaps. Skipped #{url}"
+            return
+          end
+
+          result, error = load_one(url)
+          return @warnings << "Could not read the sitemap #{url}: #{error}" if error
+
+          @sources << url
+          @entries.concat(result.entries)
+          @warnings.concat(result.warnings)
+          raise SitemapError, "The sitemaps list more than #{MAX_ENTRIES} pages" if @entries.size > MAX_ENTRIES
+
+          follow(result.children, depth)
+        end
+
+        # Returns [parsed sitemap, nil], or [nil, reason] when it could not be read.
+        def load_one(url)
+          response = @client.get(url)
+          return [nil, "HTTP #{response.status}"] unless response.success?
+
+          [Parser.parse(response.body), nil]
+        rescue FetchError, SitemapError => e
+          [nil, e.message]
+        end
+
+        def follow(children, depth)
+          return if children.empty?
+
+          if depth >= MAX_DEPTH
+            @warnings << "Sitemap indexes are nested more than #{MAX_DEPTH} levels deep. " \
+                         "Skipped #{children.size} sitemap(s)"
+            return
+          end
+          on_site_only(children).each { |child| read(child, depth + 1) }
+        end
+
+        def on_site_only(urls)
+          kept, skipped = urls.uniq.partition { |url| on_site?(url) }
+          @warnings << "Ignored sitemap(s) on another host: #{skipped.join(', ')}" unless skipped.empty?
+          kept
+        end
+
+        def on_site?(url)
+          host_of(url) == host_of(@site.website)
+        end
+
+        def host_of(url)
+          URI.parse(url).host.to_s.downcase.delete_prefix("www.")
+        rescue URI::InvalidURIError
+          ""
+        end
+
+        def unique(entries)
+          kept = entries.uniq(&:url)
+          dropped = entries.size - kept.size
+          @warnings << "#{dropped} repeated sitemap entries were ignored" if dropped.positive?
+          kept
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/sitemap/mismatch.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Sitemap
+      # When the language lists differ in length, names the pages that most likely have no counterpart, so the
+      # person knows what to put under default_language_only_paths or skip_paths.
+      class Mismatch
+        SHOWN = 10
+        MIN_WORD = 3
+
+        # groups: the entries of each language, in sitemap order. bare: turns a path into the path without its
+        # language prefix.
+        def self.lines(groups, default, bare) = new(groups, default, bare).lines
+
+        def initialize(groups, default, bare)
+          @groups = groups
+          @default = default
+          @bare = bare
+        end
+
+        def lines
+          base = @groups.fetch(@default)
+          others = @groups.reject { |language, _list| language == @default }
+          gaps = others.to_h { |language, list| [language, Aligner.call(shape(base), shape(list))] }
+          lonely = gaps.values.map(&:first).reduce { |common, found| common & found } || []
+          [*only_default(base, lonely), *missing(base, gaps, lonely), *only_other(others, gaps)]
+        end
+
+        private
+
+        def only_default(base, lonely)
+          return [] if lonely.empty?
+
+          ["Probably only in #{@default}, with no counterpart in any other language: #{listed(base, lonely)} " \
+           "(add them to default_language_only_paths)"]
+        end
+
+        def missing(base, gaps, lonely)
+          gaps.filter_map do |language, (left, _right)|
+            rest = left - lonely
+            unless rest.empty?
+              "Probably missing in #{language}, so these #{@default} pages have no partner there: " \
+                "#{listed(base, rest)}"
+            end
+          end
+        end
+
+        def only_other(others, gaps)
+          others.filter_map do |language, list|
+            right = gaps.fetch(language).last
+            "Probably only in #{language}: #{listed(list, right)} (add them to skip_paths)" unless right.empty?
+          end
+        end
+
+        def listed(entries, indices)
+          paths = indices.map { |index| entries[index].path }
+          shown = paths.first(SHOWN).join(", ")
+          paths.size > SHOWN ? "#{shown} and #{paths.size - SHOWN} more" : shown
+        end
+
+        def shape(entries)
+          entries.map do |entry|
+            segments = @bare.call(entry.path).split("/").reject(&:empty?)
+            [segments.size, words(segments.last.to_s)]
+          end
+        end
+
+        def words(segment)
+          plain = segment.unicode_normalize(:nfd).gsub(/\p{Mn}/, "").downcase
+          plain.split(/[^a-z0-9]+/).select { |word| word.length >= MIN_WORD }
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/sitemap/pairer.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Yuno
+  module Proofreader
+    module Sitemap
+      Page = Data.define(:name, :urls, :lastmod, :notes) do
+        # The address that identifies the page: its default-language address, or the first one it has.
+        def key(default_language) = urls[default_language] || urls.values.first
+      end
+
+      # The outcome of pairing. resolved is false when the person has to fix the settings first.
+      Pairing = Data.define(:strategy, :pages, :warnings, :resolved) do
+        def resolved? = resolved
+      end
+
+      # Works out which sitemap addresses are language versions of the same page.
+      # Strategies: hreflang alternates inside the sitemap, otherwise language prefixes plus position
+      # (all pages of one language, then the next, in the same order).
+      class Pairer
+        HREFLANG_SHARE = 0.5
+        CHECK = "CHECK pairing"
+
+        def self.call(entries, site) = new(entries, site).call
+
+        def initialize(entries, site)
+          @site = site
+          @languages = site.languages
+          # Longest prefix first, so the empty prefix (a language without one) only matches as a fallback.
+          @by_prefix = site.prefixes.sort_by { |_language, prefix| -prefix.length }
+          wanted = entries.reject { |entry| listed?(entry.path, site.skip_paths) }
+          @unassigned, @entries = wanted.partition { |entry| language_of(entry.path).nil? }
+          @lastmods = @entries.to_h { |entry| [entry.url, entry.lastmod] }
+          @order = @entries.each_with_index.to_h { |entry, index| [entry.url, index] }
+        end
+
+        def call
+          if @entries.empty?
+            return pairing(:none, [], ["The sitemap lists no pages that belong to a configured language."],
+                           resolved: false)
+          end
+          return by_single_language if @languages.one?
+
+          hreflang_share >= HREFLANG_SHARE ? by_hreflang : by_position
+        end
+
+        private
+
+        def pairing(strategy, pages, warnings, resolved: true)
+          Pairing.new(strategy:, pages: sort(pages), warnings: unassigned_warning + warnings, resolved:)
+        end
+
+        def unassigned_warning
+          return [] if @unassigned.empty?
+
+          count = @unassigned.size
+          sample = @unassigned.first(3).map(&:path).join(", ")
+          ["Ignored #{count} sitemap address#{'es' unless count == 1} that match no language prefix " \
+           "(e.g. #{sample}). Add them to skip_paths to silence this."]
+        end
+
+        def by_single_language
+          pages = @entries.map { |entry| page_from({ language_of(entry.path) => entry.url }) }
+          pairing(:single, pages, [])
+        end
+
+        # ---- strategy: hreflang alternates in the sitemap ----
+
+        def hreflang_share
+          @entries.count { |entry| configured_alternates(entry).size >= 2 }.fdiv(@entries.size)
+        end
+
+        def by_hreflang
+          warnings = []
+          owner = {}
+          pages = hreflang_groups(owner, warnings) + hreflang_leftovers(owner)
+          pairing(:hreflang, pages, warnings.uniq)
+        end
+
+        def hreflang_groups(owner, warnings)
+          @entries.filter_map { |entry| group_for(entry, owner, warnings) }
+        end
+
+        def group_for(entry, owner, warnings)
+          urls = configured_alternates(entry)
+          return if urls.size < 2
+
+          warnings << "#{entry.url} is not part of its own hreflang set" unless urls.value?(entry.url)
+          page_from(urls) if claim(urls, owner, warnings)
+        end
+
+        # Records which set owns each address. Returns nil when another set already owns one of them.
+        def claim(urls, owner, warnings)
+          clash = urls.values.find { |url| owner[url] && owner[url] != urls }
+          if clash
+            warnings << "Conflicting hreflang sets around #{clash}"
+            return
+          end
+          return if urls.values.all? { |url| owner[url] }
+
+          urls.each_value { |url| owner[url] = urls }
+          urls
+        end
+
+        def hreflang_leftovers(owner)
+          @entries.reject { |entry| owner[entry.url] }.map do |entry|
+            page_from({ language_of(entry.path) => entry.url }, notes: ["no hreflang alternates"])
+          end
+        end
+
+        def configured_alternates(entry)
+          entry.alternates.each_with_object({}) do |(code, href), found|
+            lang = language_for(code)
+            found[lang] ||= href if lang
+          end
+        end
+
+        def language_for(code)
+          @languages.find { |lang| code == lang || code.start_with?("#{lang}-") }
+        end
+
+        # ---- strategy: language prefix plus position ----
+
+        def by_position
+          groups = @languages.to_h { |lang| [lang, []] }
+          default_only = []
+          @entries.each do |entry|
+            lang = language_of(entry.path)
+            if lang == @site.default_language && listed?(entry.path, @site.default_language_only_paths)
+              default_only << entry
+            else
+              groups[lang] << entry
+            end
+          end
+          paired(groups, default_only)
+        end
+
+        def paired(groups, default_only)
+          sizes = groups.transform_values(&:size)
+          return length_mismatch(groups) unless sizes.values.uniq.one?
+
+          pages = groups[@site.default_language].each_index.map { |index| page_at(groups, index) }
+          pages += default_only.map do |entry|
+            page_from({ @site.default_language => entry.url }, notes: ["default language only"])
+          end
+          pairing(:positional, pages, depth_warnings(pages))
+        end
+
+        def depth_warnings(pages)
+          flagged = pages.count { |page| page.notes.include?(CHECK) }
+          return [] if flagged.zero?
+
+          ["#{flagged} pages have a different depth across languages. Check the pairing."]
+        end
+
+        def page_at(groups, index)
+          members = groups.transform_values { |list| list[index] }
+          notes = members.values.map { |entry| depth(entry.path) }.uniq.one? ? [] : [CHECK]
+          page_from(members.transform_values(&:url), notes:)
+        end
+
+        def length_mismatch(groups)
+          counts = groups.map { |lang, list| "#{lang}=#{list.size}" }.join(", ")
+          warning = "Language lists differ in length (#{counts}). Put pages that exist in only one language " \
+                    "under default_language_only_paths or skip_paths."
+          guesses = Mismatch.lines(groups, @site.default_language, method(:bare_path))
+          note = "These are guesses from the order and the addresses of the pages. Check them before you change " \
+                 "the settings."
+          pairing(:positional, [], [warning, *guesses, *(guesses.empty? ? [] : [note])], resolved: false)
+        end
+
+        # ---- shared helpers ----
+
+        def listed?(path, list)
+          list.any? { |item| path == item || path.start_with?("#{item}/") }
+        end
+
+        # The language whose URL prefix the path starts with, or nil when none matches.
+        def language_of(path)
+          lowered = path.downcase
+          @by_prefix.each do |lang, prefix|
+            prefix = prefix.downcase
+            return lang if prefix.empty? || lowered == prefix || lowered.start_with?("#{prefix}/")
+          end
+          nil
+        end
+
+        def bare_path(path)
+          lang = language_of(path)
+          prefix = lang ? @site.prefixes[lang] : ""
+          "/#{path[prefix.length..].to_s.split('/').reject(&:empty?).join('/')}"
+        end
+
+        def depth(path)
+          bare_path(path).split("/").reject(&:empty?).size
+        end
+
+        def page_from(urls, notes: [])
+          ordered = @languages.filter_map { |lang| [lang, urls[lang]] if urls[lang] }.to_h
+          lastmod = ordered.values.filter_map { |url| @lastmods[url] }.reject(&:empty?).max.to_s
+          Page.new(name: name_for(ordered), urls: ordered, lastmod:, notes:)
+        end
+
+        def name_for(urls)
+          url = urls[@site.default_language] || urls.values.first
+          bare = bare_path(URI.parse(url).path.to_s)
+          bare == "/" ? "home" : bare.delete_prefix("/")
+        end
+
+        def sort(pages)
+          pages.each_with_index
+               .sort_by do |page, index|
+            [page.urls.values.filter_map do |url|
+              @order[url]
+            end.min || @entries.size, index]
+          end
+               .map(&:first)
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/sitemap/parser.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+require "uri"
+
+module Yuno
+  module Proofreader
+    module Sitemap
+      # One <url> of a sitemap. alternates maps an hreflang code (lowercase) to its address.
+      Entry = Data.define(:url, :path, :lastmod, :alternates)
+
+      # Reads sitemap XML. It only parses text and never fetches anything.
+      class Parser
+        # children: addresses of child sitemaps, when the document is a sitemap index.
+        Result = Data.define(:entries, :children, :warnings)
+
+        def self.parse(xml) = new.parse(xml)
+
+        def parse(xml)
+          doc = read(xml)
+          warnings = []
+          children = doc.css("sitemapindex > sitemap > loc").map { |node| node.text.strip }.reject(&:empty?)
+          entries = doc.css("urlset > url").filter_map { |node| entry_from(node, warnings) }
+          Result.new(entries:, children:, warnings:)
+        end
+
+        private
+
+        def read(xml)
+          doc = Nokogiri::XML(xml.to_s) { |config| config.strict.nonet }
+          doc.remove_namespaces!
+          name = doc.root&.name
+          unless %w[urlset sitemapindex].include?(name)
+            raise SitemapError, "This is not a sitemap (root element: #{name.inspect})"
+          end
+
+          doc
+        rescue Nokogiri::XML::SyntaxError => e
+          raise SitemapError, "The sitemap is not valid XML: #{e.message}"
+        end
+
+        def entry_from(node, warnings)
+          url = node.at_xpath("loc")&.text.to_s.strip
+          uri = web_uri(url)
+          unless uri
+            warnings << "Skipped an entry with a missing or invalid address: #{url.inspect}"
+            return nil
+          end
+
+          Entry.new(
+            url:,
+            path: uri.path.to_s.empty? ? "/" : uri.path,
+            lastmod: node.at_xpath("lastmod")&.text.to_s.strip,
+            alternates: alternates_of(node)
+          )
+        end
+
+        def web_uri(url)
+          uri = URI.parse(url)
+          %w[http https].include?(uri.scheme) && !uri.host.to_s.empty? ? uri : nil
+        rescue URI::InvalidURIError
+          nil
+        end
+
+        def alternates_of(node)
+          node.xpath("link[@rel='alternate'][@hreflang][@href]").to_h do |link|
+            [link["hreflang"].strip.downcase, link["href"].strip]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/sitemap/report.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Sitemap
+      # Turns a loaded sitemap into the lines the sitemap command prints.
+      class Report
+        PREVIEW = 3
+        NOTED_LIMIT = 10
+
+        def initialize(site, result)
+          @site = site
+          @result = result
+          @pairing = result.pairing
+        end
+
+        def lines(all: false)
+          [*summary, *warning_lines, *page_lines(all), *verdict]
+        end
+
+        private
+
+        def pages
+          @pairing.pages
+        end
+
+        def noted
+          @noted ||= pages.reject { |page| page.notes.empty? }
+        end
+
+        def summary
+          complete = pages.count { |page| page.urls.size == @site.languages.size }
+          [
+            "Website: #{@site.website}",
+            "Sitemaps read: #{@result.sources.join(', ')}",
+            "Entries in the sitemaps: #{@result.entries.size}",
+            "Strategy: #{@pairing.strategy}, resolved: #{@pairing.resolved? ? 'yes' : 'no'}",
+            "Pages: #{pages.size} (#{complete} in all #{@site.languages.size} languages, #{noted.size} with notes)"
+          ]
+        end
+
+        def warning_lines
+          return [] if @result.warnings.empty?
+
+          ["", "Warnings:", *@result.warnings.map { |warning| "  - #{warning}" }]
+        end
+
+        def page_lines(all)
+          return [] if pages.empty?
+          return listing("Pages:", pages) if all
+
+          [*listing("First pages:", pages.first(PREVIEW)), *noted_lines]
+        end
+
+        def noted_lines
+          return [] if noted.empty?
+
+          shown = noted.first(NOTED_LIMIT)
+          more = noted.size - shown.size
+          [*listing("Pages with notes:", shown), *(more.positive? ? ["  ... and #{more} more (use --all)"] : [])]
+        end
+
+        def listing(title, list)
+          ["", title, *list.flat_map { |page| page_block(page) }]
+        end
+
+        def page_block(page)
+          note = page.notes.empty? ? "" : " (#{page.notes.join(', ')})"
+          ["  #{page.name}#{note}", *page.urls.map { |lang, url| "    #{lang}: #{url}" }]
+        end
+
+        def verdict
+          return [] if @pairing.resolved?
+
+          ["", "The pages could not be paired. Fix the settings or the sitemap as described above."]
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/store/cells.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Store
+      # Converts values to and from what is stored in a sheet cell.
+      module Cells
+        EPOCH = Time.utc(1899, 12, 30)
+        DATETIME = /\A(\d{4})-(\d{2})-(\d{2})(?:[ T](\d{2}):(\d{2})(?::(\d{2}))?)?\z/
+
+        module_function
+
+        def to_cell(value, type)
+          case type
+          when :datetime then value.nil? ? "" : serial(value)
+          when :integer then value.nil? ? "" : Integer(value)
+          when :number then value.nil? ? "" : Float(value).round(4)
+          else value.to_s
+          end
+        end
+
+        def from_cell(text, type)
+          case type
+          when :datetime then parse_time(text)
+          when :integer then integer(text)
+          when :number then number(text)
+          else text.to_s
+          end
+        end
+
+        # A time as a sheet date: days since 1899-12-30, counted in UTC.
+        def serial(time) = (time.getutc - EPOCH) / 86_400.0
+
+        # Reads "2026-10-24 14:30" (seconds optional, date alone allowed) as UTC. Anything else gives nil.
+        def parse_time(text)
+          match = DATETIME.match(text.to_s.strip)
+          return unless match
+
+          year, month, day, hour, minute, second = match.captures.map(&:to_i)
+          Time.utc(year, month, day, hour, minute, second)
+        rescue ArgumentError
+          nil
+        end
+
+        def integer(text)
+          Integer(text.to_s.strip, 10)
+        rescue ArgumentError
+          nil
+        end
+
+        # A decimal number. A decimal comma, as a sheet in a German locale shows it, is read too. Anything else
+        # gives nil.
+        def number(text) = Float(text.to_s.strip.tr(",", "."), exception: false)
+
+        # The A1 name of a cell: column 1 is A, 27 is AA. Row and column start at 1.
+        def a1(row, column)
+          letters = +""
+          while column.positive?
+            column, remainder = (column - 1).divmod(26)
+            letters.prepend((65 + remainder).chr)
+          end
+          "#{letters}#{row}"
+        end
+      end
+    end
+  end
+end