yuno-proofreader 0.1.0

data/lib/yuno/proofreader/analysis/signals.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "date"
+
+module Yuno
+  module Proofreader
+    module Analysis
+      # The measurements of one page. values holds each signal from 0 to 1, or nil when it could not be measured.
+      # mentioned_by is the number of pages that list or link it, and changed the date it last changed, if known.
+      PageSignals = Data.define(:key, :page, :path, :values, :mentioned_by, :changed)
+
+      # Measures every page of the pairing: how well the homepage reaches it, whether the homepage shows it, how many
+      # pages list or link it, and how recently it changed.
+      class Signals
+        FOOTER_PRESENCE = 0.4
+
+        def self.call(**inputs) = new(**inputs).call
+
+        def initialize(inventory:, pages:, weights:, language:, today: Date.today, order: "editorial")
+          @inventory = inventory
+          @pages = pages
+          @language = language
+          @today = today
+          @half_life = weights.half_life
+          @graph = Graph.new(inventory, weights, order:)
+          @home = Links.key(inventory.home)
+          @shown, @footer = homepage_keys
+          @mentions = count_mentions
+        end
+
+        def call
+          counts = @pages.map { |page| @mentions.fetch(Links.key(url_of(page)), 0) }
+          ranks = percentile_ranks(counts)
+          @pages.each_with_index.map { |page, index| measure(page, counts[index], ranks[index]) }
+        end
+
+        private
+
+        def url_of(page) = page.urls[@language] || page.urls.values.first
+
+        def measure(page, count, rank)
+          url = url_of(page)
+          key = Links.key(url)
+          changed = changed_on(page, url)
+          values = { reach: @graph.paths[key]&.value || 0.0, presence: presence(key), mentions: rank,
+                     freshness: changed && freshness(changed) }
+          PageSignals.new(key:, page:, path: @graph.paths[key], values:, mentioned_by: count, changed:)
+        end
+
+        # The pages the homepage shows in its navigation or lists, and the ones only its footer links.
+        def homepage_keys
+          facts = @inventory.facts_for(@inventory.home)
+          return [Set.new, Set.new] unless facts
+
+          links = facts.chrome_links
+          listed = facts.lists.flat_map { |list| list.list_members.map(&:url) }
+          shown = (links[:header] + links[:nav] + listed).to_set { |url| Links.key(url) }
+          [shown, links[:footer].to_set { |url| Links.key(url) }]
+        end
+
+        def presence(key)
+          return 1.0 if key == @home || @shown.include?(key)
+
+          @footer.include?(key) ? FOOTER_PRESENCE : 0.0
+        end
+
+        # How many of the pages read list a page or link to it in their text. Each page counts once.
+        def count_mentions
+          counts = Hash.new(0)
+          @inventory.facts.each_value do |facts|
+            listed = facts.lists.flat_map { |list| list.list_members.map(&:url) }
+            (listed + facts.main_links).map { |url| Links.key(url) }.uniq.each { |key| counts[key] += 1 }
+          end
+          counts
+        end
+
+        # Where each value stands among the others, from 0 to 1. Equal values share the middle of their places.
+        def percentile_ranks(values)
+          values.map do |value|
+            below = values.count { |other| other < value }
+            (below + (0.5 * values.count(value))).fdiv(values.size)
+          end
+        end
+
+        def changed_on(page, url)
+          [page.lastmod, @inventory.facts_for(url)&.date_modified].filter_map { |value| date_of(value) }.max
+        end
+
+        def date_of(value)
+          return value if value.is_a?(Date)
+
+          text = value.to_s.strip[/\A\d{4}-\d{2}-\d{2}/]
+          text && Date.iso8601(text)
+        rescue ArgumentError
+          nil
+        end
+
+        def freshness(date) = 0.5**([(@today - date).to_i, 0].max / @half_life)
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/analysis/structured_data.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Yuno
+  module Proofreader
+    module Analysis
+      # Reads the JSON-LD blocks of a page. roots are the top-level objects (a block, or each entry of its @graph).
+      # nodes are every object with an @type, nested ones included, so a list is found wherever a site puts it:
+      # at the top, in an @graph, or under mainEntity or hasPart.
+      module StructuredData
+        SCHEMA = %r{\Ahttps?://schema\.org/}i
+        SELECTOR = 'script[type="application/ld+json"]'
+        Result = Data.define(:roots, :nodes, :problems)
+
+        module_function
+
+        def read(doc)
+          roots = []
+          problems = []
+          doc.css(SELECTOR).each_with_index do |script, index|
+            roots.concat(roots_of(JSON.parse(script.text)))
+          rescue JSON::ParserError
+            problems << "JSON-LD block #{index + 1} is not valid JSON"
+          end
+          Result.new(roots:, nodes: roots.flat_map { |root| typed_nodes(root) }, problems:)
+        end
+
+        # The types of a node, without the schema.org prefix.
+        def types(node)
+          return [] unless node.is_a?(Hash)
+
+          list = node["@type"].is_a?(Array) ? node["@type"] : [node["@type"]]
+          list.compact.map { |type| type.to_s.sub(SCHEMA, "") }
+        end
+
+        def type?(node, name) = types(node).include?(name)
+
+        def roots_of(data)
+          list = data.is_a?(Array) ? data : [data]
+          list.grep(Hash).flat_map { |item| item["@graph"].is_a?(Array) ? item["@graph"].grep(Hash) : [item] }
+        end
+
+        # The value and everything below it that has an @type, depth first.
+        def typed_nodes(value)
+          case value
+          when Array then value.flat_map { |item| typed_nodes(item) }
+          when Hash
+            own = value.key?("@type") ? [value] : []
+            own + value.values.flat_map { |child| typed_nodes(child) }
+          else []
+          end
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/analysis/url_tree.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Analysis
+      # The pages of the sitemap arranged by their addresses: /a/b is under /a when /a is a page too. Pages with
+      # several pages under them are likely index pages.
+      class UrlTree
+        MIN_CHILDREN = 2
+
+        attr_reader :urls
+
+        def initialize(urls)
+          @urls = urls.uniq { |url| Links.key(url) }
+          @by_key = @urls.to_h { |url| [Links.key(url), url] }
+          @children = Hash.new { |hash, key| hash[key] = [] }
+          @urls.each do |url|
+            above = parent(url)
+            @children[Links.key(above)] << url if above
+          end
+        end
+
+        def include?(url) = @by_key.key?(Links.key(url))
+
+        def children(url) = @children.fetch(Links.key(url), [])
+
+        def depth(url) = Links.segments(url).size
+
+        # The nearest page above this one, or nil.
+        def parent(url)
+          parts = Links.segments(url)
+          host = Links.host(url)
+          until parts.empty?
+            parts.pop
+            found = @by_key["#{host}/#{parts.join('/')}"]
+            return found if found
+          end
+          nil
+        end
+
+        # Pages with at least MIN_CHILDREN pages under them, the shallowest first.
+        def hubs
+          found = @urls.each_with_index.select { |url, _index| children(url).size >= MIN_CHILDREN }
+          found.sort_by { |url, index| [depth(url), index] }.map(&:first)
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/analysis/weights.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Yuno
+  module Proofreader
+    module Analysis
+      # How the importance of a page is weighed, read from weights.yml.
+      class Weights < Data.define(:signals, :edges, :half_life, :tiers, :min_spread, :ai_share_min, :ai_share_max,
+                                  :disagreement)
+        PATH = File.join(__dir__, "weights.yml")
+        SIGNALS = %w[reach presence mentions freshness].freeze
+        EDGES = %w[navigation footer list cards list_floor text address].freeze
+
+        def self.default = @default ||= load
+
+        def self.load(path = PATH)
+          data = YAML.safe_load_file(path)
+          raise Error, "#{path} must hold the weights" unless data.is_a?(Hash)
+
+          new(signals: numbers(data["signals"], SIGNALS, "signals", path),
+              edges: numbers(data["edges"], EDGES, "edges", path, max: 1.0),
+              half_life: half_life(data["freshness_half_life_days"], path),
+              tiers: tiers(data["tiers"], path),
+              min_spread: Float(data.fetch("min_spread", 0.01), exception: false) || 0.01,
+              **ai_settings(data, path))
+        end
+
+        def self.numbers(value, keys, label, path, max: nil)
+          raise Error, "#{path}: #{label} must be a mapping" unless value.is_a?(Hash)
+
+          missing = keys - value.keys.map(&:to_s)
+          raise Error, "#{path}: #{label} lacks #{missing.join(', ')}" unless missing.empty?
+
+          keys.to_h { |key| [key.to_sym, number(value[key], "#{label}.#{key}", path, max:)] }
+        end
+
+        def self.number(value, label, path, max: nil)
+          number = Float(value, exception: false)
+          return number if number && number >= 0 && (max.nil? || number <= max)
+
+          raise Error, "#{path}: #{label} must be a number from 0#{" to #{max}" if max}"
+        end
+
+        def self.half_life(value, path)
+          days = Float(value, exception: false)
+          return days if days&.positive?
+
+          raise Error, "#{path}: freshness_half_life_days must be a number of days above 0"
+        end
+
+        def self.tiers(value, path)
+          shares = Array(value).map { |share| Float(share, exception: false) }
+          return shares if rising_shares?(shares)
+
+          raise Error, "#{path}: tiers must be rising shares between 0 and 1, such as [0.1, 0.3, 0.7]"
+        end
+
+        def self.rising_shares?(shares)
+          !shares.empty? && shares.all? { |share| share&.positive? && share < 1 } &&
+            shares.each_cons(2).all? { |low, high| low < high }
+        end
+
+        # The AI's share and when two models disagree. A file without these keys gets the defaults.
+        def self.ai_settings(data, path)
+          low = number(data.fetch("ai_share_min", 0.35), "ai_share_min", path, max: 1.0)
+          high = number(data.fetch("ai_share_max", 0.65), "ai_share_max", path, max: 1.0)
+          raise Error, "#{path}: ai_share_min must not be above ai_share_max" if low > high
+
+          { ai_share_min: low, ai_share_max: high,
+            disagreement: number(data.fetch("disagreement", 35), "disagreement", path, max: 100.0) }
+        end
+        private_class_method :numbers, :number, :half_life, :tiers, :rising_shares?, :ai_settings
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/analysis/weights.yml

@@ -0,0 +1,39 @@
+# How the importance of a page is weighed. This is data, not code: change it to retune the ranking.
+#
+# signals                   How much each measurement counts. Only the ratios matter. A signal measured on few pages counts less,
+#                     and one that is the same on every page counts nothing.
+# reach                     How the homepage leads to the page: through its navigation and the lists of the index pages, where
+#                     earlier entries weigh more. The steps multiply, so pages deep in the structure end lower.
+# presence                  Shown on the homepage: in its navigation or its lists (1), only in its footer (0.4).
+# mentions                  How many pages list it or link to it in their text, compared with the other pages.
+# freshness                 How recently it changed, from the sitemap's lastmod or the page's own date.
+# edges                     The weight of each kind of step from one page to the next, from 0 to 1.
+# list_floor                What the last entry of a long list keeps of the list's weight. The first keeps all of it.
+# cards                     A list read from the headings of a page instead of from structured data.
+# address                   A page under another one in its address, for pages nothing else leads to.
+# freshness_half_life_days  After how many days freshness has halved.
+# tiers                     Where the tiers end, as shares of the ranking: [0.1, 0.3, 0.7] puts the top 10% in tier 1, the next
+#                     20% in tier 2, the next 40% in tier 3 and the rest in tier 4.
+signals:
+  reach: 50
+  presence: 20
+  mentions: 15
+  freshness: 15
+edges:
+  navigation: 0.9
+  footer: 0.5
+  list: 1.0
+  cards: 0.85
+  list_floor: 0.6
+  text: 0.35
+  address: 0.25
+freshness_half_life_days: 180
+tiers: [0.1, 0.3, 0.7]
+min_spread: 0.01
+
+# ai_share_min, ai_share_max  The share of the score the AI's judgement gets: ai_share_min when the navigation and
+#            lists of the site lead to every page, growing to ai_share_max as they lead to fewer.
+# disagreement  When two models' judgements of a page differ by this many points (of 100) or more, it is reported.
+ai_share_min: 0.35
+ai_share_max: 0.65
+disagreement: 35

data/lib/yuno/proofreader/audit/agenda.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Audit
+      # One page in the order it is to be audited, and why. state is :never (never audited), :overdue (its time has
+      # come) or :waiting (audited recently enough). tier is the one that counted: the page's own, or the default.
+      Pick = Data.define(:record, :state, :tier, :days_since, :interval, :reason) do
+        def due? = state != :waiting
+
+        # How far into its interval the page is: 1.0 means it comes due now. Nil for a page that was never audited.
+        def ratio = days_since && (days_since / interval)
+      end
+
+      # Puts the pages in the order they are to be audited: first the pages never audited, the most important
+      # first; then the pages whose time has come, the furthest past their own interval first (the more important
+      # first on ties); and last the pages that are not due, the nearest to due first. A page whose last audit was
+      # incomplete counts from the retry period and not from the interval of its tier.
+      class Agenda
+        RANKS = { never: 0, overdue: 1, waiting: 2 }.freeze
+        DAY = 86_400.0
+
+        def self.call(records:, cadence:, now:) = new(records, cadence, now).call
+
+        def initialize(records, cadence, now)
+          @records = records
+          @cadence = cadence
+          @now = now
+        end
+
+        def call
+          picks = @records.map { |record| pick_for(record) }
+          picks.each_with_index.sort_by { |pick, index| [*order_key(pick), index] }.map(&:first)
+        end
+
+        private
+
+        def counted_tier(record) = record.tier || @cadence.default_tier
+
+        def tier_part(record)
+          record.tier ? "tier #{record.tier}" : "tier not set, counted as #{@cadence.default_tier}"
+        end
+
+        def plural(number, word) = "#{number} #{word}#{'s' unless number == 1}"
+
+        def pick_for(record)
+          record.last_audited ? audited(record, record.last_audited) : never_audited(record)
+        end
+
+        def never_audited(record)
+          Pick.new(record:, state: :never, tier: counted_tier(record), days_since: nil, interval: nil,
+                   reason: "never audited (#{tier_part(record)})")
+        end
+
+        def audited(record, last)
+          days = [(@now - last) / DAY, 0.0].max
+          retrying = record.fingerprint.to_s.strip.empty?
+          interval = retrying ? @cadence.retry_days : @cadence.interval_for(record.tier)
+          state = days >= interval ? :overdue : :waiting
+          Pick.new(record:, state:, tier: counted_tier(record), days_since: days, interval:,
+                   reason: "#{timing(state, days, interval)} (#{note(record, interval, retrying)})")
+        end
+
+        def note(record, interval, retrying)
+          days = plural(interval, "day")
+          retrying ? "last audit incomplete, retry after #{days}" : "#{tier_part(record)}, every #{days}"
+        end
+
+        def timing(state, days, interval)
+          if state == :waiting
+            return "audited #{plural(days.floor, 'day')} ago, next in #{plural((interval - days).ceil, 'day')}"
+          end
+
+          over = (days - interval).floor
+          over.zero? ? "due now" : "#{plural(over, 'day')} overdue"
+        end
+
+        def order_key(pick)
+          importance = -pick.record.importance.to_i
+          rank = RANKS.fetch(pick.state)
+          pick.state == :never ? [rank, importance, 0] : [rank, -pick.ratio, importance]
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/audit/agenda_report.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Audit
+      # Turns the order of the pages into the lines the queue command prints.
+      class AgendaReport
+        SHOWN = 20
+
+        def initialize(picks, shown: SHOWN, everything: false)
+          @picks = picks
+          @shown = shown
+          @everything = everything
+        end
+
+        def lines
+          return ["The Pages tab is empty. Run: yuno-proofreader pages import"] if @picks.empty?
+
+          [*summary, *listing, *untiered]
+        end
+
+        private
+
+        def due = @picks.select(&:due?)
+
+        def pool = @everything ? @picks : due
+
+        def summary
+          return nothing_due if due.empty?
+
+          never = due.count { |pick| pick.state == :never }
+          ["#{@picks.size} page(s): #{due.size} due (#{never} never audited, #{due.size - never} overdue), " \
+           "#{@picks.size - due.size} waiting."]
+        end
+
+        def nothing_due
+          soonest = @picks.first
+          ["Nothing is due: all #{@picks.size} page(s) were audited within their cadence.",
+           "Next to come due: #{soonest.record.name}, #{soonest.reason}"]
+        end
+
+        def listing
+          shown = pool.first(@shown)
+          return [] if shown.empty?
+
+          rows = shown.each_with_index.map do |pick, index|
+            format("  %3<place>d. %-40<name>s %<reason>s", place: index + 1, name: pick.record.name,
+                                                           reason: pick.reason)
+          end
+          hidden = pool.size - shown.size
+          more = hidden.positive? ? ["  ... and #{hidden} more (--count=N for more, --all includes pages not due)"] : []
+          ["", *rows, *more]
+        end
+
+        def untiered
+          missing = @picks.select { |pick| pick.record.tier.nil? }
+          return [] if missing.empty?
+
+          ["", "#{missing.size} page(s) have no tier yet and count as tier #{missing.first.tier}. " \
+               "Run: yuno-proofreader analyze"]
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/audit/answers.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Audit
+      # Turns what a model answered into findings, and says what was missing or unusable in the answer.
+      module Answers
+        # missing_steps: the numbers of audit steps the answer left out.
+        # discarded: a line for everything in the answer that was ignored, and why.
+        Result = Data.define(:findings, :missing_steps, :discarded)
+
+        module_function
+
+        # The answer of the step-by-step audit: { "steps": [{ "step": 1, "issues": [...] }, ...] }.
+        def audit(data, steps:, languages:, limit: Prompt::MAX_ISSUES_PER_STEP)
+          listed = data.is_a?(Hash) ? data["steps"] : nil
+          return no_steps(steps) unless listed.is_a?(Array)
+
+          by_number = steps.to_h { |step| [step.number, step] }
+          reads = listed.each_with_index.map { |entry, index| read_entry(entry, index, by_number, languages, limit) }
+          combine(reads, steps)
+        end
+
+        def no_steps(steps)
+          Result.new(findings: [], missing_steps: steps.map(&:number), discarded: ["the answer has no list of steps"])
+        end
+
+        # reads: [step or nil, result] for each entry of the answer.
+        def combine(reads, steps)
+          results = reads.map(&:last)
+          answered = reads.filter_map { |step, _result| step&.number }
+          Result.new(findings: results.flat_map(&:findings), missing_steps: steps.map(&:number) - answered,
+                     discarded: results.flat_map(&:discarded))
+        end
+
+        # The answer of the second review: { "issues": [...] }.
+        def review(data, languages:, limit: Prompt::MAX_REVIEW_ISSUES)
+          items = data.is_a?(Hash) ? data["issues"] : nil
+          return nothing("the answer has no list of issues") unless items.is_a?(Array)
+
+          parsed = Findings::Finding.parse_all(items.first(limit), languages:)
+          notes = parsed.discarded.dup
+          notes << "#{items.size - limit} findings beyond the limit of #{limit} were ignored" if items.size > limit
+          Result.new(findings: parsed.findings, missing_steps: [], discarded: notes)
+        end
+
+        def read_entry(entry, index, by_number, languages, limit)
+          step = entry.is_a?(Hash) ? by_number[number_of(entry["step"])] : nil
+          return [nil, nothing("entry #{index + 1} of the answer is not a step of this audit")] unless step
+
+          [step, read_step(entry["issues"], step, languages, limit)]
+        end
+
+        def read_step(items, step, languages, limit)
+          return nothing("step #{step.number}: the issues are not a list") unless items.nil? || items.is_a?(Array)
+
+          items = Array(items)
+          parsed = Findings::Finding.parse_all(items.first(limit), languages:, categories: step.categories,
+                                                                   fallback: step.categories.first)
+          notes = parsed.discarded.map { |note| "step #{step.number}: #{note}" }
+          if items.size > limit
+            notes << "step #{step.number}: #{items.size - limit} findings beyond the limit of #{limit} were ignored"
+          end
+          Result.new(findings: parsed.findings, missing_steps: [], discarded: notes)
+        end
+
+        def number_of(value) = value.to_s.strip[/\A\d+\z/]&.to_i
+
+        def nothing(note) = Result.new(findings: [], missing_steps: [], discarded: [note])
+        private_class_method :no_steps, :combine, :read_entry, :read_step, :number_of, :nothing
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/audit/attempt.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Audit
+      # One model call: which model answered, how long it took and what it cost.
+      Call = Data.define(:role, :model, :seconds, :input_tokens, :output_tokens, :thinking_tokens)
+
+      # A call that was made, with its answer.
+      Asked = Data.define(:answer, :call)
+
+      # The second review: the call, or why there is none.
+      Review = Data.define(:asked, :error)
+
+      # The two model calls for one page and what was read from them. audit_read and review_read are
+      # Answers::Result values. review_read is nil when the review failed or its answer could not be used.
+      Attempt = Data.define(:first, :review, :audit_read, :review_read) do
+        def calls = [first.call, review.asked&.call].compact
+
+        def reviewed? = !review_read.nil?
+
+        def missing_steps = audit_read.missing_steps
+
+        def discarded = [*audit_read.discarded, *Array(review_read&.discarded)]
+
+        # What the person should know about this attempt, beyond the numbers.
+        def notes(fetched)
+          [*unusable_versions(fetched), steps_note, *review_notes, cut_note(fetched)].compact
+        end
+
+        private
+
+        def unusable_versions(fetched)
+          fetched.snapshots.reject(&:ok?).map do |snapshot|
+            "The #{snapshot.language} version could not be used (#{snapshot.problem}), " \
+              "so it was left out of the review."
+          end
+        end
+
+        def steps_note
+          return if missing_steps.empty?
+
+          "The audit left out step #{missing_steps.join(', ')}, so it does not count as complete."
+        end
+
+        def review_notes
+          if review.error
+            ["The second review failed (#{review.error}). New findings are marked 'audit (unreviewed)'."]
+          elsif review_read.nil?
+            ["The second review answered in a way that could not be used. " \
+             "New findings are marked 'audit (unreviewed)'."]
+          elsif review.asked.call.model == first.call.model
+            ["The second review used the same model as the audit, because no other model was available. " \
+             "It is not independent."]
+          else
+            []
+          end
+        end
+
+        def cut_note(fetched)
+          cut = fetched.snapshots.select(&:truncated).map(&:language)
+          return if cut.empty?
+
+          "The #{cut.join(', ')} version was cut at the size limit, so its end was not reviewed."
+        end
+      end
+    end
+  end
+end