yuno-proofreader 0.1.0

data/lib/yuno/proofreader/audit/report.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Audit
+      # Turns the result of an audit into the lines the audit command prints.
+      class Report
+        SHOWN = 30
+        TEXT = 60
+        KINDS = { added: "added", reappeared: "reappeared", seen_again: "seen again", closed: "closed",
+                  suppressed: "suppressed", ignored: "ignored", duplicate: "duplicates",
+                  close_skipped: "close skipped" }.freeze
+        QUIET = %i[seen_again suppressed ignored duplicate close_skipped].freeze
+
+        def initialize(result, verbose: false)
+          @result = result
+          @verbose = verbose
+        end
+
+        def lines
+          @result.status == :skipped ? skipped_lines : audited_lines
+        end
+
+        private
+
+        # ---- a page that has not changed ----
+
+        def skipped_lines
+          since = @result.skipped_since ? " since #{stamp(@result.skipped_since)}" : ""
+          ["Skipped #{@result.page}: unchanged#{since}. No model was called.", "Use --force to audit it anyway.",
+           @result.dry_run ? "Dry run: nothing was written." : "Its last_audited date was updated."]
+        end
+
+        # ---- an audit ----
+
+        def audited_lines
+          [headline, *call_lines, findings_line, "", summary_line, *verified_line, *new_lines, *closed_lines,
+           *quiet_lines, *discarded_lines, *note_lines, "", footer]
+        end
+
+        def headline
+          versions = @result.fetched.snapshots.map(&:language).join(", ")
+          "#{'[dry run] ' if @result.dry_run}Audited #{@result.page} (#{versions})"
+        end
+
+        def call_lines
+          audit = @result.calls.find { |call| call.role == :audit }
+          review = @result.calls.find { |call| call.role == :review }
+          ["  #{call_line('Audit:  ', audit)}", "  #{call_line('Review: ', review)}"]
+        end
+
+        def call_line(label, call)
+          return "#{label}not run" unless call
+
+          "#{label}#{call.model}, #{call.seconds.round} s, tokens in/out/thinking " \
+            "#{delimited(call.input_tokens)} / #{delimited(call.output_tokens)} / #{delimited(call.thinking_tokens)}"
+        end
+
+        def findings_line
+          review = @result.review_count.nil? ? "no review" : "#{@result.review_count} from the review"
+          "Findings: #{@result.audit_count} from the audit, #{review}."
+        end
+
+        def summary_line
+          counts = @result.plan.counts
+          parts = KINDS.filter_map { |kind, label| "#{counts[kind]} #{label}" if counts[kind] }
+          "Suggestions: #{parts.empty? ? 'no changes' : parts.join(', ')}."
+        end
+
+        def verified_line
+          additions = @result.plan.additions
+          return [] if additions.empty?
+
+          tally = additions.map(&:verified).tally
+          ["Of the #{additions.size} new suggestions, #{tally.fetch('yes', 0)} quote text that is on the page " \
+           "exactly, #{tally.fetch('loose', 0)} loosely and #{tally.fetch('no', 0)} not at all."]
+        end
+
+        def new_lines
+          additions = @result.plan.additions
+          return [] if additions.empty?
+
+          shown = @verbose ? additions : additions.first(SHOWN)
+          more = additions.size - shown.size
+          ["", "New suggestions:", *shown.map { |record| addition_line(record) },
+           *(more.positive? ? ["  ... and #{more} more (use --verbose)"] : [])]
+        end
+
+        def addition_line(record)
+          mark = record.reason.start_with?("[Reappeared") ? "!" : "+"
+          place = record.approximate_place.to_s.empty? ? "" : "  (#{record.approximate_place})"
+          "  #{mark} [#{record.found_by}] #{record.category}, #{record.language}: " \
+            "#{quoted(record.original)} -> #{quoted(record.suggested)}#{place}"
+        end
+
+        def closed_lines
+          closed = decisions { |decision| decision.kind == :closed }
+          return [] if closed.empty?
+
+          ["", "Closed:", *closed.map { |decision| "  - #{decision.summary}  (#{decision.reason})" }]
+        end
+
+        def quiet_lines
+          quiet = decisions { |decision| QUIET.include?(decision.kind) }
+          return [] if !@verbose || quiet.empty?
+
+          ["", "Also decided:", *quiet.map { |d| "  = #{d.kind.to_s.tr('_', ' ')}: #{d.summary} (#{d.reason})" }]
+        end
+
+        def discarded_lines
+          items = @result.discarded
+          return [] if items.empty?
+          unless @verbose
+            return ["",
+                    "#{items.size} items in the model answers could not be used (--verbose lists them)."]
+          end
+
+          ["", "Items in the model answers that could not be used:", *items.map { |item| "  - #{item}" }]
+        end
+
+        def note_lines
+          return [] if @result.notes.empty?
+
+          ["", "Notes:", *@result.notes.map { |note| "  - #{note}" }]
+        end
+
+        def footer
+          return "Dry run: nothing was written to the sheet." if @result.dry_run
+          if @result.complete
+            return "Written to the sheet. The audit is complete, so the page is skipped until it changes."
+          end
+
+          "Written to the sheet. The audit was not complete, so the page's fingerprint was not recorded and " \
+            "it will be audited again."
+        end
+
+        # ---- helpers ----
+
+        def decisions(&) = @result.plan.decisions.select(&)
+
+        def quoted(text)
+          flat = Findings::Text.collapse(text)
+          flat = "#{flat[0, TEXT - 1]}…" if flat.length > TEXT
+          %("#{flat}")
+        end
+
+        def stamp(time) = time.strftime("%Y-%m-%d %H:%M")
+
+        def delimited(number) = number.to_i.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/audit/runner.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Audit
+      # Audits one page from start to finish: fetches it, asks the model, asks a second model for an independent
+      # review, merges the answers, compares them with the suggestions already in the store and writes the changes.
+      # Everything it needs is passed in, so it runs with fakes in tests. A page that has not changed since its
+      # last complete audit is skipped without calling a model.
+      class Runner
+        Result = Data.define(:status, :page, :fetched, :plan, :calls, :audit_count, :review_count, :missing_steps,
+                             :discarded, :notes, :complete, :dry_run, :skipped_since)
+
+        def initialize(site:, store:, fetcher:, audit_client:, review_client:, prompt: Prompt.new(site),
+                       clock: -> { Time.now.utc }, progress: ->(_message) {}, log: Log.null)
+          @site = site
+          @store = store
+          @fetcher = fetcher
+          @audit_client = audit_client
+          @review_client = review_client
+          @prompt = prompt
+          @clock = clock
+          @progress = progress
+          @log = log
+        end
+
+        # query: a page name from the Pages tab, or any address of the page.
+        def run(query, dry_run: false, force: false)
+          record = find_record(query)
+          say("Fetching #{record.name} (#{record.urls.keys.join(', ')})...")
+          fetched = @fetcher.fetch(record)
+          raise Error, nothing_fetched(fetched) unless fetched.snapshots.any?(&:ok?)
+
+          now = @clock.call
+          return skip(record, fetched, now, dry_run) if unchanged?(record, fetched, force)
+
+          audit_page(record, fetched, now, dry_run)
+        end
+
+        private
+
+        def say(message) = @progress.call(message)
+
+        def find_record(query)
+          records = @store.pages
+          raise Error, "The Pages tab is empty. Run: yuno-proofreader pages import" if records.empty?
+
+          wanted = query.to_s.strip
+          records.find { |record| record.name == wanted || record.urls.value?(wanted) } ||
+            raise(Error, "No page #{wanted.inspect} in the Pages tab. Page names are in its first column.")
+        end
+
+        def nothing_fetched(fetched)
+          problems = fetched.snapshots.map { |snapshot| "#{snapshot.language}: #{snapshot.problem}" }.join("; ")
+          "None of the language versions of #{fetched.name} could be fetched (#{problems}). " \
+            "Nothing was sent to the model and nothing was written."
+        end
+
+        # ---- a page that has not changed ----
+
+        def unchanged?(record, fetched, force)
+          current = fetched.fingerprint
+          known = record.fingerprint.to_s.strip
+          !force && !current.nil? && !known.empty? && current == known
+        end
+
+        # The date is updated, so an unchanged page moves to the back of the queue. The fingerprint stays.
+        def skip(record, fetched, now, dry_run)
+          @store.record_audit(record, fingerprint: record.fingerprint.to_s.strip, at: now) unless dry_run
+          Result.new(status: :skipped, page: record.name, fetched:, plan: nil, calls: [], audit_count: 0,
+                     review_count: 0, missing_steps: [], discarded: [], notes: [], complete: true, dry_run:,
+                     skipped_since: record.last_audited)
+        end
+
+        # ---- an audit ----
+
+        def audit_page(record, fetched, now, dry_run)
+          attempt = attempt_for(fetched)
+          plan = plan_for(record, fetched, attempt, now)
+          complete = complete?(fetched, attempt)
+          write(record, fetched, plan, complete, now) unless dry_run
+          audited_result(record:, fetched:, plan:, attempt:, complete:, dry_run:)
+        end
+
+        def attempt_for(fetched)
+          user = @prompt.user_message(fetched)
+          first = ask(:audit, @audit_client, @prompt.audit_system, user)
+          review = try_review(user, first.call.model)
+          audit_read = Answers.audit(first.answer.data, steps: @prompt.steps, languages: @site.languages)
+          Attempt.new(first:, review:, audit_read:, review_read: read_review(review))
+        end
+
+        def ask(role, client, system, user, avoid: [])
+          say("Asking the #{role} model (this can take a minute or two)...")
+          answer = client.generate(system:, user:, avoid:)
+          say("The #{role} model answered: #{answer.model}, #{answer.seconds.round} s.")
+          Asked.new(answer:, call: Call.new(role:, model: answer.model, seconds: answer.seconds,
+                                            input_tokens: answer.input_tokens, output_tokens: answer.output_tokens,
+                                            thinking_tokens: answer.thinking_tokens))
+        end
+
+        # The review asks a model other than the audit's, when there is one. If it fails, the audit stands alone.
+        def try_review(user, audit_model)
+          asked = ask(:review, @review_client, @prompt.review_system, user, avoid: [audit_model])
+          Review.new(asked:, error: nil)
+        rescue LlmError => e
+          say("The second review failed: #{e.message}")
+          @log.warn("second review failed", message: e.message)
+          Review.new(asked: nil, error: e.message)
+        end
+
+        def read_review(review)
+          data = review.asked&.answer&.data
+          return unless data.is_a?(Hash) && data["issues"].is_a?(Array)
+
+          Answers.review(data, languages: @site.languages)
+        end
+
+        def plan_for(record, fetched, attempt, now)
+          merged = Findings::Merger.call(attempt.audit_read.findings, attempt.review_read&.findings)
+          Findings::Reconciler.call(page: fetched, findings: merged, existing: @store.suggestions(record.name),
+                                    ignore_terms: @site.context.ignore_terms, now:)
+        end
+
+        # Only a complete audit records the page's fingerprint, because that is what lets the next run skip it.
+        def complete?(fetched, attempt)
+          fetched.ok? && fetched.snapshots.none?(&:truncated) && attempt.missing_steps.empty? && attempt.reviewed?
+        end
+
+        def write(record, fetched, plan, complete, now)
+          plan.apply(@store)
+          @store.record_audit(record, fingerprint: complete ? fetched.fingerprint : "", at: now)
+        end
+
+        def audited_result(record:, fetched:, plan:, attempt:, complete:, dry_run:)
+          Result.new(status: :audited, page: record.name, fetched:, plan:, calls: attempt.calls,
+                     audit_count: attempt.audit_read.findings.size, review_count: attempt.review_read&.findings&.size,
+                     missing_steps: attempt.missing_steps, discarded: attempt.discarded,
+                     notes: attempt.notes(fetched), complete:, dry_run:, skipped_since: nil)
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/audit/steps.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Yuno
+  module Proofreader
+    module Audit
+      # One check of the step-by-step audit: what to look for, and which categories its findings may have.
+      Step = Data.define(:number, :name, :categories, :text)
+
+      # The steps of the audit, read from prompts/steps.yml.
+      module Steps
+        PATH = File.join(__dir__, "prompts", "steps.yml")
+
+        module_function
+
+        def default = @default ||= load
+
+        def load(path = PATH)
+          entries = YAML.safe_load_file(path)
+          raise Error, "#{path} must hold a list of steps" unless entries.is_a?(Array) && !entries.empty?
+
+          steps = entries.each_with_index.map { |entry, index| build(entry, index + 1, path) }
+          numbers = steps.map(&:number)
+          raise Error, "#{path}: step numbers must be unique" unless numbers.uniq.size == numbers.size
+
+          steps.sort_by(&:number)
+        end
+
+        def build(entry, position, path)
+          raise Error, "#{path}: step #{position} must be a mapping" unless entry.is_a?(Hash)
+
+          number = Integer(entry["number"], exception: false)
+          name = entry["name"].to_s.strip
+          text = entry["text"].to_s.strip
+          categories = Array(entry["categories"]).map { |category| category.to_s.strip.downcase }.reject(&:empty?).uniq
+          problem = problem_with(number, name, text, categories)
+          raise Error, "#{path}: step #{position} #{problem}" if problem
+
+          Step.new(number:, name:, categories:, text:)
+        end
+
+        def problem_with(number, name, text, categories)
+          return "needs a number from 1" unless number&.positive?
+          return "needs a name" if name.empty?
+          return "needs a text" if text.empty?
+          return "needs at least one category" if categories.empty?
+
+          unknown = categories - Findings::Finding::CATEGORIES
+          "has unknown categories: #{unknown.join(', ')}" unless unknown.empty?
+        end
+        private_class_method :build, :problem_with
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/cli/analysis_commands.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    # The command that ranks the pages of the site by importance: analyze.
+    module AnalysisCommands
+      private
+
+      def analyze(args)
+        options, rest, unknown = split_args(args, %w[dry-run no-ai reuse-ai])
+        problem = analyze_args_problem(options, rest, unknown)
+        return usage_error(problem) if problem
+
+        mode = ai_mode(options)
+        site = Site.load(rest.first || CLI::DEFAULT_SITE_FILE)
+        site.ready_for_audit! if mode == :fresh
+        result = analysis_runner(site, mode).run(judgement: mode, dry_run: options.key?("dry-run"))
+        Analysis::Report.new(result).lines.each { |line| @out.puts line }
+        0
+      end
+
+      def analyze_args_problem(options, rest, unknown)
+        return "Unknown option: #{unknown.first}" unless unknown.empty?
+        return "--no-ai and --reuse-ai cannot be used together" if options.key?("no-ai") && options.key?("reuse-ai")
+
+        "Too many arguments for analyze" if rest.size > 1
+      end
+
+      def ai_mode(options)
+        return :none if options.key?("no-ai")
+
+        options.key?("reuse-ai") ? :reuse : :fresh
+      end
+
+      # The model client is built first, so a missing API key stops the command before anything is read.
+      def analysis_runner(site, mode)
+        client = mode == :fresh ? Llm.client_for(site, env: @env, transport: @llm_transport, log:) : nil
+        Analysis::Runner.new(site:, store: store_for(site), http: http_client, client:, log:,
+                             progress: ->(message) { @out.puts message })
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/cli/audit_commands.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    # The commands that work with the prompts and the audit: prompt and audit. The queue and the batch form of
+    # audit (audit --next) are in QueueCommands.
+    module AuditCommands
+      DISPLAY_LIMIT = 300
+
+      private
+
+      def prompt(args)
+        options, rest, unknown = split_args(args, %w[review full])
+        problem = prompt_args_problem(rest, unknown)
+        return usage_error(problem) if problem
+
+        site = Site.load(rest[1] || CLI::DEFAULT_SITE_FILE).ready_for_audit!
+        page = find_page(Sitemap::Loader.call(site, http_client).pairing, rest.first)
+        fetched = Pages::Fetcher.new(site, http_client).fetch(page)
+        prompt_lines(Audit::Prompt.new(site), fetched, options).each { |line| @out.puts line }
+        0
+      end
+
+      def prompt_args_problem(rest, unknown)
+        return "Unknown option: #{unknown.first}" unless unknown.empty?
+        return "prompt needs a page name or address" if rest.empty?
+
+        "Too many arguments for prompt" if rest.size > 2
+      end
+
+      def prompt_lines(builder, fetched, options)
+        kind = options.key?("review") ? "review" : "audit"
+        system = kind == "review" ? builder.review_system : builder.audit_system
+        message = builder.user_message(fetched)
+        shown = options.key?("full") ? message : builder.user_message(fetched, html_limit: DISPLAY_LIMIT)
+        [*prompt_summary(kind, fetched, system, message), "", "--- system prompt (#{kind}) ---", system, "",
+         "--- user message ---", shown]
+      end
+
+      def prompt_summary(kind, fetched, system, message)
+        languages = fetched.snapshots.select(&:ok?).map(&:language).join(", ")
+        tokens = ((system.length + message.length) / 4.0).round
+        [%(#{kind.capitalize} prompt for page "#{fetched.name}" (versions: #{languages})),
+         "System prompt: #{count(system.length)} characters. User message: #{count(message.length)} characters.",
+         "Roughly #{count(tokens)} tokens in all (a very rough estimate of 4 characters per token).",
+         "Nothing was sent to any model."]
+      end
+
+      def count(number) = number.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
+
+      def audit(args)
+        count, remaining, problem = take_count(args)
+        return usage_error(problem) if problem
+
+        options, rest, unknown = split_args(remaining, %w[dry-run force verbose next include-not-due])
+        problem = audit_args_problem(options, rest, unknown, count)
+        return usage_error(problem) if problem
+
+        batch = options.key?("next")
+        site = Site.load(rest[batch ? 0 : 1] || CLI::DEFAULT_SITE_FILE).ready_for_audit!
+        batch ? audit_batch(site, options, count) : audit_page(site, rest.first, options)
+      end
+
+      def audit_page(site, page, options)
+        result = audit_runner(site).run(page, dry_run: options.key?("dry-run"), force: options.key?("force"))
+        Audit::LogFields.record(log, result)
+        Audit::Report.new(result, verbose: options.key?("verbose")).lines.each { |line| @out.puts line }
+        0
+      end
+
+      def audit_args_problem(options, rest, unknown, count)
+        return "Unknown option: #{unknown.first}" unless unknown.empty?
+        return batch_args_problem(rest) if options.key?("next")
+        return "--count and --include-not-due only go with --next" if count || options.key?("include-not-due")
+        return "audit needs a page name or address, or --next" if rest.empty?
+
+        "Too many arguments for audit" if rest.size > 2
+      end
+
+      def batch_args_problem(rest)
+        "audit --next takes no page name, only a settings file" if rest.size > 1
+      end
+
+      def audit_runner(site, store = store_for(site))
+        audit_client = Llm.client_for(site, env: @env, transport: @llm_transport, log:)
+        review_client = Llm.client_for(site, env: @env, review: true, transport: @llm_transport, log:)
+        Audit::Runner.new(site:, store:, fetcher: Pages::Fetcher.new(site, http_client), audit_client:,
+                          review_client:, progress: ->(message) { @out.puts message }, log:)
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/cli/command_log.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    # Puts a line in the log when a command starts and when it ends, and makes the log available to the commands.
+    # It wraps CLI#run, so every command is covered without each one doing it. A run that fails always ends with
+    # an error line, which is what an alert on the log can look for.
+    module CommandLog
+      def run(argv)
+        begin
+          log
+        rescue ConfigError => e
+          @err.puts "Error: #{e.message}"
+          return 1
+        end
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        log.info("run started", command: argv.first, args: argv.drop(1).join(" "), version: VERSION)
+        finish(super, started)
+      rescue StandardError => e
+        log.error("run failed", error: e.class.name, message: e.message, seconds: elapsed(started))
+        raise
+      end
+
+      private
+
+      # Reads LOG_LEVEL and LOG_FORMAT the first time it is needed.
+      def log = @log ||= Log.from_env(@env, @err)
+
+      def finish(code, started)
+        seconds = elapsed(started)
+        if code.zero?
+          log.info("run finished", result: "ok", exit_code: code, seconds:)
+        else
+          log.error("run failed", result: "failed", exit_code: code, seconds:)
+        end
+        code
+      end
+
+      def elapsed(started) = started ? (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round(1) : 0.0
+    end
+  end
+end

data/lib/yuno/proofreader/cli/flags.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    # Reads the options of a command: switches (--yes), single values (--website ADDRESS or --website=ADDRESS) and
+    # values that may be given several times (--language de --language es:/es).
+    module Flags
+      Parsed = Data.define(:switches, :values, :lists, :rest, :problem)
+      KINDS = { switches: :switch, values: :value, lists: :list }.freeze
+
+      module_function
+
+      def parse(args, switches: [], values: [], lists: [])
+        kinds = { switches:, values:, lists: }.flat_map do |group, names|
+          names.map do |name|
+            [name, KINDS.fetch(group)]
+          end
+        end
+        state = { switches: {}, values: {}, lists: lists.to_h { |name| [name, []] }, rest: [], problem: nil }
+        queue = args.dup
+        consume(queue.shift, queue, state, kinds.to_h) until queue.empty? || state[:problem]
+        Parsed.new(**state)
+      end
+
+      def consume(arg, queue, state, kinds)
+        return state[:rest] << arg unless arg.start_with?("--")
+
+        name, inline = arg.delete_prefix("--").split("=", 2)
+        kind = kinds[name]
+        return state[:problem] = "Unknown option: --#{name}" unless kind
+
+        kind == :switch ? take_switch(name, inline, state) : take_value(name, kind, inline, queue, state)
+      end
+
+      def take_switch(name, inline, state)
+        state[:problem] = "--#{name} takes no value" if inline
+        state[:switches][name] = true
+      end
+
+      def take_value(name, kind, inline, queue, state)
+        value = inline || (queue.shift unless queue.first.to_s.start_with?("--"))
+        return state[:problem] = "--#{name} needs a value" if value.nil?
+
+        kind == :list ? state[:lists][name] << value : state[:values][name] = value
+      end
+    end
+  end
+end