yuno-proofreader 0.1.0

data/lib/yuno/proofreader/init/web.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Init
+      # The two things init looks up on the web: the homepage of the site, and its sitemap. Both honor robots.txt
+      # and the pace of the fetcher.
+      class Web
+        def initialize(http:)
+          @http = http
+        end
+
+        # [html, address] of the homepage. Raises FetchError when it cannot be read.
+        def homepage(origin)
+          site = Site.from_hash("website" => origin, "languages" => [{ "code" => "en", "prefix" => "" }])
+          url = "#{origin}/"
+          page = Sitemap::Page.new(name: url, urls: { "en" => url }, lastmod: "", notes: [])
+          snapshot = Pages::Fetcher.new(site, @http).fetch(page).snapshots.first
+          raise FetchError, snapshot.problem.to_s unless snapshot.ok?
+
+          [snapshot.raw, snapshot.final_url]
+        end
+
+        # [how many pages the sitemap lists, whether they could be paired by language]. Raises an Error when no
+        # sitemap can be used.
+        def sitemap(settings)
+          pairing = Sitemap::Loader.call(Site.from_hash(settings), @http).pairing
+          [pairing.pages.size, pairing.resolved?]
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/llm/client.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Llm
+      # Asks a list of models in turn until one answers with JSON. A model that is over its limit or does not
+      # exist is skipped for the rest of the run, and a round in which only overloaded models were met is
+      # repeated after a pause. It knows nothing about a particular provider or model. What goes wrong is written
+      # to the log as warnings, and each answer, with its tokens, at debug level.
+      class Client
+        RETRY_ROUNDS = 3
+        RETRY_WAIT = 20
+        SKIP_MINUTE = 65
+        SKIP_DAY = 21_600
+        MAX_OUTPUT_TOKENS = 24_000
+        REAL_SLEEP = ->(seconds) { sleep(seconds) }
+        REAL_CLOCK = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+        Failure = Data.define(:model, :message, :kind)
+
+        attr_reader :models
+
+        def initialize(provider:, models:, sleeper: REAL_SLEEP, clock: REAL_CLOCK, retry_wait: RETRY_WAIT,
+                       max_output_tokens: MAX_OUTPUT_TOKENS, log: Log.null)
+          raise ArgumentError, "The client needs at least one model" if models.empty?
+
+          @provider = provider
+          @models = models.dup.freeze
+          @sleeper = sleeper
+          @clock = clock
+          @retry_wait = retry_wait
+          @max_output_tokens = max_output_tokens
+          @log = log
+          @skipped = {}
+        end
+
+        # Whether there is a model other than this one, for a second opinion that is independent of the first.
+        def independent_of?(model) = @models.any? { |name| name != model }
+
+        # avoid: models to try last, such as the one that gave the answer this one should be independent of.
+        def generate(system:, user:, avoid: [])
+          failures = []
+          RETRY_ROUNDS.times do |round|
+            outcome = round_of(order(avoid), system, user, failures)
+            return outcome if outcome.is_a?(Answer)
+            break unless outcome == :transient && round < RETRY_ROUNDS - 1
+
+            pause(round)
+          end
+          raise UnavailableError, unavailable_message(failures)
+        end
+
+        private
+
+        def pause(round)
+          seconds = @retry_wait * (round + 1)
+          @log.warn("waiting before trying the models again", seconds:, round: round + 1)
+          @sleeper.call(seconds)
+        end
+
+        def order(avoid)
+          preferred = @models.reject { |model| avoid.include?(model) }
+          preferred + (@models - preferred)
+        end
+
+        # Tries every model once. Returns the Answer, or :transient when waiting might help, or nil.
+        def round_of(models, system, user, failures)
+          transient = false
+          models.each do |model|
+            next if skipped?(model)
+
+            result = attempt(model, system, user, failures)
+            return result if result.is_a?(Answer)
+
+            transient ||= result == :transient
+          end
+          transient ? :transient : nil
+        end
+
+        def attempt(model, system, user, failures)
+          started = @clock.call
+          completion = @provider.complete(model:, system:, user:, max_output_tokens: @max_output_tokens)
+          log_answer(answer_from(model, completion, @clock.call - started))
+        rescue AnswerError => e
+          failures << Failure.new(model:, message: e.message, kind: :answer)
+          @log.warn("model answer unusable", model:, message: e.message)
+          nil
+        rescue ProviderError => e
+          failures << Failure.new(model:, message: e.message, kind: e.kind)
+          @log.warn("model failed", model:, kind: e.kind, message: e.message)
+          react(model, e)
+        end
+
+        def answer_from(model, completion, seconds)
+          data = JsonText.parse(completion.text, finish: completion.finish)
+          Answer.new(data:, model:, finish: completion.finish, input_tokens: completion.input_tokens,
+                     output_tokens: completion.output_tokens, thinking_tokens: completion.thinking_tokens, seconds:)
+        end
+
+        def log_answer(answer)
+          @log.debug("model answered", model: answer.model, seconds: answer.seconds, finish: answer.finish,
+                                       input_tokens: answer.input_tokens, output_tokens: answer.output_tokens,
+                                       thinking_tokens: answer.thinking_tokens)
+          answer
+        end
+
+        def react(model, error)
+          case error.kind
+          when :auth then raise error
+          when :rate_limited then skip(model, error.daily ? SKIP_DAY : SKIP_MINUTE)
+          when :not_found, :forbidden then skip(model, SKIP_DAY)
+          when :overloaded, :network then :transient
+          end
+        end
+
+        def skip(model, seconds)
+          @skipped[model] = @clock.call + seconds
+          @log.warn("model paused", model:, seconds:)
+          nil
+        end
+
+        def skipped?(model)
+          limit = @skipped[model]
+          !limit.nil? && @clock.call < limit
+        end
+
+        def unavailable_message(failures)
+          detail = failures.last(6).map { |failure| "#{failure.model}: #{failure.message}" }.join(" | ")
+          detail = "every model is paused after limits" if detail.empty?
+          "No model could answer: #{detail}.#{choose_hint(failures)}"
+        end
+
+        def choose_hint(failures)
+          return "" unless failures.any? { |failure| %i[not_found forbidden].include?(failure.kind) }
+
+          " Run yuno-proofreader models to see which models your key can use, then set models in proofreader.yml."
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/llm/defaults.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Yuno
+  module Proofreader
+    module Llm
+      # The built-in model lists, read from defaults.yml.
+      module Defaults
+        PATH = File.join(__dir__, "defaults.yml")
+
+        module_function
+
+        def models_for(provider_id)
+          Array(table[provider_id.to_s]).map { |name| name.to_s.strip }.reject(&:empty?)
+        end
+
+        def table = @table ||= YAML.safe_load_file(PATH) || {}
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/llm/defaults.yml

@@ -0,0 +1,6 @@
+# Fallback model lists, used only when proofreader.yml does not set `models`.
+# Model names change often, so look this list over at every release. Used models can be replace at any time in proofreader.yml.
+gemini:
+  - gemini-3.8-flash
+  - gemini-3.7-flash
+  - gemini-3.6-flash

data/lib/yuno/proofreader/llm/errors.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Llm
+      # A model call that failed. kind says what to do about it: :rate_limited (daily says whether the limit
+      # is a daily one), :overloaded, :network, :empty, :blocked, :not_found, :forbidden, :bad_request,
+      # :auth (the key is wrong, so nothing will work) or :other.
+      class ProviderError < LlmError
+        attr_reader :kind, :daily
+
+        def initialize(message, kind: :other, daily: false)
+          super(message)
+          @kind = kind
+          @daily = daily
+        end
+      end
+
+      # A model answered, but not with JSON that can be used.
+      class AnswerError < LlmError; end
+
+      # None of the models could answer.
+      class UnavailableError < LlmError; end
+    end
+  end
+end

data/lib/yuno/proofreader/llm/factory.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    # Builds the pieces for calling models from the site settings and the environment.
+    module Llm
+      # The models in use and where the list came from (:settings or :defaults).
+      Selection = Data.define(:names, :source)
+
+      class << self
+        # The provider of the site, with its API key from the environment.
+        def provider_for(site, env: ENV, transport: nil)
+          klass = Providers.resolve(site.provider)
+          key = env[klass.api_key_env].to_s.strip
+          if key.empty?
+            raise ConfigError,
+                  "Missing environment variable #{klass.api_key_env}: your #{klass.id} API key."
+          end
+
+          klass.new(api_key: key, **(transport ? { transport: } : {}))
+        end
+
+        # The settings win. For the second review, review_models win over models. Without any, the provider's
+        # built-in list is used.
+        def models_for(site, provider_class, review: false)
+          listed = review && !site.review_models.empty? ? site.review_models : site.models
+          return Selection.new(names: listed, source: :settings) unless listed.empty?
+
+          defaults = Defaults.models_for(provider_class.id)
+          if defaults.empty?
+            raise ConfigError, "Site: set models in proofreader.yml. #{provider_class.id} has no built-in list " \
+                               "(yuno-proofreader models shows what your key can use)."
+          end
+          Selection.new(names: defaults, source: :defaults)
+        end
+
+        # options go to the client: log:, sleeper:, clock:, retry_wait:, max_output_tokens:
+        def client_for(site, env: ENV, review: false, transport: nil, **)
+          provider = provider_for(site, env:, transport:)
+          Client.new(provider:, models: models_for(site, provider.class, review:).names, **)
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/llm/http_transport.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+
+module Yuno
+  module Proofreader
+    module Llm
+      # Sends one request to a model API and returns the answer whatever its status. Network errors are raised
+      # as they are. The provider turns them into ProviderError.
+      class HttpTransport
+        CONNECT_TIMEOUT = 15
+
+        def call(method, url, headers:, timeout:, body: nil)
+          uri = URI.parse(url)
+          response = connection(uri, timeout).request(build(method, uri, headers, body))
+          Http::RawResponse.new(status: response.code.to_i, headers: response.to_hash.transform_values do |v|
+            v.join(", ")
+          end,
+                                body: response.body.to_s.dup.force_encoding(Encoding::UTF_8).scrub)
+        end
+
+        private
+
+        def connection(uri, timeout)
+          Net::HTTP.new(uri.host, uri.port).tap do |http|
+            http.use_ssl = uri.scheme == "https"
+            http.open_timeout = CONNECT_TIMEOUT
+            http.read_timeout = timeout
+          end
+        end
+
+        def build(method, uri, headers, body)
+          request = (method == :post ? Net::HTTP::Post : Net::HTTP::Get).new(uri, headers)
+          request.body = body if body
+          request
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/llm/json_text.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Yuno
+  module Proofreader
+    module Llm
+      # Reads the JSON a model wrote. Models sometimes wrap it in a markdown code fence.
+      module JsonText
+        module_function
+
+        def parse(text, finish: nil)
+          clean = text.to_s.strip.sub(/\A```(?:json)?\s*/i, "").sub(/\s*```\z/, "").strip
+          JSON.parse(clean)
+        rescue JSON::ParserError
+          cut = finish.to_s == "MAX_TOKENS" ? " The answer was cut off at the output limit." : ""
+          raise AnswerError, "The answer is not valid JSON (finish: #{finish.inspect}).#{cut} " \
+                             "It starts: #{clean[0, 120].inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/llm/model_commands.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    # The command that works with the model provider: models.
+    module ModelCommands
+      private
+
+      def models(args)
+        file, problem = single_file(args, "models")
+        return usage_error(problem) if problem
+
+        site = Site.load(file)
+        provider = Llm.provider_for(site, env: @env, transport: @llm_transport)
+        selection = Llm.models_for(site, provider.class)
+        used = selection.names.map { |name| provider.normalize(name) }
+        print_models(provider, provider.models, used, selection.source)
+        0
+      end
+
+      def print_models(provider, available, used, source)
+        @out.puts "Models your #{provider.class.id} key can use for text generation:"
+        available.each { |name| @out.puts "  #{name}#{'  (in use)' if used.include?(name)}" }
+        origin = source == :settings ? "from your settings" : "built-in default"
+        @out.puts "", "In use (#{origin}): #{used.join(', ')}"
+        missing = used - available
+        return if missing.empty?
+
+        @out.puts "Not available to this key: #{missing.join(', ')}. A model that fails is skipped, " \
+                  "so set models in proofreader.yml to the ones you want."
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/llm/provider.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "openssl"
+require "timeout"
+
+module Yuno
+  module Proofreader
+    module Llm
+      # What every model provider offers. A provider is a light wrapper around one family of models: it knows
+      # the address, how the request is written and how errors are reported. The rest (model list, retries,
+      # JSON handling) is shared. To add a provider, subclass this, define the two class methods and the two
+      # public methods, and register the class (see providers/gemini.rb).
+      class Provider
+        NETWORK_ERRORS = [SocketError, SystemCallError, IOError, Timeout::Error, OpenSSL::SSL::SSLError,
+                          Net::ProtocolError, Net::HTTPBadResponse].freeze
+        TIMEOUT = 180
+
+        # The name used for it in the settings file, in lowercase.
+        def self.id
+          raise NotImplementedError, "#{name} must define .id"
+        end
+
+        # The environment variable that holds its API key.
+        def self.api_key_env
+          raise NotImplementedError, "#{name} must define .api_key_env"
+        end
+
+        def initialize(api_key:, transport: HttpTransport.new, timeout: TIMEOUT)
+          @api_key = api_key
+          @transport = transport
+          @timeout = timeout
+        end
+
+        # Sends one request. Returns a Completion, or raises ProviderError saying what kind of failure it was.
+        def complete(model:, system:, user:, max_output_tokens:)
+          raise NotImplementedError, "#{self.class.name} must define #complete"
+        end
+
+        # The names of the models this key can use for text generation.
+        def models
+          raise NotImplementedError, "#{self.class.name} must define #models"
+        end
+
+        # A model name as the provider's API wants it. Providers that write names differently override this.
+        def normalize(name) = name.to_s
+
+        private
+
+        def request(method, url, headers: {}, body: nil)
+          @transport.call(method, url, headers:, timeout: @timeout, body:)
+        rescue *NETWORK_ERRORS => e
+          raise ProviderError.new("network problem (#{e.class}: #{e.message})", kind: :network)
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/llm/providers/gemini.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "json"
+require "uri"
+
+module Yuno
+  module Proofreader
+    module Llm
+      # The model providers that are installed. Each wrapper registers itself when it is loaded.
+      module Providers
+        # The wrapper for Google's Gemini models.
+        class Gemini < Provider
+          BASE = "https://generativelanguage.googleapis.com/v1beta"
+          MAX_PAGES = 10
+          OVERLOADED = [500, 502, 503, 504].freeze
+          KINDS = { 401 => :auth, 403 => :forbidden, 404 => :not_found, 429 => :rate_limited }.freeze
+
+          def self.id = "gemini"
+
+          def self.api_key_env = "GEMINI_API_KEY"
+
+          def complete(model:, system:, user:, max_output_tokens:)
+            body = JSON.generate(payload(system, user, max_output_tokens))
+            url = "#{BASE}/models/#{normalize(model)}:generateContent"
+            response = request(:post, url, headers: auth_headers.merge("Content-Type" => "application/json"), body:)
+            raise failure(response) unless response.status == 200
+
+            completion_from(parse_body(response))
+          end
+
+          def models
+            names = []
+            token = nil
+            MAX_PAGES.times do
+              page = list_page(token)
+              names.concat(usable_names(page))
+              token = page["nextPageToken"].to_s
+              break if token.empty?
+            end
+            names.uniq.sort
+          end
+
+          def normalize(name) = name.to_s.delete_prefix("models/")
+
+          private
+
+          def auth_headers = { "x-goog-api-key" => @api_key }
+
+          def payload(system, user, max_output_tokens)
+            {
+              systemInstruction: { parts: [{ text: system }] },
+              contents: [{ role: "user", parts: [{ text: user }] }],
+              generationConfig: { temperature: 0, responseMimeType: "application/json",
+                                  maxOutputTokens: max_output_tokens }
+            }
+          end
+
+          def list_page(token)
+            url = "#{BASE}/models?pageSize=200"
+            url += "&pageToken=#{URI.encode_www_form_component(token)}" if token
+            response = request(:get, url, headers: auth_headers)
+            raise failure(response) unless response.status == 200
+
+            parse_body(response)
+          end
+
+          def usable_names(page)
+            Array(page["models"]).filter_map do |model|
+              next unless Array(model["supportedGenerationMethods"]).include?("generateContent")
+
+              normalize(model["name"])
+            end
+          end
+
+          def parse_body(response)
+            data = JSON.parse(response.body)
+            return data if data.is_a?(Hash)
+
+            raise ProviderError.new("The answer is not a JSON object.", kind: :other)
+          rescue JSON::ParserError
+            raise ProviderError.new("The answer is not JSON (HTTP #{response.status}).", kind: :other)
+          end
+
+          def completion_from(data)
+            candidate = Array(data["candidates"]).first
+            raise no_candidate(data) unless candidate
+
+            finish = candidate["finishReason"].to_s
+            text = candidate_text(candidate)
+            if text.strip.empty?
+              raise ProviderError.new("The answer is empty (finish: #{finish.inspect}).",
+                                      kind: :empty)
+            end
+
+            input, output, thinking = token_counts(data["usageMetadata"])
+            Completion.new(text:, finish:, input_tokens: input, output_tokens: output, thinking_tokens: thinking)
+          end
+
+          def candidate_text(candidate)
+            parts = Array(candidate.dig("content", "parts")).reject { |part| part["thought"] }
+            parts.map { |part| part["text"].to_s }.join
+          end
+
+          def token_counts(usage)
+            usage ||= {}
+            [usage["promptTokenCount"].to_i, usage["candidatesTokenCount"].to_i, usage["thoughtsTokenCount"].to_i]
+          end
+
+          def no_candidate(data)
+            reason = data.dig("promptFeedback", "blockReason")
+            return ProviderError.new("The request was blocked (#{reason}).", kind: :blocked) if reason
+
+            ProviderError.new("The answer has no candidates.", kind: :empty)
+          end
+
+          def failure(response)
+            body = response.body.to_s
+            kind = classify(response.status, body)
+            ProviderError.new("HTTP #{response.status}: #{api_message(body)}", kind:,
+                                                                               daily: daily?(kind, body))
+          end
+
+          def classify(status, body)
+            return :overloaded if OVERLOADED.include?(status)
+            return KINDS[status] if KINDS.key?(status)
+            return :other unless status == 400
+
+            body.match?(/API key not valid/i) ? :auth : :bad_request
+          end
+
+          # A limit per day is not worth waiting for. A limit per minute passes.
+          def daily?(kind, body) = kind == :rate_limited && body.include?("PerDay")
+
+          def api_message(body)
+            text = JSON.parse(body).dig("error", "message")
+            (text || body).to_s.strip[0, 200]
+          rescue JSON::ParserError, TypeError, NoMethodError
+            body.to_s.strip[0, 200]
+          end
+        end
+
+        register(Gemini)
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/llm/providers.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Llm
+      # The providers that are installed. Each wrapper registers itself when it is loaded.
+      module Providers
+        @registry = {}
+
+        class << self
+          def register(provider)
+            @registry[provider.id] = provider
+          end
+
+          def ids = @registry.keys
+
+          # The provider named in the settings. Without a name, the only installed one.
+          def resolve(id)
+            name = id.to_s.strip.downcase
+            return only_provider if name.empty?
+
+            @registry.fetch(name) do
+              raise ConfigError, "Site: unknown provider #{name.inspect}. Available: #{ids.join(', ')}"
+            end
+          end
+
+          private
+
+          def only_provider
+            return @registry.values.first if @registry.size == 1
+
+            raise ConfigError, "Site: set provider to one of: #{ids.join(', ')}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/llm/types.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Llm
+      # What a provider returns for one call.
+      Completion = Data.define(:text, :finish, :input_tokens, :output_tokens, :thinking_tokens)
+
+      # What the client returns: the parsed JSON, which model answered, and what it cost.
+      Answer = Data.define(:data, :model, :finish, :input_tokens, :output_tokens, :thinking_tokens, :seconds)
+    end
+  end
+end

data/lib/yuno/proofreader/llm/usage.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Llm
+      # What model calls used, for the log. calls: anything with seconds and input, output and thinking tokens.
+      module Usage
+        module_function
+
+        def of(calls)
+          { model_calls: calls.size, input_tokens: calls.sum(&:input_tokens), output_tokens: calls.sum(&:output_tokens),
+            thinking_tokens: calls.sum(&:thinking_tokens), model_seconds: calls.sum(&:seconds).round(1) }
+        end
+      end
+    end
+  end
+end