yuno-proofreader 0.1.0

data/lib/yuno/proofreader/log.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "time"
+
+module Yuno
+  module Proofreader
+    # Writes one line per event: what happened, when, and the facts about it, such as the page and the tokens
+    # used. A line is "key=value" pairs, or with LOG_FORMAT=json one JSON object that Google Cloud Logging reads
+    # as a structured entry (its severity and message fields become the entry's severity and text). Which lines
+    # are written depends on LOG_LEVEL.
+    class Log
+      LEVELS = Config::LOG_LEVELS
+      FORMATS = %w[text json].freeze
+      RANK = LEVELS.each_with_index.to_h { |name, index| [name.to_sym, index] }.freeze
+      SEVERITIES = { debug: "DEBUG", info: "INFO", warn: "WARNING", error: "ERROR" }.freeze
+      DEFAULT_LEVEL = :warn
+      MAX_VALUE = 500
+      SECRET = /api_?key|secret|password|credential|authorization|\A(access_|auth_|bearer_)?token\z/i
+      PLAIN = %r{\A[\w.:/@+,%-]+\z}
+
+      # A log that writes nothing, for code that was not given one.
+      def self.null = new(io: nil)
+
+      # Reads LOG_LEVEL and LOG_FORMAT. Every line carries a short id of the run, so the lines of one run can be
+      # picked out of a shared log.
+      def self.from_env(env = ENV, io = $stderr, **)
+        new(io:, level: level_from(env["LOG_LEVEL"]), format: format_from(env["LOG_FORMAT"]),
+            fields: { run: SecureRandom.hex(4) }, **)
+      end
+
+      def self.level_from(value)
+        level = value.to_s.strip.downcase
+        return DEFAULT_LEVEL if level.empty?
+        return level.to_sym if LEVELS.include?(level)
+
+        raise ConfigError, "LOG_LEVEL must be one of #{LEVELS.join(', ')} (got #{value.inspect})"
+      end
+
+      def self.format_from(value)
+        kind = value.to_s.strip.downcase
+        return "text" if kind.empty?
+        return kind if FORMATS.include?(kind)
+
+        raise ConfigError, "LOG_FORMAT must be one of #{FORMATS.join(', ')} (got #{value.inspect})"
+      end
+
+      def initialize(io:, level: DEFAULT_LEVEL, format: "text", fields: {}, clock: -> { Time.now.utc })
+        @io = io
+        @level = level
+        @format = format
+        @fields = fields
+        @clock = clock
+      end
+
+      # The same log with more fields on every line it writes, such as the page being worked on.
+      def with(**fields)
+        self.class.new(io: @io, level: @level, format: @format, fields: @fields.merge(fields), clock: @clock)
+      end
+
+      def enabled?(level) = !@io.nil? && RANK.fetch(level) >= RANK.fetch(@level)
+
+      def debug(message, **fields) = write(:debug, message, fields)
+
+      def info(message, **fields) = write(:info, message, fields)
+
+      def warn(message, **fields) = write(:warn, message, fields)
+
+      def error(message, **fields) = write(:error, message, fields)
+
+      private
+
+      # One write per line, so lines from different threads or processes do not mix. A log that cannot be written
+      # (a closed pipe) never stops the work.
+      def write(level, message, fields)
+        return unless enabled?(level)
+
+        @io.write("#{line(level, shorten(message.to_s), clean(@fields.merge(fields)))}\n")
+      rescue IOError, SystemCallError
+        nil
+      end
+
+      def line(level, message, data)
+        time = @clock.call.utc.iso8601
+        @format == "json" ? json_line(level, message, time, data) : text_line(level, message, time, data)
+      end
+
+      def text_line(level, message, time, data)
+        head = { "time" => time, "level" => level.to_s, "msg" => message }
+        head.merge(data.except(*head.keys)).map { |key, value| "#{key}=#{text_value(value)}" }.join(" ")
+      end
+
+      def json_line(level, message, time, data)
+        head = { "severity" => SEVERITIES.fetch(level), "message" => message, "time" => time }
+        JSON.generate(head.merge(data.except(*head.keys)))
+      end
+
+      def text_value(value)
+        text = value.to_s
+        text.match?(PLAIN) ? text : JSON.generate(text)
+      end
+
+      # Fields as the log writes them: empty ones left out, secrets hidden, values on one line and not too long.
+      def clean(fields)
+        fields.each_with_object({}) do |(key, value), found|
+          next if value.nil?
+
+          found[key.to_s] = SECRET.match?(key.to_s) ? "[redacted]" : normalize(value)
+        end
+      end
+
+      def normalize(value)
+        case value
+        when true, false, Integer then value
+        when Float then value.round(2)
+        when Time then value.utc.iso8601
+        when Array then shorten(value.join(","))
+        else shorten(value.to_s)
+        end
+      end
+
+      def shorten(text)
+        flat = text.gsub(/\s+/, " ").strip
+        flat.length > MAX_VALUE ? "#{flat[0, MAX_VALUE - 1]}…" : flat
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/pages/fetcher.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Pages
+      # Downloads every language version of a page politely: it reads robots.txt first, goes through the
+      # client's pacing and size limits, and reports a problem per version instead of failing the whole page.
+      class Fetcher
+        MAX_HTML_CHARS = 200_000
+
+        def initialize(site, client)
+          @site = site
+          @client = client
+        end
+
+        # Raises FetchError when robots.txt cannot be read, because then nothing may be fetched.
+        def fetch(page)
+          rules = robots
+          FetchedPage.new(name: page.name,
+                          snapshots: page.urls.map { |language, url| snapshot(rules, language, url) })
+        end
+
+        private
+
+        def robots
+          @robots ||= load_robots
+        end
+
+        def load_robots
+          response = @client.get("#{@site.website}/robots.txt")
+          case response.status
+          when 200..299 then Http::Robots.parse(response.body, agent: Http::PRODUCT_TOKEN)
+          when 400..499 then Http::Robots.allow_all
+          else raise FetchError,
+                     "robots.txt answered HTTP #{response.status}, so no pages are fetched. Try again later."
+          end
+        end
+
+        def snapshot(rules, language, url)
+          return Snapshot.failed(language, url, "blocked by robots.txt") unless rules.allowed?(url)
+
+          build(language, url, @client.get(url))
+        rescue FetchError => e
+          Snapshot.failed(language, url, "could not be fetched (#{e.message})")
+        end
+
+        def build(language, url, response)
+          problem = problem_with(response)
+          return Snapshot.failed(language, url, problem, status: response.status) if problem
+
+          trimmed = Trimmer.call(response.body)
+          cut = trimmed.length > MAX_HTML_CHARS
+          Snapshot.new(language:, url:, final_url: response.url, status: response.status,
+                       content_type: response.content_type, raw: response.body,
+                       html: cut ? trimmed[0, MAX_HTML_CHARS] : trimmed,
+                       fingerprint: Trimmer.fingerprint(trimmed), truncated: cut, problem: nil)
+        end
+
+        def problem_with(response)
+          return "HTTP #{response.status}" unless response.status == 200
+          return if html?(response)
+
+          "not an HTML page (content type #{response.content_type.inspect})"
+        end
+
+        def html?(response)
+          type = response.content_type.to_s
+          return type.match?(/html/i) unless type.empty?
+
+          response.body.match?(/<html|<!doctype html/i)
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/pages/report.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Pages
+      # Turns a fetched page into the lines the fetch command prints.
+      class Report
+        def initialize(fetched)
+          @fetched = fetched
+        end
+
+        def lines
+          ["Page: #{@fetched.name}", *@fetched.snapshots.flat_map { |snapshot| block(snapshot) }, "", fingerprint_line]
+        end
+
+        private
+
+        def block(snapshot)
+          status = snapshot.ok? ? "ok     " : "PROBLEM"
+          ["  #{snapshot.language.ljust(3)} #{status} #{snapshot.url}", *details(snapshot)]
+        end
+
+        def details(snapshot)
+          return ["      - #{snapshot.problem}"] unless snapshot.ok?
+
+          facts = [
+            "raw #{snapshot.raw.bytesize} bytes", "trimmed #{snapshot.html.length} chars",
+            "complete: #{snapshot.complete? ? 'yes' : 'no'}", "fingerprint #{snapshot.fingerprint[0, 12]}"
+          ]
+          notes = []
+          notes << "redirected to #{snapshot.final_url}" if snapshot.redirected?
+          notes << "cut at #{Fetcher::MAX_HTML_CHARS} characters" if snapshot.truncated
+          ["      #{facts.join(', ')}", *notes.map { |note| "      - #{note}" }]
+        end
+
+        def fingerprint_line
+          fingerprint = @fetched.fingerprint
+          return "Combined fingerprint: none (not every version could be fetched)" unless fingerprint
+
+          "Combined fingerprint: #{fingerprint[0, 12]}"
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/pages/snapshot.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Yuno
+  module Proofreader
+    module Pages
+      # A trimmed page this short, or without a closing </html>, is treated as incomplete.
+      MIN_COMPLETE_CHARS = 3_000
+
+      # One language version of a page as fetched. problem is nil when the page could be used.
+      # raw is the HTML exactly as received, html the trimmed (and possibly cut) text for the model.
+      Snapshot = Data.define(:language, :url, :final_url, :status, :content_type, :raw, :html,
+                             :fingerprint, :truncated, :problem) do
+        def self.failed(language, url, problem, status: nil)
+          new(language:, url:, final_url: url, status:, content_type: nil, raw: "", html: "",
+              fingerprint: nil, truncated: false, problem:)
+        end
+
+        def ok? = problem.nil?
+
+        def redirected? = ok? && final_url != url
+
+        def complete?
+          ok? && !truncated && html.length >= MIN_COMPLETE_CHARS && raw.match?(%r{</html\s*>}i)
+        end
+      end
+
+      # All language versions of one page.
+      FetchedPage = Data.define(:name, :snapshots) do
+        def snapshot(language) = snapshots.find { |snapshot| snapshot.language == language }
+
+        def ok? = snapshots.all?(&:ok?)
+
+        def complete? = snapshots.all?(&:complete?)
+
+        # One fingerprint for the whole page, or nil when a version is missing (then nothing can be called unchanged).
+        def fingerprint
+          return unless ok?
+
+          Digest::SHA256.hexdigest(snapshots.map do |snapshot|
+            "#{snapshot.language}:#{snapshot.fingerprint}"
+          end.join("\n"))
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/pages/trimmer.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Yuno
+  module Proofreader
+    module Pages
+      # Cleans a page's HTML for the model without rewriting it. What stays is exactly what the server sent
+      # (only runs of whitespace become one space), so a text the model quotes can still be found in the page.
+      module Trimmer
+        LONG_DATA_URI = /data:[^"')\s]{50,}/
+        # Values that change on every request. They are ignored when deciding whether a page changed.
+        VOLATILE = [
+          /<meta\b[^>]*\bname=["']?(?:csrf-token|csrf-param|csp-nonce)["']?[^>]*>/i,
+          /<input\b[^>]*\bname=["']authenticity_token["'][^>]*>/i,
+          /\snonce=(?:"[^"]*"|'[^']*')/i
+        ].freeze
+
+        module_function
+
+        def call(html)
+          html.to_s
+              .gsub(%r{<(script|style|svg)\b.*?</\1\s*>}mi, "")
+              .gsub(/<!--.*?-->/m, "")
+              .gsub(LONG_DATA_URI, "data:...")
+              .gsub(/\s+/, " ")
+              .strip
+        end
+
+        def fingerprint(trimmed)
+          Digest::SHA256.hexdigest(VOLATILE.reduce(trimmed) { |text, pattern| text.gsub(pattern, "") })
+        end
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/settings_file.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Yuno
+  module Proofreader
+    # Writes settings as YAML in the style of proofreader.example.yml: text in double quotes, lists with the dash
+    # indented, and the settings in a fixed order. Settings with nothing in them are left out.
+    module SettingsFile
+      ORDER = %w[website languages sitemap_url default_language_only_paths skip_paths provider models review_models
+                 business audience tone rules ignore_terms intentional_differences analysis cadence].freeze
+      PLAIN_KEY = %r{\A[\w/.-]+\z}
+      INDENT = "  "
+
+      module_function
+
+      # settings: a hash with string keys. header: comment lines for the top of the file.
+      def render(settings, header: [])
+        present = settings.select { |_key, value| filled?(value) }
+        keys = ORDER.select { |key| present.key?(key) } + (present.keys - ORDER)
+        body = keys.map { |key| emit(key, present[key], 0).join("\n") }.join("\n\n")
+        comments = header.map { |line| "# #{line}".rstrip }.join("\n")
+        "#{comments}#{"\n\n" unless comments.empty?}#{body}\n"
+      end
+
+      def filled?(value) = !(value.nil? || (value.respond_to?(:empty?) && value.empty?))
+
+      # The lines of one setting. level is how deep it is nested.
+      def emit(key, value, level)
+        pad = INDENT * level
+        case value
+        when Hash then ["#{pad}#{name(key)}:", *value.flat_map { |inner, item| emit(inner, item, level + 1) }]
+        when Array then ["#{pad}#{name(key)}:", *value.flat_map { |item| list_item(item, level + 1) }]
+        else ["#{pad}#{name(key)}: #{scalar(value)}"]
+        end
+      end
+
+      def list_item(item, level)
+        pad = INDENT * level
+        return ["#{pad}- #{scalar(item)}"] unless item.is_a?(Hash)
+
+        lines = item.flat_map { |key, value| emit(key, value, level + 1) }
+        ["#{pad}- #{lines.first.lstrip}", *lines.drop(1)]
+      end
+
+      def name(key) = key.to_s.match?(PLAIN_KEY) ? key.to_s : JSON.generate(key.to_s)
+
+      def scalar(value) = value.is_a?(String) ? JSON.generate(value) : value.to_s
+    end
+  end
+end

data/lib/yuno/proofreader/site.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "uri"
+require "yaml"
+
+module Yuno
+  module Proofreader
+    # What you tell the tool about your website. Read from a YAML file.
+    Site = Data.define(:website, :prefixes, :language_notes, :sitemap_url, :default_language_only_paths,
+                       :skip_paths, :provider, :models, :review_models, :context, :analysis, :cadence) do
+      def self.load(path)
+        raise ConfigError, "Site file not found: #{path}" unless File.file?(path)
+
+        from_hash(YAML.safe_load_file(path))
+      rescue Psych::Exception => e
+        raise ConfigError, "Site file #{path} could not be read as YAML: #{e.message}"
+      end
+
+      def self.from_hash(hash) = SiteReader.call(hash)
+
+      def languages = prefixes.keys
+
+      def default_language = languages.first
+
+      # How each setting that describes the site stands: :set, :empty or :placeholder.
+      def states
+        notes = language_notes.to_h { |code, note| ["languages: #{code} notes", Context.state_of(note)] }
+        context.states.merge(notes)
+      end
+
+      # What stops an audit from starting: no business description, or example text that was never replaced.
+      def audit_problems
+        problems = []
+        problems << "business is not set. Describe the site in one or two sentences." if context.business.empty?
+        unfinished = states.select { |_name, state| state == :placeholder }.keys
+        unless unfinished.empty?
+          problems << "These settings still hold the [FILL IN] example text: #{unfinished.join(', ')}. " \
+                      "Replace it with your own text, or delete the setting."
+        end
+        problems
+      end
+
+      def ready_for_audit!
+        problems = audit_problems
+        raise ConfigError, "Site settings are not ready for an audit. #{problems.join(' ')}" unless problems.empty?
+
+        self
+      end
+
+      def validate!
+        unless http_url?(website)
+          raise ConfigError,
+                "Site: website must be a full address, e.g. https://example.com"
+        end
+        unless prefixes.values.uniq.size == prefixes.size
+          raise ConfigError,
+                "Site: two languages share the same URL prefix"
+        end
+        raise ConfigError, "Site: sitemap_url must be a full address" if sitemap_url && !http_url?(sitemap_url)
+
+        self
+      end
+
+      private
+
+      def http_url?(value)
+        uri = URI.parse(value)
+        %w[http https].include?(uri.scheme) && !uri.host.to_s.empty?
+      rescue URI::InvalidURIError
+        false
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/site_reader.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    # Turns the parsed YAML of a settings file into a Site, with a clear error for anything that is wrong.
+    class SiteReader
+      SITE_KEYS = %w[website languages sitemap_url default_language_only_paths skip_paths provider models
+                     review_models analysis cadence].freeze
+      PROVIDER_ID = /\A[a-z][a-z0-9_-]{0,39}\z/
+      MODEL_NAME = %r{\A[A-Za-z0-9][A-Za-z0-9._:/-]{0,99}\z}
+      MAX_MODELS = 8
+      KEYS = (SITE_KEYS + Context::KEYS).freeze
+      LANGUAGE_KEYS = %w[code prefix notes].freeze
+      LANGUAGE_CODE = /\A[a-z]{2,3}(-[a-z0-9]+)*\z/
+
+      def self.call(hash) = new(hash).call
+
+      def initialize(hash)
+        raise ConfigError, "Site settings must be a list of key: value pairs" unless hash.nil? || hash.is_a?(Hash)
+
+        @raw = (hash || {}).transform_keys(&:to_s)
+      end
+
+      def call
+        check_keys(@raw.keys, KEYS, "Site")
+        prefixes, notes = languages
+        Site.new(
+          website: @raw["website"].to_s.strip.chomp("/"), prefixes:, language_notes: notes,
+          sitemap_url: sitemap_url, default_language_only_paths: paths("default_language_only_paths"),
+          skip_paths: paths("skip_paths"), provider: provider_name, models: model_list("models"),
+          review_models: model_list("review_models"), context: Context.from_hash(@raw),
+          analysis: Analysis::Settings.from_hash(@raw["analysis"]), cadence: Audit::Cadence.from_hash(@raw["cadence"])
+        ).validate!
+      end
+
+      private
+
+      def check_keys(found, allowed, where)
+        unknown = found - allowed
+        return if unknown.empty?
+
+        raise ConfigError, "#{where}: unknown setting(s): #{unknown.join(', ')}. Allowed: #{allowed.join(', ')}"
+      end
+
+      def sitemap_url
+        value = @raw["sitemap_url"].to_s.strip
+        value.empty? ? nil : value
+      end
+
+      # Returns the URL prefix and the notes of each language, both keyed by language code in the order given.
+      def languages
+        value = @raw["languages"]
+        unless value.is_a?(Array) && !value.empty? && value.all?(Hash)
+          raise ConfigError, "Site: languages must be a list with one entry per language, " \
+                             "each with a code and a prefix (see proofreader.example.yml)"
+        end
+
+        prefixes = {}
+        notes = {}
+        value.each_with_index do |item, index|
+          code, prefix, note = language(item, index + 1)
+          raise ConfigError, "Site: language #{code} is listed twice" if prefixes.key?(code)
+
+          prefixes[code] = prefix
+          notes[code] = note
+        end
+        [prefixes, notes]
+      end
+
+      def language(item, number)
+        entry = item.transform_keys(&:to_s)
+        check_keys(entry.keys, LANGUAGE_KEYS, "Site: language ##{number}")
+        [language_code(entry["code"], number), normalize_prefix(entry["prefix"]),
+         Context.text(entry["notes"], "language ##{number} notes", Context::MAX_ITEM)]
+      end
+
+      def language_code(value, number)
+        if [true, false].include?(value)
+          raise ConfigError, "Site: language ##{number}: YAML read the code as #{value}. " \
+                             "Put quotes around language codes, for example code: \"no\""
+        end
+
+        code = value.to_s.strip.downcase
+        return code if code.match?(LANGUAGE_CODE)
+
+        raise ConfigError, "Site: language ##{number} needs a code such as \"de\" or \"pt-br\" (got #{value.inspect})"
+      end
+
+      def normalize_prefix(value)
+        prefix = value.to_s.strip
+        return "" if prefix.empty? || prefix == "/"
+
+        prefix = "/#{prefix}" unless prefix.start_with?("/")
+        prefix.chomp("/")
+      end
+
+      def paths(key)
+        items = Array(@raw[key])
+        raise ConfigError, "Site: #{key} must be a list of paths, written as text" unless items.all?(String)
+
+        items.filter_map do |item|
+          path = item.strip
+          next if path.empty?
+
+          path = "/#{path}" unless path.start_with?("/")
+          path = path.chomp("/")
+          path unless path.empty?
+        end
+      end
+
+      def provider_name
+        name = Context.text(@raw["provider"], "provider", 40).downcase
+        return name if name.empty? || name.match?(PROVIDER_ID)
+
+        raise ConfigError, "Site: provider must be a short lowercase name (got #{@raw['provider'].inspect})"
+      end
+
+      def model_list(key)
+        items = Array(@raw[key])
+        raise ConfigError, "Site: #{key} must be a list of model names, written as text" unless items.all?(String)
+
+        names = items.map(&:strip).reject(&:empty?).uniq
+        bad = names.find { |name| !name.match?(MODEL_NAME) }
+        raise ConfigError, "Site: #{key} has an entry that is not a model name: #{bad.inspect}" if bad
+        raise ConfigError, "Site: #{key} has more than #{MAX_MODELS} entries" if names.size > MAX_MODELS
+
+        names
+      end
+    end
+  end
+end

data/lib/yuno/proofreader/sitemap/aligner.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Yuno
+  module Proofreader
+    module Sitemap
+      # Lines up two lists of pages so that as many pages as possible get a partner, and returns the ones that did
+      # not. Pages match better when their addresses have the same depth, and better still when their last segment
+      # shares a word. It is a guess for lists whose order is mostly the same.
+      module Aligner
+        GAP = -1
+        SAME_DEPTH = 2
+        OTHER_DEPTH = -1
+        SHARED_WORD = 1
+
+        module_function
+
+        # left, right: [depth, words] for each page. Returns [indices of left pages without a partner,
+        # indices of right pages without a partner].
+        def call(left, right)
+          table = fill(left, right)
+          trace(table, left, right)
+        end
+
+        def fill(left, right)
+          table = Array.new(left.size + 1) { |row| Array.new(right.size + 1) { |col| (row + col) * GAP } }
+          (1..left.size).each do |row|
+            (1..right.size).each { |col| table[row][col] = best(table, left, right, row, col) }
+          end
+          table
+        end
+
+        def best(table, left, right, row, col)
+          [table[row - 1][col - 1] + similarity(left[row - 1], right[col - 1]),
+           table[row - 1][col] + GAP, table[row][col - 1] + GAP].max
+        end
+
+        def trace(table, left, right)
+          row = left.size
+          col = right.size
+          alone = [[], []]
+          while row.positive? || col.positive?
+            move = step(table, row, col)
+            note_alone(alone, move, row, col)
+            row, col = after(move, row, col)
+          end
+          alone.map(&:reverse)
+        end
+
+        # A page passed on its own, without the other side, has no partner.
+        def note_alone(alone, move, row, col)
+          alone[0] << (row - 1) if move == :left
+          alone[1] << (col - 1) if move == :right
+        end
+
+        # A pair moves past a page on both sides, a gap past the page on its own side only.
+        def after(move, row, col) = [move == :right ? row : row - 1, move == :left ? col : col - 1]
+
+        # On a tie the later page is left alone, because a page without a partner is most often one that was added
+        # at the end of a list, such as the pages of a second sitemap. Any move this allows still gives the best
+        # alignment, so the tie only decides which of two equally good guesses is shown.
+        def step(table, row, col)
+          return :right if row.zero?
+          return :left if col.zero?
+          return :left if table[row][col] == table[row - 1][col] + GAP
+          return :right if table[row][col] == table[row][col - 1] + GAP
+
+          :both
+        end
+
+        def similarity(first, second)
+          score = first[0] == second[0] ? SAME_DEPTH : OTHER_DEPTH
+          first[1].intersect?(second[1]) ? score + SHARED_WORD : score
+        end
+        private_class_method :fill, :best, :trace, :step, :similarity
+      end
+    end
+  end
+end