datalog-theme 0.6.1 → 0.8.0

data/_plugins/csp_generator.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "securerandom"
-require "digest"
 
 module Datalog
   module Security
@@ -18,7 +17,6 @@ module Datalog
       def generate(site)
         site.data["csp"] ||= {}
         site.data["csp"]["nonces"] ||= {}
-        site.data["csp"]["hashes"] ||= {}
 
         assign_nonces(site, site.pages)
         site.collections.each_value do |collection|
@@ -40,7 +38,6 @@ module Datalog
 
         nonce = SecureRandom.base64(nonce_bytes)
         document.data["csp_nonce"] = nonce
-        document.data["csp_hashes"] ||= []
 
         registry_site = site || (document.respond_to?(:site) ? document.site : nil)
         if registry_site.respond_to?(:data)
@@ -61,7 +58,12 @@ module Datalog
         end
       end
 
-      def self.compute_hashes(document)
+      # Gives every inline script in the rendered page the page's nonce. The
+      # generator also took a SHA-256 of each inline script, but the policy is
+      # written into the head while the page renders, so the hashes of the
+      # finished page never reached it; with every script nonced they are not
+      # needed.
+      def self.add_nonces(document)
         return unless document.respond_to?(:output)
 
         output = document.output
@@ -77,39 +79,27 @@ module Datalog
           attributes = Regexp.last_match(1)
           "<script nonce=\"#{nonce}\"#{attributes}>"
         end
-
-        document.output = output
-
-        hashes = []
-        output.scan(%r{<script(?![^>]*\bsrc=)[^>]*>(.*?)</script>}mi) do |match|
-          content = match.first
-          next if content.nil? || content.empty?
-
-          hashes << Digest::SHA256.base64digest(content)
+        # A template that printed page.csp_nonce before the page had one left
+        # nonce="", which authorises nothing.
+        document.output = output.gsub(/(<(?:script|style)\b[^>]*\bnonce=)""/i) do
+          "#{Regexp.last_match(1)}\"#{nonce}\""
         end
-
-        hashes.uniq!
-        document.data["csp_hashes"] = hashes
-
-        return unless site.respond_to?(:data)
-
-        site.data["csp"] ||= {}
-        site.data["csp"]["hashes"] ||= {}
-
-        key = document_key(document)
-
-        site.data["csp"]["hashes"][key] = hashes if key
       end
     end
   end
 end
 
 %i[pages documents].each do |target|
-  Jekyll::Hooks.register target, :pre_render do |document|
-    Datalog::Security::CspGenerator.assign_nonce(document.respond_to?(:site) ? document.site : nil, document)
+  Jekyll::Hooks.register target, :pre_render do |document, payload|
+    nonce = Datalog::Security::CspGenerator.assign_nonce(document.respond_to?(:site) ? document.site : nil, document)
+    # A page another generator creates after this one runs, such as a notebook
+    # page, only gets its nonce here. Jekyll has already copied a page's data
+    # into the hash its templates read, so page.csp_nonce rendered empty there.
+    page_data = payload && payload["page"]
+    page_data["csp_nonce"] ||= nonce if nonce && page_data.is_a?(Hash)
   end
 
   Jekyll::Hooks.register target, :post_render do |document|
-    Datalog::Security::CspGenerator.compute_hashes(document)
+    Datalog::Security::CspGenerator.add_nonces(document)
   end
 end

data/_plugins/datalog_bibliography.rb

@@ -1,17 +1,19 @@
 # frozen_string_literal: true
 
 module Jekyll
+  # Stands in for the `datalog_bibliography` tag when the datalog-citations
+  # plugin is not enabled, so a layout that uses the tag still builds. The plugin
+  # registers the real tag and sets `datalog_bibliography` on the pages it
+  # handles; a page that sets the key by hand without the plugin gets nothing and
+  # a build warning, where it used to get a "coming soon" notice.
   class DatalogBibliographyTag < Liquid::Tag
     def render(context)
       page = context.registers[:page]
-      bib_data = page["datalog_bibliography"]
-      return "" unless bib_data
+      return "" unless page["datalog_bibliography"]
 
-      <<~HTML
-        <div class="datalog-bibliography-placeholder">
-          <p>Bibliography feature coming soon.</p>
-        </div>
-      HTML
+      Jekyll.logger.warn("datalog-citations", "#{page['path']} sets datalog_bibliography, " \
+                                              "but datalog_plugins.enabled does not list datalog-citations")
+      ""
     end
   end
 end

data/_plugins/datalog_comments.rb

@@ -1,16 +1,19 @@
 # frozen_string_literal: true
 
 module Jekyll
+  # Stands in for the `datalog_comments` tag when the datalog-comments plugin is
+  # not enabled, so a layout that uses the tag still builds. The plugin registers
+  # the real tag and sets `datalog_comments` on the pages it handles; a page that
+  # sets the key by hand without the plugin gets nothing and a build warning,
+  # where it used to get a "coming soon" notice.
   class DatalogCommentsTag < Liquid::Tag
     def render(context)
       page = context.registers[:page]
       return "" unless page["datalog_comments"]
 
-      <<~HTML
-        <div class="datalog-comments-placeholder">
-          <p>Comments feature coming soon.</p>
-        </div>
-      HTML
+      Jekyll.logger.warn("datalog-comments", "#{page['path']} sets datalog_comments, " \
+                                             "but datalog_plugins.enabled does not list datalog-comments")
+      ""
     end
   end
 end

data/_plugins/datalog_slides.rb

@@ -1,18 +1,19 @@
 # frozen_string_literal: true
 
 module Jekyll
+  # Stands in for the `datalog_slides` tag when the datalog-slides plugin is not
+  # enabled, so a layout that uses the tag still builds. The plugin registers the
+  # real tag and sets `datalog_slides` on the pages it handles; a page that sets
+  # the key by hand without the plugin gets nothing and a build warning, where it
+  # used to get a "coming soon" notice showing its raw configuration.
   class DatalogSlidesTag < Liquid::Tag
     def render(context)
       page = context.registers[:page]
-      slides_data = page["datalog_slides"]
-      return "" unless slides_data
+      return "" unless page["datalog_slides"]
 
-      # Render a simple placeholder or embed
-      <<~HTML
-        <div class="datalog-slides-placeholder">
-          <p>Slides feature coming soon. Configured: #{slides_data}</p>
-        </div>
-      HTML
+      Jekyll.logger.warn("datalog-slides", "#{page['path']} sets datalog_slides, " \
+                                           "but datalog_plugins.enabled does not list datalog-slides")
+      ""
     end
   end
 end

data/_plugins/i18n.rb

@@ -116,6 +116,11 @@ class TranslateTag < Liquid::Tag
   # quote into the first option name, or interpolation silently fails.
   SYNTAX = /\A\s*(['"]?)(\w[\w.-]*)\1(.*)?\z/m
 
+  # `name: value` pairs, separated by commas or spaces. A quoted value may
+  # contain commas: splitting the markup on every comma cut `name: "Doe, Jane"`
+  # in two.
+  OPTION = /(\w+)\s*:\s*("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|[^\s,]+)/
+
   def initialize(tag_name, markup, tokens)
     super
     raise Liquid::SyntaxError, "Syntax Error in 't' - Valid syntax: t key [arg: value]" unless markup.strip =~ SYNTAX
@@ -134,14 +139,8 @@ class TranslateTag < Liquid::Tag
   def parse_options(markup, context)
     return {} unless markup && !markup.strip.empty?
 
-    tokens = markup.strip.split(",").map(&:strip)
-    tokens.each_with_object({}) do |token, memo|
-      next if token.empty?
-
-      if token.include?(":")
-        key, value = token.split(":", 2)
-        memo[key.strip.to_sym] = context.evaluate(Liquid::Expression.parse(value.strip))
-      end
+    markup.scan(OPTION).to_h do |key, value|
+      [key.to_sym, context.evaluate(Liquid::Expression.parse(value))]
     end
   end
 end

data/_plugins/image_optimizer.rb

@@ -34,6 +34,18 @@ module Jekyll
     }.freeze
     RASTER_EXTENSIONS = %w[.jpg .jpeg .png].freeze
 
+    # The one image a page fetches early: the first in its post or page content,
+    # unless the author made it lazy or the page already preloads an image, as
+    # the hero does. The first <img> anywhere used to be marked both lazy and
+    # high priority; on the home page that was a post card below the hero, and
+    # on tutorials the thumbnail of a related post at the bottom.
+    def priority_image(fragment)
+      return if fragment.at_css('link[rel="preload"][as="image"]')
+
+      image = fragment.css(".post-content img, .page-content img").find { |img| img["data-no-optimize"] != "true" }
+      image unless image.nil? || image["loading"] == "lazy"
+    end
+
     def process(document)
       return unless document.output_ext == ".html"
       return if document.output.nil? || document.output.empty?
@@ -46,18 +58,17 @@ module Jekyll
       manifest = site&.data&.fetch("datalog_responsive_images", {}) || {}
       image_config = site&.config&.fetch("datalog_image_config", {}) || {}
       optimized = false
-      first_priority_assigned = false
+      priority = priority_image(fragment)
 
       fragment.css("img").each do |img|
         next if img["data-no-optimize"] == "true"
 
-        img["loading"] ||= "lazy"
-        img["decoding"] ||= "async"
-
-        unless first_priority_assigned
+        if img == priority
           img["fetchpriority"] ||= "high"
-          first_priority_assigned = true
+        else
+          img["loading"] ||= "lazy"
         end
+        img["decoding"] ||= "async"
 
         normalized_src = normalize_src(img["src"], site)
         picture_entry = manifest[normalized_src]

data/_plugins/math_preprocessor.rb

@@ -19,8 +19,11 @@ module MathPreprocessor
   ].freeze
 
   INLINE_PATTERNS = [
+    # Pandoc's rule for inline math, so prices and shell variables stay text:
+    # the opening $ is followed by a non-space, the closing $ follows a
+    # non-space and is not followed by a digit, and a blank line ends it.
     {
-      regex: /(?<![\\$])(?<open>\$)(?!\$)(?<body>[^$]+?)(?<close>\$)(?!\$)/m,
+      regex: /(?<![\\$])(?<open>\$)(?![\s$])(?<body>(?:[^$\\\n]|\\.|\n(?![ \t]*\n))+?)(?<![\s\\])(?<close>\$)(?![$\d])/m,
       tag: "span"
     },
     {
@@ -29,6 +32,20 @@ module MathPreprocessor
     }
   ].freeze
 
+  # Code shows dollar signs literally (shell and R variables, amounts in SQL),
+  # so fenced blocks, highlight tags, <pre>/<code> elements and inline code
+  # spans are set aside before looking for math and put back afterwards.
+  CODE_PATTERNS = [
+    /^([ \t]*)(`{3,}|~{3,})[^\n]*\n.*?(?:^\1\2[ \t]*$|\z)/m,
+    /\{%-?\s*highlight\b.*?\{%-?\s*endhighlight\s*-?%\}/m,
+    %r{<(pre|code)\b[^>]*>.*?</\1>}mi,
+    /(?<!`)(`+)(?!`)(?:(?!\n[ \t]*\n).)+?(?<!`)\1(?!`)/m
+  ].freeze
+
+  # NUL marks masked code, since page content never contains it. It is written as
+  # an escape: a raw NUL byte in the source stopped RuboCop from parsing the file.
+  PLACEHOLDER = /\x00(\d+)\x00/
+
   class Processor
     attr_reader :expressions
 
@@ -40,9 +57,18 @@ module MathPreprocessor
     def process
       return @content unless @content&.match?(/\$|\\\(|\\\[|\\begin\{/)
 
-      processed = @content.dup
+      code = []
+      processed = CODE_PATTERNS.reduce(@content.dup) do |text, pattern|
+        text.gsub(pattern) do |match|
+          code << match
+          "\x00#{code.size - 1}\x00"
+        end
+      end
       processed = apply_patterns(processed, DISPLAY_PATTERNS, display: true)
-      apply_patterns(processed, INLINE_PATTERNS, display: false)
+      processed = apply_patterns(processed, INLINE_PATTERNS, display: false)
+      # A segment set aside can contain the placeholder of an earlier one.
+      processed = processed.gsub(PLACEHOLDER) { code[Regexp.last_match(1).to_i] } while processed.match?(PLACEHOLDER)
+      processed
     end
 
     private
@@ -66,8 +92,12 @@ module MathPreprocessor
       cleaned_source = cleanup_source(latex)
       record_expression(cleaned_source, alt_text)
 
+      # ARIA forbids aria-label on an element with no role, such as a plain span.
+      # axe let it pass while the span held the raw LaTeX as text, and failed it
+      # once MathJax rendered the expression; the math role allows the label.
       attributes = {
         "class" => display ? "math-expression math-expression--source" : "math-expression-inline math-expression--source",
+        "role" => "math",
         "data-math-alt" => alt_text,
         "data-math-source" => cleaned_source,
         "aria-label" => alt_text,
@@ -165,6 +195,9 @@ module MathPreprocessor
   def apply(document)
     return unless document.respond_to?(:content)
     return unless document.respond_to?(:output_ext) && document.output_ext == ".html"
+    # A page that opts out of math rendering (`math: false` or `mathjax: false`)
+    # keeps its dollar signs and TeX-looking text verbatim.
+    return if document.respond_to?(:data) && (document.data["math"] == false || document.data["mathjax"] == false)
 
     content = document.content
     return unless content&.match?(/\$|\\\(|\\\[|\\begin\{/)

data/_plugins/notebook_converter.rb

@@ -7,6 +7,7 @@ require "fileutils"
 require "cgi"
 require "loofah"
 require "base64"
+require_relative "rouge_highlight_filter"
 
 module Datalog
   module NotebookRenderer
@@ -94,14 +95,30 @@ module Datalog
                                       metadata: build_sanitization_metadata(meta, cell_index: cell_index))
       return if sanitized.to_s.strip.empty?
 
-      %(<section class="notebook-cell notebook-cell--markdown">\n#{sanitized}\n</section>)
+      %(<section class="notebook-cell notebook-cell--markdown">\n#{demote_headings(sanitized.to_s)}\n</section>)
+    end
+
+    # The notebook layout gives the page its <h1>, and a notebook's first
+    # markdown cell usually repeats the title as `# Title`. Each heading moves
+    # down a level (h1 to h2, and so on to h6), which keeps one <h1> on the
+    # page and the cells' own outline under it.
+    def demote_headings(html)
+      html.gsub(%r{<(/?)h([1-5])(?=[\s>])}i) { "<#{Regexp.last_match(1)}h#{Regexp.last_match(2).to_i + 1}" }
     end
 
     # Class names mirror the theme stylesheet (`.notebook-cell--input`,
     # `.notebook-cell__code` and `.notebook-cell__outputs` in _sass/_components.scss).
     def render_code(cell, source, metadata, site, cell_index)
       language = cell.dig("metadata", "language") || metadata[:language] || "text"
-      code_html = %(<pre class="notebook-cell__code"><code class="language-#{language}">#{CGI.escapeHTML(source)}</code></pre>)
+      # The language comes from the notebook file, so it is cut down to the
+      # characters a class name can hold before it goes into the attribute.
+      language_class = language.to_s.gsub(/[^\w+#.-]/, "")
+      # Rouge highlights the cell as the site builds, as kramdown does for code
+      # blocks, and escapes it.
+      highlighted = Jekyll::RougeHighlightFilter.highlight(source, language_class)
+      pre_attributes = %(class="highlight notebook-cell__code" tabindex="0")
+      code_attributes = %(class="language-#{language_class}")
+      code_html = %(<pre #{pre_attributes}><code #{code_attributes}>#{highlighted}</code></pre>)
       base_metadata = metadata.respond_to?(:merge) ? metadata.merge(language: language) : { language: language }
       outputs_html = render_outputs(Array(cell["outputs"]), site: site, cell_index: cell_index, metadata: base_metadata)
       outputs_html = %(\n<div class="notebook-cell__outputs">\n#{outputs_html}\n</div>) unless outputs_html.empty?
@@ -157,11 +174,13 @@ module Datalog
     def image_output_html(output)
       data = output["data"] || {}
 
+      # Jupyter writes base64 image data split over lines or ending in a newline,
+      # and the data URI check rejects whitespace, which dropped those images.
       if (png = data["image/png"])
-        html = %(<img src="data:image/png;base64,#{Array(png).join}" alt="Notebook output" />)
+        html = %(<img src="data:image/png;base64,#{Array(png).join.gsub(/\s+/, '')}" alt="Notebook output" />)
         return [html, "image/png"]
       elsif (jpeg = data["image/jpeg"])
-        html = %(<img src="data:image/jpeg;base64,#{Array(jpeg).join}" alt="Notebook output" />)
+        html = %(<img src="data:image/jpeg;base64,#{Array(jpeg).join.gsub(/\s+/, '')}" alt="Notebook output" />)
         return [html, "image/jpeg"]
       elsif (svg = data["image/svg+xml"])
         encoded = Base64.strict_encode64(Array(svg).join)
@@ -629,31 +648,9 @@ module Datalog
       nil
     end
 
-    def patch_jupyter_converter!
-      return unless defined?(JekyllJupyterNotebook::Converter)
-      return if @converter_fallback_applied
-
-      fallback = Module.new do
-        def convert(content)
-          super
-        rescue Errno::ENOENT, StandardError => e
-          Jekyll.logger.warn("notebook converter", "primary conversion failed: #{e.message}; using fallback renderer")
-          Datalog::NotebookRenderer.render_from_raw(content) || ""
-        end
-      end
-
-      JekyllJupyterNotebook::Converter.prepend(fallback)
-      @converter_fallback_applied = true
-    rescue StandardError => e
-      Jekyll.logger.warn("notebook converter", "failed to apply converter fallback: #{e.message}")
-    end
   end
 end
 
-Jekyll::Hooks.register :site, :after_init do |_site|
-  Datalog::NotebookRenderer.patch_jupyter_converter!
-end
-
 module Jekyll
   # Converts Jupyter notebooks into HTML pages and downloadable assets.
   class NotebookConverter < Generator
@@ -671,8 +668,6 @@ module Jekyll
         return
       end
 
-      ensure_dependency
-
       files = notebook_files
       logger.debug("notebook converter", "located #{files.size} notebooks")
       return if files.empty?
@@ -722,21 +717,6 @@ module Jekyll
       value.sub(%r{/+$}, "")
     end
 
-    # Notebook pages do not need the gem (see #convert_notebook). When it is
-    # present, its converter is given the same renderer as a fallback so the
-    # gem's own `.ipynb` handling and `{% jupyter_notebook %}` tag keep working
-    # on machines without a `jupyter` executable.
-    def ensure_dependency
-      return true if defined?(JekyllJupyterNotebook::Converter)
-
-      require "jekyll-jupyter-notebook"
-      Datalog::NotebookRenderer.patch_jupyter_converter!
-      true
-    rescue LoadError => e
-      logger.debug("notebook converter", "jekyll-jupyter-notebook not loaded: #{e.message}")
-      false
-    end
-
     def notebook_files
       glob = File.join(site.source, config["source"], "**", "*.ipynb")
       Dir.glob(glob)

data/_plugins/plugin_loader.rb

@@ -149,7 +149,9 @@ end
 
 module Datalog
   module PluginLoaderHooks
-    HOOK_SCOPES = %i[pages documents posts].freeze
+    # Posts are documents: Jekyll fires a post's `posts` hooks and then its
+    # `documents` hooks, so registering both ran every plugin hook twice a post.
+    HOOK_SCOPES = %i[pages documents].freeze
 
     module_function
 

data/_plugins/publications_generator.rb

@@ -6,7 +6,13 @@ module Jekyll
     priority :low
 
     def generate(site)
-      publications_data = site.data["publications"] ||= {}
+      publications_data = site.data["publications"]
+      # _data/publications.yml is normally a map with `settings` and
+      # `manual_entries`; a plain list of entries is read as the manual
+      # entries rather than stopping the build with a TypeError.
+      publications_data = { "manual_entries" => publications_data } if publications_data.is_a?(Array)
+      publications_data = {} unless publications_data.is_a?(Hash)
+      site.data["publications"] = publications_data
       settings = publications_data["settings"] || {}
       config_source = site.config.dig("theme_options", "publications", "bibtex_source")
       bibtex_source = settings["bibtex_source"] || config_source
@@ -22,7 +28,7 @@ module Jekyll
         end
       end
 
-      manual_entries = publications_data["manual_entries"] || []
+      manual_entries = Array(publications_data["manual_entries"]).select { |entry| entry.is_a?(Hash) }
       combined = (imported_entries + manual_entries).map { |entry| normalize_entry(entry) }
 
       academic_citations = site.data.dig("academic", "citations", "per_publication") || {}

data/_plugins/rouge_highlight_filter.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "rouge"
+
+# `rouge_highlight` highlights code with Rouge as the site builds, the way
+# kramdown highlights fenced code blocks. Includes that print code passed to
+# them, such as `components/api-function.html`, call it as
+# `code | rouge_highlight: language` inside `<pre class="highlight"><code>`;
+# the notebook converter calls `RougeHighlightFilter.highlight` for code cells.
+# The result is escaped; a language Rouge does not know comes back as plain
+# text.
+module Jekyll
+  module RougeHighlightFilter
+    # kramdown's opening tag for a block Rouge highlighted.
+    KRAMDOWN_CODE_BLOCK = '<pre class="highlight">'
+
+    def self.highlight(code, language = nil)
+      return "" if code.nil?
+
+      source = code.to_s
+      lexer = Rouge::Lexer.find_fancy(language.to_s.strip.downcase, source) || Rouge::Lexers::PlainText
+      Rouge::Formatters::HTML.new.format(lexer.lex(source))
+    end
+
+    def rouge_highlight(code, language = nil)
+      RougeHighlightFilter.highlight(code, language)
+    end
+  end
+end
+
+Liquid::Template.register_filter(Jekyll::RougeHighlightFilter)
+
+# A code block wider than the page scrolls, and a keyboard user can only scroll
+# it once it takes focus. Prism made every block focusable in the browser; the
+# blocks kramdown highlights get the attribute here, and the includes and the
+# notebook converter write it themselves.
+Jekyll::Hooks.register %i[pages documents], :post_convert do |document|
+  block = Jekyll::RougeHighlightFilter::KRAMDOWN_CODE_BLOCK
+  next unless document.content&.include?(block)
+
+  document.content = document.content.gsub(block, '<pre class="highlight" tabindex="0">')
+end

data/_plugins/search_code_blocks.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Datalog
+  # Collects the fenced code blocks of every page and document for the search
+  # index. The index template used to split `doc.content` on backticks, but
+  # Jekyll renders documents before pages, so by the time search.json rendered
+  # that content was HTML and every document's code list came out empty.
+  module SearchCodeBlocks
+    # An opening fence of three or more backticks or tildes with an optional
+    # language, the code, and a closing fence of the same characters.
+    FENCE = /^ {0,3}(`{3,}|~{3,})[ \t]*([^\s`~{]*)[^\n]*\n(.*?)^ {0,3}\1[ \t]*$/m
+
+    module_function
+
+    def extract(source)
+      text = source.to_s.gsub("\r\n", "\n")
+      text.scan(FENCE).map do |_fence, language, code|
+        { "language" => language.empty? ? "text" : language.downcase, "code" => code.chomp }
+      end
+    end
+  end
+end
+
+# Content is still the author's source before rendering starts. Collection
+# docs only: site.documents also lists a collection's static files.
+Jekyll::Hooks.register :site, :pre_render do |site|
+  (site.pages + site.collections.values.flat_map(&:docs)).each do |item|
+    item.data["search_code"] = Datalog::SearchCodeBlocks.extract(item.content)
+  end
+end

data/_plugins/search_normalizer.rb

@@ -1,66 +1,28 @@
 # frozen_string_literal: true
 
-begin
-  require "unicode_normalize"
-  UNICODE_NORMALIZE_SUPPORTED = true
-rescue LoadError
-  UNICODE_NORMALIZE_SUPPORTED = false
-  warn "[search_normalizer] unicode_normalize gem not available; falling back to basic normalization"
-end
-
 module Datalog
   module SearchFilters
     module_function
 
+    # Lower-cases text and strips combining marks, so "Café" and "cafe" match
+    # while letters outside ASCII ("ß", "ł", Cyrillic, CJK) are kept. The
+    # browser normalizes queries the same way (assets/js/search/utils.js), so
+    # the index and the query agree. String#unicode_normalize is part of Ruby:
+    # this file used to require it as if it were a gem, fail, and fall back to
+    # dropping every character outside ASCII.
     def normalize_search(input)
-      value = input.to_s
+      # Invalid byte sequences are dropped first: unicode_normalize raises on them.
+      value = input.to_s.scrub("")
       return "" if value.empty?
 
-      normalized = if UNICODE_NORMALIZE_SUPPORTED && value.respond_to?(:unicode_normalize)
-                     value.unicode_normalize(:nfkd).gsub(/\p{Mn}/, "")
-                   else
-                     transliterate(value)
-                   end
-
-      normalized.downcase.strip
-    rescue StandardError
-      input.to_s.downcase
-    end
-
-    def transliterate(value)
-      value.encode("ASCII", fallback: lambda { |char|
-        approximate_character(char)
-      }, invalid: :replace, undef: :replace, replace: "")
-    rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
-      value
-    end
-
-    def approximate_character(char)
-      @transliteration_map ||= build_transliteration_map
-      @transliteration_map.fetch(char, "")
-    end
-
-    def build_transliteration_map
-      basic_map = {}
-
-      accents = {
-        "ÀÁÂÃÄÅàáâãäå" => "a",
-        "ÈÉÊËèéêë" => "e",
-        "ÌÍÎÏìíîï" => "i",
-        "ÒÓÔÕÖØòóôõöø" => "o",
-        "ÙÚÛÜùúûü" => "u",
-        "Çç" => "c",
-        "Ññ" => "n",
-        "Ýýÿ" => "y",
-        "Ææ" => "ae",
-        "Œœ" => "oe"
-      }
-
-      accents.each do |chars, replacement|
-        chars.each_char { |char| basic_map[char] = replacement }
+      stripped = begin
+        value.unicode_normalize(:nfkd).gsub(/\p{Mn}/, "")
+      rescue Encoding::CompatibilityError
+        # Text in an encoding other than Unicode cannot be normalized, so it is
+        # indexed as it is.
+        value
       end
-
-      basic_map
+      stripped.downcase.strip
     end
 
     def normalize_search_array(values)