jektex 0.1.0 → 0.2.0

data/lib/jektex/page_processor.rb

@@ -0,0 +1,304 @@
+require 'digest'
+require 'execjs'
+require 'htmlentities'
+
+module Jektex
+  # Finds math expressions in a page, decides whether the page should be
+  # processed at all, and answers each expression from the cache or by
+  # rendering it. This is the orchestration layer between Jekyll hooks
+  # and the renderer/cache.
+  #
+  # Rendering happens in two phases. At pre_render every LaTeX-notation
+  # expression in the raw content is swapped for an inert token, so Liquid
+  # and kramdown treat math like ordinary text and cannot mangle it. At
+  # post_render the final HTML structure is known, so tokens inside code
+  # markup turn back into their original source text (a code sample about
+  # LaTeX must show the LaTeX, see github issue #5) while all other math
+  # is rendered.
+  #
+  # Kramdown notation is converted to LaTeX notation by kramdown itself —
+  # except inside raw HTML blocks, which kramdown does not process; the
+  # $$..$$ expressions left there are found and rendered too (issue #7),
+  # but only for markdown source files, where $$ has a defined meaning.
+  #
+  # All expressions a page needs are collected first and rendered in one
+  # batched KaTeX call, because each call to an external ExecJS runtime
+  # costs far more than the rendering itself.
+  class PageProcessor
+    # The (?<!\\) lookbehind skips escaped delimiters. It is also what
+    # prevents the post_render pass from re-matching \\[..] sequences
+    # inside the TeX source that KaTeX embeds in its own output.
+    INLINE_MATH = /(\\\()(.*?)(?<!\\)\\\)/m
+    DISPLAY_MATH = /(\\\[)(.*?)(?<!\\)\\\]/m
+
+    TOKEN = /jektexprotected[0-9a-f]{32}/
+
+    # everything resolve_math reacts to, found in one pass
+    MATH_PATTERN = /(?<inline>\\\((?<inline_body>.*?)(?<!\\)\\\))|(?<display>\\\[(?<display_body>.*?)(?<!\\)\\\])|(?<token>#{TOKEN.source})/m
+
+    # kramdown converts its $$..$$ notation everywhere except inside raw
+    # HTML blocks; these leftovers in the output of markdown pages are
+    # rendered too (github issue #7). (?<!\\) honors \$$ as an opt-out
+    # and keeps \$ inside a body from closing the expression.
+    DOLLAR_MATH = /(?<dollars>(?<!\\)\$\$(?<dollars_body>.+?)(?<!\\)\$\$)/m
+
+    # never used on protected segments: the dollars alternative could
+    # swallow a protection token between two dollar signs there
+    MATH_PATTERN_WITH_DOLLARS = /#{MATH_PATTERN.source}|#{DOLLAR_MATH.source}/m
+
+    # markup whose contents must never be rendered
+    # (the same set KaTeX's own auto-render extension ignores)
+    PROTECTED_ELEMENTS = %r{<(pre|code|script|style|textarea)\b[^>]*>.*?</\1\s*>}mi
+
+    ProtectedExpression = Struct.new(:source, :body, :display_mode)
+
+    # a single substitution: replace text[position, length] with either the
+    # rendering of expression body (kind :expression, source is the fallback
+    # when rendering is impossible) or a literal string (kind :text)
+    Instruction = Struct.new(:position, :length, :kind, :body, :display_mode, :source)
+
+    attr_reader :rendered_count, :cache_hit_count
+
+    def initialize(config:, cache:, renderer:, reporter:)
+      @config = config
+      @cache = cache
+      @renderer = renderer
+      @reporter = reporter
+      @entity_parser = HTMLEntities.new
+      # tokens must stay resolvable across watch-mode rebuilds, because other
+      # pages can embed token-carrying content of pages that are not rebuilt
+      @protected_math = Hash.new
+      # batch results waiting for their first substitution: [expression, mode] => html
+      @freshly_rendered = Hash.new
+      # failed expressions of the current page: [expression, mode] => fallback html or nil
+      @error_fallbacks = Hash.new
+      reset_counters
+    end
+
+    # pre_render: protect LaTeX-notation math in the raw content
+    def process_content(page)
+      return page.content if !page.data || ignored?(page)
+      text = page.content.to_s
+      return text unless text.include?('\(') || text.include?('\[')
+      return protect_math(text)
+    end
+
+    # post_render: render the protected tokens and the kramdown-notation
+    # math (kramdown turns its $$..$$ into \(..\)/\[..\] during conversion,
+    # except inside raw HTML blocks, where the $$..$$ survives verbatim)
+    def process_output(page)
+      return restore_protected_math(page.output) if !page.data || ignored?(page)
+      return resolve_math(page.output.to_s, page.relative_path,
+                          dollars: markdown_page?(page))
+    end
+
+    def protect_math(text)
+      text = text.gsub(INLINE_MATH) { protect_expression($~[0], $2, false) }
+      return text.gsub(DISPLAY_MATH) { protect_expression($~[0], $2, true) }
+    end
+
+    def resolve_math(text, doc_path, dollars: false)
+      return text unless contains_math?(text, dollars)
+      instructions = collect_instructions(text, dollars)
+      render_missing_expressions(instructions, doc_path)
+      return apply_instructions(text, instructions, doc_path)
+    end
+
+    # immediate rendering in a plain string, without code protection
+    def render_math(text, doc_path)
+      instructions = Array.new
+      scan_segment(text, 0, :plain, instructions)
+      render_missing_expressions(instructions, doc_path)
+      return apply_instructions(text, instructions, doc_path)
+    end
+
+    def ignored?(page)
+      front_matter_flag = page.data[@config.front_matter_tag]
+      return true if front_matter_flag == false || front_matter_flag == "false"
+      return @config.ignore.any? do |pattern|
+        File.fnmatch?(pattern, page.relative_path, File::FNM_DOTMATCH)
+      end
+    end
+
+    def reset_counters
+      @rendered_count = 0
+      @cache_hit_count = 0
+    end
+
+    private
+
+    # An ignored page must not render math, but it can still carry
+    # protection tokens — feeds embed the content of processed posts.
+    # Those tokens must never leak into the published site, so they
+    # are turned back into their original source text.
+    def restore_protected_math(output)
+      return output unless output.is_a?(String) && output.include?("jektexprotected")
+      return output.gsub(TOKEN) do |token|
+        expression = @protected_math[token]
+        expression ? expression.source : token
+      end
+    end
+
+    def contains_math?(text, dollars)
+      return true if text.include?('\(') || text.include?('\[') || text.include?("jektexprotected")
+      return dollars && text.include?("$$")
+    end
+
+    def markdown_page?(page)
+      return @config.markdown_extensions.include?(File.extname(page.relative_path.to_s).downcase)
+    end
+
+    def protect_expression(source, body, display_mode)
+      token = "jektexprotected" + Digest::SHA2.hexdigest(source)[0, 32]
+      @protected_math[token] = ProtectedExpression.new(source, body, display_mode)
+      return token
+    end
+
+    ##### scanning
+
+    def collect_instructions(text, dollars)
+      instructions = Array.new
+      position = 0
+      text.scan(PROTECTED_ELEMENTS) do
+        match = Regexp.last_match
+        scan_segment(text[position...match.begin(0)], position, :plain, instructions,
+                     text: text, dollars: dollars)
+        scan_segment(match[0], match.begin(0), :protected, instructions)
+        position = match.end(0)
+      end
+      scan_segment(text[position..].to_s, position, :plain, instructions,
+                   text: text, dollars: dollars)
+      return instructions
+    end
+
+    def scan_segment(segment, offset, kind, instructions, text: segment, dollars: false)
+      pattern = dollars ? MATH_PATTERN_WITH_DOLLARS : MATH_PATTERN
+      segment.scan(pattern) do
+        match = Regexp.last_match
+        position = offset + match.begin(0)
+        if match[:token]
+          expression = @protected_math[match[0]]
+          next unless expression
+          if kind == :protected
+            # a code sample must show the LaTeX source, not the rendering
+            instructions.append(Instruction.new(position, match[0].length, :text,
+                                                nil, nil, expression.source))
+          else
+            instructions.append(Instruction.new(position, match[0].length, :expression,
+                                                expression.body, expression.display_mode,
+                                                expression.source))
+          end
+        elsif kind == :plain
+          # the dollars guard is required: asking a plain MATH_PATTERN
+          # match for the group raises IndexError
+          if dollars && match[:dollars]
+            instructions.append(Instruction.new(position, match[0].length, :expression,
+                                                match[:dollars_body],
+                                                standalone_line?(text, position, position + match[0].length),
+                                                match[0]))
+          else
+            body = match[:inline_body] || match[:display_body]
+            instructions.append(Instruction.new(position, match[0].length, :expression,
+                                                body, !match[:display].nil?, match[0]))
+          end
+        end
+      end
+    end
+
+    # kramdown renders $$..$$ standing alone on its line(s) as display math
+    # and mid-text occurrences as inline math; mirror that on the output text
+    def standalone_line?(text, start, stop)
+      line_start = start.zero? ? 0 : ((text.rindex("\n", start - 1) || -1) + 1)
+      line_end = text.index("\n", stop) || text.length
+      return text[line_start...start].match?(/\A\s*\z/) &&
+             text[stop...line_end].match?(/\A\s*\z/)
+    end
+
+    ##### rendering
+
+    # renders everything the instructions need but the cache cannot answer,
+    # in a single call to the JavaScript runtime
+    def render_missing_expressions(instructions, doc_path)
+      @error_fallbacks = Hash.new
+      needed = instructions.filter_map do |instruction|
+        next unless instruction.kind == :expression
+        [@entity_parser.decode(instruction.body), instruction.display_mode]
+      end.uniq
+      missing = needed.reject { |expression, mode| @cache.fetch(expression, mode) }
+      return if missing.empty?
+
+      begin
+        results = @renderer.render_batch(missing)
+      rescue SystemExit, Interrupt
+        # save the work done so far, then let Jekyll die properly
+        @cache.save
+        raise
+      end
+
+      missing.zip(results) do |(expression, mode), result|
+        if result.error
+          @reporter.error(result.error, doc_path)
+          @error_fallbacks[[expression, mode]] = result.html
+        else
+          @cache.store(expression, mode, result.html)
+          @freshly_rendered[[expression, mode]] = result.html
+        end
+      end
+    end
+
+    def apply_instructions(text, instructions, doc_path)
+      result = +""
+      position = 0
+      instructions.each do |instruction|
+        result << text[position...instruction.position]
+        result << if instruction.kind == :text
+                    instruction.source
+                  else
+                    render_expression(instruction.body,
+                                      display_mode: instruction.display_mode,
+                                      doc_path: doc_path) || instruction.source
+                  end
+        position = instruction.position + instruction.length
+      end
+      result << text[position..].to_s
+      return result
+    end
+
+    # Returns nil only for expressions that failed so hard that not even
+    # error highlighting could be rendered — callers keep the source text.
+    def render_expression(expression, display_mode:, doc_path:)
+      expression = @entity_parser.decode(expression)
+      key = [expression, display_mode]
+
+      return @error_fallbacks[key] if @error_fallbacks.key?(key)
+
+      if (batched_html = @freshly_rendered.delete(key))
+        @rendered_count += 1
+        @reporter.progress(@rendered_count, @cache_hit_count)
+        return batched_html
+      end
+
+      if (cached_html = @cache.fetch(expression, display_mode))
+        @cache_hit_count += 1
+        @reporter.progress(@rendered_count, @cache_hit_count)
+        return cached_html
+      end
+
+      # not covered by the batch — render it individually
+      begin
+        html = @renderer.render(expression, display_mode: display_mode)
+      rescue SystemExit, Interrupt
+        @cache.save
+        raise
+      rescue ExecJS::ProgramError => error
+        @reporter.error(error.message, doc_path)
+        return @renderer.render_with_error_fallback(expression, display_mode: display_mode)
+      end
+
+      @cache.store(expression, display_mode, html)
+      @rendered_count += 1
+      @reporter.progress(@rendered_count, @cache_hit_count)
+      return html
+    end
+  end
+end

data/lib/jektex/renderer.rb

@@ -0,0 +1,81 @@
+require 'execjs'
+
+module Jektex
+  # Wraps the bundled KaTeX JavaScript. Knows nothing about pages,
+  # caching or Jekyll — it turns expressions into HTML.
+  class Renderer
+    # With an external ExecJS runtime (node, bun) every call spawns a fresh
+    # process that has to parse the whole KaTeX bundle (~70 ms), while the
+    # rendering itself takes single milliseconds. Rendering a whole batch
+    # of expressions in one call amortizes that overhead.
+    BATCH_HELPER = <<~JS
+      function jektexRenderBatch(items) {
+        return items.map(function (item) {
+          var expression = item[0];
+          var options = item[1];
+          try {
+            return { html: katex.renderToString(expression, options) };
+          } catch (error) {
+            var message = String(error.message || error);
+            options.throwOnError = false;
+            try {
+              return { error: message, html: katex.renderToString(expression, options) };
+            } catch (unrenderableError) {
+              return { error: message, html: null };
+            }
+          }
+        });
+      }
+    JS
+
+    # error is nil for successful renders; html is the rendered expression,
+    # for failed ones rendered with the error highlighted in place — or nil
+    # when even that was impossible
+    RenderedExpression = Struct.new(:html, :error)
+
+    def initialize(config)
+      @path_to_katex_js = config.path_to_katex_js
+      @global_macros = config.global_macros
+      @katex_options = config.katex_options
+    end
+
+    # Renders [[expression, display_mode], ...] in one JavaScript call.
+    # Returns a RenderedExpression for every item, in the same order.
+    # A parse error in one expression does not affect the others.
+    def render_batch(expressions)
+      return [] if expressions.empty?
+      items = expressions.map do |expression, display_mode|
+        [expression, base_options(display_mode)]
+      end
+      results = katex.call("jektexRenderBatch", items)
+      return results.map { |result| RenderedExpression.new(result["html"], result["error"]) }
+    end
+
+    # Raises ExecJS::ProgramError when the expression is invalid LaTeX.
+    def render(expression, display_mode:)
+      katex.call("katex.renderToString", expression, base_options(display_mode))
+    end
+
+    # Never raises on invalid LaTeX; KaTeX renders the error into the
+    # document instead (red highlighting).
+    def render_with_error_fallback(expression, display_mode:)
+      katex.call("katex.renderToString", expression,
+                 base_options(display_mode).merge(throwOnError: false))
+    end
+
+    private
+
+    # compiling the KaTeX bundle takes ~100 ms, so defer it to the first
+    # render instead of paying for it at plugin load time
+    def katex
+      @katex ||= ExecJS.compile(File.read(@path_to_katex_js) + BATCH_HELPER)
+    end
+
+    # user options first, so the keys jektex decides itself always win
+    # (reserved keys are already stripped by the config, this is a backstop)
+    def base_options(display_mode)
+      @katex_options.merge(displayMode: display_mode,
+                           macros: @global_macros)
+    end
+  end
+end

data/lib/jektex/reporter.rb

@@ -0,0 +1,70 @@
+module Jektex
+  # All console output of the plugin. Every method is a no-op when the
+  # silent option is set, so callers never have to check it themselves.
+  class Reporter
+    def initialize(config, out: $stdout, err: $stderr)
+      @out = out
+      @err = err
+      @silent = config.silent
+      @indent = config.console_indent
+      @last_progress_time = nil
+    end
+
+    # Configuration problems must stay visible even with the silent
+    # option set — a broken config could be the reason silence was
+    # requested by accident. They go to stderr, so redirecting the
+    # build output does not hide them either.
+    def warn(message)
+      @err.puts "\e[33m#{@indent}JekTeX: #{message}\e[0m"
+    end
+
+    def info(message)
+      return if @silent
+      @out.puts "#{@indent}LaTeX: #{message}"
+    end
+
+    def macro_summary(macro_count, updated_count)
+      return if @silent
+      if macro_count == 0
+        info("no macros loaded")
+      else
+        message = "#{macro_count} macro#{macro_count == 1 ? "" : "s"} loaded"
+        message += " (#{updated_count} updated)" if updated_count > 0
+        info(message)
+      end
+    end
+
+    def error(message, doc_path)
+      return if @silent
+      @out.puts "\e[31m #{message.gsub("ParseError: ", "")}\n\t#{doc_path}\e[0m"
+    end
+
+    # more frequent updates than this are invisible anyway, and printing
+    # one per expression measurably slows down large cached builds
+    PROGRESS_UPDATE_INTERVAL = 0.1
+
+    def progress(rendered, from_cache)
+      return if @silent
+      now = Time.now
+      return if @last_progress_time && now - @last_progress_time < PROGRESS_UPDATE_INTERVAL
+      @last_progress_time = now
+      print_progress_line(rendered, from_cache)
+    end
+
+    def finish(rendered, from_cache)
+      return if @silent
+      print_progress_line(rendered, from_cache)
+      @out.print "\n"
+    end
+
+    private
+
+    # single line overwritten in place with a carriage return;
+    # padded to a fixed width so a shorter update fully covers the previous one
+    def print_progress_line(rendered, from_cache)
+      line = "#{@indent}LaTeX: #{rendered} expressions rendered (#{from_cache} loaded from cache)"
+      @out.print line.ljust(72) + "\r"
+      @out.flush
+    end
+  end
+end

data/lib/jektex/version.rb

@@ -1,3 +1,3 @@
 module Jektex
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/jektex.rb

@@ -1,5 +1,10 @@
-module Jektex
-  autoload :VERSION, "jektex/version.rb"
-end
+require "jektex/version"
+require "jektex/configuration"
+require "jektex/reporter"
+require "jektex/renderer"
+require "jektex/cache"
+require "jektex/page_processor"
 
-require "jektex/jektex.rb"
+# when loaded as a Jekyll plugin, Jekyll is always defined; the guard
+# lets the gem be required standalone (for tests and other tooling)
+require "jektex/hooks" if defined?(Jekyll)

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: jektex
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Jan Černý
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-28 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: execjs
@@ -16,40 +15,34 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '2.9'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.1
+        version: 2.9.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '2.9'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.1
+        version: 2.9.1
 - !ruby/object:Gem::Dependency
   name: digest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 3.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 3.1.1
 - !ruby/object:Gem::Dependency
   name: htmlentities
   requirement: !ruby/object:Gem::Requirement
@@ -76,7 +69,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.6'
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0
@@ -86,10 +79,38 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.6'
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
 description: Highly optimized and cached latex server side rendering for Jekyll with
   macros and dynamic output
 email: jc@ucw.cz
@@ -100,8 +121,13 @@ files:
 - LICENSE
 - README.md
 - lib/jektex.rb
-- lib/jektex/jektex.rb
+- lib/jektex/cache.rb
+- lib/jektex/configuration.rb
+- lib/jektex/hooks.rb
 - lib/jektex/katex.min.js
+- lib/jektex/page_processor.rb
+- lib/jektex/renderer.rb
+- lib/jektex/reporter.rb
 - lib/jektex/version.rb
 homepage: https://github.com/yagarea/jektex
 licenses:
@@ -109,10 +135,8 @@ licenses:
 metadata:
   bug_tracker_uri: https://github.com/yagarea/jektex/issues
   documentation_uri: https://github.com/yagarea/jektex/blob/master/README.md
-  homepage_uri: https://github.com/yagarea/jektex
   source_code_uri: https://github.com/yagarea/jektex
   changelog_uri: https://github.com/yagarea/jektex/blob/master/changelog.md
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -120,15 +144,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.25
-signing_key:
+rubygems_version: 4.0.9
 specification_version: 4
 summary: Highly optimized latex rendering for Jekyll
 test_files: []