latexmath 0.1.4 → 1.0.0

data/lib/latexmath/model.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module Latexmath
+  # The typed LaTeX language model, declared with LutaML Model.
+  #
+  # Latexmath.parse returns a Model::Document. Its classes
+  # (lib/latexmath/model/nodes.rb) are generated by scripts/generate_model.rb
+  # from the capture shapes of the normalized grammar: every capture holds
+  # one kind, so each class has one attribute per capture key and
+  # lutaml-model builds the whole tree from the declared attribute types
+  # (Document.new on the grammar's model binding). No code here walks the
+  # parse tree. See docs/model-contract.adoc.
+  module Model
+  end
+end
+
+require_relative "model/nodes"
+
+module Latexmath
+  module Model
+    # The symbol table entry of a symbol is looked up from its spelling,
+    # stripped, as in the legacy transform (data/latex-symbols.json, read by
+    # Latexmath::Symbols). A spelling the table lacks is a plain symbol of
+    # its own text, kind "symbol".
+    class MathSymbol
+      # The Symbols::Entry for #spelling.
+      def entry
+        Latexmath::Symbols.resolve(spelling.to_s)
+      end
+
+      # The Unicode text the symbol stands for.
+      def value
+        entry.value
+      end
+
+      # "symbol", "open" or "close".
+      def kind
+        entry.kind
+      end
+    end
+  end
+end

data/lib/latexmath/parser/artifact.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Latexmath
+  # The compiled grammar and its declared Ruby bindings, loaded, verified
+  # and paired once per pair of paths.
+  class Artifact
+    Loaded = Struct.new(:parse, :bindings)
+
+    # Loads the Parsanol::PARG::Artifact at +path+ and checks it has an
+    # entry for every decimal marker.
+    def self.load_parse(path)
+      raise ArtifactError, "artifact not found: #{path}" unless File.file?(path)
+
+      artifact = Parsanol::PARG::Artifact.load(path)
+      missing = Configuration::DECIMAL_ENTRIES.values - artifact.entries
+      unless missing.empty?
+        raise ArtifactError, "artifact #{path} lacks required entries: #{missing.join(', ')}"
+      end
+
+      artifact
+    rescue JSON::ParserError => e
+      raise ArtifactError, "artifact #{path} is not valid JSON: #{e.message.lines.first&.strip}"
+    rescue Parsanol::PARG::Error, SystemCallError, KeyError, TypeError, NoMethodError => e
+      raise ArtifactError, "artifact #{path} is invalid: #{e.class}: #{e.message}"
+    end
+
+    # Loads the Ruby bindings artifact at +path+: kind, checksum, target and
+    # binding version are verified, then it is paired with +parse+ (same
+    # grammar, version and source hash). Every entry must declare the tree
+    # capture.
+    def self.load_bindings(path, parse)
+      raise ArtifactError, "bindings artifact not found: #{path}" unless File.file?(path)
+
+      bindings = Parsanol::PARG::Bindings::Artifact.from_json(File.read(path, encoding: "UTF-8"),
+                                                              target: "ruby")
+      bindings.pair_with!(parse)
+      Configuration::DECIMAL_ENTRIES.each_value do |entry|
+        declared = bindings.bindings(entry).map { |b| b["capture"] }
+        next if declared.include?(Configuration::TREE_CAPTURE.to_s)
+
+        raise ArtifactError, "bindings artifact #{path} does not bind #{Configuration::TREE_CAPTURE} for entry #{entry}"
+      end
+      bindings
+    rescue JSON::ParserError => e
+      raise ArtifactError, "bindings artifact #{path} is not valid JSON: #{e.message.lines.first&.strip}"
+    rescue Parsanol::PARG::Error, SystemCallError, KeyError, TypeError, NoMethodError => e
+      raise ArtifactError, "bindings artifact #{path} is invalid: #{e.class}: #{e.message}"
+    end
+
+    attr_reader :path, :bindings_path
+
+    def initialize(path, bindings_path)
+      @path = path
+      @bindings_path = bindings_path
+      @loaded = nil
+      @lock = Mutex.new
+    end
+
+    # Points at other artifacts (mainly for tests); the next #fetch loads them.
+    def use(path, bindings_path)
+      @lock.synchronize do
+        @path = path.to_s
+        @bindings_path = bindings_path.to_s
+        @loaded = nil
+      end
+    end
+
+    # The loaded pair, loaded on first use.
+    def fetch
+      @lock.synchronize do
+        @loaded ||= begin
+          parse = self.class.load_parse(@path)
+          Loaded.new(parse, self.class.load_bindings(@bindings_path, parse))
+        end
+      end
+    end
+  end
+end

data/lib/latexmath/parser/configuration.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Latexmath
+  # Fixed settings: the artifact entries, the engine names and where the
+  # bundled artifacts live.
+  module Configuration
+    # Decimal marker => artifact entry name.
+    DECIMAL_ENTRIES = {
+      "." => "dot",
+      "," => "comma",
+      "٫" => "arabic", # ARABIC DECIMAL SEPARATOR
+    }.freeze
+
+    ENGINES = %i[auto native ruby].freeze
+
+    ARTIFACT_DIR = File.expand_path("../../../artifacts", __dir__)
+    DEFAULT_ARTIFACT_PATH = File.join(ARTIFACT_DIR, "latex.json")
+    DEFAULT_BINDINGS_PATH = File.join(ARTIFACT_DIR, "latex.bindings.ruby.json")
+
+    # The capture under which the declared bindings carry the legacy tree.
+    TREE_CAPTURE = :latex_tree
+    # The binding target the typed model is built from.
+    MODEL_CAPTURE = :model
+
+    # Capture names the PARG grammar spells differently from the legacy
+    # grammar, and the key the legacy raw tree has instead. The legacy
+    # grammar captures a matrix row break as :"\\\\" and then renames that
+    # key to the String "\\\\"; PARG names must be identifiers.
+    CAPTURE_RENAMES = { row_break: "\\\\" }.freeze
+
+    # Deepest bracket nesting sent to the native engine off the main thread,
+    # where the machine stack is small.
+    THREAD_NATIVE_DEPTH = 25
+  end
+end

data/lib/latexmath/parser/engine.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module Latexmath
+  # Runs the grammar on the native or Ruby Parsanol engine with the legacy
+  # grammar's settings (Parsanol's default, strict memo policy), maps
+  # failures to Latexmath errors and records the engine that produced the
+  # result per thread.
+  #
+  # Internal.
+  module Engine
+    LAST_ENGINE_KEY = :latexmath_last_engine
+    OPENING = "([{".chars.freeze
+    CLOSING = ")]}".chars.freeze
+
+    module_function
+
+    def last_engine
+      Thread.current[LAST_ENGINE_KEY]
+    end
+
+    def native_available?
+      Parsanol::Native.available?
+    rescue LoadError, StandardError
+      false
+    end
+
+    # The engine (:native or :ruby) that +engine+ (one of ENGINES) runs.
+    def resolve(engine)
+      case engine
+      when :auto then native_available? ? :native : :ruby
+      when :ruby then :ruby
+      when :native
+        return :native if native_available?
+
+        raise EngineUnavailable,
+              "engine: :native requested but the Parsanol native extension " \
+              "is not available on this platform (#{RUBY_PLATFORM}, Ruby #{RUBY_VERSION})"
+      else
+        raise ArgumentError,
+              "unknown engine #{engine.inspect} (use #{Configuration::ENGINES.map(&:inspect).join(', ')})"
+      end
+    end
+
+    # Parses +text+ from +entry+ of the parse artifact +artifact+ on +mode+
+    # and returns the parse shape. Under +fallback+ (engine: :auto) input
+    # too deep for the native engine is rerun on Ruby. +source+ is the
+    # normalized caller input +text+ was preprocessed from; a ParseError
+    # reports its position in +source+ (and in +text+).
+    def run(artifact, entry, text, mode, fallback:, source:)
+      shape, ran = run_with_fallback(artifact, entry, text, mode, fallback: fallback)
+      record(ran)
+      shape
+    rescue ParseError => e
+      raise in_source(e, source)
+    end
+
+    # +error+ (positioned in the grammar input) as a ParseError of the same
+    # class positioned in +source+.
+    def in_source(error, source)
+      text = error.input
+      char = error.char_offset && SourceMap.caller_char(source, text, error.char_offset)
+      byte = char && source[0, char].bytesize
+      message = error.message
+      if error.byte_offset
+        where = char ? "at character #{char} (byte #{byte})" : "at an unmapped position"
+        message = message.sub("LaTeX input rejected at", "LaTeX input rejected #{where}; preprocessed text")
+      end
+      error.class.new(message, input: source, byte_offset: byte, char_offset: char,
+                               grammar_input: text, grammar_byte_offset: error.byte_offset,
+                               grammar_char_offset: error.char_offset)
+    end
+
+    def run_with_fallback(artifact, entry, text, mode, fallback:)
+      if mode == :native && too_deep_for_native_thread?(text)
+        record(:native)
+        unless fallback
+          raise native_too_deep(text, "nesting depth over #{Configuration::THREAD_NATIVE_DEPTH} off the main thread")
+        end
+
+        return [run_mode(artifact, entry, text, :ruby), :ruby]
+      end
+
+      [run_mode(artifact, entry, text, mode), mode]
+    rescue NativeRecursionLimit => e
+      record(:native)
+      raise native_too_deep(text, e.message) unless fallback
+
+      [run_mode(artifact, entry, text, :ruby), :ruby]
+    rescue NestingTooDeep
+      raise unless fallback && mode == :native
+
+      [run_mode(artifact, entry, text, :ruby), :ruby]
+    end
+
+    def too_deep_for_native_thread?(text)
+      return false if Thread.current.equal?(Thread.main)
+
+      depth = 0
+      max = 0
+      text.each_char do |char|
+        if OPENING.include?(char)
+          depth += 1
+          max = depth if depth > max
+        elsif CLOSING.include?(char)
+          depth -= 1 if depth.positive?
+        end
+      end
+      max > Configuration::THREAD_NATIVE_DEPTH
+    end
+
+    def native_too_deep(text, detail)
+      NestingTooDeep.new(
+        "LaTeX input nests deeper than the native engine's recursion limit " \
+        "(#{detail}); use engine: :ruby or engine: :auto",
+        input: text, byte_offset: nil, char_offset: nil,
+      )
+    end
+
+    def run_mode(artifact, entry, text, mode)
+      root = artifact.root_atom(entry)
+      reporter = nil
+      if mode == :native
+        NativeRecursionGuard.watch { Parsanol::Native.parse(root, text) }
+      else
+        reporter = Parsanol::ErrorReporter::Deepest.new
+        root.parse(text, reporter: reporter)
+      end
+    rescue Parsanol::ParseFailed => e
+      record(mode)
+      raise parse_error(e, reporter, text)
+    rescue SystemStackError
+      record(mode)
+      raise NestingTooDeep.new(
+        "LaTeX input nests too deeply for the #{mode} engine (Ruby stack exhausted)",
+        input: text, byte_offset: nil, char_offset: nil,
+      )
+    rescue Parsanol::PARG::Error, Parsanol::GrammarError, Parsanol::Native::UnsupportedGrammar => e
+      raise ArtifactError, "the grammar artifact cannot be used: #{e.class}: #{e.message}"
+    end
+
+    # The legacy grammar runs on the native engine, whose failure reports
+    # the furthest position any branch reached. The Ruby engine's raised
+    # cause is the root failure; its Deepest reporter holds the furthest
+    # leaf, which is the same position.
+    def parse_error(error, reporter, text)
+      cause = reporter&.deepest_cause || error.parse_failure_cause
+      byte = cause&.position
+      char = byte && text.byteslice(0, byte).force_encoding(Encoding::UTF_8).scrub.length
+      where = byte ? " at character #{char} (byte #{byte})" : ""
+      ParseError.new("LaTeX input rejected#{where}: #{error.message.lines.first.to_s.strip[0, 200]}",
+                     input: text, byte_offset: byte, char_offset: char)
+    end
+
+    def record(mode)
+      Thread.current[LAST_ENGINE_KEY] = mode
+    end
+
+    private_class_method :in_source, :run_with_fallback, :native_too_deep, :run_mode, :parse_error,
+                         :record, :too_deep_for_native_thread?
+  end
+end

data/lib/latexmath/parser/errors.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Latexmath
+  # Base class for every error Latexmath raises on purpose, except
+  # ArgumentError for invalid arguments.
+  class Error < StandardError; end
+
+  # The grammar refused the input.
+  #
+  # +input+ is the caller's input, as normalized UTF-8, and the offsets are
+  # zero-based positions in it: +byte_offset+ in UTF-8 bytes, +char_offset+
+  # in characters. They are the position the legacy Plurimath grammar
+  # reports, mapped back through the preprocessing (see SourceMap): an
+  # entity or other text the preprocessing wrote maps to the start of the
+  # caller text it stands for.
+  #
+  # +grammar_input+ is the text the grammar ran on (the preprocessed text);
+  # +grammar_byte_offset+ and +grammar_char_offset+ are the same position in
+  # it, as the legacy grammar reports it on either engine. They are for
+  # parity replay and diagnostics.
+  class ParseError < Error
+    attr_reader :input, :byte_offset, :char_offset,
+                :grammar_input, :grammar_byte_offset, :grammar_char_offset
+
+    def initialize(message, input:, byte_offset:, char_offset:,
+                   grammar_input: nil, grammar_byte_offset: nil, grammar_char_offset: nil)
+      super(message)
+      @input = input
+      @byte_offset = byte_offset
+      @char_offset = char_offset
+      @grammar_input = grammar_input
+      @grammar_byte_offset = grammar_byte_offset
+      @grammar_char_offset = grammar_char_offset
+    end
+  end
+
+  # The engine that ran cannot handle the input's nesting depth: the native
+  # engine's recursion limit, or an exhausted Ruby stack. +input+ and
+  # +grammar_input+ are set; every offset is nil. This does not claim that
+  # the input is invalid.
+  class NestingTooDeep < ParseError; end
+
+  # A parse or bindings artifact is missing, malformed, fails its checksum,
+  # declares the wrong target or kind, or does not pair with the other.
+  # Latexmath never falls back to another artifact.
+  class ArtifactError < Error; end
+
+  # engine: :native was requested but Parsanol's native extension is not
+  # available on this platform.
+  class EngineUnavailable < Error; end
+end

data/lib/latexmath/parser/native_recursion_guard.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Latexmath
+  # Raised inside Parsanol's native error path when the native engine hit
+  # its recursion limit, before Parsanol's interpreter reparse runs (that
+  # reparse can take minutes or exhaust the Ruby stack on deep input).
+  class NativeRecursionLimit < Error; end
+  private_constant :NativeRecursionLimit
+
+  # Parsanol::Native.raise_native_parse_error translates a native failure.
+  # While Latexmath's own native call runs on this thread, this hook turns a
+  # recursion-limit failure into NativeRecursionLimit; every other failure,
+  # and every other caller of Parsanol, is unaffected. (The same guard as
+  # amath's.)
+  module NativeRecursionGuard
+    MESSAGE = /\A(?:Maximum recursion depth \(\d+\) exceeded|Recursion limit exceeded\b)/
+    KEY = :latexmath_native_recursion_guard
+
+    def self.watch
+      outer = Thread.current[KEY]
+      Thread.current[KEY] = true
+      yield
+    ensure
+      Thread.current[KEY] = outer
+    end
+
+    def self.installed?
+      Parsanol::Native.singleton_class.ancestors.include?(self)
+    end
+
+    def raise_native_parse_error(error, grammar, input)
+      if Thread.current[KEY] && error.message.match?(MESSAGE)
+        raise NativeRecursionLimit, error.message.lines.first.strip, cause: nil
+      end
+
+      super
+    end
+  end
+
+  if Parsanol::Native.singleton_class.method_defined?(:raise_native_parse_error)
+    Parsanol::Native.singleton_class.prepend(NativeRecursionGuard)
+  end
+end

data/lib/latexmath/parser/preprocessor.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "htmlentities"
+
+module Latexmath
+  # The legacy LaTeX input preprocessing (Plurimath::Latex::Parser at the
+  # pinned commit, see docs/parser-contract.adoc), step for step:
+  #
+  # 1. remember every \mbox{...} and \text{...} group of the input;
+  # 2. decode HTML entities, then encode the whole text with hexadecimal
+  #    entities;
+  # 3. remove spaces that do not follow a backslash and runs of newlines;
+  #    turn "\\ " into "\\"; turn & and " back into & and ";
+  #    turn a backslash before an encoded newline into "\ " unless it ends
+  #    a row break; drop the remaining encoded newlines;
+  # 4. put the remembered groups back as the caller wrote them.
+  #
+  # Internal.
+  module Preprocessor
+    TEXT_OPEN = /\\(?:mbox|text)\{/
+    TEXT_OPEN_LENGTH = "\\mbox{".length
+    ENTITIES = HTMLEntities.new
+
+    module_function
+
+    # The grammar input for the normalized UTF-8 caller +input+.
+    def call(input)
+      text_functions = text_groups(input).map { |range| input[range] }
+      text = ENTITIES.encode(ENTITIES.decode(input), :hexadecimal)
+      text = gsub_space_and_unicodes(text)
+      replace_text_groups(text) { text_functions.shift }
+    end
+
+    # Legacy: the matches of %r(\\(?:mbox|text)\{[^}]+\}), as index ranges.
+    # Each match runs from an opener to the first "}" after it, with at
+    # least one character between; an opener followed directly by "}" does
+    # not match there and the search goes on one character later. Written
+    # out without the backtracking regexp, which is polynomial on input
+    # with many unclosed openers.
+    def text_groups(text)
+      groups = []
+      pos = 0
+      while (start = text.index(TEXT_OPEN, pos))
+        body = start + TEXT_OPEN_LENGTH
+        close = text.index("}", body) or break
+        if close == body
+          pos = start + 1
+        else
+          groups << (start..close)
+          pos = close + 1
+        end
+      end
+      groups
+    end
+
+    # Legacy: text.gsub(TEXT_REGEX) { ... }, keeping its result encoding.
+    def replace_text_groups(text)
+      out = String.new(encoding: text.encoding)
+      pos = 0
+      text_groups(text).each do |range|
+        out << text[pos...range.begin] << yield.to_s
+        pos = range.end + 1
+      end
+      out << text[pos..]
+    end
+
+    def gsub_space_and_unicodes(text)
+      text
+        .gsub(/((?<!\\) )|\n+/, "")
+        .gsub("\\\\ ", "\\\\\\\\")
+        .gsub("&#x26;", "&")
+        .gsub("&#x22;", "\"")
+        .gsub(/(?<!\\\\)\\&#xa;/, "\\ ")
+        .gsub("&#xa;", "")
+    end
+  end
+end

data/lib/latexmath/parser/raw_parser.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Latexmath
+  # The internal raw-tree boundary: the legacy preprocessing, then the
+  # grammar, then the declared syntax bindings, which carry the legacy tree.
+  #
+  # Internal and for tests and parity replay only; not public API. The
+  # public Latexmath.parse, which returns the typed LaTeX model, is built on
+  # top of it.
+  module RawParser
+    # +input+: the normalized UTF-8 caller input; +text+: the grammar input
+    # (Preprocessor.call); +tree+: the legacy raw tree (Hashes, Arrays and
+    # Parsanol::Slice leaves, with the row-break key restored), which is
+    # Unwrap of +raw+; +raw+: the grammar's own tree, with the wrappers its
+    # shape normalization adds (grammars/latex.wrappers.json); +model+: the
+    # same tree as the model binding gives it, the attributes the typed
+    # model is built from; +engine+: the engine that produced it.
+    Result = Struct.new(:input, :text, :raw, :model, :engine, keyword_init: true) do
+      def tree
+        @tree ||= RawParser.restore_capture_names(Unwrap.call(raw))
+      end
+    end
+
+    module_function
+
+    def parse(input, decimal: ".", engine: :auto)
+      entry = entry_for(decimal)
+      mode = Engine.resolve(engine)
+      normalized = normalize_input(input)
+      text = Preprocessor.call(normalized)
+
+      loaded = Latexmath.artifacts
+      shape = Engine.run(loaded.parse, entry, grammar_input(text), mode,
+                         fallback: engine == :auto, source: normalized)
+      bound = loaded.bindings.apply(entry, shape)
+      raw, model = [Configuration::TREE_CAPTURE, Configuration::MODEL_CAPTURE].map do |name|
+        bound.fetch(name) { raise ArtifactError, "the declared bindings did not produce #{name}" }
+      end
+      Result.new(input: normalized, text: text, raw: raw, model: model, engine: Engine.last_engine)
+    end
+
+    def entry_for(decimal)
+      Configuration::DECIMAL_ENTRIES.fetch(decimal) do
+        raise ArgumentError,
+              "unsupported decimal marker #{decimal.inspect} " \
+              "(supported: #{Configuration::DECIMAL_ENTRIES.keys.map(&:inspect).join(', ')})"
+      end
+    end
+
+    # The preprocessed text as UTF-8. Preprocessing returns US-ASCII text
+    # when nothing outside \text and \mbox groups is non-ASCII (as
+    # the legacy preprocessing does); the native engine gives UTF-8 slices
+    # either way, and the Ruby engine's slices take the input's encoding.
+    def grammar_input(text)
+      text.encoding == Encoding::UTF_8 ? text : text.dup.force_encoding(Encoding::UTF_8)
+    end
+
+    # +input+ as valid UTF-8, or an ArgumentError.
+    def normalize_input(input)
+      raise ArgumentError, "input must be a String, got #{input.class}" unless input.is_a?(String)
+
+      text = input.encoding == Encoding::UTF_8 ? input : input.encode(Encoding::UTF_8)
+      raise ArgumentError, "input is not valid UTF-8" unless text.valid_encoding?
+
+      text
+    rescue EncodingError => e
+      raise ArgumentError, "input cannot be converted to UTF-8: #{e.message}"
+    end
+
+    # Renames the captures the grammar had to spell differently
+    # (Configuration::CAPTURE_RENAMES) back to their legacy keys, keeping
+    # key order.
+    def restore_capture_names(tree)
+      case tree
+      when Hash
+        tree.to_h do |key, value|
+          [Configuration::CAPTURE_RENAMES.fetch(key, key), restore_capture_names(value)]
+        end
+      when Array then tree.map { |value| restore_capture_names(value) }
+      else tree
+      end
+    end
+  end
+end