latexmath 0.1.4 → 1.0.0

data/lib/latexmath/parser/source_map.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "htmlentities"
+
+module Latexmath
+  # Maps positions in the preprocessed text back to the caller's text.
+  #
+  # It runs the steps of Preprocessor.call again, each as a scan that
+  # records, for every character it writes, the caller character it came
+  # from: a character that passes through keeps its origin; the characters
+  # of a replacement (a decoded or encoded entity, "\\ " for an encoded
+  # newline, a restored \text group) start at the origin of the text they
+  # replace, and a restored group's characters map one to one onto the
+  # caller's group. Removed text leaves no characters behind, so a position
+  # after it maps to the caller character that follows it.
+  #
+  # Only used to report refusals; the grammar input itself always comes
+  # from Preprocessor.call. ParseError positions are mapped only when this
+  # scan writes exactly the bytes Preprocessor.call wrote.
+  #
+  # Internal.
+  module SourceMap
+    FLAVOR = "xhtml1" # HTMLEntities.new's default, as Preprocessor::ENTITIES
+    DECODE = HTMLEntities::Decoder.new(FLAVOR).instance_variable_get(:@entity_regexp)
+    NAMES = HTMLEntities::MAPPINGS.fetch(FLAVOR)
+    # HTMLEntities' :hexadecimal encoding: the basic characters and every
+    # character outside U+0020..U+007E.
+    ENCODE = /[<>'"&]|[^\u{20}-\u{7E}]/
+
+    # Preprocessor.gsub_space_and_unicodes, as [pattern, literal replacement].
+    STEPS = [
+      [/((?<!\\) )|\n+/, ""],
+      [/\\\\ /, "\\\\"],
+      [/&#x26;/, "&"],
+      [/&#x22;/, "\""],
+      [/(?<!\\\\)\\&#xa;/, "\\ "],
+      [/&#xa;/, ""],
+    ].freeze
+
+    module_function
+
+    # [preprocessed text, origins] for the normalized UTF-8 caller +input+:
+    # origins[i] is the caller character offset of preprocessed character i,
+    # and origins[text.length] is input.length.
+    def call(input)
+      originals = Preprocessor.text_groups(input).map { |range| [input[range], range.begin] }
+      text = input
+      map = (0..input.length).to_a
+      text, map = scan(text, map, DECODE) { |m| decode(m) }
+      text, map = scan(text, map, ENCODE) { |m| encode(m[0]) }
+      STEPS.each { |pattern, replacement| text, map = scan(text, map, pattern) { replacement } }
+      restore_groups(text, map, originals)
+    end
+
+    # The caller character offset of character +char+ of +text+ (the
+    # preprocessed text of +input+), or nil when the mapping does not
+    # reproduce +text+.
+    def caller_char(input, text, char)
+      mapped, map = call(input)
+      return nil unless mapped.b == text.b && char && char <= mapped.length
+
+      map[char]
+    end
+
+    def scan(text, map, pattern)
+      out = +""
+      origins = []
+      pos = 0
+      while (m = pattern.match(text, pos))
+        start = m.begin(0)
+        out << text[pos...start]
+        origins.concat(map[pos...start])
+        replacement = yield(m)
+        out << replacement
+        origins.concat([map[start]] * replacement.length)
+        pos = m.end(0)
+      end
+      out << text[pos..]
+      origins.concat(map[pos..])
+      [out, origins]
+    end
+
+    # HTMLEntities::Decoder#decode for one match.
+    def decode(match)
+      if match[1] && (codepoint = NAMES[match[1]])
+        codepoint.chr(Encoding::UTF_8)
+      elsif match[2]
+        match[2].to_i(10).chr(Encoding::UTF_8)
+      elsif match[3]
+        match[3].to_i(16).chr(Encoding::UTF_8)
+      else
+        match[0]
+      end
+    end
+
+    def encode(char)
+      "&#x#{char.codepoints.first.to_s(16)};"
+    end
+
+    # Preprocessor.replace_text_groups: the n-th group found in +text+ gives
+    # way to the n-th group of the caller's input, character for character.
+    def restore_groups(text, map, originals)
+      out = +""
+      origins = []
+      pos = 0
+      Preprocessor.text_groups(text).each do |range|
+        out << text[pos...range.begin]
+        origins.concat(map[pos...range.begin])
+        group, start = originals.shift
+        if group
+          out << group
+          origins.concat((start...(start + group.length)).to_a)
+        end
+        pos = range.end + 1
+      end
+      out << text[pos..]
+      origins.concat(map[pos..])
+      [out, origins]
+    end
+  end
+end

data/lib/latexmath/parser/unwrap.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Latexmath
+  # The inverse of the grammar's shape normalization (option F), for the
+  # raw-tree boundary and parity replay only.
+  #
+  # The generator only ADDS captures, listed in grammars/latex.wrappers.json
+  # as {name => kind}. Unwrap replaces each wrapper by the value legacy
+  # produced there and re-runs Parslet's own flattening algebra
+  # (Parsanol::Atoms::CanFlatten#merge_fold / #flatten_repetition) over the
+  # parts, bottom-up:
+  #
+  #   hash   entries are split, in order, into runs of ordinary keys (one hash
+  #          part each) and wrapper keys (one part each, by kind):
+  #            text       -> the wrapped value (nil / [] -> "", legacy unnamed)
+  #            empty      -> "" (String)
+  #            list       -> the list ([] -> "")
+  #            nil        -> nil
+  #            text_list  -> the text; "" (or []) -> [] (legacy named, empty)
+  #            items      -> the list, unchanged
+  #          then folded with merge_fold, as legacy folded the sequence.
+  #          A hash of nil-kind wrappers only is nil.
+  #   array  elements are unwrapped, then flatten_repetition (named).
+  module Unwrap
+    extend Parsanol::Atoms::CanFlatten
+
+    PATH = File.expand_path("../../../grammars/latex.wrappers.json", __dir__)
+
+    @wrappers = nil
+    @lock = Mutex.new
+
+    module_function
+
+    # The wrapper table of the bundled grammar.
+    def wrappers
+      @lock.synchronize { @wrappers ||= load(PATH).freeze }
+    end
+
+    def load(path)
+      JSON.parse(File.read(path, encoding: "UTF-8")).fetch("wrappers")
+    end
+
+    def call(value, wrappers = self.wrappers)
+      case value
+      when Hash then unwrap_hash(value, wrappers)
+      when Array then Unwrap.send(:flatten_repetition, value.map { |v| call(v, wrappers) }, true)
+      else value
+      end
+    end
+
+    def unwrap_hash(value, wrappers)
+      return value.to_h { |k, v| [k, call(v, wrappers)] } if value.keys.none? { |k| wrappers.key?(k.to_s) }
+
+      parts = []
+      run = nil
+      only_nil = true
+      value.each do |k, v|
+        kind = wrappers[k.to_s]
+        if kind
+          parts << run if run
+          run = nil
+          only_nil &&= kind == "nil"
+          parts << part(kind, call(v, wrappers))
+        else
+          only_nil = false
+          (run ||= {})[k] = call(v, wrappers)
+        end
+      end
+      parts << run if run
+      return nil if only_nil
+
+      Unwrap.send(:foldl, parts) { |l, r| Unwrap.send(:merge_fold, l, r) }
+    end
+
+    def part(kind, v)
+      case kind
+      when "text" then v.nil? || v == [] ? +"" : v
+      when "empty" then +""
+      when "list" then v == [] ? +"" : v
+      when "nil" then nil
+      when "text_list" then v == [] || v == "" ? [] : v
+      when "items" then v
+      else raise ArgumentError, "unknown wrapper kind #{kind}"
+      end
+    end
+
+    private_class_method :unwrap_hash, :part
+  end
+end

data/lib/latexmath/parser.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "parsanol"
+require_relative "version"
+
+# The LaTeX stage parser, backed by a compiled Parsanol (PARG) grammar
+# artifact generated from the pinned legacy Plurimath grammar.
+#
+# Latexmath.parse(input, decimal:, engine:) returns the typed LaTeX model, a
+# Latexmath::Model::Document. Underneath are the grammar artifact, its
+# declared syntax bindings and the internal raw-tree boundary
+# (Latexmath::RawParser) that parity replay compares with the legacy
+# Plurimath parser. See docs/parser-contract.adoc and
+# docs/model-contract.adoc.
+#
+# lib/latexmath.rb requires this file.
+module Latexmath
+  autoload :Artifact, "latexmath/parser/artifact"
+  autoload :Configuration, "latexmath/parser/configuration"
+  autoload :Engine, "latexmath/parser/engine"
+  autoload :Model, "latexmath/model"
+  autoload :Preprocessor, "latexmath/parser/preprocessor"
+  autoload :RawParser, "latexmath/parser/raw_parser"
+  autoload :SourceMap, "latexmath/parser/source_map"
+  autoload :Symbols, "latexmath/symbols"
+  autoload :Unwrap, "latexmath/parser/unwrap"
+
+  @artifacts = nil
+  @artifacts_lock = Mutex.new
+
+  class << self
+    # Parses LaTeX math +input+ (a String, valid UTF-8 or convertible to it)
+    # into the typed model: a Model::Document whose +expression+ is the
+    # formula's tree and +decimal+ the decimal marker it was parsed with
+    # ("." , "," or "\u066B"). Input with no formula (empty, or only
+    # spaces and newlines, which the preprocessing removes) gives a
+    # Document whose expression is nil.
+    #
+    # +engine+: :auto (native when available, else Ruby), :native or :ruby.
+    # Raises ParseError (offsets into +input+), NestingTooDeep,
+    # ArgumentError, ArtifactError or EngineUnavailable.
+    def parse(input, decimal: ".", engine: :auto)
+      text = RawParser.normalize_input(input)
+      RawParser.entry_for(decimal)
+      return Model::Document.new(decimal: decimal) if Preprocessor.call(text).empty?
+
+      raw = RawParser.parse(text, decimal: decimal, engine: engine)
+      build(raw, decimal, text)
+    end
+
+    # The verified, paired parse and Ruby bindings artifacts (internal).
+    def artifacts
+      artifact_store.fetch
+    end
+
+    # Points Latexmath at other artifacts (for tests). The next parse loads
+    # and verifies them.
+    def use_artifacts(path, bindings_path)
+      artifact_store.use(path, bindings_path)
+    end
+
+    def artifact_paths
+      [artifact_store.path, artifact_store.bindings_path]
+    end
+
+    # The engine (:native or :ruby) of the last parser run on this thread,
+    # or nil before the first one.
+    def last_engine
+      Engine.last_engine
+    end
+
+    def native_available?
+      Engine.native_available?
+    end
+
+    private
+
+    def build(raw, decimal, text)
+      Model::Document.new(expression: raw.model, decimal: decimal)
+    rescue SystemStackError
+      raise NestingTooDeep.new("LaTeX input nests too deeply to build its model (Ruby stack exhausted)",
+                               input: text, byte_offset: nil, char_offset: nil, grammar_input: raw.text)
+    end
+
+    def artifact_store
+      @artifacts_lock.synchronize do
+        @artifacts ||= Artifact.new(Configuration::DEFAULT_ARTIFACT_PATH,
+                                    Configuration::DEFAULT_BINDINGS_PATH)
+      end
+    end
+  end
+end
+
+# Installs a hook into Parsanol at load time, so these are required, not
+# autoloaded.
+require_relative "parser/errors"
+require_relative "parser/native_recursion_guard"

data/lib/latexmath/symbols.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Latexmath
+  # The legacy LaTeX symbol table, exported from the pinned Plurimath by
+  # scripts/export_symbols.rb into data/latex-symbols.json: for every
+  # spelling a symbol capture can have, its Unicode value, the MathML
+  # element the legacy writer uses and its kind. Loading it needs no
+  # Plurimath.
+  #
+  # The legacy transform looks a captured symbol up by its text, stripped;
+  # text the table lacks stays a plain symbol of that text (#resolve).
+  module Symbols
+    DATA_PATH = File.expand_path("../../data/latex-symbols.json", __dir__)
+    FORMAT = "latexmath-symbols/v1"
+
+    # +spelling+: the captured text ("alpha", "α", "("); +value+: its
+    # Unicode text; +mathml+: "mi" or "mo", nil for text the table lacks;
+    # +kind+: "symbol", "open", "close" or "paren"; +plurimath_class+: the
+    # legacy class relative to Plurimath::Math::Symbols.
+    Entry = Struct.new(:spelling, :value, :mathml, :kind, :plurimath_class, keyword_init: true)
+
+    @table = nil
+    @lock = Mutex.new
+
+    module_function
+
+    # The Entry for the captured +text+, or nil when the table lacks it.
+    def lookup(text)
+      table[:entries][text.to_s.strip]
+    end
+
+    # The Entry for the captured +text+; text the table lacks becomes a
+    # plain symbol of its own (stripped) text, as in the legacy transform.
+    def resolve(text)
+      key = text.to_s.strip
+      table[:entries][key] ||
+        Entry.new(spelling: key, value: key, mathml: nil, kind: "symbol", plurimath_class: "Symbol").freeze
+    end
+
+    def size
+      table[:entries].size
+    end
+
+    # Where the table comes from: the oracle repository, commit, lib tree and
+    # Ruby of the export.
+    def provenance
+      table[:oracle]
+    end
+
+    def table
+      @lock.synchronize { @table ||= load_table(DATA_PATH) }
+    end
+
+    def load_table(path)
+      data = JSON.parse(File.read(path, encoding: "UTF-8"))
+      raise ArtifactError, "#{path} is #{data['format'].inspect}, not #{FORMAT}" unless data["format"] == FORMAT
+
+      entries = data.fetch("symbols").to_h do |r|
+        entry = Entry.new(spelling: r.fetch("spelling"), value: r.fetch("value"), mathml: r.fetch("mathml"),
+                          kind: r.fetch("kind"), plurimath_class: r.fetch("class")).freeze
+        [entry.spelling, entry]
+      end
+      raise ArtifactError, "#{path} repeats a spelling" unless entries.size == data["symbols"].size
+
+      { oracle: data.fetch("oracle").freeze, entries: entries.freeze }.freeze
+    rescue JSON::ParserError, KeyError, SystemCallError => e
+      raise ArtifactError, "symbol table #{path} cannot be loaded: #{e.class}: #{e.message}"
+    end
+
+    private_class_method :table, :load_table
+  end
+end

data/lib/latexmath/version.rb

@@ -1,3 +1,3 @@
 module Latexmath
-  VERSION = '0.1.4'.freeze
+  VERSION = '1.0.0'.freeze
 end

data/lib/latexmath.rb

@@ -1,88 +1,6 @@
-#require 'byebug' unless RUBY_ENGINE == 'opal'
-require 'json'
-require 'htmlentities'
-require 'ox'
-require_relative 'latexmath/ext'
-require_relative 'latexmath/version'
-require_relative 'latexmath/constants/symbols'
-require_relative 'latexmath/aggregator'
-require_relative 'latexmath/converter'
-require_relative 'latexmath/symbol'
-require_relative 'latexmath/tokenizer'
-require_relative 'latexmath/xml/element'
-require_relative 'latexmath/equation'
+# frozen_string_literal: true
 
-module Latexmath
-  MATRICES = [
-    '\\matrix',
-    '\\matrix*',
-    '\\pmatrix',
-    '\\pmatrix*',
-    '\\bmatrix',
-    '\\bmatrix*',
-    '\\Bmatrix',
-    '\\Bmatrix*',
-    '\\vmatrix',
-    '\\vmatrix*',
-    '\\Vmatrix',
-    '\\Vmatrix*',
-    '\\array',
-    '\\split',
-    '\\substack'
-  ].freeze
-
-  SPACES = ['\\,', '\\:', '\\;', '\\\\'].freeze
-  STYLES = {
-    '\\bf' => 'mathbf'
-  }.freeze
-
-  LIMITS = ['\\lim', '\\sup', '\\inf', '\\max', '\\min'].freeze
-  COMMANDS = {
-    # command: [params_count, mathml_equivalent, attributes]
-    '_' => [2, 'msub', {}],
-    '^' => [2, 'msup', {}],
-    '_^' => [3, 'msubsup', {}],
-    '\\frac' => [2, 'mfrac', {}],
-    '\\sqrt' => [1, 'msqrt', {}],
-    '\\root' => [2, 'mroot', {}],
-    '\\binom' => [2, 'mfrac', { "linethickness": '0' }],
-    '\\left' => [
-      1,
-      'mo',
-      [%w[stretchy true], %w[fence true], %w[form prefix]]
-    ],
-    '\\right' => [
-      1,
-      'mo',
-      [%w[stretchy true], %w[fence true], %w[form postfix]]
-    ],
-    '\\overline' => [1, 'mover', {}],
-    '\\bar' => [1, 'mover', {}],
-    '\\underline' => [1, 'munder', {}],
-    '\\limits' => [3, 'munderover', {}],
-    '\\overrightarrow' => [1, 'mover', {}]
-  }
-
-  COMMANDS['\\quad'] = [0, 'mo', { "mathvariant": 'italic', separator: 'true' }]
-  COMMANDS['\\qquad'] = [0, 'mo', { "mathvariant": 'italic', separator: 'true' }]
-  SPACES.each do |space|
-    COMMANDS[space] = [0, 'mspace', { "width": '0.167em' }]
-  end
-
-  MATRICES.each do |matrix|
-    COMMANDS[matrix] = [1, 'mtable', {}]
-  end
-
-  LIMITS.each do |limit|
-    COMMANDS[limit] = [1, 'munder', {}]
-  end
-
-  def self.parse(string)
-    lxm_input = HTMLEntities.new.decode(string)
-
-    # parse
-    Equation.new(lxm_input)
-  end
-
-  class Error < StandardError; end
-end
+# LaTeX math parser for Plurimath. Latexmath.parse returns the typed LaTeX
+# model; see lib/latexmath/parser.rb, docs/parser-contract.adoc and
+# docs/model-contract.adoc.
+require_relative "latexmath/parser"

data/model/latex.lml.checksums.txt

@@ -0,0 +1,36 @@
+# Written by scripts/verify_lml.rb: the LML checksum of every model
+# document, then the sha256 of the documents file.
+Latexmath::Model::Args sha256:5462960711eec1726badec07cd0a0697463c8fc35df4769e10bd23323bfabaa6
+Latexmath::Model::Base sha256:3ce2dcae82ec2dc51287195a3cee13977f9325889a8523da2643f56c9d3ef1f4
+Latexmath::Model::Binary sha256:10ab6af22512dc6e3b794d02d715b4b45e23c7a448766233d09e12cc96e5fb1f
+Latexmath::Model::BinaryFunctions sha256:91faa02cd9931ba27287f393594d7b1312d8324f99fe7f07a64e3bbc15b2ef17
+Latexmath::Model::Dividend sha256:b3e9b1d3d03d9640d83b3268ced1204f7d8ccde89453115823df34bf43aa0dd8
+Latexmath::Model::Divisor sha256:8830d0ca59e31ba6645cde667705156f73d02fdc60d87745f8d8beac81725f0a
+Latexmath::Model::Document sha256:c770d3c85caedc6dd289102e426cc8cf5cc7327e54f021c36ec9d277c5fbe77a
+Latexmath::Model::Ending sha256:4e6b05b7cc3e0cf383082b5b91eab692d4fcbe7aa9f3709b015b80c8eedaec8b
+Latexmath::Model::Environment sha256:4ca449f28a047f5c233571f18c73a53ad40fc283aedb7ab7876a25866f3fce10
+Latexmath::Model::Expression sha256:e5cb9286cf37e399d9fe11058e517d5dd4a11e3076fb7d08141ca4689252ec07
+Latexmath::Model::FirstValue sha256:f23a8ce39e0e1a4800e8f46ccef2cc39b1f0f50333aab254c41cee80ad423638
+Latexmath::Model::IntermediateExp sha256:124f5a4368fc210f21361fbc54629c6da4c44c4e9185c354c78a4aac97b3cee7
+Latexmath::Model::ItemsItem sha256:28625103e5527d530dec8971463a4186aaa1fa31a4960be1e44103e1199cf601
+Latexmath::Model::LatexTree sha256:ab699f312f98a337d0443fb0e46e901ec3a5f02d3b1dcc1a1e41edf2db8a8c96
+Latexmath::Model::LeftParen sha256:9948524e67fa426033187ebc1c9e015e2c5e3a4ace59b81ca3d223de9e849484
+Latexmath::Model::LeftRight sha256:d219a07366ebf63b556957d75d0a2ef9a7b0b4d69db9d5a47cef3ba198673072
+Latexmath::Model::Limits sha256:04d4af34438ccab20272798c0d23877dd1a14cc95c01290ffc37ba156d1a8a0c
+Latexmath::Model::MathSymbol sha256:655eefa4badd24c9960f2ba8728f4a1981b808c9fbe076bfdc7875b8703cea3a
+Latexmath::Model::Options sha256:09363eec7dc62f50a06ea6a1065280fbeaef1385e2e84f1da92f7df4e9a01e6c
+Latexmath::Model::Over sha256:5ec5c3ccd41a5c65149887cbed15632ac8dd2a05bd66ba337cde4b6cb1cee7c1
+Latexmath::Model::Power sha256:6bf73b69d65980865f2c1d8922e0a2ab6c43c76d96f9135b94f9ffb2936da81a
+Latexmath::Model::PowerBase sha256:31dae32f1de56ab2203d7b649c760a1e25ab170b6847dbec3286d5c9cd24ecae
+Latexmath::Model::RightParen sha256:b6f7510f3a8bf4cf8b61e15074e483f921c1ec51c512a333a7eb68cc53e24569
+Latexmath::Model::SecondValue sha256:94bef28cac697a6d37d1e84bf8441202effabfb540ae017790faba5c83c59a3c
+Latexmath::Model::Sequence sha256:3e7caee1c9af131b51beb87a8d666b03961c0885bba79f9db865c2c5eb9cca61
+Latexmath::Model::Subscript sha256:ad5250291647cc6ec2caaecf3571a2a0244d77e2a3320b2513f00050a747c08b
+Latexmath::Model::Supscript sha256:2143627f3829bcf53abfdebbdbab2824de4ddfcbdd77530075d0edbcfa07d48f
+Latexmath::Model::SymbolNode sha256:e01f443c419fdcca1b2c3c70e21532d9aea1be13a4304558e01bb08644ae5fa1
+Latexmath::Model::TableData sha256:b2e2b1c329b4017897fde5d61958487c3e88da33581c414480e15d8746fb7572
+Latexmath::Model::TernaryClass sha256:bd741ce478aa270bd90cd24c9e2e8a073afc750e9796113a79e79284c5ae4f8b
+Latexmath::Model::ThirdValue sha256:2ce323686afb2f7ad053ac49a62eb1dc092cbddd126f94f5ac2bc39cf043c4ac
+Latexmath::Model::UnaryFunctions sha256:df5b690a991e6963faecbfdd2bebb06deb319bd0485d20c48379a3d844a67326
+Latexmath::Model::UnderOver sha256:a6e7aa8d2ae55610b8473eed4fa39481c75bbfc09e1a44f4d14fd2df63ce8713
+model/latex.lml.json sha256:312b1de981c4aa6435fba996c4fc34d34f401e36742a6cdc1c4e9f1cf1f20d60