umath 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +27 -0
- data/LICENSE.md +33 -0
- data/README.adoc +155 -0
- data/artifacts/umath.bindings.ruby.json +1 -0
- data/artifacts/umath.json +1 -0
- data/data/umath-symbols.json +13145 -0
- data/docs/model-contract.adoc +184 -0
- data/docs/parser-contract.adoc +341 -0
- data/docs/testsuite-pin.adoc +95 -0
- data/lib/umath/artifact.rb +84 -0
- data/lib/umath/configuration.rb +32 -0
- data/lib/umath/engine.rb +163 -0
- data/lib/umath/errors.rb +38 -0
- data/lib/umath/model/nodes.rb +737 -0
- data/lib/umath/model.rb +30 -0
- data/lib/umath/native_recursion_guard.rb +43 -0
- data/lib/umath/parser.rb +50 -0
- data/lib/umath/preprocessor.rb +91 -0
- data/lib/umath/raw_parser.rb +59 -0
- data/lib/umath/source_map.rb +146 -0
- data/lib/umath/symbols.rb +71 -0
- data/lib/umath/version.rb +5 -0
- data/lib/umath.rb +73 -0
- metadata +110 -0
data/lib/umath/model.rb
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "lutaml/model"
|
|
4
|
+
|
|
5
|
+
module Umath
|
|
6
|
+
# The typed UnicodeMath language model, declared with LutaML Model.
|
|
7
|
+
#
|
|
8
|
+
# Umath.parse returns a Model::Document. Its tree is built by
|
|
9
|
+
# lutaml-model from the declared attribute types (Document.new on the
|
|
10
|
+
# grammar's bound tree, see Parser); no code here walks the parse tree.
|
|
11
|
+
# The node classes in model/nodes.rb are generated from the normalized
|
|
12
|
+
# grammar's capture shapes (scripts/generate_model.rb), one class per
|
|
13
|
+
# capture; see docs/model-contract.adoc.
|
|
14
|
+
module Model
|
|
15
|
+
# The root: one parsed UnicodeMath formula.
|
|
16
|
+
#
|
|
17
|
+
# +expression+ is the formula's tree, nil for input without a formula
|
|
18
|
+
# (empty, or only "#" label separators). +label+ is the equation label,
|
|
19
|
+
# the text after the last "#" (nil when the input has none; the legacy
|
|
20
|
+
# facade rule). +decimal+ is the decimal marker the input was parsed
|
|
21
|
+
# with; numbers keep their digits and marker as written.
|
|
22
|
+
class Document < Lutaml::Model::Serializable
|
|
23
|
+
attribute :expression, "Umath::Model::Expression"
|
|
24
|
+
attribute :label, :string
|
|
25
|
+
attribute :decimal, :string
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
require_relative "model/nodes"
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Umath
|
|
4
|
+
# Raised inside Parsanol's native error path when the native engine hit
|
|
5
|
+
# its recursion limit, before Parsanol's interpreter reparse runs (that
|
|
6
|
+
# reparse can take minutes or exhaust the Ruby stack on deep input).
|
|
7
|
+
class NativeRecursionLimit < Error; end
|
|
8
|
+
private_constant :NativeRecursionLimit
|
|
9
|
+
|
|
10
|
+
# Parsanol::Native.raise_native_parse_error translates a native failure.
|
|
11
|
+
# While Umath's own native call runs on this thread, this hook turns
|
|
12
|
+
# a recursion-limit failure into NativeRecursionLimit; every other failure,
|
|
13
|
+
# and every other caller of Parsanol, is unaffected. (The same guard as
|
|
14
|
+
# amath's.)
|
|
15
|
+
module NativeRecursionGuard
|
|
16
|
+
MESSAGE = /\A(?:Maximum recursion depth \(\d+\) exceeded|Recursion limit exceeded\b)/
|
|
17
|
+
KEY = :umath_native_recursion_guard
|
|
18
|
+
|
|
19
|
+
def self.watch
|
|
20
|
+
outer = Thread.current[KEY]
|
|
21
|
+
Thread.current[KEY] = true
|
|
22
|
+
yield
|
|
23
|
+
ensure
|
|
24
|
+
Thread.current[KEY] = outer
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def self.installed?
|
|
28
|
+
Parsanol::Native.singleton_class.ancestors.include?(self)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def raise_native_parse_error(error, grammar, input)
|
|
32
|
+
if Thread.current[KEY] && error.message.match?(MESSAGE)
|
|
33
|
+
raise NativeRecursionLimit, error.message.lines.first.strip, cause: nil
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
super
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
if Parsanol::Native.singleton_class.method_defined?(:raise_native_parse_error)
|
|
41
|
+
Parsanol::Native.singleton_class.prepend(NativeRecursionGuard)
|
|
42
|
+
end
|
|
43
|
+
end
|
data/lib/umath/parser.rb
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Umath
|
|
4
|
+
# The public parser behind Umath.parse: preprocessing, grammar and
|
|
5
|
+
# bindings (RawParser), typed model construction by lutaml-model from the
|
|
6
|
+
# declared attribute types, then symbol resolution (Symbols.resolve).
|
|
7
|
+
module Parser
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
def parse(input, decimal: ".", engine: :ruby)
|
|
11
|
+
text = RawParser.normalize_input(input)
|
|
12
|
+
raw = begin
|
|
13
|
+
RawParser.parse(text, decimal: decimal, engine: engine)
|
|
14
|
+
rescue NestingTooDeep => e
|
|
15
|
+
raise NestingTooDeep.new(e.message, input: text, byte_offset: nil, char_offset: nil)
|
|
16
|
+
rescue ParseError => e
|
|
17
|
+
raise relocate(e, text)
|
|
18
|
+
end
|
|
19
|
+
# Input without a formula part (empty, or only "#" label separators):
|
|
20
|
+
# an empty Document. The legacy facade fails there with NoMethodError,
|
|
21
|
+
# so there is no legacy result to keep; amath returns an empty result
|
|
22
|
+
# for empty input too.
|
|
23
|
+
return Model::Document.new(label: raw.label, decimal: decimal) if raw.text.nil?
|
|
24
|
+
|
|
25
|
+
build(raw, decimal, text)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def build(raw, decimal, text)
|
|
29
|
+
document = Model::Document.new(expression: raw.model_data, label: raw.label, decimal: decimal)
|
|
30
|
+
Symbols.resolve(document)
|
|
31
|
+
rescue SystemStackError
|
|
32
|
+
raise NestingTooDeep.new("UnicodeMath input nests too deeply to build its model (Ruby stack exhausted)",
|
|
33
|
+
input: text, byte_offset: nil, char_offset: nil)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# The grammar's error, with its offsets moved from the grammar input to
|
|
37
|
+
# the caller's text.
|
|
38
|
+
def relocate(error, text)
|
|
39
|
+
char = byte = nil
|
|
40
|
+
if error.char_offset
|
|
41
|
+
char, byte = SourceMap.build(text).caller_position(error.char_offset)
|
|
42
|
+
end
|
|
43
|
+
where = char ? " at character #{char} (byte #{byte})" : ""
|
|
44
|
+
detail = error.message.sub(/\AUnicodeMath input rejected( at character \d+ \(byte \d+\))?: /, "")
|
|
45
|
+
ParseError.new("UnicodeMath input rejected#{where}: #{detail}", input: text, byte_offset: byte, char_offset: char)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private_class_method :build, :relocate
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Umath
|
|
4
|
+
# The legacy UnicodeMath preprocessing (Plurimath::UnicodeMath::Parser at
|
|
5
|
+
# the pinned oracle commit), step for step. The grammar runs on its
|
|
6
|
+
# result, never on the caller's text. See docs/parser-contract.adoc,
|
|
7
|
+
# "Preprocessing".
|
|
8
|
+
#
|
|
9
|
+
# Internal: the public parser owns this step.
|
|
10
|
+
module Preprocessor
|
|
11
|
+
# +text+ is the grammar input, or nil when the input has no formula part
|
|
12
|
+
# (an empty input, or one made only of label separators; the legacy code
|
|
13
|
+
# raises NoMethodError there). +label+ is the text after the last "#",
|
|
14
|
+
# set only when +labeled+.
|
|
15
|
+
Result = Struct.new(:text, :label, :labeled, keyword_init: true)
|
|
16
|
+
|
|
17
|
+
PEN_OPEN = "✎("
|
|
18
|
+
PROTECTED_HASH = "\"replacement\""
|
|
19
|
+
# HTMLEntities (xhtml1 flavour) with the :hexadecimal instruction: the
|
|
20
|
+
# basic characters first, then everything outside printable ASCII.
|
|
21
|
+
BASIC = /[<>'"&]/
|
|
22
|
+
EXTENDED = /[^\u{20}-\u{7E}]/
|
|
23
|
+
COMMENT_OPEN = "⫷"
|
|
24
|
+
COMMENT_CLOSE = "⫸"
|
|
25
|
+
U_ESCAPE = /\\u([\da-fA-F]{1,5})\w{0,5}/
|
|
26
|
+
|
|
27
|
+
module_function
|
|
28
|
+
|
|
29
|
+
# +input+ must be a valid UTF-8 String.
|
|
30
|
+
def call(input)
|
|
31
|
+
formula, label, labeled = split_label(input)
|
|
32
|
+
return Result.new(text: nil, label: label, labeled: labeled) if formula.nil?
|
|
33
|
+
|
|
34
|
+
text = encode(formula)
|
|
35
|
+
text = text.gsub("&", "&")
|
|
36
|
+
text = text.gsub(""", "\"")
|
|
37
|
+
text = remove_comment(text)
|
|
38
|
+
text = text.gsub("\\\\", "\\")
|
|
39
|
+
text = text.gsub(U_ESCAPE) { "&#x#{::Regexp.last_match(1)};" }
|
|
40
|
+
Result.new(text: text.strip, label: label, labeled: labeled)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# A "#" inside a ✎(...) group is protected; the text is then split on
|
|
44
|
+
# "#": the first part is the formula and, if there is more than one
|
|
45
|
+
# part, the last one is the label. Parts in between are dropped.
|
|
46
|
+
def split_label(input)
|
|
47
|
+
text = input.split("\n", -1).map { |line| protect_pen_group(line) }.join("\n")
|
|
48
|
+
parts = text.split("#")
|
|
49
|
+
return [nil, nil, false] if parts.empty?
|
|
50
|
+
|
|
51
|
+
formula = parts.first.gsub(PROTECTED_HASH, "#")
|
|
52
|
+
parts.size > 1 ? [formula, parts.last, true] : [formula, nil, false]
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Legacy: text.gsub(/✎\(.*(\#).*\)/) { |group| group.gsub("#", ...) }.
|
|
56
|
+
# Per line ("." stops at newlines) the only match runs from the first
|
|
57
|
+
# "✎(" to the last ")" when a "#" lies between them. Written out
|
|
58
|
+
# without the backtracking regexp, which is polynomial on long input.
|
|
59
|
+
def protect_pen_group(line)
|
|
60
|
+
start = line.index(PEN_OPEN) or return line
|
|
61
|
+
close = line.rindex(")")
|
|
62
|
+
return line unless close && close >= start + PEN_OPEN.length
|
|
63
|
+
|
|
64
|
+
hash = line.index("#", start + PEN_OPEN.length)
|
|
65
|
+
return line unless hash && hash < close
|
|
66
|
+
|
|
67
|
+
line[0...start] + line[start..close].gsub("#", PROTECTED_HASH) + line[(close + 1)..]
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Legacy: text.gsub(/⫷.*⫸/, ""): from the first opener to
|
|
71
|
+
# the last closer after it. The encoded text has no newlines.
|
|
72
|
+
def remove_comment(text)
|
|
73
|
+
start = text.index(COMMENT_OPEN) or return text
|
|
74
|
+
close = text.rindex(COMMENT_CLOSE)
|
|
75
|
+
return text unless close && close >= start + COMMENT_OPEN.length
|
|
76
|
+
|
|
77
|
+
text[0...start] + text[(close + COMMENT_CLOSE.length)..]
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def encode(text)
|
|
81
|
+
text = text.gsub(BASIC) { |char| hex(char) }
|
|
82
|
+
text.gsub(EXTENDED) { |char| hex(char) }
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def hex(char)
|
|
86
|
+
"&#x#{char.ord.to_s(16)};"
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
private_class_method :split_label, :protect_pen_group, :remove_comment, :encode, :hex
|
|
90
|
+
end
|
|
91
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Umath
|
|
4
|
+
# The internal raw-tree boundary: the legacy preprocessing, then the
|
|
5
|
+
# grammar, then the declared syntax bindings, which carry the raw tree.
|
|
6
|
+
#
|
|
7
|
+
# Internal and for tests and parity replay only; not public API. The
|
|
8
|
+
# public Umath.parse, which returns the typed language model, is
|
|
9
|
+
# built on top of it.
|
|
10
|
+
module RawParser
|
|
11
|
+
# +text+: the grammar input (see Preprocessor::Result), nil when the
|
|
12
|
+
# input has no formula part; +label+/+labeled+: the label split off by
|
|
13
|
+
# preprocessing; +tree+: the raw tree of the normalized grammar (Hashes,
|
|
14
|
+
# Arrays and Parsanol::Slice leaves; scripts/unwrap.rb turns it into the
|
|
15
|
+
# legacy tree), nil when +text+ is nil; +model_data+: the same
|
|
16
|
+
# tree as plain Hashes, Arrays and Strings with String keys (the typed
|
|
17
|
+
# model's construction input); +engine+: the engine that produced it.
|
|
18
|
+
Result = Struct.new(:text, :label, :labeled, :tree, :model_data, :engine, keyword_init: true)
|
|
19
|
+
|
|
20
|
+
module_function
|
|
21
|
+
|
|
22
|
+
def parse(input, decimal: ".", engine: :ruby)
|
|
23
|
+
entry = entry_for(decimal)
|
|
24
|
+
mode = Engine.resolve(engine)
|
|
25
|
+
pre = Preprocessor.call(normalize_input(input))
|
|
26
|
+
result = Result.new(text: pre.text, label: pre.label, labeled: pre.labeled)
|
|
27
|
+
return result if pre.text.nil?
|
|
28
|
+
|
|
29
|
+
loaded = Umath.artifacts
|
|
30
|
+
shape = Engine.run(loaded.parse, entry, pre.text, mode, fallback: engine == :auto)
|
|
31
|
+
bound = loaded.bindings.apply(entry, shape)
|
|
32
|
+
result.tree, result.model_data = [Configuration::TREE_CAPTURE, Configuration::MODEL_BINDING].map do |key|
|
|
33
|
+
bound.fetch(key) { raise ArtifactError, "the declared bindings did not produce #{key}" }
|
|
34
|
+
end
|
|
35
|
+
result.engine = Engine.last_engine
|
|
36
|
+
result
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def entry_for(decimal)
|
|
40
|
+
Configuration::DECIMAL_ENTRIES.fetch(decimal) do
|
|
41
|
+
raise ArgumentError,
|
|
42
|
+
"unsupported decimal marker #{decimal.inspect} " \
|
|
43
|
+
"(supported: #{Configuration::DECIMAL_ENTRIES.keys.map(&:inspect).join(', ')})"
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# +input+ as valid UTF-8, or an ArgumentError.
|
|
48
|
+
def normalize_input(input)
|
|
49
|
+
raise ArgumentError, "input must be a String, got #{input.class}" unless input.is_a?(String)
|
|
50
|
+
|
|
51
|
+
text = input.encoding == Encoding::UTF_8 ? input : input.encode(Encoding::UTF_8)
|
|
52
|
+
raise ArgumentError, "input is not valid UTF-8" unless text.valid_encoding?
|
|
53
|
+
|
|
54
|
+
text
|
|
55
|
+
rescue EncodingError => e
|
|
56
|
+
raise ArgumentError, "input cannot be converted to UTF-8: #{e.message}"
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Umath
|
|
4
|
+
# Maps positions in the grammar input back to the caller's text.
|
|
5
|
+
#
|
|
6
|
+
# The grammar runs on the preprocessed text (Preprocessor), which differs
|
|
7
|
+
# from the caller's: non-ASCII characters become character references, a
|
|
8
|
+
# comment and the label are removed, escapes are folded. SourceMap repeats
|
|
9
|
+
# each Preprocessor step on the caller's text while recording, for every
|
|
10
|
+
# character it produces, the caller character it came from. A step that
|
|
11
|
+
# replaces a match gives every produced character the position of the
|
|
12
|
+
# match's first character.
|
|
13
|
+
#
|
|
14
|
+
# Internal: the public parser uses it to report ParseError offsets in the
|
|
15
|
+
# caller's text.
|
|
16
|
+
class SourceMap
|
|
17
|
+
attr_reader :text
|
|
18
|
+
|
|
19
|
+
# +input+: the caller's text as valid UTF-8.
|
|
20
|
+
def self.build(input)
|
|
21
|
+
new(input)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def initialize(input)
|
|
25
|
+
@input = input
|
|
26
|
+
chars, origins = formula(input)
|
|
27
|
+
chars, origins = encode(chars, origins)
|
|
28
|
+
text = chars.join
|
|
29
|
+
text, origins = replace(text, origins, "&") { "&" }
|
|
30
|
+
text, origins = replace(text, origins, """) { "\"" }
|
|
31
|
+
text, origins = remove_comment(text, origins)
|
|
32
|
+
text, origins = replace(text, origins, "\\\\") { "\\" }
|
|
33
|
+
text, origins = replace(text, origins, Preprocessor::U_ESCAPE) { |m| "&#x#{m[1]};" }
|
|
34
|
+
@text, @origins = strip(text, origins)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# The caller position, as [char_offset, byte_offset], of grammar input
|
|
38
|
+
# character +char_offset+ (the end of the grammar input maps to the end
|
|
39
|
+
# of the formula part of the caller's text).
|
|
40
|
+
def caller_position(char_offset)
|
|
41
|
+
char = if char_offset < @origins.size
|
|
42
|
+
@origins[char_offset]
|
|
43
|
+
elsif @origins.empty?
|
|
44
|
+
@text_end || 0
|
|
45
|
+
else
|
|
46
|
+
@origins.last + 1
|
|
47
|
+
end
|
|
48
|
+
[char, @input[0, char].bytesize]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
# The formula part as characters with their caller positions:
|
|
54
|
+
# Preprocessor.split_label with the protection of "#" inside ✎(...)
|
|
55
|
+
# undone by position.
|
|
56
|
+
def formula(input)
|
|
57
|
+
chars = []
|
|
58
|
+
origins = []
|
|
59
|
+
position = 0
|
|
60
|
+
input.split("\n", -1).each_with_index do |line, index|
|
|
61
|
+
unless index.zero?
|
|
62
|
+
chars << "\n"
|
|
63
|
+
origins << (position - 1)
|
|
64
|
+
end
|
|
65
|
+
protected_line = protect(line)
|
|
66
|
+
protected_line.each { |c, offset| chars << c; origins << (position + offset) }
|
|
67
|
+
position += line.length + 1
|
|
68
|
+
end
|
|
69
|
+
text = chars.join
|
|
70
|
+
cut = text.index("#") || text.length
|
|
71
|
+
chars = chars.first(cut)
|
|
72
|
+
origins = origins.first(cut)
|
|
73
|
+
@text_end = cut.zero? ? 0 : nil
|
|
74
|
+
text, origins = replace(chars.join, origins, Preprocessor::PROTECTED_HASH) { "#" }
|
|
75
|
+
[text.chars, origins]
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# The line's characters with their offsets in it; a protected "#"
|
|
79
|
+
# becomes Preprocessor::PROTECTED_HASH, every character of which keeps
|
|
80
|
+
# the "#"'s offset.
|
|
81
|
+
def protect(line)
|
|
82
|
+
chars = line.chars.each_with_index.to_a
|
|
83
|
+
protected_text = Preprocessor.send(:protect_pen_group, line)
|
|
84
|
+
return chars if protected_text == line
|
|
85
|
+
|
|
86
|
+
start = line.index(Preprocessor::PEN_OPEN)
|
|
87
|
+
close = line.rindex(")")
|
|
88
|
+
chars.flat_map do |c, offset|
|
|
89
|
+
if c == "#" && offset > start && offset < close
|
|
90
|
+
Preprocessor::PROTECTED_HASH.chars.map { |p| [p, offset] }
|
|
91
|
+
else
|
|
92
|
+
[[c, offset]]
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def encode(chars, origins)
|
|
98
|
+
out = []
|
|
99
|
+
out_origins = []
|
|
100
|
+
chars.each_with_index do |c, i|
|
|
101
|
+
piece = c.match?(Preprocessor::BASIC) || c.match?(Preprocessor::EXTENDED) ? "&#x#{c.ord.to_s(16)};" : c
|
|
102
|
+
piece.each_char { |p| out << p; out_origins << origins[i] }
|
|
103
|
+
end
|
|
104
|
+
[out, out_origins]
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def replace(text, origins, pattern)
|
|
108
|
+
regexp = pattern.is_a?(Regexp) ? pattern : Regexp.new(Regexp.escape(pattern))
|
|
109
|
+
out = +""
|
|
110
|
+
out_origins = []
|
|
111
|
+
last = 0
|
|
112
|
+
text.scan(regexp) do
|
|
113
|
+
match = Regexp.last_match
|
|
114
|
+
from, to = match.offset(0)
|
|
115
|
+
out << text[last...from]
|
|
116
|
+
out_origins.concat(origins[last...from])
|
|
117
|
+
piece = yield(match)
|
|
118
|
+
out << piece
|
|
119
|
+
out_origins.concat([origins[from]] * piece.length)
|
|
120
|
+
last = to
|
|
121
|
+
end
|
|
122
|
+
out << text[last..]
|
|
123
|
+
out_origins.concat(origins[last..])
|
|
124
|
+
[out, out_origins]
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def remove_comment(text, origins)
|
|
128
|
+
start = text.index(Preprocessor::COMMENT_OPEN) or return [text, origins]
|
|
129
|
+
close = text.rindex(Preprocessor::COMMENT_CLOSE)
|
|
130
|
+
return [text, origins] unless close && close >= start + Preprocessor::COMMENT_OPEN.length
|
|
131
|
+
|
|
132
|
+
stop = close + Preprocessor::COMMENT_CLOSE.length
|
|
133
|
+
[text[0...start] + text[stop..], origins[0...start] + origins[stop..]]
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# String#strip: leading whitespace, trailing whitespace and NULs.
|
|
137
|
+
def strip(text, origins)
|
|
138
|
+
stripped = text.strip
|
|
139
|
+
return [text, origins] if stripped.length == text.length
|
|
140
|
+
|
|
141
|
+
lead = text.length - text.lstrip.length
|
|
142
|
+
@text_end ||= origins[lead] if stripped.empty?
|
|
143
|
+
[stripped, origins[lead, stripped.length]]
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Umath
|
|
6
|
+
# Symbol resolution: the separate pass after model construction that
|
|
7
|
+
# gives every Model::MathSymbol its Unicode value and kind from the
|
|
8
|
+
# exported symbol catalog (data/umath-symbols.json, written by
|
|
9
|
+
# scripts/export_symbols.rb from the pinned Plurimath commit).
|
|
10
|
+
#
|
|
11
|
+
# Lookup is the legacy one: the spelling with surrounding whitespace
|
|
12
|
+
# removed, exactly as the grammar matched it.
|
|
13
|
+
module Symbols
|
|
14
|
+
FORMAT = "umath-symbol-catalog/1"
|
|
15
|
+
CHARACTER_REFERENCE = /&#(?:x(\h{1,6})|(\d{1,7}));/
|
|
16
|
+
|
|
17
|
+
@catalog = nil
|
|
18
|
+
@lock = Mutex.new
|
|
19
|
+
|
|
20
|
+
module_function
|
|
21
|
+
|
|
22
|
+
# The catalog: spelling => [value, kind].
|
|
23
|
+
def catalog
|
|
24
|
+
@lock.synchronize { @catalog ||= load(Configuration::SYMBOLS_PATH) }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def load(path)
|
|
28
|
+
doc = JSON.parse(File.read(path, encoding: "UTF-8"))
|
|
29
|
+
raise ArtifactError, "symbol catalog #{path} has format #{doc['format'].inspect}, not #{FORMAT}" unless doc["format"] == FORMAT
|
|
30
|
+
|
|
31
|
+
doc.fetch("symbols").transform_values { |entry| [entry.fetch("value").freeze, entry.fetch("kind").freeze].freeze }.freeze
|
|
32
|
+
rescue SystemCallError, JSON::ParserError, KeyError, TypeError, NoMethodError => e
|
|
33
|
+
raise ArtifactError, "symbol catalog #{path} cannot be used: #{e.class}: #{e.message.lines.first&.strip}"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# [value, kind] for +spelling+.
|
|
37
|
+
def lookup(spelling)
|
|
38
|
+
catalog.fetch(spelling.strip) { [decode(spelling), "unlisted"] }
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Fills value and kind of every node under +node+ (a model instance)
|
|
42
|
+
# that holds a spelling: each Model::MathSymbol, and the symbol nodes
|
|
43
|
+
# (SymbolNode, OpenParen, ...) whose text the grammar captured as
|
|
44
|
+
# spelling. Iterative over the declared attributes; returns +node+.
|
|
45
|
+
def resolve(node)
|
|
46
|
+
stack = [node]
|
|
47
|
+
until stack.empty?
|
|
48
|
+
current = stack.pop
|
|
49
|
+
attributes = current.class.attributes
|
|
50
|
+
current.value, current.kind = lookup(current.spelling) if attributes.key?(:spelling) && current.spelling
|
|
51
|
+
attributes.each_key do |name|
|
|
52
|
+
value = current.public_send(name)
|
|
53
|
+
case value
|
|
54
|
+
when Lutaml::Model::Serializable then stack << value
|
|
55
|
+
when ::Array then value.each { |v| stack << v if v.is_a?(Lutaml::Model::Serializable) }
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
node
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# +text+ with its numeric character references decoded.
|
|
63
|
+
def decode(text)
|
|
64
|
+
text.gsub(CHARACTER_REFERENCE) do
|
|
65
|
+
code = ::Regexp.last_match(1) ? ::Regexp.last_match(1).to_i(16) : ::Regexp.last_match(2).to_i
|
|
66
|
+
valid = code <= 0x10FFFF && !(0xD800..0xDFFF).cover?(code)
|
|
67
|
+
valid ? [code].pack("U") : ::Regexp.last_match(0)
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
data/lib/umath.rb
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "parsanol"
|
|
4
|
+
|
|
5
|
+
# UnicodeMath parser backed by a compiled Parsanol (PARG) grammar artifact.
|
|
6
|
+
#
|
|
7
|
+
# Umath.parse returns the typed language model (Umath::Model,
|
|
8
|
+
# declared with LutaML Model). The raw parse tree (Umath::RawParser)
|
|
9
|
+
# is an internal boundary for parity replay with the legacy Plurimath
|
|
10
|
+
# parser. See docs/parser-contract.adoc and docs/model-contract.adoc.
|
|
11
|
+
module Umath
|
|
12
|
+
autoload :VERSION, "umath/version"
|
|
13
|
+
autoload :Error, "umath/errors"
|
|
14
|
+
autoload :ParseError, "umath/errors"
|
|
15
|
+
autoload :NestingTooDeep, "umath/errors"
|
|
16
|
+
autoload :ArtifactError, "umath/errors"
|
|
17
|
+
autoload :EngineUnavailable, "umath/errors"
|
|
18
|
+
autoload :Artifact, "umath/artifact"
|
|
19
|
+
autoload :Configuration, "umath/configuration"
|
|
20
|
+
autoload :Engine, "umath/engine"
|
|
21
|
+
autoload :Preprocessor, "umath/preprocessor"
|
|
22
|
+
autoload :RawParser, "umath/raw_parser"
|
|
23
|
+
autoload :Parser, "umath/parser"
|
|
24
|
+
autoload :Model, "umath/model"
|
|
25
|
+
autoload :SourceMap, "umath/source_map"
|
|
26
|
+
autoload :Symbols, "umath/symbols"
|
|
27
|
+
|
|
28
|
+
@artifacts = Artifact.new(Configuration::DEFAULT_ARTIFACT_PATH,
|
|
29
|
+
Configuration::DEFAULT_BINDINGS_PATH)
|
|
30
|
+
|
|
31
|
+
class << self
|
|
32
|
+
# Parses +input+ (a String) and returns its Umath::Model::Document.
|
|
33
|
+
#
|
|
34
|
+
# +decimal+: the decimal marker, "." (default), "," or "٫" (U+066B).
|
|
35
|
+
# +engine+: :ruby (default), :native, or :auto (native where available,
|
|
36
|
+
# else Ruby).
|
|
37
|
+
#
|
|
38
|
+
# Raises ParseError (with offsets into +input+) when the input is
|
|
39
|
+
# refused, NestingTooDeep when the selected engine cannot handle its
|
|
40
|
+
# depth, EngineUnavailable for an unavailable engine: :native,
|
|
41
|
+
# ArtifactError for unusable bundled artifacts and ArgumentError for
|
|
42
|
+
# invalid arguments.
|
|
43
|
+
def parse(input, decimal: ".", engine: :ruby)
|
|
44
|
+
Parser.parse(input, decimal: decimal, engine: engine)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# The engine (:native or :ruby) of this thread's last parse or
|
|
48
|
+
# rejection, nil before any.
|
|
49
|
+
def last_engine
|
|
50
|
+
Engine.last_engine
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# The verified, paired parse and Ruby bindings artifacts (internal).
|
|
54
|
+
def artifacts
|
|
55
|
+
@artifacts.fetch
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Points Umath at other artifacts (for tests). The next parse
|
|
59
|
+
# loads and verifies them.
|
|
60
|
+
def use_artifacts(path, bindings_path)
|
|
61
|
+
@artifacts.use(path, bindings_path)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def artifact_paths
|
|
65
|
+
[@artifacts.path, @artifacts.bindings_path]
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Installs a hook into Parsanol at load time, so it is required, not
|
|
71
|
+
# autoloaded.
|
|
72
|
+
require_relative "umath/errors"
|
|
73
|
+
require_relative "umath/native_recursion_guard"
|
metadata
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: umath
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Ribose Inc.
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: lutaml-model
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - "~>"
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '0.8'
|
|
19
|
+
- - ">="
|
|
20
|
+
- !ruby/object:Gem::Version
|
|
21
|
+
version: 0.8.162
|
|
22
|
+
type: :runtime
|
|
23
|
+
prerelease: false
|
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
25
|
+
requirements:
|
|
26
|
+
- - "~>"
|
|
27
|
+
- !ruby/object:Gem::Version
|
|
28
|
+
version: '0.8'
|
|
29
|
+
- - ">="
|
|
30
|
+
- !ruby/object:Gem::Version
|
|
31
|
+
version: 0.8.162
|
|
32
|
+
- !ruby/object:Gem::Dependency
|
|
33
|
+
name: parsanol
|
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
|
35
|
+
requirements:
|
|
36
|
+
- - "~>"
|
|
37
|
+
- !ruby/object:Gem::Version
|
|
38
|
+
version: '1.3'
|
|
39
|
+
- - ">="
|
|
40
|
+
- !ruby/object:Gem::Version
|
|
41
|
+
version: 1.3.131
|
|
42
|
+
type: :runtime
|
|
43
|
+
prerelease: false
|
|
44
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
45
|
+
requirements:
|
|
46
|
+
- - "~>"
|
|
47
|
+
- !ruby/object:Gem::Version
|
|
48
|
+
version: '1.3'
|
|
49
|
+
- - ">="
|
|
50
|
+
- !ruby/object:Gem::Version
|
|
51
|
+
version: 1.3.131
|
|
52
|
+
description: |
|
|
53
|
+
UnicodeMath parser extracted from Plurimath: a compiled Parsanol grammar
|
|
54
|
+
whose parse result is a typed UnicodeMath model declared with LutaML
|
|
55
|
+
Model.
|
|
56
|
+
email:
|
|
57
|
+
- open.source@ribose.com
|
|
58
|
+
executables: []
|
|
59
|
+
extensions: []
|
|
60
|
+
extra_rdoc_files: []
|
|
61
|
+
files:
|
|
62
|
+
- CHANGELOG.md
|
|
63
|
+
- LICENSE.md
|
|
64
|
+
- README.adoc
|
|
65
|
+
- artifacts/umath.bindings.ruby.json
|
|
66
|
+
- artifacts/umath.json
|
|
67
|
+
- data/umath-symbols.json
|
|
68
|
+
- docs/model-contract.adoc
|
|
69
|
+
- docs/parser-contract.adoc
|
|
70
|
+
- docs/testsuite-pin.adoc
|
|
71
|
+
- lib/umath.rb
|
|
72
|
+
- lib/umath/artifact.rb
|
|
73
|
+
- lib/umath/configuration.rb
|
|
74
|
+
- lib/umath/engine.rb
|
|
75
|
+
- lib/umath/errors.rb
|
|
76
|
+
- lib/umath/model.rb
|
|
77
|
+
- lib/umath/model/nodes.rb
|
|
78
|
+
- lib/umath/native_recursion_guard.rb
|
|
79
|
+
- lib/umath/parser.rb
|
|
80
|
+
- lib/umath/preprocessor.rb
|
|
81
|
+
- lib/umath/raw_parser.rb
|
|
82
|
+
- lib/umath/source_map.rb
|
|
83
|
+
- lib/umath/symbols.rb
|
|
84
|
+
- lib/umath/version.rb
|
|
85
|
+
homepage: https://github.com/plurimath/umath
|
|
86
|
+
licenses:
|
|
87
|
+
- BSD-2-Clause
|
|
88
|
+
metadata:
|
|
89
|
+
homepage_uri: https://github.com/plurimath/umath
|
|
90
|
+
source_code_uri: https://github.com/plurimath/umath
|
|
91
|
+
changelog_uri: https://github.com/plurimath/umath/blob/main/CHANGELOG.md
|
|
92
|
+
rubygems_mfa_required: 'true'
|
|
93
|
+
rdoc_options: []
|
|
94
|
+
require_paths:
|
|
95
|
+
- lib
|
|
96
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
97
|
+
requirements:
|
|
98
|
+
- - ">="
|
|
99
|
+
- !ruby/object:Gem::Version
|
|
100
|
+
version: 3.2.0
|
|
101
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
102
|
+
requirements:
|
|
103
|
+
- - ">="
|
|
104
|
+
- !ruby/object:Gem::Version
|
|
105
|
+
version: '0'
|
|
106
|
+
requirements: []
|
|
107
|
+
rubygems_version: 4.0.23
|
|
108
|
+
specification_version: 4
|
|
109
|
+
summary: UnicodeMath parser by Plurimath.
|
|
110
|
+
test_files: []
|