parsanol 3.0.0

data/lib/parsanol.rb

@@ -0,0 +1,408 @@
+# frozen_string_literal: true
+
+# A simple parser generator library. Typical usage would look like this:
+#
+#   require 'parslet'
+#
+#   class MyParser < Parsanol::Parser
+#     rule(:a) { str('a').repeat }
+#     root(:a)
+#   end
+#
+#   pp MyParser.new.parse('aaaa')   # => 'aaaa'@0
+#   pp MyParser.new.parse('bbbb')   # => Parsanol::Atoms::ParseFailed:
+#                                   #    Don't know what to do with bbbb at line 1 char 1.
+#
+# The simple DSL allows you to define grammars in PEG-style. This kind of
+# grammar construction does away with the ambiguities that usually comes with
+# parsers; instead, it allows you to construct grammars that are easier to
+# debug, since less magic is involved.
+#
+# Parslet is typically used in stages:
+#
+#
+# * Parsing the input string; this yields an intermediary tree, see
+#   Parslet.any, Parslet.match, Parslet.str, Parsanol::ClassMethods#rule and
+#   Parsanol::ClassMethods#root.
+# * Transformation of the tree into something useful to you, see
+#   Parsanol::Transform, Parslet.simple, Parslet.sequence and Parslet.subtree.
+#
+# The first stage is traditionally intermingled with the second stage; output
+# from the second stage is usually called the 'Abstract Syntax Tree' or AST.
+#
+# The stages are completely decoupled; You can change your grammar around and
+# use the second stage to isolate the rest of your code from the changes
+# you've effected.
+#
+# == Further reading
+#
+# All parslet atoms are subclasses of {Parsanol::Atoms::Base}. You might want to
+# look at all of those: {Parsanol::Atoms::Re}, {Parsanol::Atoms::Str},
+# {Parsanol::Atoms::Repetition}, {Parsanol::Atoms::Sequence},
+# {Parsanol::Atoms::Alternative}.
+#
+# == When things go wrong
+#
+# A parse that fails will raise {Parsanol::ParseFailed}. This exception contains
+# all the details of what went wrong, including a detailed error trace that
+# can be printed out as an ascii tree. ({Parsanol::Cause})
+#
+module Parsanol
+  # Extends classes that include Parslet with the module
+  # {Parsanol::ClassMethods}.
+  #
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  # Raised when the parse failed to match. It contains the message that should
+  # be presented to the user. More details can be extracted from the
+  # exceptions #parse_failure_cause member: It contains an instance of {Parsanol::Cause} that
+  # stores all the details of your failed parse in a tree structure.
+  #
+  #   begin
+  #     parslet.parse(str)
+  #   rescue Parsanol::ParseFailed => failure
+  #     puts failure.parse_failure_cause.ascii_tree
+  #   end
+  #
+  # Alternatively, you can just require 'parsanol/convenience' and call the
+  # method #parse_with_debug instead of #parse. This method will never raise
+  # and print error trees to stdout.
+  #
+  #   require 'parsanol/convenience'
+  #   parslet.parse_with_debug(str)
+  #
+  class ParseFailed < StandardError
+    def initialize(message, parse_failure_cause=nil)
+      super(message)
+      @parse_failure_cause = parse_failure_cause
+    end
+
+    # Why the parse failed.
+    #
+    # @return [Parsanol::Cause]
+    attr_reader :parse_failure_cause
+  end
+
+  module ClassMethods
+    # Enable automatic rule optimization for all rules in this parser.
+    # This includes quantifier simplification, sequence optimization,
+    # choice optimization, and lookahead optimization.
+    #
+    # NOTE: As of v3.1.0, optimizations are DISABLED BY DEFAULT to avoid
+    # overhead on tiny/small inputs. Use optimize_rules! to opt-in for
+    # complex parsers (JSON, ERB, large grammars) where benefits outweigh costs.
+    #
+    #   class SimpleParser < Parsanol::Parser
+    #     # Optimizations disabled by default - good for tiny inputs
+    #     rule(:fast) { str('a').repeat(1, 1) }  # remains as repeat
+    #   end
+    #
+    #   class ComplexParser < Parsanol::Parser
+    #     optimize_rules!  # Opt-in for complex grammars
+    #     rule(:optimized) { str('a').repeat(1, 1) }  # becomes str('a')
+    #   end
+    #
+    def optimize_rules!(enable = true)
+      @optimize_rules = enable
+    end
+    
+    # Disable automatic rule optimization.
+    # Use this for compatibility with parsers that rely on specific
+    # unoptimized behavior or for debugging purposes.
+    #
+    # @example Disable optimization
+    #   class MyParser < Parsanol::Parser
+    #     disable_optimization!
+    #     # rules will not be optimized
+    #   end
+    #
+    def disable_optimization!
+      @optimize_rules = false
+    end
+
+    # Check if rule optimization is enabled.
+    # As of v3.1.0, defaults to FALSE for compatibility.
+    # Use optimize_rules! to opt-in for complex parsers that benefit.
+    #
+    # @return [Boolean] true if optimization enabled
+    def optimize_rules?
+      # Default to false (disabled) for compatibility
+      # Use optimize_rules! to opt-in for performance
+      @optimize_rules = false if @optimize_rules.nil?
+      @optimize_rules
+    end
+
+    # Define an entity for the parser. This generates a method of the same
+    # name that can be used as part of other patterns. Those methods can be
+    # freely mixed in your parser class with real ruby methods.
+    #
+    #   class MyParser
+    #     include Parslet
+    #
+    #     rule(:bar) { str('bar') }
+    #     rule(:twobar) do
+    #       bar >> bar
+    #     end
+    #
+    #     root :twobar
+    #   end
+    #
+    # To enable automatic quantifier simplification:
+    #
+    #   class OptimizedParser
+    #     include Parslet
+    #     optimize_rules!
+    #
+    #     rule(:bar) { str('a').repeat(1, 1) }  # becomes str('a')
+    #   end
+    #
+    def rule(name, opts={}, &definition)
+      undef_method name if method_defined? name
+      define_method(name) do
+        @rules ||= {}     # <name, rule> memoization
+        return @rules[name] if @rules.has_key?(name)
+
+        # Capture the self of the parser class along with the definition.
+        definition_closure = proc {
+          result = self.instance_eval(&definition)
+
+      # Apply optimizations if enabled (only for classes that support it)
+      if self.class.respond_to?(:optimize_rules?) && self.class.optimize_rules?
+        # Apply all optimizers: quantifiers, sequences, choices, and lookaheads
+        result = Parsanol::Optimizer.simplify_quantifiers(result)
+        result = Parsanol::Optimizer.simplify_sequences(result)
+        result = Parsanol::Optimizer.simplify_choices(result)
+        result = Parsanol::Optimizer.simplify_lookaheads(result)
+      end
+
+          result
+        }
+
+        @rules[name] = Atoms::Entity.new(name, opts[:label], &definition_closure)
+      end
+    end
+  end
+
+  # Allows for delayed construction of #match. See also Parslet.match.
+  #
+  # @api private
+  class DelayedMatchConstructor
+    def [](str)
+      Atoms::Re.new("[" + str + "]")
+    end
+  end
+
+  # Returns an atom matching a character class. All regular expressions can be
+  # used, as long as they match only a single character at a time.
+  #
+  #   match('[ab]')     # will match either 'a' or 'b'
+  #   match('[\n\s]')   # will match newlines and spaces
+  #
+  # There is also another (convenience) form of this method:
+  #
+  #   match['a-z']      # synonymous to match('[a-z]')
+  #   match['\n']       # synonymous to match('[\n]')
+  #
+  # @overload match(str)
+  #   @param str [String] character class to match (regexp syntax)
+  #   @return [Parsanol::Atoms::Re] a parslet atom
+  #
+  def match(str=nil)
+    return DelayedMatchConstructor.new unless str
+
+    return Atoms::Re.new(str)
+  end
+  module_function :match
+
+  # Returns an atom matching the +str+ given:
+  #
+  #   str('class')      # will match 'class'
+  #
+  # @param str [String] string to match verbatim
+  # @return [Parsanol::Atoms::Str] a parslet atom
+  #
+  def str(str)
+    Atoms::Str.new(str)
+  end
+  module_function :str
+
+  # Returns an atom matching any character. It acts like the '.' (dot)
+  # character in regular expressions.
+  #
+  #   any.parse('a')    # => 'a'
+  #
+  # @return [Parsanol::Atoms::Re] a parslet atom
+  #
+  def any
+    Atoms::Re.new('.')
+  end
+  module_function :any
+
+  # Introduces a new capture scope. This means that all old captures stay
+  # accessible, but new values stored will only be available during the block
+  # given and the old values will be restored after the block.
+  #
+  # Example:
+  #   # :a will be available until the end of the block. Afterwards,
+  #   # :a from the outer scope will be available again, if such a thing
+  #   # exists.
+  #   scope { str('a').capture(:a) }
+  #
+  def scope(&block)
+    Parsanol::Atoms::Scope.new(block)
+  end
+  module_function :scope
+
+  # Designates a piece of the parser as being dynamic. Dynamic parsers can
+  # either return a parser at runtime, which will be applied on the input, or
+  # return a result from a parse.
+  #
+  # Dynamic parse pieces are never cached and can introduce performance
+  # abnormalitites - use sparingly where other constructs fail.
+  #
+  # Example:
+  #   # Parses either 'a' or 'b', depending on the weather
+  #   dynamic { rand() < 0.5 ? str('a') : str('b') }
+  #
+  def dynamic(&block)
+    Parsanol::Atoms::Dynamic.new(block)
+  end
+  module_function :dynamic
+
+  # Returns a parslet atom that parses infix expressions. Operations are
+  # specified as a list of <atom, precedence, associativity> tuples, where
+  # atom is simply the parslet atom that matches an operator, precedence is
+  # a number and associativity is either :left or :right.
+  #
+  # Higher precedence indicates that the operation should bind tighter than
+  # other operations with lower precedence. In common algebra, '+' has
+  # lower precedence than '*'. So you would have a precedence of 1 for '+' and
+  # a precedence of 2 for '*'. Only the order relation between these two
+  # counts, so any number would work.
+  #
+  # Associativity is what decides what interpretation to take for strings that
+  # are ambiguous like '1 + 2 + 3'. If '+' is specified as left associative,
+  # the expression would be interpreted as '(1 + 2) + 3'. If right
+  # associativity is chosen, it would be interpreted as '1 + (2 + 3)'. Note
+  # that the hash trees output reflect that choice as well.
+  #
+  # An optional block can be provided in order to manipulate the generated tree.
+  # The block will be called on each operator and passed 3 arguments: the left
+  # operand, the operator, and the right operand.
+  #
+  # Examples:
+  #   infix_expression(integer, [add_op, 1, :left])
+  #   # would parse things like '1 + 2'
+  #
+  #   infix_expression(integer, [add_op, 1, :left]) { |l,o,r| { :plus => [l, r] } }
+  #   # would parse '1 + 2 + 3' as:
+  #   # { :plus => [1, { :plus => [2, 3] }] }
+  #
+  # @param element [Parsanol::Atoms::Base] elements that take the NUMBER position
+  #    in the expression
+  # @param operations [Array<(Parsanol::Atoms::Base, Integer, {:left, :right})>]
+  #
+  # @see Parsanol::Atoms::Infix
+  #
+  def infix_expression(element, *operations, &reducer)
+    Parsanol::Atoms::Infix.new(element, operations, &reducer)
+  end
+  module_function :infix_expression
+
+  # A special kind of atom that allows embedding whole treetop expressions
+  # into parslet construction.
+  #
+  #   # the same as str('a') >> str('b').maybe
+  #   exp(%Q("a" "b"?))
+  #
+  # @param str [String] a treetop expression
+  # @return [Parsanol::Atoms::Base] the corresponding parslet parser
+  #
+  def exp(str)
+    Parsanol::Expression.new(str).to_parslet
+  end
+  module_function :exp
+
+  # Returns a placeholder for a tree transformation that will only match a
+  # sequence of elements. The +symbol+ you specify will be the key for the
+  # matched sequence in the returned dictionary.
+  #
+  #   # This would match a body element that contains several declarations.
+  #   { :body => sequence(:declarations) }
+  #
+  # The above example would match <code>:body => ['a', 'b']</code>, but not
+  # <code>:body => 'a'</code>.
+  #
+  # see {Parsanol::Transform}
+  #
+  def sequence(symbol)
+    Pattern::SequenceBind.new(symbol)
+  end
+  module_function :sequence
+
+  # Returns a placeholder for a tree transformation that will only match
+  # simple elements. This matches everything that <code>#sequence</code>
+  # doesn't match.
+  #
+  #   # Matches a single header.
+  #   { :header => simple(:header) }
+  #
+  # see {Parsanol::Transform}
+  #
+  def simple(symbol)
+    Pattern::SimpleBind.new(symbol)
+  end
+  module_function :simple
+
+  # Returns a placeholder for tree transformation patterns that will match
+  # any kind of subtree.
+  #
+  #   { :expression => subtree(:exp) }
+  #
+  def subtree(symbol)
+    Pattern::SubtreeBind.new(symbol)
+  end
+  module_function :subtree
+
+  autoload :Expression, 'parsanol/expression'
+end
+
+require 'parsanol/version'
+require 'parsanol/result'
+require 'parsanol/slice'
+require 'parsanol/string_view'
+require 'parsanol/rope'
+require 'parsanol/pool'
+require 'parsanol/pools/slice_pool'
+require 'parsanol/pools/array_pool'
+require 'parsanol/pools/position_pool'
+require 'parsanol/buffer'
+require 'parsanol/pools/buffer_pool'
+require 'parsanol/lazy_result'
+require 'parsanol/result_builder'
+require 'parsanol/first_set'
+require 'parsanol/cause'
+require 'parsanol/source'
+require 'parsanol/atoms'
+require 'parsanol/pattern'
+require 'parsanol/pattern/binding'
+require 'parsanol/transform'
+require 'parsanol/parser'
+require 'parsanol/error_reporter'
+require 'parsanol/scope'
+require 'parsanol/optimizer'
+require 'parsanol/options'
+require 'parsanol/native'
+
+# New features (require native extension for full functionality)
+require 'parsanol/source_location'
+require 'parsanol/grammar_builder'
+require 'parsanol/streaming_parser'
+require 'parsanol/incremental_parser'
+require 'parsanol/builder_callbacks'
+require 'parsanol/parallel'
+
+# Add GrammarBuilder DSL to Parsanol module
+Parsanol.extend(Parsanol::GrammarBuilderDSL)

data/parsanol-ruby.gemspec

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative 'lib/parsanol/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'parsanol'
+  spec.version = Parsanol::VERSION
+  spec.platform = Gem::Platform::RUBY
+
+  spec.authors = ['Ribose Inc.']
+  spec.email = ['open.source@ribose.com']
+
+  spec.summary = 'Parser construction library with great error reporting in Ruby.'
+  spec.description = 'A small Ruby library for constructing parsers in the PEG (Parsing Expression Grammar) fashion. ' \
+                     'Parsanol provides Parslet-compatible API with additional features including ' \
+                     'static frozen parsers and dynamic parsers, with optional Rust native extension for improved performance.'
+  spec.homepage = 'https://github.com/parsanol/parsanol-ruby'
+  spec.license = 'MIT'
+
+  spec.metadata = {
+    'bug_tracker_uri' => 'https://github.com/parsanol/parsanol-ruby/issues',
+    'changelog_uri' => 'https://github.com/parsanol/parsanol-ruby/blob/main/HISTORY.txt',
+    'documentation_uri' => 'https://parsanol.github.io/parsanol-ruby/',
+    'homepage_uri' => 'https://github.com/parsanol/parsanol-ruby',
+    'source_code_uri' => 'https://github.com/parsanol/parsanol-ruby',
+    'rubygems_mfa_required' => 'true',
+  }
+
+  # Rust extension
+  spec.extensions = ['ext/parsanol_native/extconf.rb']
+
+  spec.files = Dir.glob('{lib,spec,example}/**/*') + %w[
+    HISTORY.txt
+    LICENSE
+    Rakefile
+    README.adoc
+    parsanol-ruby.gemspec
+  ]
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = '>= 2.7.0'
+
+  # Required for Rust extension
+  spec.add_dependency 'rb_sys', '~> 0.9.39'
+
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rake-compiler', '~> 1.2.0'
+  spec.add_development_dependency 'rdoc', '~> 6.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+
+  # For Parslet compatibility verification
+  spec.add_development_dependency 'parslet', '~> 2.0.0'
+
+  # For benchmarking
+  spec.add_development_dependency 'benchmark-ips', '~> 2.0'
+end

data/spec/acceptance/examples_spec.rb

@@ -0,0 +1,96 @@
+require 'spec_helper'
+require 'open3'
+
+# Strip positions from the output, so that we can compare it with the expected output.
+# (Some other specs utilize the inspect of inner objects outside example/*.rb expections)
+# The behavior of comparison of #inspect as done in parselet behaves differently
+# in Opal and MRI.
+
+describe 'Regression on' do
+  Dir['example/*.rb'].each do |example|
+    context example do
+      # Generates a product path for a given example file.
+      def product_path(str, ext)
+        str
+          .gsub('.rb', ".#{ext}")
+          .gsub('example/', 'example/output/')
+      end
+
+      it 'runs successfully' do
+        # Skip if output files don't exist (new examples may not have expected outputs yet)
+        unless Dir["example/output/#{File.basename(example, '.rb')}.*"].any?
+          skip "No output file found for #{example}"
+        end
+
+        # Skip examples that have missing dependencies
+        skip_missing_deps = %w{
+          example/optimized_erb.rb
+        }
+        if skip_missing_deps.include?(example)
+          skip "Missing dependencies"
+        end
+
+        if RUBY_ENGINE == 'opal'
+
+          skip_examples = %w{
+            example/calc.rb
+            example/empty.rb
+            example/erb.rb
+            example/ip_address.rb
+            example/mathn.rb
+            example/nested_errors.rb
+            example/optimized_erb.rb
+            example/prec_calc.rb
+          }
+          if skip_examples.include?(example)
+            skip "Opal does not support #{example} yet"
+          end
+
+          begin
+            system("opal -srubygems -ropal-parser -rnodejs -Ilib -I. #{example} >_stdout 2>_stderr")
+
+            handle_map = {
+              '_stdout' => :out,
+              '_stderr' => :err,
+            }
+            expectation_found = handle_map.any? do |io, ext|
+              name = product_path(example, ext)
+
+              if File.exist?(name)
+                actual_output = File.read(io).strip
+                expected_output = File.read(name).strip.gsub(/:(\w+)(=>|,|\]|\})/, '"\1"\2').gsub('1.0e+23', '1e+23').gsub(/@\d+/, '').strip
+                expect(strip_positions(actual_output)).to eq(strip_positions(expected_output))
+                true
+              end
+            end
+          ensure
+            File.unlink('_stdout')
+            File.unlink('_stderr')
+          end
+        else
+          _, stdout, stderr = Open3.popen3("ruby #{example}")
+
+          handle_map = {
+            stdout => :out,
+            stderr => :err,
+          }
+          expectation_found = handle_map.any? do |io, ext|
+            name = product_path(example, ext)
+
+            if File.exist?(name)
+              actual_output = io.read.strip
+              expected_output = File.read(name).strip
+              expect(actual_output).to eq(expected_output)
+              true
+            end
+          end
+        end
+
+        unless expectation_found
+          raise "Example doesn't have either an .err or an .out file. " +
+                'Please create in examples/output!'
+        end
+      end
+    end
+  end
+end

data/spec/acceptance/infix_parser_spec.rb

@@ -0,0 +1,145 @@
+require 'spec_helper'
+
+describe 'Infix expression parsing' do
+  class InfixExpressionParser < Parsanol::Parser
+    rule(:space) { match['\s'] }
+
+    def cts(atom)
+      atom >> space.repeat
+    end
+
+    def infix(*args)
+      Infix.new(*args)
+    end
+
+    rule(:mul_op) { match['*/'] >> str(' ').maybe }
+    rule(:add_op) { match['+-'] >> str(' ').maybe }
+    rule(:digit) { match['0-9'] }
+    rule(:integer) { cts digit.repeat(1) }
+
+    rule(:expression) do
+      infix_expression(integer,
+                       [mul_op, 2, :left],
+                       [add_op, 1, :right])
+    end
+  end
+
+  let(:p) { InfixExpressionParser.new }
+
+  describe '#integer' do
+    let(:i) { p.integer }
+
+    it 'parses integers' do
+      i.should parse('1')
+      i.should parse('123')
+    end
+
+    it 'consumes trailing white space' do
+      i.should parse('1   ')
+      i.should parse('134   ')
+    end
+
+    it "doesn't parse floats" do
+      i.should_not parse('1.3')
+    end
+  end
+
+  describe '#multiplication' do
+    let(:m) { p.expression }
+
+    it 'parses simple multiplication' do
+      m.should parse('1*2').as(l: '1', o: '*', r: '2')
+    end
+
+    it 'parses simple multiplication with spaces' do
+      m.should parse('1 * 2').as(l: '1 ', o: '* ', r: '2')
+    end
+
+    it 'parses division' do
+      m.should parse('1/2')
+    end
+  end
+
+  describe '#addition' do
+    let(:a) { p.expression }
+
+    it 'parses simple addition' do
+      a.should parse('1+2')
+    end
+
+    it 'parses complex addition' do
+      a.should parse('1+2+3-4')
+    end
+
+    it 'parses a single element' do
+      a.should parse('1')
+    end
+  end
+
+  describe 'mixed operations' do
+    let(:mo) { p.expression }
+
+    describe 'inspection' do
+      it 'produces useful expressions' do
+        p.expression.parslet.inspect.should ==
+          'infix_expression(INTEGER, [MUL_OP, ADD_OP])'
+      end
+    end
+
+    describe 'right associativity' do
+      it 'produces trees that lean right' do
+        mo.should parse('1+2+3').as(
+          l: '1', o: '+', r: { l: '2', o: '+', r: '3' },
+        )
+      end
+    end
+
+    describe 'left associativity' do
+      it 'produces trees that lean left' do
+        mo.should parse('1*2*3').as(
+          l: { l: '1', o: '*', r: '2' }, o: '*', r: '3',
+        )
+      end
+    end
+
+    describe 'error handling' do
+      describe 'incomplete expression' do
+        it 'produces the right error' do
+          cause = catch_failed_parse do
+            mo.parse('1+')
+          end
+
+          cause.ascii_tree.to_s.should == <<~ERROR
+            INTEGER was expected at line 1 char 3.
+            `- Failed to match sequence (DIGIT{1, } SPACE{0, }) at line 1 char 3.
+               `- Expected at least 1 of DIGIT at line 1 char 3.
+                  `- Premature end of input at line 1 char 3.
+          ERROR
+        end
+      end
+
+      describe 'invalid operator' do
+        it 'produces the right error' do
+          cause = catch_failed_parse do
+            mo.parse('1%')
+          end
+
+          cause.ascii_tree.to_s.should == <<~ERROR
+            Don't know what to do with "%" at line 1 char 2.
+          ERROR
+        end
+      end
+    end
+  end
+
+  describe 'providing a reducer block' do
+    class InfixExpressionReducerParser < Parsanol::Parser
+      rule(:top) { infix_expression(str('a'), [str('-'), 1, :right]) { |l, _o, r| { and: [l, r] } } }
+    end
+
+    it 'applies the reducer' do
+      result = InfixExpressionReducerParser.new.top.parse('a-a-a')
+      strip_positions(result).should == { and: ['a', { and: %w[a a] }] }
+    end
+  end
+end