parsanol 3.0.0

data/lib/parsanol/error_reporter/contextual.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module ErrorReporter
+
+    # A reporter that tries to improve on the deepest error reporter by
+    # using heuristics to find the most relevant error and provide more
+    # context.
+    # The heuristic chooses the deepest error when parsing a sequence for which
+    # no alternative parsed successfully.
+    #
+    # Given the following parser:
+    #
+    # root(:call)
+    #
+    # rule(:call, label: 'call') {
+    #   identifier >> str('.') >> method
+    # }
+    #
+    # rule(:method, label: 'method call') {
+    #   identifier >> str('(') >> arguments.maybe >> str(')')
+    # }
+    #
+    # rule(:identifier, label: 'identifier') {
+    #   match['[:alnum:]'].repeat(1)
+    # }
+    #
+    # rule(:arguments, label: 'method call arguments') {
+    #   argument >> str(',') >> arguments | argument
+    # }
+    #
+    # rule(:argument) {
+    #    call | identifier
+    # }
+    #
+    # and the following source:
+    #
+    #   foo.bar(a,goo.baz(),c,)
+    #
+    # The contextual reporter returns the following causes:
+    #
+    # 0: Failed to match sequence (identifier '.' method call) at line 1 char 5
+    #    when parsing method call arguments.
+    # 1: Failed to match sequence (identifier '(' method call arguments? ')') at
+    #    line 1 char 22 when parsing method call arguments.
+    # 2: Failed to match [[:alnum:]] at line 1 char 23 when parsing method call
+    #    arguments.
+    #
+    # (where 2 is a child cause of 1 and 1 a child cause of 0)
+    #
+    # The last piece used by the reporter is the (newly introduced) ability
+    # to attach a label to rules that describe a sequence in the grammar. The
+    # labels are used in two places:
+    #   - In the "to_s" of Atom::Base so that any error message uses labels to
+    #     refer to atoms
+    #   - In the cause error messages to give information about which expression
+    #     failed to parse
+    #
+    class Contextual < Deepest
+
+      def initialize
+        @last_reset_pos = 0
+        reset
+      end
+
+      # A sequence expression successfully parsed, reset all errors reported
+      # for previous expressions in the sequence (an alternative matched)
+      # Only reset errors if the position of the source that matched is higher
+      # than the position of the source that was last successful (so we keep
+      # errors that are the "deepest" but for which no alternative succeeded)
+      #
+      def succ(source)
+        source_pos = source.pos.bytepos
+        return if source_pos < @last_reset_pos
+        @last_reset_pos = source_pos
+        reset
+      end
+
+      # Reset deepest error and its position and sequence index
+      #
+      def reset
+        @deepest_cause = nil
+        @label_pos = -1
+      end
+
+      # Produces an error cause that combines the message at the current level
+      # with the errors that happened at a level below (children).
+      # Compute and set label used by Cause to produce error message.
+      #
+      # @param atom [Parsanol::Atoms::Base] parslet that failed
+      # @param source [Source] Source that we're using for this parse. (line
+      #   number information...)
+      # @param message [String, Array] Error message at this level.
+      # @param children [Array] A list of errors from a deeper level (or nil).
+      # @return [Cause] An error tree combining children with message.
+      #
+      def err(atom, source, message, children=nil)
+        cause = super(atom, source, message, children)
+        if (label = atom.respond_to?(:label) && atom.label)
+          update_label(label, source.pos.bytepos)
+          cause.set_label(@label)
+        end
+        cause
+      end
+
+      # Update error message label if given label is more relevant.
+      # A label is more relevant if the position of the matched source is
+      # bigger.
+      #
+      # @param label [String] label to apply if more relevant
+      # @param bytepos [Integer] position in source code of matched source
+      #
+      def update_label(label, bytepos)
+        if bytepos >= @label_pos
+          @label_pos = bytepos
+          @label = label
+        end
+      end
+
+    end
+  end
+end

data/lib/parsanol/error_reporter/deepest.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module ErrorReporter
+    # Instead of reporting the latest error that happens like {Tree} does,
+    # this class reports the deepest error. Depth is defined here as how
+    # advanced into the input an error happens. The errors close to the
+    # greatest depth tend to be more relevant to the end user, since they
+    # specify what could be done to make them go away.
+    #
+    # More specifically, errors produced by this reporter won't be related to
+    # the structure of the grammar at all. The positions of the errors will
+    # be advanced and convey at every grammar level what the deepest rule
+    # was to fail.
+    #
+    # @example Using Deepest reporter
+    #   parser.parse(input, reporter: Parsanol::ErrorReporter::Deepest.new)
+    #
+    class Deepest < Base
+      def initialize
+        @deepest_cause = nil
+      end
+
+      # Produces an error cause that combines the message at the current level
+      # with the errors that happened at a level below (children).
+      #
+      # @param atom [Parsanol::Atoms::Base] parslet that failed
+      # @param source [Source] Source that we're using for this parse. (line
+      #   number information...)
+      # @param message [String, Array] Error message at this level.
+      # @param children [Array] A list of errors from a deeper level (or nil).
+      # @return [Cause] An error tree combining children with message.
+      #
+      def err(atom, source, message, children = nil)
+        position = source.pos
+        cause = Cause.format(source, position, message, children)
+        return deepest(cause)
+      end
+
+      # Produces an error cause that combines the message at the current level
+      # with the errors that happened at a level below (children).
+      #
+      # @param atom [Parsanol::Atoms::Base] parslet that failed
+      # @param source [Source] Source that we're using for this parse. (line
+      #   number information...)
+      # @param message [String, Array] Error message at this level.
+      # @param pos [Fixnum] The real position of the error.
+      # @param children [Array] A list of errors from a deeper level (or nil).
+      # @return [Cause] An error tree combining children with message.
+      #
+      def err_at(atom, source, message, pos, children = nil)
+        position = pos
+        cause = Cause.format(source, position, message, children)
+        return deepest(cause)
+      end
+
+      # Returns the cause that is currently deepest. Mainly for specs.
+
+      # Notification that an expression successfully parsed
+      # not used, see ErrorReporter::Contextual
+      def succ(source)
+        # No-op for Deepest reporter
+      end
+      #
+      attr_reader :deepest_cause
+
+      # Checks to see if the lineage of the cause given includes a cause with
+      # an error position deeper than the current deepest cause stored. If
+      # yes, it passes the cause through to the caller. If no, it returns the
+      # current deepest error that was saved as a reference.
+      #
+      def deepest(cause)
+        _, leaf = deepest_child(cause)
+
+        if !deepest_cause || leaf.pos >= deepest_cause.pos
+          # This error reaches deeper into the input, save it as reference.
+          @deepest_cause = leaf
+          return cause
+        end
+
+        return deepest_cause
+      end
+
+    private
+      # Returns the leaf from a given error tree with the biggest rank.
+      #
+      def deepest_child(cause, rank = 0)
+        max_child = cause
+        max_rank  = rank
+
+        if cause.children && !cause.children.empty?
+          cause.children.each do |child|
+            c_rank, c_cause = deepest_child(child, rank + 1)
+
+            if c_rank > max_rank
+              max_rank = c_rank
+              max_child = c_cause
+            end
+          end
+        end
+
+        return max_rank, max_child
+      end
+    end
+  end
+end

data/lib/parsanol/error_reporter/tree.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module ErrorReporter
+    # An error reporter has two central methods, one for reporting errors at
+    # the current parse position (#err) and one for reporting errors at a
+    # given parse position (#err_at). The reporter can return an object (a
+    # 'cause') that will be returned to the caller along with the information
+    # that the parse failed.
+    #
+    # When reporting errors on the outer levels of your parser, these methods
+    # get passed a list of error objects ('causes') from the inner levels. In
+    # this default implementation, the inner levels are considered error
+    # subtrees and are appended to the generated tree node at each level,
+    # thereby constructing an error tree.
+    #
+    # This error tree will report in parallel with the grammar structure that
+    # failed. A one-to-one correspondence exists between each error in the
+    # tree and the parslet atom that produced that error.
+    #
+    # The implementor is really free to use these return values as he sees
+    # fit. One example would be to return an error state object from these
+    # methods that is then updated as errors cascade up the parse derivation
+    # tree.
+    #
+    # @example Using Tree reporter
+    #   parser.parse(input, reporter: Parsanol::ErrorReporter::Tree.new)
+    #
+    class Tree < Base
+      # Produces an error cause that combines the message at the current level
+      # with the errors that happened at a level below (children).
+      #
+      # @param atom [Parsanol::Atoms::Base] parslet that failed
+      # @param source [Source] Source that we're using for this parse. (line
+      #   number information...)
+      # @param message [String, Array] Error message at this level.
+      # @param children [Array] A list of errors from a deeper level (or nil).
+      # @return [Cause] An error tree combining children with message.
+      #
+      def err(atom, source, message, children = nil)
+        position = source.pos
+        Cause.format(source, position, message, children)
+      end
+
+      # Produces an error cause that combines the message at the current level
+      # with the errors that happened at a level below (children).
+      #
+      # @param atom [Parsanol::Atoms::Base] parslet that failed
+      # @param source [Source] Source that we're using for this parse. (line
+      #   number information...)
+      # @param message [String, Array] Error message at this level.
+      # @param pos [Fixnum] The real position of the error.
+      # @param children [Array] A list of errors from a deeper level (or nil).
+      # @return [Cause] An error tree combining children with message.
+      #
+      def err_at(atom, source, message, pos, children = nil)
+        position = pos
+        Cause.format(source, position, message, children)
+      end
+
+      # Notification that an expression successfully parsed
+      # not used, see ErrorReporter::Contextual
+      def succ(source)
+        # No-op for Tree reporter
+      end
+    end
+  end
+end

data/lib/parsanol/error_reporter.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# A namespace for all error reporters.
+#
+# Error reporters collect and format parse errors. The parsing engine
+# calls reporter methods as it attempts to match atoms, building up
+# an error structure that can be presented to the user.
+#
+# @example Using a specific error reporter
+#   parser = MyParser.new
+#   parser.parse(input, reporter: Parsanol::ErrorReporter::Deepest.new)
+#
+# @example Creating a custom error reporter
+#   class MyReporter < Parsanol::ErrorReporter::Base
+#     def initialize
+#       @errors = []
+#     end
+#
+#     def err(atom, source, message, children = nil)
+#       @errors << { position: source.pos, message: message }
+#       @errors.last
+#     end
+#
+#     def err_at(atom, source, message, pos, children = nil)
+#       @errors << { position: pos, message: message }
+#       @errors.last
+#     end
+#   end
+#
+module Parsanol::ErrorReporter
+  # Base class for error reporters.
+  #
+  # Error reporters collect and format parse errors. The parsing engine
+  # calls reporter methods as it attempts to match atoms, building up
+  # an error structure that can be presented to the user.
+  #
+  # Subclasses must implement {#err} and {#err_at} methods.
+  #
+  class Base
+    # Report an error at the current parse position.
+    #
+    # @param atom [Parsanol::Atoms::Base] The atom that failed to match
+    # @param source [Parsanol::Source] The input source
+    # @param message [String, Array<String>] Error message(s)
+    # @param children [Array<Cause>, nil] Child errors from deeper levels
+    # @return [Object] An error cause object (implementation-specific)
+    #
+    # @abstract Subclasses must implement this method
+    #
+    def err(atom, source, message, children = nil)
+      raise NotImplementedError,
+        "Error reporters must implement #err(atom, source, message, children)"
+    end
+
+    # Report an error at a specific position.
+    #
+    # @param atom [Parsanol::Atoms::Base] The atom that failed to match
+    # @param source [Parsanol::Source] The input source
+    # @param message [String, Array<String>] Error message(s)
+    # @param pos [Integer] The byte position of the error
+    # @param children [Array<Cause>, nil] Child errors from deeper levels
+    # @return [Object] An error cause object (implementation-specific)
+    #
+    # @abstract Subclasses must implement this method
+    #
+    def err_at(atom, source, message, pos, children = nil)
+      raise NotImplementedError,
+        "Error reporters must implement #err_at(atom, source, message, pos, children)"
+    end
+
+    # Called when an expression successfully parses.
+    #
+    # This method allows reporters to track successful parses for
+    # better error context. The default implementation does nothing.
+    #
+    # @param source [Parsanol::Source] The input source at success position
+    # @return [void]
+    #
+    def succ(source)
+      # Default: no-op
+    end
+
+    # Called after parse completes for finalization.
+    #
+    # Override this method to perform cleanup or generate final reports.
+    # The default implementation does nothing.
+    #
+    # @return [void]
+    #
+    def finalize
+      # Default: no-op
+    end
+  end
+end
+
+require 'parsanol/error_reporter/tree'
+require 'parsanol/error_reporter/deepest'
+require 'parsanol/error_reporter/contextual'

data/lib/parsanol/export.rb

@@ -0,0 +1,163 @@
+# Allows exporting parslet grammars to other lingos. 
+
+require 'set'
+require 'parsanol/atoms/visitor'
+
+class Parsanol::Parser
+  module Visitors
+    class Citrus
+      attr_reader :context, :output
+      def initialize(context)
+        @context = context
+      end
+      
+      def visit_str(str)
+        "\"#{str.inspect[1..-2]}\""
+      end
+      def visit_re(match)
+        match.to_s
+      end
+
+      def visit_entity(name, block)
+        context.deferred(name, block)
+
+        "(#{context.mangle_name(name)})"
+      end
+      def visit_named(name, parslet)
+        parslet.accept(self)
+      end
+
+      def visit_sequence(parslets)
+        '(' +
+        parslets.
+          map { |el| el.accept(self) }.
+          join(' ') +
+        ')'
+      end
+      def visit_repetition(tag, min, max, parslet)
+        parslet.accept(self) + "#{min}*#{max}"
+      end
+      def visit_alternative(alternatives)
+        '(' +
+        alternatives.
+          map { |el| el.accept(self) }.
+          join(' | ') +
+        ')'
+      end
+
+      def visit_lookahead(positive, bound_parslet)
+        (positive ? '&' : '!') +
+        bound_parslet.accept(self)
+      end
+    end
+
+    class Treetop < Citrus
+      def visit_repetition(tag, min, max, parslet)
+        parslet.accept(self) + "#{min}..#{max}"
+      end
+
+      def visit_alternative(alternatives)
+        '(' +
+        alternatives.
+          map { |el| el.accept(self) }.
+          join(' / ') +
+        ')'
+      end
+    end
+  end
+
+  # A helper class that formats Citrus and Treetop grammars as a string. 
+  #
+  class PrettyPrinter
+    attr_reader :visitor
+    def initialize(visitor_klass)
+      @visitor = visitor_klass.new(self)
+    end
+
+    # Pretty prints the given parslet using the visitor that has been
+    # configured in initialize. Returns the string representation of the
+    # Citrus or Treetop grammar.
+    #
+    def pretty_print(name, parslet)
+      output = ["grammar #{name}\n"]
+      
+      output << rule('root', parslet)
+      
+      seen = Set.new
+      loop do
+        # @todo is constantly filled by the visitor (see #deferred). We 
+        # keep going until it is empty.
+        break if @todo.empty?
+        name, block = @todo.shift
+
+        # Track what rules we've already seen. This breaks loops.
+        next if seen.include?(name)
+        seen << name
+
+        output << rule(name, block.call)
+      end
+      
+      output << "end\n"
+      output.join
+    end
+    
+    # Formats a rule in either dialect. 
+    #
+    def rule(name, parslet)
+      "  rule #{mangle_name name}\n" +
+      "    " + parslet.accept(visitor) + "\n" +
+      "  end\n"
+    end
+    
+    # Whenever the visitor encounters an rule in a parslet, it defers the
+    # pretty printing of the rule by calling this method. 
+    #
+    def deferred(name, content)
+      @todo ||= []
+      @todo << [name, content]
+    end
+
+    # Mangles names so that Citrus and Treetop can live with it. This mostly
+    # transforms some of the things that Ruby allows into other patterns. If
+    # there is collision, we will not detect it for now. 
+    #
+    def mangle_name(str)
+      str.to_s.sub(/\?$/, '_p')
+    end
+  end
+
+  # Exports the current parser instance as a string in the Citrus dialect. 
+  #
+  # Example: 
+  #
+  #   require 'parsanol/export'
+  #   class MyParser < Parsanol::Parser
+  #     root(:expression)
+  #     rule(:expression) { str('foo') }
+  #   end
+  #   
+  #   MyParser.new.to_citrus # => a citrus grammar as a string
+  #
+  def to_citrus
+    PrettyPrinter.new(Visitors::Citrus).
+      pretty_print(self.class.name, root)
+  end
+
+  # Exports the current parser instance as a string in the Treetop dialect. 
+  #
+  # Example: 
+  #
+  #   require 'parsanol/export'
+  #   class MyParser < Parsanol::Parser
+  #     root(:expression)
+  #     rule(:expression) { str('foo') }
+  #   end
+  #   
+  #   MyParser.new.to_treetop # => a treetop grammar as a string
+  #
+  def to_treetop
+    PrettyPrinter.new(Visitors::Treetop).
+      pretty_print(self.class.name, root)
+  end
+end
+

data/lib/parsanol/expression/treetop.rb

@@ -0,0 +1,94 @@
+require 'parsanol'
+
+class Parsanol::Expression::Treetop
+  class Parser < Parsanol::Parser
+    root(:expression)
+    
+    rule(:expression) { alternatives }
+    
+    # alternative 'a' / 'b'
+    rule(:alternatives) {
+      (simple >> (spaced('/') >> simple).repeat).as(:alt)
+    }
+    
+    # sequence by simple concatenation 'a' 'b'
+    rule(:simple) { occurrence.repeat(1).as(:seq) }
+    
+    # occurrence modifiers
+    rule(:occurrence) {
+      atom.as(:repetition) >> spaced('*').as(:sign) |
+      atom.as(:repetition) >> spaced('+').as(:sign) |
+      atom.as(:repetition) >> repetition_spec |
+      
+      atom.as(:maybe) >> spaced('?') | 
+      atom
+    }
+        
+    rule(:atom) { 
+      spaced('(') >> expression.as(:unwrap) >> spaced(')') |
+      dot |
+      string |
+      char_class
+    }
+    
+    # a character class
+    rule(:char_class) {
+      (str('[') >>
+        (str('\\') >> any |
+        str(']').absent? >> any).repeat(1) >>
+      str(']')).as(:match) >> space?
+    }
+    
+    # anything at all
+    rule(:dot) { spaced('.').as(:any) }
+    
+    # recognizing strings
+    rule(:string) {
+      str('\'') >> 
+      (
+        (str('\\') >> any) |
+        (str("'").absent? >> any)
+      ).repeat.as(:string) >> 
+      str('\'') >> space?
+    }
+    
+    # repetition specification like {1, 2}
+    rule(:repetition_spec) {
+      spaced('{') >> 
+        integer.maybe.as(:min) >> spaced(',') >> 
+        integer.maybe.as(:max) >> spaced('}')
+    }
+    rule(:integer) {
+      match['0-9'].repeat(1)
+    }
+    
+    # whitespace handling
+    rule(:space) { match("\s").repeat(1) }
+    rule(:space?) { space.maybe }
+    
+    def spaced(str)
+      str(str) >> space?
+    end
+  end
+  
+  class Transform < Parsanol::Transform
+    
+    rule(:repetition => simple(:rep), :sign => simple(:sign)) { 
+      min = sign=='+' ? 1 : 0
+      Parsanol::Atoms::Repetition.new(rep, min, nil) }
+    rule(:repetition => simple(:rep), :min => simple(:min), :max => simple(:max)) { 
+      Parsanol::Atoms::Repetition.new(rep, 
+        Integer(min || 0), 
+        max && Integer(max) || nil) }
+      
+    rule(:alt => subtree(:alt))       { Parsanol::Atoms::Alternative.new(*alt) }
+    rule(:seq => sequence(:s))        { Parsanol::Atoms::Sequence.new(*s) }
+    rule(:unwrap => simple(:u))       { u }
+    rule(:maybe => simple(:m))        { |d| d[:m].maybe }
+    rule(:string => simple(:s))       { Parsanol::Atoms::Str.new(s) }
+    rule(:match => simple(:m))        { Parsanol::Atoms::Re.new(m) }
+    rule(:any => simple(:a))          { Parsanol::Atoms::Re.new('.') }
+  end
+  
+end
+

data/lib/parsanol/expression.rb

@@ -0,0 +1,51 @@
+
+# Allows specifying rules as strings using the exact same grammar that treetop
+# does, minus the actions. This is on one hand a good example of a fully
+# fledged parser and on the other hand might even turn out really useful. 
+#
+# This can be viewed as an extension to parslet and might even be hosted in
+# its own gem one fine day. 
+# 
+class Parsanol::Expression
+  include Parsanol
+  
+  autoload :Treetop, 'parsanol/expression/treetop'
+  
+  # Creates a parslet from a foreign language expression. 
+  #
+  # Example: 
+  #   
+  #   Parsanol::Expression.new("'a' 'b'")
+  #
+  def initialize(str, opts={}, context=self)
+    @type = opts[:type] || :treetop
+    @exp = str
+    @parslet = transform(
+      parse(str))
+  end
+  
+  # Transforms the parse tree into a parslet expression. 
+  #
+  def transform(tree)
+    transform = Treetop::Transform.new
+    
+    # pp tree
+    transform.apply(tree)
+  rescue 
+    warn "Could not transform: " + tree.inspect
+    raise
+  end
+  
+  # Parses the string and returns a parse tree.
+  #
+  def parse(str)
+    parser = Treetop::Parser.new
+    parser.parse(str)
+  end
+
+  # Turns this expression into a parslet.
+  #
+  def to_parslet
+    @parslet
+  end
+end