dolos 0.1.3 → 0.2.1

data/benchmarks/letter.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+require 'bundler/setup'
+require 'dolos'
+require 'dolos_common_parsers/common_parsers'
+require 'benchmark/ips'
+
+include Dolos
+
+# Include common parsers
+# In future this can be more structured, moved them to separate module to prevent breaking changes
+include Dolos::CommonParsers
+
+# Library usage example
+# Parse out a name and address from a letter
+# For higher difficulty, we will not split this into multiple lines, but instead parse it all at once
+letter = <<-LETTER
+        Mr. Vardeniui Pavardeniui
+        AB „Lietuvos Paštas“
+        Totorių g. 8
+        01121 Vilnius
+LETTER
+
+# Combine with 'or'
+honorific = c("Mr. ") | c("Mrs. ") | c("Ms. ")
+
+# Can be parsed any_char which will include needed letters
+# Or combine LT letters with latin alphabet
+alpha_with_lt = char_in("ąčęėįšųūžĄČĘĖĮŠŲŪŽ") | alpha
+
+# Capture all letters in a row and join them,
+# because they are captured as elements of array by each alpha_with_lt parser.
+first_name = alpha_with_lt.rep.map(&:join).capture!
+last_name = alpha_with_lt.rep.map(&:join).capture!
+
+# Combine first line parsers
+# Consume zero or more whitespace, after that honorific must follow and so on
+name_line = ws_rep0 & honorific & first_name & ws & last_name & eol
+
+# Next line is company info
+# We could choose to accept UAB and AB or just AB and etc.
+# 'c("AB")' is for case-sensitive string. 'string' can also be used
+company_type = c("AB")
+quote_open = c("„")
+quote_close = c("“")
+
+# Consume LT alphabet with whitespace
+company_name = (alpha_with_lt | ws).rep.map(&:join).capture!
+company_info = company_type & ws_rep0 & quote_open & company_name & quote_close
+second_line = ws_rep0 & company_info & eol
+
+# Address line
+# 'char_while' will consume characters while passed predicate is true
+# This could be an alternative to previous 'alpha_with_lt' approach
+# After that result is captured and mapped to hash
+# Mapping to hash so at the end its easy to tell tuples apart
+# Also while mapping, doing some cleaning with '.strip'
+street_name = char_while(->(char) { !char.match(/\d/) }).map { |s| { street: s.strip } }.capture!
+building = digits.map { |s| { building: s.strip } }.capture!
+address_line = ws_rep0 & street_name & building & eol
+
+# City line
+# All digits can be matched here or 'digits.rep(5)' could be used. Also joining with map.
+postcode = digits.map { |s| { postcode: s.strip } }.capture!
+city = alpha_with_lt.rep.map(&:join).map { |s| { city: s.strip } }.capture!
+city_line = ws_rep0 & postcode & ws & city & eol
+
+# Full letter parser which is combined from all previous parsers. All previous parsers can be ran separately.
+letter_parser = name_line & second_line & address_line & city_line
+result = letter_parser.run(letter)
+
+puts result.success?
+
+Benchmark.ips do |x|
+  x.report('letter benchmark') do
+    letter_parser.run(letter)
+  end
+  x.compare!
+end

data/{lib/example.rb → examples/letter.rb}

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
-require_relative 'dolos'
-require_relative 'dolos_common_parsers/common_parsers'
+require 'dolos'
+require 'dolos_common_parsers/common_parsers'
 
 include Dolos
 
@@ -27,12 +27,12 @@ alpha_with_lt = char_in("ąčęėįšųūžĄČĘĖĮŠŲŪŽ") | alpha
 
 # Capture all letters in a row and join them,
 # because they are captured as elements of array by each alpha_with_lt parser.
-first_name = alpha_with_lt.rep.capture!.map(&:join)
-last_name = alpha_with_lt.rep.capture!.map(&:join)
+first_name = alpha_with_lt.rep.map(&:join).capture!
+last_name = alpha_with_lt.rep.map(&:join).capture!
 
 # Combine first line parsers
 # Consume zero or more whitespace, after that honorific must follow and so on
-name_line = ws.rep0 >> honorific >> first_name >> ws >> last_name >> eol
+name_line = ws.rep0 & honorific & first_name & ws & last_name & eol
 
 # Next line is company info
 # We could choose to accept UAB and AB or just AB and etc.
@@ -42,9 +42,9 @@ quote_open = c("„")
 quote_close = c("“")
 
 # Consume LT alphabet with whitespace
-company_name = (alpha_with_lt | ws).rep.capture!.map(&:join)
-company_info = company_type >> ws.rep0 >> quote_open >> company_name >> quote_close
-second_line = ws.rep0 >> company_info >> eol
+company_name = (alpha_with_lt | ws).rep.map(&:join).capture!
+company_info = company_type & ws.rep0 & quote_open & company_name & quote_close
+second_line = ws.rep0 & company_info & eol
 
 # Address line
 # 'char_while' will consume characters while passed predicate is true
@@ -52,18 +52,18 @@ second_line = ws.rep0 >> company_info >> eol
 # After that result is captured and mapped to hash
 # Mapping to hash so at the end its easy to tell tuples apart
 # Also while mapping, doing some cleaning with '.strip'
-street_name = char_while(->(char) { !char.match(/\d/) }).capture!.map(&:first).map { |s| { street: s.strip } }
-building = digits.capture!.map(&:first).map { |s| { building: s.strip } }
-address_line = ws.rep0 >> street_name >> building >> eol
+street_name = char_while(->(char) { !char.match(/\d/) }).map { |s| { street: s.strip } }.capture!
+building = digits.map { |s| { building: s.strip } }.capture!
+address_line = ws.rep0 & street_name & building & eol
 
 # City line
 # All digits can be matched here or 'digits.rep(5)' could be used. Also joining with map.
-postcode = digits.capture!.map(&:join).map { |s| { postcode: s.strip } }
-city = alpha_with_lt.rep.capture!.map(&:join).map { |s| { city: s.strip } }
-city_line = ws.rep0 >> postcode >> ws >> city >> eol
+postcode = digits.map { |s| { postcode: s.strip } }.capture!
+city = alpha_with_lt.rep.map(&:join).map { |s| { city: s.strip } }.capture!
+city_line = ws.rep0 & postcode & ws & city & eol
 
 # Full letter parser which is combined from all previous parsers. All previous parsers can be ran separately.
-letter_parser = name_line >> second_line >> address_line >> city_line
+letter_parser = name_line & second_line & address_line & city_line
 result = letter_parser.run(letter)
 
 pp result.captures

data/lib/dolos/parsers.rb

@@ -3,9 +3,10 @@
 module Dolos
   module Parsers
     def string(str)
+      utf8_str = str.encode('UTF-8')
+
       Parser.new do |state|
         state.input.mark_offset
-        utf8_str = str.encode('UTF-8')
         if state.input.matches?(utf8_str)
           Success.new(utf8_str, str.bytesize)
         else
@@ -13,7 +14,7 @@ module Dolos
           got_error = state.input.io.string.byteslice(state.input.backup, advanced)
           state.input.rollback
           Failure.new(
-            "Expected #{str.inspect} but got #{got_error.inspect}",
+            -> { "Expected #{str.inspect} but got #{got_error.inspect}" },
             advanced,
             state
           )
@@ -32,7 +33,7 @@ module Dolos
           advanced = state.input.offset
           state.input.rollback
           Failure.new(
-            "Expected pattern #{pattern.inspect} but got #{state.input.io.string.inspect}",
+            -> { "Expected pattern #{pattern.inspect} but got #{state.input.io.string.inspect}" },
             advanced,
             state
           )
@@ -52,7 +53,7 @@ module Dolos
           advanced = state.input.offset
           state.input.rollback
           Failure.new(
-            'Expected any character but got end of input',
+            -> { 'Expected any character but got end of input' },
             advanced,
             state
           )
@@ -64,20 +65,20 @@ module Dolos
     # Example:
     #  char_in('abc').run('b') # => Success.new('b', 1)
     def char_in(characters_string)
-      characters_array = characters_string.chars
+      characters_set = characters_string.chars
 
       Parser.new do |state|
         state.input.mark_offset
 
         char, bytesize = state.input.peek(1)
 
-        if char && characters_array.include?(char)
+        if char && characters_set.include?(char)
           Success.new(char, bytesize)
         else
           advanced = state.input.offset
           state.input.rollback
           Failure.new(
-            "Expected one of #{characters_array.inspect} but got #{char.inspect}",
+            -> { "Expected one of #{characters_set.to_a.inspect} but got #{char.inspect}" },
             advanced,
             state
           )
@@ -90,18 +91,18 @@ module Dolos
         state.input.mark_offset
 
         buffer = String.new
-        loop do
-          char, bytesize = state.input.peek(1)
-          break if char.nil? || !predicate.call(char)
+        char, bytesize = state.input.peek(1)
 
+        while char && predicate.call(char)
           buffer << char
           state.input.advance(bytesize)
+          char, bytesize = state.input.peek(1)
         end
 
         if buffer.empty?
           advanced = state.input.offset
           Failure.new(
-            "Predicate never returned true",
+            -> { "Predicate never returned true" },
             advanced,
             state
           )
@@ -111,5 +112,23 @@ module Dolos
       end
     end
 
+    def recursive(&block)
+      recursive_parser = nil
+
+      placeholder = Parser.new do |state|
+        raise "Recursive parser accessed before it was initialized!" if recursive_parser.nil?
+
+        recursive_parser.call.run_with_state(state).tap do |result|
+          if result.failure?
+            error_msg = -> { "Error in recursive structure around position #{state.input.offset}: #{result.message}" }
+            Failure.new(error_msg, state.input.offset, state)
+          end
+        end
+      end
+
+      recursive_parser = -> { block.call(placeholder) }
+      placeholder
+    end
+
   end
 end

data/lib/dolos/result.rb

@@ -10,20 +10,21 @@ module Dolos
     def initialize(value, length, captures = [])
       @value = value
       @length = length
-      # @captures = captures || value
       @captures = captures
     end
 
-    def capture!
-      if value.is_a?(Array)
-        value.each do |v|
-          captures << v
-        end
+    # can be some named capture, :street, {:street => capture }
+    # or an array, [], [capture]
+    def capture!(wrap_in = nil)
+      mapped_value = self.value  # use the transformed value here
+
+      if wrap_in.is_a?(Array)
+        save_capture([mapped_value])
+      elsif wrap_in.is_a?(Symbol)
+        save_capture({ wrap_in => mapped_value })
       else
-        captures << value
+        save_capture(mapped_value)
       end
-
-      Success.new(value, length, captures)
     end
 
     def inspect
@@ -37,15 +38,38 @@ module Dolos
     def failure?
       false
     end
+
+    private
+
+    def save_capture(val)
+      if val.is_a?(Array)
+        val.each do |v|
+          captures << v
+        end
+      else
+        captures << val
+      end
+
+      Success.new(val, length, captures)
+    end
   end
 
   class Failure < Result
-    attr_reader :message, :error_position, :state
+    attr_reader  :error_position, :state
 
-    def initialize(message, error_position, state)
-      @message = message
+    def initialize(message_proc, error_position, state)
+      @message_proc = message_proc
       @error_position = error_position
       @state = state
+      @message_evaluated = false
+    end
+
+    def message
+      unless @message_evaluated
+        @message_value = @message_proc.call
+        @message_evaluated = true
+      end
+      @message_value
     end
 
     def inspect

data/lib/dolos/string_io_wrapper.rb

@@ -22,12 +22,7 @@ module Dolos
 
     def matches?(utf8_str)
       read = io.read(utf8_str.bytesize)
-
-      if read.nil?
-        false
-      else
-        read.force_encoding('UTF-8') == utf8_str
-      end
+      !read.nil? && read.force_encoding('UTF-8') == utf8_str
     end
 
     def advance(bytesize)
@@ -61,8 +56,8 @@ module Dolos
       remaining_data = io.read
       io.seek(current_position)
 
-      if (match_data = remaining_data.match(/\A#{pattern}/))
-        matched_string = match_data[0]
+      if remaining_data =~ /\A#{pattern}/
+        matched_string = $&
         io.seek(current_position + matched_string.bytesize)
         return matched_string
       end

data/lib/dolos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Dolos
-  VERSION = "0.1.3"
+  VERSION = "0.2.1"
 end

data/lib/dolos.rb

@@ -10,9 +10,7 @@ module Dolos
   include Parsers
 
   class Parser
-
     attr_accessor :parser_proc
-
     def initialize(&block)
       @parser_proc = block
     end
@@ -22,54 +20,42 @@ module Dolos
     end
 
     def run_with_state(state)
-      result = parser_proc.call(state)
-      if result.success?
-        state.last_success_position = state.input.offset
-      end
+      result = @parser_proc.call(state)
+      state.last_success_position = state.input.offset if result.success?
       result
     end
 
-    def capture!
+    def capture!(wrap_in = nil)
       Parser.new do |state|
         result = run_with_state(state)
-        if result.success?
-          result.capture!
-        else
-          result
-        end
+        result.success? ? result.capture!(wrap_in) : result
       end
     end
 
-    def map(&block)
+    # Will call `map` on captures
+    def map_captures(&block)
       Parser.new do |state|
         result = run_with_state(state)
-        if result.success?
-          Success.new(result.value, result.length, block.call(result.captures))
-        else
-          result
-        end
+        result.success? ? Success.new(result.value, result.length, block.call(result.captures)) : result
       end
     end
 
-    def map_value(&block)
+    # Will call block on tuple of value
+    def map(&block)
       Parser.new do |state|
         result = run_with_state(state)
-        if result.success?
-          Success.new(block.call(result.value), result.length, result.captures)
-        else
-          result
-        end
+        result.success? ? Success.new(block.call(result.value), result.length, result.captures) : result
       end
     end
 
-    def flat_map(&block)
+    def combine(&block)
       Parser.new do |state|
         result = run_with_state(state)
+
         if result.success?
+          state.input.advance(result.length)
           new_parser = block.call(result.value, result.captures)
-          new_state = state.dup
-          new_state.input.advance(result.length)
-          new_parser.run_with_state(new_state)
+          new_parser.run_with_state(state)
         else
           result
         end
@@ -77,22 +63,44 @@ module Dolos
     end
 
     def flatten
-      map do |captures|
+      map_captures do |captures|
         captures.flatten
       end
     end
 
     def product(other_parser)
-      flat_map do |value1, capture1|
-        other_parser.map_value do |value2|
+      combine do |value1, capture1|
+        other_parser.map do |value2|
           [value1, value2]
-        end.map do |capture2|
+        end.map_captures do |capture2|
           [capture1, capture2].flatten
         end
       end
     end
+    alias_method :&, :product
 
-    alias_method :>>, :product
+    def product_l(other_parser)
+      combine do |value1, capture1|
+        other_parser.map do |_|
+          value1
+        end.map_captures do |capture2|
+          [capture1, capture2].flatten
+        end
+      end
+    end
+
+    def product_r(other_parser)
+      combine do |_, capture1|
+        other_parser.map do |value2|
+          value2
+        end.map_captures do |capture2|
+          [capture1, capture2].flatten
+        end
+      end
+    end
+
+    alias_method :<<, :product_l
+    alias_method :>>, :product_r
 
     def choice(other_parser)
       Parser.new do |state|
@@ -111,29 +119,36 @@ module Dolos
     # rep(n = 2)   # exactly 2
     # repeat(n_min: 2, n_max: 4) # 2 to 4
     # repeat(n_min: 2) # 2 or more
-    def repeat(n_min:, n_max: Float::INFINITY)
+    def repeat(n_min:, n_max: Float::INFINITY, separator: nil)
       Parser.new do |state|
         values = []
         captures = []
         count = 0
-        state.input.mark_offset
 
-        while count < n_max
-          result = run_with_state(state.dup)
+        loop do
+          result = run_with_state(state) # Removing .dup for performance. Be cautious of side effects.
 
-          break if result.failure?
+          if result.failure? || count >= n_max
+            break
+          end
 
           values << result.value
           captures.concat(result.captures)
           state.input.advance(result.length)
           count += 1
+
+          if separator && count < n_max
+            sep_result = separator.run_with_state(state) # Removing .dup for performance. Be cautious of side effects.
+            break if sep_result.failure?
+
+            state.input.advance(sep_result.length)
+          end
         end
 
         if count < n_min
-          error_pos = state.input.offset
           Failure.new(
-            "Expected parser to match at least #{n_min} times but matched only #{count} times",
-            error_pos,
+            -> { "Expected parser to match at least #{n_min} times but matched only #{count} times" },
+            state.input.offset,
             state
           )
         else
@@ -141,7 +156,6 @@ module Dolos
         end
       end
     end
-
     def zero_or_more
       repeat(n_min: 0, n_max: Float::INFINITY)
     end
@@ -168,5 +182,15 @@ module Dolos
     end
     alias_method :opt, :optional
 
+    # Used to declare lazy parser to avoid infinite loops in recursive parsers
+    def lazy
+      parser_memo = nil
+
+      Parser.new do |state|
+        parser_memo ||= self
+        parser_memo.run_with_state(state)
+      end
+    end
+
   end
 end

data/lib/dolos_common_parsers/common_parsers.rb

@@ -6,13 +6,20 @@ module Dolos
       regex(/\s/)
     end
 
+    def ws_rep0
+      regex(/\s*/)
+    end
+
     def eol
       regex(/\n|\r\n|\r/)
     end
 
-    # Capture as String and convert to integer
     def digit
-      regex(/\d/).capture!.map { |capt| capt.map(&:to_i) }
+      regex(/\d/)
+    end
+
+    def int
+      digit.map(&:to_i)
     end
 
     # Capture as string

data/sig/dolos/common_parsers.rbs

@@ -1,5 +1,16 @@
 module Dolos
   module CommonParsers
+    def digit: -> Parser[String]
+    def digits: -> Parser[String]
+
+    def int: -> Parser[Integer]
+
+    def eol: -> Parser[String]
+
     def ws: -> Parser[String]
+    def ws_rep0: -> Parser[String]
+
+    def alpha: -> Parser[String]
+    def alphanum: -> Parser[String]
   end
 end

data/sig/dolos/parser.rbs

@@ -4,16 +4,20 @@ module Dolos
     def initialize: (^(ParserState) -> Result[A]) -> Parser[A]
     def capture!: -> Parser[A]
     def choice: [B](Parser[B])-> Parser[A | B]
+    def combine: [B](^(A, B) -> Parser[B]) -> Parser[B]
     def flat_map: [B](Parser[A], ^(A) -> Parser[B]) -> Parser[B]
     def flatten: -> Parser[A]
     def map: [B](^(A) -> B) -> Parser[B]
-    def map_value: [B](^(A) -> B) -> Parser[B]
+    def map_captures: [B](^(A) -> B) -> Parser[B]
     def optional: -> Parser[A?]
     def product: [B](Parser[A]) -> Parser[B]
+    def product_l: [B](Parser[B]) -> Parser[B]
+    def product_r: [B](Parser[B]) -> Parser[A]
     def run: (String) -> Result[A]
     def run_with_state: (ParserState) -> Result[A]
-    def repeat: (Integer, Integer)-> Parser[Array[A]]
+    def repeat: [B](Integer, Integer, Parser[B]?)-> Parser[Array[A]]
     def zero_or_more: -> Parser[Array[A]]
     def one_or_more: (Integer?) -> Parser[Array[A]]
+    def lazy: -> Parser[A]
   end
 end

data/sig/dolos/parser_state.rbs

@@ -1,6 +1,6 @@
 module Dolos
   class ParserState
-    attr_reader input: Dolos::StringIOWrapper
+    attr_reader input: StringIOWrapper
     attr_accessor last_success_position: Integer
 
     def initialize: (String) -> void

data/sig/dolos/parsers.rbs

@@ -1,6 +1,10 @@
 module Dolos
   module Parsers
     def any_char: -> Parser[String]
+    def char_in: -> Parser[String]
+    def char_while : -> Parser[String]
+    def recursive: [A,B,C]() { (Parser[A]) -> Parser[B] } -> Parser[C]
+
     def regex: (Regexp) -> Parser[String]
     def string: (String)-> Parser[String]
   end

data/sig/dolos/result.rbs

@@ -15,6 +15,11 @@ module Dolos
   end
 
   class Failure < Result[bot]
+    @message_proc: ^-> String
+    @message_evaluated: bool
+    @message_value: String
+    @state: ParserState
+
     attr_reader committed: bool
     attr_reader error_position: Integer
     attr_reader message: String
@@ -25,6 +30,8 @@ module Dolos
 
     def map: [B](^(bot) -> B) -> Result[B]
 
+    def pretty_print: -> String
+
     def success?: -> bool
   end
 end