dolos 0.2.0 → 0.3.0

data/benchmarks/letter.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+require 'bundler/setup'
+require 'dolos'
+require 'dolos_common_parsers/common_parsers'
+require 'benchmark/ips'
+
+include Dolos
+
+# Include common parsers
+# In future this can be more structured, moved them to separate module to prevent breaking changes
+include Dolos::CommonParsers
+
+# Library usage example
+# Parse out a name and address from a letter
+# For higher difficulty, we will not split this into multiple lines, but instead parse it all at once
+letter = <<-LETTER
+        Mr. Vardeniui Pavardeniui
+        AB „Lietuvos Paštas“
+        Totorių g. 8
+        01121 Vilnius
+LETTER
+
+# Combine with 'or'
+honorific = c("Mr. ") | c("Mrs. ") | c("Ms. ")
+
+# Can be parsed any_char which will include needed letters
+# Or combine LT letters with latin alphabet
+alpha_with_lt = char_in("ąčęėįšųūžĄČĘĖĮŠŲŪŽ") | alpha
+
+# Capture all letters in a row and join them,
+# because they are captured as elements of array by each alpha_with_lt parser.
+first_name = alpha_with_lt.rep.map(&:join).capture!
+last_name = alpha_with_lt.rep.map(&:join).capture!
+
+# Combine first line parsers
+# Consume zero or more whitespace, after that honorific must follow and so on
+name_line = ws_rep0 & honorific & first_name & ws & last_name & eol
+
+# Next line is company info
+# We could choose to accept UAB and AB or just AB and etc.
+# 'c("AB")' is for case-sensitive string. 'string' can also be used
+company_type = c("AB")
+quote_open = c("„")
+quote_close = c("“")
+
+# Consume LT alphabet with whitespace
+company_name = (alpha_with_lt | ws).rep.map(&:join).capture!
+company_info = company_type & ws_rep0 & quote_open & company_name & quote_close
+second_line = ws_rep0 & company_info & eol
+
+# Address line
+# 'char_while' will consume characters while passed predicate is true
+# This could be an alternative to previous 'alpha_with_lt' approach
+# After that result is captured and mapped to hash
+# Mapping to hash so at the end its easy to tell tuples apart
+# Also while mapping, doing some cleaning with '.strip'
+street_name = char_while(->(char) { !char.match(/\d/) }).map { |s| { street: s.strip } }.capture!
+building = digits.map { |s| { building: s.strip } }.capture!
+address_line = ws_rep0 & street_name & building & eol
+
+# City line
+# All digits can be matched here or 'digits.rep(5)' could be used. Also joining with map.
+postcode = digits.map { |s| { postcode: s.strip } }.capture!
+city = alpha_with_lt.rep.map(&:join).map { |s| { city: s.strip } }.capture!
+city_line = ws_rep0 & postcode & ws & city & eol
+
+# Full letter parser which is combined from all previous parsers. All previous parsers can be ran separately.
+letter_parser = name_line & second_line & address_line & city_line
+result = letter_parser.run(letter)
+
+puts result.success?
+
+Benchmark.ips do |x|
+  x.report('letter benchmark') do
+    letter_parser.run(letter)
+  end
+  x.compare!
+end

data/docs/README.md

@@ -0,0 +1,22 @@
+# Dolos
+
+## What is Dolos?
+Dolos is parser combinator library for Ruby. It is inspired by FastParse and Scala Parser Combinators.
+
+## What are parser combinators?
+Parser combinators are a way to build parsers from smaller parsers. For example, you can build a parser for a number from a parser for a digit.
+This is a very simple example, but it can be used to build more complex parsers.
+Parsers are lazy and only run when needed. This allows to build complex parsers before passing input to them.
+```ruby
+hello = string("Hello")
+greeting = hello >> c(" ") >> string("Ruby developer!")
+greeting.run("Hello Ruby developer!") # => Success
+```
+
+## What's different from alternatives?
+This library focuses on two things:
+- Parsers integrate well into Ruby code. There is no need to keep them in separate classes.
+- Fine grained control over parsers. You can `map` and adjust each parser separately
+- Two ways of capturing values: traditional `>>`, other product operators to construct value and `capture!`
+  - For simple parsers `capture!` can be used to very quickly capture values into flat arrays 
+- Running parsers will not throw exceptions and instead return a result object. Exceptions don't play well with parsing.

data/docs/_sidebar.md

@@ -0,0 +1,4 @@
+* [Home](/)
+* [Getting started](getting_started.md)
+  * [Installation](getting_started.md#installation)
+  * [Usage](getting_started.md#usage)

data/docs/getting_started.md

@@ -0,0 +1,52 @@
+# Getting started 
+
+## Installation
+
+Install the gem and add it to your Gemfile:
+```shell
+$ bundle add dolos
+```
+Or manually:
+```ruby
+gem 'dolos'
+```
+
+## Usage
+
+Two things to do:
+- require library
+- include module `Dolos` and `Dolos::Common`
+
+```ruby
+require 'dolos'
+
+include Dolos
+include Dolos::Common # Common parsers
+```
+
+### Basic parsers
+
+A simple parser which matches one word.
+
+```ruby
+require 'dolos'
+include Dolos
+
+hello = c("Hello") # c("") is an alias for string(""). Can be read as: case-sensitive string match
+
+hello.run("Hello").success? # => true
+
+hello.run("hello").success? # => failure
+```
+
+After defining parser, it can be ran with `run('my-input')` method. It returns a `Result` object.
+
+### Result
+
+Result can be either `Success` or `Failure`. It can be checked with `success?` or `failure?` methods.
+
+Success will also have `value` property which will contain the result of the parser. There is also `captures`, but 
+that's for later.
+
+
+Failure will have `inspect` method which will return a string with the error message. It will show error position as well.

data/docs/index.html

@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>Document</title>
+  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
+  <meta name="description" content="Description">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0">
+  <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/docsify@4/lib/themes/vue.css">
+</head>
+<body>
+  <div id="app"></div>
+  <script>
+    window.$docsify = {
+      name: '',
+      repo: ''
+    }
+    window.$docsify = {
+      loadSidebar: true
+    }
+  </script>
+  <!-- Docsify v4 -->
+  <script src="//cdn.jsdelivr.net/npm/docsify@4"></script>
+  <script src="//cdn.jsdelivr.net/npm/prismjs@1/components/prism-ruby.min.js"></script>
+</body>
+</html>

data/examples/letter.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
-require_relative 'dolos'
-require_relative 'dolos_common_parsers/arsers/common_parsers'
+require 'dolos'
+require 'dolos_common_parsers/common_parsers'
 
 include Dolos
 
 # Include common parsers
 # In future this can be more structured, moved them to separate module to prevent breaking changes
-include Dolos::CommonParsers
+include Dolos::Common
 
 # Library usage example
 # Parse out a name and address from a letter

data/lib/dolos/common.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Dolos
+  # Common parsers
+  # Separated from the main library to improve them later on
+  # These will change, new ones will be added. Once API stabilises, we will see what to do
+  # We have to be careful what is in the scope when we include this main module
+  # Probably a package of parsers following some RFC will be added as well.
+  # Keeping them separate for now
+  module Common
+    def ws
+      regex(/\s/)
+    end
+
+    def ws_rep0
+      regex(/\s*/)
+    end
+
+    def eol
+      regex(/\n|\r\n|\r/)
+    end
+
+    def digit
+      regex(/\d/)
+    end
+
+    def int
+      digit.map(&:to_i)
+    end
+
+    # Capture as string
+    def digits
+      regex(/\d+/)
+    end
+
+    def alpha_num
+      regex(/[a-zA-Z0-9]/)
+    end
+
+    def alpha
+      regex(/[a-zA-Z]/)
+    end
+  end
+end

data/lib/dolos/parsers.rb

@@ -2,10 +2,16 @@
 
 module Dolos
   module Parsers
+
+    # String parser
+    # Matches exactly the given string
+    # string('hello').run('hello') => Success.new('hello', 5)
+    # Alias: c, for case-sensitive. Ex: c('hello').run('hello') => Success.new('hello', 5)
     def string(str)
+      utf8_str = str.encode('UTF-8')
+
       Parser.new do |state|
         state.input.mark_offset
-        utf8_str = str.encode('UTF-8')
         if state.input.matches?(utf8_str)
           Success.new(utf8_str, str.bytesize)
         else
@@ -13,16 +19,19 @@ module Dolos
           got_error = state.input.io.string.byteslice(state.input.backup, advanced)
           state.input.rollback
           Failure.new(
-            "Expected #{str.inspect} but got #{got_error.inspect}",
+            -> { "Expected #{str.inspect} but got #{got_error.inspect}" },
             advanced,
             state
           )
         end
       end
     end
-
     alias_method :c, :string
 
+    # Regex parser
+    # Accepts a regex, matches the regex against the input
+    # parser = regex(/\d+/)
+    # result = parser.run('123') # => Success.new('123', 3)
     def regex(pattern)
       Parser.new do |state|
         state.input.mark_offset
@@ -32,7 +41,7 @@ module Dolos
           advanced = state.input.offset
           state.input.rollback
           Failure.new(
-            "Expected pattern #{pattern.inspect} but got #{state.input.io.string.inspect}",
+            -> { "Expected pattern #{pattern.inspect} but got #{state.input.io.string.inspect}" },
             advanced,
             state
           )
@@ -40,6 +49,8 @@ module Dolos
       end
     end
 
+    # Matches any character
+    # any_char.run('a') # => Success.new('a', 1)
     def any_char
       Parser.new do |state|
         state.input.mark_offset
@@ -52,7 +63,7 @@ module Dolos
           advanced = state.input.offset
           state.input.rollback
           Failure.new(
-            'Expected any character but got end of input',
+            -> { 'Expected any character but got end of input' },
             advanced,
             state
           )
@@ -61,23 +72,24 @@ module Dolos
     end
 
     # Matches any character in a string
+    # Passed string can be imagined as a set of characters
     # Example:
     #  char_in('abc').run('b') # => Success.new('b', 1)
     def char_in(characters_string)
-      characters_array = characters_string.chars
+      characters_set = characters_string.chars
 
       Parser.new do |state|
         state.input.mark_offset
 
         char, bytesize = state.input.peek(1)
 
-        if char && characters_array.include?(char)
+        if char && characters_set.include?(char)
           Success.new(char, bytesize)
         else
           advanced = state.input.offset
           state.input.rollback
           Failure.new(
-            "Expected one of #{characters_array.inspect} but got #{char.inspect}",
+            -> { "Expected one of #{characters_set.to_a.inspect} but got #{char.inspect}" },
             advanced,
             state
           )
@@ -90,18 +102,18 @@ module Dolos
         state.input.mark_offset
 
         buffer = String.new
-        loop do
-          char, bytesize = state.input.peek(1)
-          break if char.nil? || !predicate.call(char)
+        char, bytesize = state.input.peek(1)
 
+        while char && predicate.call(char)
           buffer << char
           state.input.advance(bytesize)
+          char, bytesize = state.input.peek(1)
         end
 
         if buffer.empty?
           advanced = state.input.offset
           Failure.new(
-            "Predicate never returned true",
+            -> { "Predicate never returned true" },
             advanced,
             state
           )
@@ -111,7 +123,6 @@ module Dolos
       end
     end
 
-    # Unstable API
     def recursive(&block)
       recursive_parser = nil
 
@@ -120,7 +131,7 @@ module Dolos
 
         recursive_parser.call.run_with_state(state).tap do |result|
           if result.failure?
-            error_msg = "Error in recursive structure around position #{state.input.offset}: #{result.message}"
+            error_msg = -> { "Error in recursive structure around position #{state.input.offset}: #{result.message}" }
             Failure.new(error_msg, state.input.offset, state)
           end
         end
@@ -130,7 +141,5 @@ module Dolos
       placeholder
     end
 
-
-
   end
 end

data/lib/dolos/result.rb

@@ -55,12 +55,21 @@ module Dolos
   end
 
   class Failure < Result
-    attr_reader :message, :error_position, :state
+    attr_reader  :error_position, :state
 
-    def initialize(message, error_position, state)
-      @message = message
+    def initialize(message_proc, error_position, state)
+      @message_proc = message_proc
       @error_position = error_position
       @state = state
+      @message_evaluated = false
+    end
+
+    def message
+      unless @message_evaluated
+        @message_value = @message_proc.call
+        @message_evaluated = true
+      end
+      @message_value
     end
 
     def inspect

data/lib/dolos/string_io_wrapper.rb

@@ -22,12 +22,7 @@ module Dolos
 
     def matches?(utf8_str)
       read = io.read(utf8_str.bytesize)
-
-      if read.nil?
-        false
-      else
-        read.force_encoding('UTF-8') == utf8_str
-      end
+      !read.nil? && read.force_encoding('UTF-8') == utf8_str
     end
 
     def advance(bytesize)
@@ -61,8 +56,8 @@ module Dolos
       remaining_data = io.read
       io.seek(current_position)
 
-      if (match_data = remaining_data.match(/\A#{pattern}/))
-        matched_string = match_data[0]
+      if remaining_data =~ /\A#{pattern}/
+        matched_string = $&
         io.seek(current_position + matched_string.bytesize)
         return matched_string
       end

data/lib/dolos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Dolos
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/dolos.rb

@@ -10,80 +10,76 @@ module Dolos
   include Parsers
 
   class Parser
-
     attr_accessor :parser_proc
-
     def initialize(&block)
       @parser_proc = block
     end
 
+    # Run the parser with the given input
+    # Returns a Result<Success|Failure>
+    # string("hello").run("hello") => Success.new("hello", 5)
     def run(input)
       run_with_state(ParserState.new(input))
     end
 
+
     def run_with_state(state)
-      result = parser_proc.call(state)
-      if result.success?
-        state.last_success_position = state.input.offset
-      end
+      result = @parser_proc.call(state)
+      state.last_success_position = state.input.offset if result.success?
       result
     end
 
+    # Capture the result of the parser
+    # p = string("hello").capture!
+    # p.run("hello").captures => ["hello"]
+    # Captures is a flat array of all captured values
     def capture!(wrap_in = nil)
       Parser.new do |state|
         result = run_with_state(state)
-        if result.success?
-          result.capture!(wrap_in)
-        else
-          result
-        end
+        result.success? ? result.capture!(wrap_in) : result
       end
     end
 
-    # Will call block on captures
+    # Map the captures of the parser
+    # p = string("hello").map_captures { |captures| captures.map(&:upcase) }
+    # p.run("hello") => Success.new("hello", 5, ["HELLO"])
+    # This only maps over captures, not the value
     def map_captures(&block)
       Parser.new do |state|
         result = run_with_state(state)
-        if result.success?
-          Success.new(result.value, result.length, block.call(result.captures))
-        else
-          result
-        end
+        result.success? ? Success.new(result.value, result.length, block.call(result.captures)) : result
       end
     end
 
-    # Will call block on tuple of value
+    # Map the result of the parser
+    # p = string("hello").map { |s| s.upcase }
+    # p.run("hello") => Success.new("HELLO", 5)
     def map(&block)
       Parser.new do |state|
         result = run_with_state(state)
-        if result.success?
-          Success.new(block.call(result.value), result.length, result.captures)
-        else
-          result
-        end
+        result.success? ? Success.new(block.call(result.value), result.length, result.captures) : result
       end
     end
 
+    # Combine the result of the parser with another parser
     def combine(&block)
       Parser.new do |state|
         result = run_with_state(state)
+
         if result.success?
+          state.input.advance(result.length)
           new_parser = block.call(result.value, result.captures)
-          new_state = state.dup
-          new_state.input.advance(result.length)
-          new_parser.run_with_state(new_state)
+          new_parser.run_with_state(state)
         else
           result
         end
       end
     end
 
-    def flatten
-      map_captures do |captures|
-        captures.flatten
-      end
-    end
-
+    # Combine the result of the parser with another parser
+    # Has an alias of `&`
+    # p = string("hello") & string("world")
+    # p.run("helloworld") => Success.new(["hello", "world"], 10)
     def product(other_parser)
       combine do |value1, capture1|
         other_parser.map do |value2|
@@ -95,6 +91,10 @@ module Dolos
     end
     alias_method :&, :product
 
+
+    # Combine the result of the parser with another parser
+    # Discards the result of the second parser
+    # p = string("hello") << string("world")
     def product_l(other_parser)
       combine do |value1, capture1|
         other_parser.map do |_|
@@ -105,6 +105,9 @@ module Dolos
       end
     end
 
+    # Combine the result of the parser with another parser
+    # Discards the result of the first parser
+    # p = string("hello") >> string("world")
     def product_r(other_parser)
       combine do |_, capture1|
         other_parser.map do |value2|
@@ -118,6 +121,10 @@ module Dolos
     alias_method :<<, :product_l
     alias_method :>>, :product_r
 
+    # Combine the result of the parser with another parser
+    # If the first parser fails, it will try the second parser
+    # p = string("hello") | string("world") | string("!")
+    # p.run("hello") => Success.new("hello", 5)
     def choice(other_parser)
       Parser.new do |state|
         result = run_with_state(state)
@@ -130,6 +137,9 @@ module Dolos
     end
     alias_method :|, :choice
 
+
+    # Repeat the parser n times
+    # Separator is optional, its another parser that will be run between each repetition
     # rep0         # 0 or more
     # rep          # 1 or more
     # rep(n = 2)   # exactly 2
@@ -140,10 +150,9 @@ module Dolos
         values = []
         captures = []
         count = 0
-        state.input.mark_offset
 
         loop do
-          result = run_with_state(state.dup)
+          result = run_with_state(state) # Removing .dup for performance. Be cautious of side effects.
 
           if result.failure? || count >= n_max
             break
@@ -155,7 +164,7 @@ module Dolos
           count += 1
 
           if separator && count < n_max
-            sep_result = separator.run_with_state(state.dup)
+            sep_result = separator.run_with_state(state) # Removing .dup for performance. Be cautious of side effects.
             break if sep_result.failure?
 
             state.input.advance(sep_result.length)
@@ -163,10 +172,9 @@ module Dolos
         end
 
         if count < n_min
-          error_pos = state.input.offset
           Failure.new(
-            "Expected parser to match at least #{n_min} times but matched only #{count} times",
-            error_pos,
+            -> { "Expected parser to match at least #{n_min} times but matched only #{count} times" },
+            state.input.offset,
             state
           )
         else
@@ -175,11 +183,16 @@ module Dolos
       end
     end
 
+    # Repeat the parser zero or more times
+    # c(" ").rep0.run("   ") => Success.new([" ", " ", " "], 3)
     def zero_or_more
       repeat(n_min: 0, n_max: Float::INFINITY)
     end
     alias_method :rep0, :zero_or_more
 
+    # Repeat the parser one or more times
+    # Same as rep0, but must match at least once
+    # c(" ").rep.run("A") => Failure.new("...")
     def one_or_more(exactly = nil)
       if exactly.nil?
         repeat(n_min: 1, n_max: Float::INFINITY)
@@ -189,6 +202,8 @@ module Dolos
     end
     alias_method :rep, :one_or_more
 
+    # Make parser optional
+    # c(" ").opt.run("A") => Success.new([], 0)
     def optional
       Parser.new do |state|
         result = run_with_state(state.dup)
@@ -201,7 +216,6 @@ module Dolos
     end
     alias_method :opt, :optional
 
-    # Unstable API
     # Used to declare lazy parser to avoid infinite loops in recursive parsers
     def lazy
       parser_memo = nil
@@ -212,11 +226,5 @@ module Dolos
       end
     end
 
-    private
-
-    def combine_and_discard_empty(*arrays)
-      arrays.compact.reject { |arr| arr.is_a?(Array) && arr.empty? }
-    end
-
   end
 end