fupeg 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e5cd2184b66fd7961603e8e21fbdf82514619001cff01cbe230f218c6f2737b1
-  data.tar.gz: 74af0b350f4f966c7c7da490609f18c982c4aed3541a9e279f5eff318c4f698f
+  metadata.gz: 7fb3eb5f497d0c4294b507b5c8681b39635b4a7d1df96644f2afd73170e552b2
+  data.tar.gz: 682bfdc3c694abeddd1dc31591ef5bd39429ec4c9b2511e71269939eedd26549
 SHA512:
-  metadata.gz: ac6a5f6588125e13597104c4c9969fbcf85825b73c76d677ea525eae81791111658733bc366a0b129af86bd85d017bec36b2b6a5c48d2677a574b63334b32bb7
-  data.tar.gz: 8988b71fe64ea7202cc08c996a58ab0c461df6bdba19e1cf7f32315fa039d3412048235e752a7ae4895847f1215b2dd7716255c8b7c03782154926bfb43fd874
+  metadata.gz: 00d7f7fc3d440d968df79df59dab77c87918437dfee730682f31728fe0e1f43c650d88a1ef8d44d92c18f107d83a14fb47db0758516cf97e520e47690e118af7
+  data.tar.gz: 1653a4e8f7a3302b42682b29fac7b18e28e61dc99f1e63b392058282d73f3f2e54963e081431507b5e9fbb52bf5da7341de72d34e2fdaf7497db25215621f8b2

data/CHANGELOG.md

@@ -1,4 +1,26 @@
-## [0.1.0] - 2023-08-11
+## [0.2.0] - 2023-08-15
+
+- Split Parser and Grammar
+- Use `_` for both literals and sequence:
+  `_("x")` , `_(/x/)`, `_{ _("x") }`
+- Use backtick "\`" for string literals
+  `x`
+- `cont?` used with block to detect uncutted alternative
+```ruby
+  cut {
+    # condition
+    _ { `if` && cut! && ... } ||
+    # loop
+    cont? { `while` && cut! && ...} ||
+    # assignment
+    cont? { (i = ident) && sp? && `=` && cut! && ... } ||
+    # function call
+    cont? { (i = ident) && sp? && `(` && cut! && ... } ||
+    ...
+  }
+```
+
+## [0.1.0] - 2023-08-14
 
 - Initial release
 - Simplest rule definition in Ruby code without magic

data/README.md

@@ -1,8 +1,16 @@
-# Fupeg
+# Fupeg - simplest parser combinator
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/fupeg`. To experiment with that code, run `bin/console` for an interactive prompt.
+PEG like parser combinator as simple as possible, but still useful.
+- backtracking, manually specified by user.
+- no memoization (yet).
+- no left recursion (yet).
+- built with StringScanner.
+- pattern sequences and alteration are implemented with logical operators.
 
-TODO: Delete this and the text above, and describe your gem
+Grammar code is pure-ruby and is executed as it is written.
+No grammar tree is built and evaluated.
+
+As bonus, "cut" operator is implemented.
 
 ## Installation
 
@@ -16,7 +24,87 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
 ## Usage
 
-TODO: Write usage instructions here
+First you should define grammar:
+
+```ruby
+require "fupeg"
+
+class Calc < FuPeg::Grammar
+  def eof
+    wont! { dot } && :eof
+  end
+
+  def lnsp?
+    # match regular expression
+    _(/[ \t]*/)
+  end
+
+  # Ruby 3.0 flavour
+  def sp? = _(/\s*/)
+
+  def number = (n = _(/\d+/)) && [:num, n]
+
+  def atom
+    # match raw string: _("(") is aliased to `(`
+    #
+    # match sequence of patterns with backtracking:
+    #   `_{ x && y && z }` will rewind position, if block returns `nil` or `false`
+    #
+    # store value, returned by subpattern: just stor it into variable
+    #
+    # use `||` for alternatives
+    number || _ { _("(") && sp? && (sub = sum) && sp? && `)` && [:sub, sub] }
+  end
+
+  def fact
+    # repetition returns array of block results
+    # it stops if block returns falsey (`nil` or `false`)
+    rep { |fst| # fst == true for first element
+      op = nil
+      # don't expect operator before first term
+      (fst || (op = `*` || _("/") || _(/%/)) && sp?) &&
+        (a = atom) && lnsp? &&
+        [op, a].compact
+      # flat AST tree, returns [:fact, at, op, at, op, at, op] if matched
+    }&.flatten(1)&.unshift(:fact)
+  end
+
+  def sum
+    _ {
+      op = rest = nil
+      (f = fact) &&
+        # optional matches pattern always succeed
+        opt { lnsp? && (op = `+` || `-`) && sp? && (rest = sum) } &&
+        # recursive AST tree
+        (rest ? [:sum, f, op, rest] : f)
+    }
+  end
+
+  def root
+    _ { sum || eof }
+  end
+end
+```
+
+Then either parse string directly, or create parser and grammar:
+
+```ruby
+# Direct parsing
+pp Calc.parse(:root, "1")
+pp Calc.parse(:root, "1 + 2")
+
+# separate parser and grammar initialization
+parser = FuPeg::Parser.new("1 - 2*4/7 + 5")
+grammar = Calc.new(parser)
+pp grammar.root
+
+# combined parser and grammar initialization
+_parser, grammar = Calc.create("(1 -
+                        2)*
+                      (4 -10) +
+11")
+pp grammar.root
+```
 
 ## Development
 
@@ -26,7 +114,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/fupeg.
+Bug reports and pull requests are welcome on GitHub at https://github.com/funny-falcon/fupeg .
 
 ## License
 

data/examples/calc.rb

@@ -0,0 +1,69 @@
+require "fupeg"
+
+class Calc < FuPeg::Grammar
+  def eof
+    wont! { dot } && :eof
+  end
+
+  def lnsp?
+    # match regular expression
+    _(/[ \t]*/)
+  end
+
+  # Ruby 3.0 flavour
+  def sp? = _(/\s*/)
+
+  def number = (n = _(/\d+/)) && [:num, n]
+
+  def atom
+    # match raw string: _("(") is aliased to `(`
+    #
+    # match sequence of patterns with backtracking:
+    #   `_{ x && y && z }` will rewind position, if block returns `nil` or `false`
+    #
+    # store value, returned by subpattern: just stor it into variable
+    number || _ { _("(") && sp? && (sub = sum) && sp? && `)` && [:sub, sub] }
+  end
+
+  def fact
+    # repetition returns array of block results
+    # it stops if block returns falsey (`nil` or `false`)
+    rep { |fst| # fst == true for first element
+      op = nil
+      (fst || (op = `*` || `/` || "%") && sp?) &&
+        (a = atom) && lnsp? &&
+        [op, a].compact
+      # flat AST tree, returns [:fact, at, op, at, op, at, op] if matched
+    }&.flatten(1)&.unshift(:fact)
+  end
+
+  def sum
+    _ {
+      op = rest = nil
+      (f = fact) &&
+        # optional matches pattern always succeed
+        opt { lnsp? && (op = `+` || `-`) && sp? && (rest = sum) } &&
+        # recursive AST tree
+        (rest ? [:sum, f, op, rest] : f)
+    }
+  end
+
+  def root
+    _ { sum || eof }
+  end
+end
+
+pp Calc.parse(:root, "1")
+pp Calc.parse(:root, "1 + 2")
+
+# separate parser and grammar initialization
+parser = FuPeg::Parser.new("1 - 2*4/7 + 5")
+grammar = Calc.new(parser)
+pp grammar.root
+
+# combined parser and grammar initialization
+_parser, grammar = Calc.create("(1 -
+                        2)*
+                      (4 -10) +
+11")
+pp grammar.root

data/lib/fupeg/grammar.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require_relative "parser"
+
+module FuPeg
+  class Grammar
+    def self.create(str, pos = 0)
+      parser = Parser.new(str, pos)
+      grammar = new(parser)
+      [parser, grammar]
+    end
+
+    def self.parse(root, str)
+      _, gr = create(str)
+      gr.__send__(root)
+    end
+
+    def initialize(parser)
+      @p = parser
+    end
+
+    def fail!
+      @p.fail!(skip: 3)
+    end
+
+    def dot
+      @p.match(/./m)
+    end
+
+    def `(str)
+      @p.match(str)
+    end
+
+    def _(lit = nil, &block)
+      @p.match(lit, &block)
+    end
+
+    def opt(arg = nil, &block)
+      @p.match(arg, &block) || true
+    end
+
+    def will?(lit = nil, &block)
+      @p.preserve(pos: true) { @p.match(lit, &block) }
+    end
+
+    def wont!(lit = nil, &block)
+      @p.preserve(pos: true, failed: true) { !@p.match(lit, &block) } || @p.fail!
+    end
+
+    def text(lit = nil, &block)
+      @p.text(lit, &block)
+    end
+
+    def bounds(lit = nil, &block)
+      @p.bounds(lit, &block)
+    end
+
+    def cut(&block)
+      @p.with_cut_point(&block)
+    end
+
+    def cut!
+      @p.current_cutpoint.cut!
+    end
+
+    def cont?(&block)
+      @p.current_cutpoint.can_continue? && (block ? @p.backtrack(&block) : true)
+    end
+
+    def rep(range = 0.., lit = nil, &block)
+      range = range..range if Integer === range
+      range = 0..range.max if range.begin.nil?
+      unless Integer === range.min && (range.end.nil? || Integer === range.max)
+        raise "Range malformed #{range}"
+      end
+      @p.backtrack do
+        max = range.end && range.max
+        ar = []
+        (1..max).each do |i|
+          res = @p.backtrack { yield i == 1 }
+          break unless res
+          ar << res
+        end
+        (ar.size >= range.min) ? ar : @p.fail!
+      end
+    end
+  end
+end

data/lib/fupeg/parser.rb

@@ -28,28 +28,31 @@ module FuPeg
       @scan.pos
     end
 
-    def charpos
-      @str_size - @str.byteslice(@scan.pos..).size
+    def charpos(pos = @scan.pos)
+      @str_size - @str.byteslice(pos..).size
     end
 
-    Fail = Struct.new(:stack, :pos, :bytepos)
+    Fail = Struct.new(:stack, :bytepos)
 
-    def fail!(skip = 2)
+    def fail!(*, skip: 2)
       if !@failed || bytepos > @failed.bytepos
         stack = caller_locations(skip)
         stack.delete_if do |loc|
-          if loc.path == __FILE__
-            loc.label =~ /\b(_bt|each|block)\b/
+          if loc.path.start_with?(__dir__)
+            loc.label =~ /\b(backtrack|each|block)\b/
           end
         end
-        pos = position_for_charpos(charpos)
-        @failed = Fail.new(stack, pos, bytepos)
+        @failed = Fail.new(stack, bytepos)
       end
       nil
     end
 
+    def failed_position
+      position_for_bytepos(@failed.bytepos)
+    end
+
     def report_failed(out)
-      pos = @failed.pos
+      pos = position_for_bytepos(@failed.bytepos)
       out << "Failed at #{pos.lineno}:#{pos.colno} :\n"
       out << pos.line + "\n"
       out << (" " * (pos.colno - 1) + "^\n")
@@ -60,75 +63,36 @@ module FuPeg
       out
     end
 
-    def dot
-      @scan.scan(/./m) || fail!
-    end
-
     begin
       StringScanner.new("x").skip("x")
-      def lit(reg_or_str)
-        @scan.scan(reg_or_str) || fail!
+      def match(lit = //, &block)
+        block ? backtrack(&block) : (@scan.scan(lit) || fail!)
       end
     rescue
-      def lit(reg_or_str)
-        if String === reg_or_str
-          @__match_lit_cache ||= Hash.new { |h, s| h[s] = Regexp.new(Regexp.escape(s)) }
-          reg_or_str = @__match_lit_cache[reg_or_str]
+      def match(lit = //, &block)
+        if String === lit
+          @_lit_cache ||= {}
+          lit = @_lit_cache[lit] ||= Regexp.new(Regexp.escape(lit))
         end
-        @scan.scan(reg_or_str) || fail!
+        block ? backtrack(&block) : (@scan.scan(lit) || fail!)
       end
     end
 
-    def seq(*args, &block)
-      _bt(&block)
-    end
-
-    def opt(&block)
-      _rewind(nil, @failed, _bt(&block) || true)
-    end
-
-    def rep(range = 0.., &block)
-      range = range..range if Integer === range
-      range = 0..range.max if range.begin.nil?
-      unless Integer === range.min && (range.end.nil? || Integer === range.max)
-        raise "Range malformed #{range}"
-      end
-      _bt do
-        max = range.end && range.max
-        ar = []
-        (1..max).each do
-          res = _bt(&block)
-          break unless res
-          ar << res
-        end
-        (ar.size >= range.min) ? ar : fail!
-      end
-    end
-
-    def text(&block)
+    def text(lit = nil, &block)
       pos = @scan.pos
-      _bt(&block) && @str.byteslice(pos, @scan.pos - pos)
+      match(lit, &block) && @str.byteslice(pos, @scan.pos - pos)
     end
 
-    def will?(&block)
-      _rewind(@scan.pos, false, _bt(&block))
-    end
-
-    def wont!(&block)
-      _rewind(@scan.pos, @failed, !_bt(&block)) || fail!
+    def bounds(lit = nil, &block)
+      pos = @scan.pos
+      match(lit, &block) && pos...@scan.pos
     end
 
-    # cut point handling
-    #   cut do
-    #     seq { lit("{") && cut! && lit("}") } ||
-    #     !cut? && seq { lit("[") && cut! && lit("]") } ||
-    #     !cut? && dot
-    #   end
     class CutPoint
       attr_accessor :next
 
       def initialize
-        @cut = false
+        @cut = nil
         @next = nil
       end
 
@@ -137,13 +101,13 @@ module FuPeg
         @cut = true
       end
 
-      def cut?
-        @cut
+      def can_continue?
+        @cut ? nil : true
       end
     end
 
-    # for use with cut! and cut?
-    def cut
+    # for use with cut! and cont?
+    def with_cut_point
       prev_cut = @cut
       @cut = CutPoint.new
       prev_cut.next = @cut
@@ -153,51 +117,40 @@ module FuPeg
       @cut = prev_cut
     end
 
-    def cut!
-      @cut.cut!
-    end
-
-    def cut?
-      @cut.cut?
+    def current_cutpoint
+      @cut
     end
 
     # Position handling for failures
 
     Position = Struct.new(:lineno, :colno, :line, :charpos)
 
-    private
-
     def init_line_ends
       @line_ends = [-1]
-      pos = 0
-      while (pos = @str.index("\n", pos))
-        @line_ends << @pos
-        pos += 1
+      scan = StringScanner.new(@str)
+      while scan.skip_until(/\n|\r\n?/)
+        @line_ends << scan.pos - 1
       end
-      @line_ends << @str.size
+      @line_ends << @str.bytesize
     end
 
-    public
-
-    def position_for_charpos(charpos)
-      lineno = @line_ends.bsearch_index { |x| x >= charpos }
+    def position_for_bytepos(pos)
+      lineno = @line_ends.bsearch_index { |x| x >= pos }
       case lineno
       when nil
-        raise "Position #{charpos} is larger than string size #{@str.size}"
+        raise "Position #{pos} is larger than string byte size #{@str.bytesize}"
       else
         prev_end = @line_ends[lineno - 1]
         line_start = prev_end + 1
-        column = charpos - prev_end
+        column = @str.byteslice(line_start, pos - prev_end).size
       end
-      line = @str[line_start..@line_ends[lineno]]
-      Position.new(lineno, column, line, charpos)
+      line = @str.byteslice(line_start..@line_ends[lineno])
+      Position.new(lineno, column, line, charpos(pos))
     end
 
     # helper methods
 
-    private
-
-    def _bt
+    def backtrack
       pos = @scan.pos
       res = yield
       if res
@@ -212,10 +165,12 @@ module FuPeg
       raise
     end
 
-    def _rewind(pos, failed, val)
-      @scan.pos = pos if pos
-      @failed = failed if failed != false
-      val
+    def preserve(pos = false, failed = false, &block)
+      p, f = @scan.pos, @failed
+      r = yield
+      @scan.pos = p if pos
+      @failed = f if failed
+      r
     end
   end
 end

data/lib/fupeg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FuPeg
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/fupeg.rb

@@ -2,10 +2,9 @@
 
 require_relative "fupeg/version"
 require_relative "fupeg/parser"
+require_relative "fupeg/grammar"
 
 module FuPeg
-  VERSION = "0.1.0"
-
   class Error < StandardError; end
   # Your code goes here...
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fupeg
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Yura Sokolov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-14 00:00:00.000000000 Z
+date: 2023-08-15 00:00:00.000000000 Z
 dependencies: []
 description: "\n    Simple backtracing parser, using ruby logical operators for primitive
   sequence/choice\n    and slim wrappers for other PEG style operators and backtrace.\n
@@ -26,7 +26,9 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/calc.rb
 - lib/fupeg.rb
+- lib/fupeg/grammar.rb
 - lib/fupeg/parser.rb
 - lib/fupeg/version.rb
 - sig/fupeg.rbs