fupeg 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e5cd2184b66fd7961603e8e21fbdf82514619001cff01cbe230f218c6f2737b1
-  data.tar.gz: 74af0b350f4f966c7c7da490609f18c982c4aed3541a9e279f5eff318c4f698f
+  metadata.gz: c08b3d48fd87bd8fc04611663270f967afdf9bab0e0e5ce9c63b38c9f00332b6
+  data.tar.gz: 93308247a0fb582957862a4c0e274a3c646c1a7cba7ffc9acade0af1f5d6d250
 SHA512:
-  metadata.gz: ac6a5f6588125e13597104c4c9969fbcf85825b73c76d677ea525eae81791111658733bc366a0b129af86bd85d017bec36b2b6a5c48d2677a574b63334b32bb7
-  data.tar.gz: 8988b71fe64ea7202cc08c996a58ab0c461df6bdba19e1cf7f32315fa039d3412048235e752a7ae4895847f1215b2dd7716255c8b7c03782154926bfb43fd874
+  metadata.gz: 42973a5d22dd77f267a94d6ec65823ab891a2d63983dc548a4818b8b77ca24f5138754e08ae1a9b9b3e42a8791215f0c9b0f546e2397729a80dcb5fbb8929447
+  data.tar.gz: 5b99feff0f26acdac764194409f2b2d137a5dc6e16a6122e78cefda3647c680944b847112e135df32b9a27cd0c50652e3c6f32387713906c0fcb4938a8712c85

data/CHANGELOG.md

@@ -1,4 +1,36 @@
-## [0.1.0] - 2023-08-11
+## [0.3.0] - 2023-08-28
+
+- Grammar.use_gram - to simplier grammar inclusion
+- Grammar.proxy - to proxy rules to included grammar
+- `_(pat)` doesn't return matched text, use `txt(pat)` instead
+- "\`" is specialized for tokens
+-- token is either ident (which is specified with `ident_only` method), or symbols,
+-- `token_sp?` is skipped after token
+- fixes for position calculation
+
+## [0.2.0] - 2023-08-15
+
+- Split Parser and Grammar
+- Use `_` for both literals and sequence:
+  `_("x")` , `_(/x/)`, `_{ _("x") }`
+- Use backtick "\`" for string literals
+  `x`
+- `cont?` used with block to detect uncutted alternative
+```ruby
+  cut {
+    # condition
+    _ { `if` && cut! && ... } ||
+    # loop
+    cont? { `while` && cut! && ...} ||
+    # assignment
+    cont? { (i = ident) && sp? && `=` && cut! && ... } ||
+    # function call
+    cont? { (i = ident) && sp? && `(` && cut! && ... } ||
+    ...
+  }
+```
+
+## [0.1.0] - 2023-08-14
 
 - Initial release
 - Simplest rule definition in Ruby code without magic

data/README.md

@@ -1,8 +1,16 @@
-# Fupeg
+# Fupeg - simplest parser combinator
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/fupeg`. To experiment with that code, run `bin/console` for an interactive prompt.
+PEG like parser combinator as simple as possible, but still useful.
+- backtracking, manually specified by user.
+- no memoization (yet).
+- no left recursion (yet).
+- built with StringScanner.
+- pattern sequences and alteration are implemented with logical operators.
 
-TODO: Delete this and the text above, and describe your gem
+Grammar code is pure-ruby and is executed as it is written.
+No grammar tree is built and evaluated.
+
+As bonus, "cut" operator is implemented.
 
 ## Installation
 
@@ -16,7 +24,87 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
 ## Usage
 
-TODO: Write usage instructions here
+First you should define grammar:
+
+```ruby
+require "fupeg"
+
+class Calc < FuPeg::Grammar
+  def eof
+    wont! { dot } && :eof
+  end
+
+  def lnsp?
+    # match regular expression
+    _(/[ \t]*/)
+  end
+
+  # Ruby 3.0 flavour
+  def sp? = _(/\s*/)
+
+  def number = (n = _(/\d+/)) && [:num, n]
+
+  def atom
+    # match raw string: _("(") is aliased to `(`
+    #
+    # match sequence of patterns with backtracking:
+    #   `_{ x && y && z }` will rewind position, if block returns `nil` or `false`
+    #
+    # store value, returned by subpattern: just stor it into variable
+    #
+    # use `||` for alternatives
+    number || _ { _("(") && sp? && (sub = sum) && sp? && `)` && [:sub, sub] }
+  end
+
+  def fact
+    # repetition returns array of block results
+    # it stops if block returns falsey (`nil` or `false`)
+    rep { |fst| # fst == true for first element
+      op = nil
+      # don't expect operator before first term
+      (fst || (op = `*` || _("/") || _(/%/)) && sp?) &&
+        (a = atom) && lnsp? &&
+        [op, a].compact
+      # flat AST tree, returns [:fact, at, op, at, op, at, op] if matched
+    }&.flatten(1)&.unshift(:fact)
+  end
+
+  def sum
+    _ {
+      op = rest = nil
+      (f = fact) &&
+        # optional matches pattern always succeed
+        opt { lnsp? && (op = `+` || `-`) && sp? && (rest = sum) } &&
+        # recursive AST tree
+        (rest ? [:sum, f, op, rest] : f)
+    }
+  end
+
+  def root
+    _ { sum || eof }
+  end
+end
+```
+
+Then either parse string directly, or create parser and grammar:
+
+```ruby
+# Direct parsing
+pp Calc.parse(:root, "1")
+pp Calc.parse(:root, "1 + 2")
+
+# separate parser and grammar initialization
+parser = FuPeg::Parser.new("1 - 2*4/7 + 5")
+grammar = Calc.new(parser)
+pp grammar.root
+
+# combined parser and grammar initialization
+_parser, grammar = Calc.create("(1 -
+                        2)*
+                      (4 -10) +
+11")
+pp grammar.root
+```
 
 ## Development
 
@@ -26,7 +114,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/fupeg.
+Bug reports and pull requests are welcome on GitHub at https://github.com/funny-falcon/fupeg .
 
 ## License
 

data/examples/calc.rb

@@ -0,0 +1,69 @@
+require "fupeg"
+
+class Calc < FuPeg::Grammar
+  def eof
+    wont! { dot } && :eof
+  end
+
+  def lnsp?
+    # match regular expression
+    _(/[ \t]*/)
+  end
+
+  # Ruby 3.0 flavour
+  def sp? = _(/\s*/)
+
+  def number = (n = _(/\d+/)) && [:num, n]
+
+  def atom
+    # match raw string: _("(") is aliased to `(`
+    #
+    # match sequence of patterns with backtracking:
+    #   `_{ x && y && z }` will rewind position, if block returns `nil` or `false`
+    #
+    # store value, returned by subpattern: just stor it into variable
+    number || _ { _("(") && sp? && (sub = sum) && sp? && `)` && [:sub, sub] }
+  end
+
+  def fact
+    # repetition returns array of block results
+    # it stops if block returns falsey (`nil` or `false`)
+    rep { |fst| # fst == true for first element
+      op = nil
+      (fst || (op = `*` || `/` || "%") && sp?) &&
+        (a = atom) && lnsp? &&
+        [op, a].compact
+      # flat AST tree, returns [:fact, at, op, at, op, at, op] if matched
+    }&.flatten(1)&.unshift(:fact)
+  end
+
+  def sum
+    _ {
+      op = rest = nil
+      (f = fact) &&
+        # optional matches pattern always succeed
+        opt { lnsp? && (op = `+` || `-`) && sp? && (rest = sum) } &&
+        # recursive AST tree
+        (rest ? [:sum, f, op, rest] : f)
+    }
+  end
+
+  def root
+    _ { sum || eof }
+  end
+end
+
+pp Calc.parse(:root, "1")
+pp Calc.parse(:root, "1 + 2")
+
+# separate parser and grammar initialization
+parser = FuPeg::Parser.new("1 - 2*4/7 + 5")
+grammar = Calc.new(parser)
+pp grammar.root
+
+# combined parser and grammar initialization
+_parser, grammar = Calc.create("(1 -
+                        2)*
+                      (4 -10) +
+11")
+pp grammar.root

data/lib/fupeg/grammar.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require_relative "parser"
+
+module FuPeg
+  class Grammar
+    def self.create(str, pos = 0)
+      parser = Parser.new(str, pos)
+      grammar = new(parser)
+      [parser, grammar]
+    end
+
+    def self.parse(root, str)
+      _, gr = create(str)
+      gr.__send__(root)
+    end
+
+    def self.use_gram(gram, *, as: nil)
+      if as.nil?
+        name = gram.name[/\w+$/]
+        name = name.gsub(/(?<!^)(?=[A-Z](?![A-Z\d_]))/, "_").downcase
+        as = :"@#{name}"
+      elsif !as.start_with?("@")
+        as = :"@#{as}"
+      end
+      @used_grams ||= {}
+      @used_grams[as] = gram
+    end
+
+    def self.proxy(*meths, to:)
+      meths.each do |meth|
+        define_method(meth) { |*args, &block|
+          instance_variable_get(to).__send__(meth, *args, &block)
+        }
+      end
+    end
+
+    def self.used_grams
+      @used_grams&.dup || {}
+    end
+
+    def initialize(parser)
+      @p = parser
+      self.class.used_grams.each do |iv, v|
+        instance_variable_set(iv, v.new(parser))
+      end
+    end
+
+    def fail!(pat: nil)
+      @p.fail!(pat: pat, skip: 3)
+    end
+
+    def dot
+      @p.dot
+    end
+
+    def _(lit = nil, &block)
+      @p.match(lit, &block)
+    end
+
+    def opt(arg = nil, &block)
+      @p.match(arg, &block) || true
+    end
+
+    def will?(lit = nil, &block)
+      @p.look_ahead(true, lit, &block)
+    end
+
+    def wont!(lit = nil, &block)
+      @p.look_ahead(false, lit, &block)
+    end
+
+    def txt(lit = nil, &block)
+      @p.text(lit, &block)
+    end
+
+    def cut(&block)
+      @p.with_cut_point(&block)
+    end
+
+    def cut!
+      @p.current_cutpoint.cut!
+    end
+
+    def cont?(&block)
+      @p.current_cutpoint.can_continue? && (block ? @p.backtrack(&block) : true)
+    end
+
+    def rep(range = 0.., lit = nil, &block)
+      @p.repetition(range, lit, &block)
+    end
+
+    # specialized matchers
+
+    def eof
+      @p.eof? && :eof
+    end
+
+    def nl
+      _(/\r\n|\r|\n/)
+    end
+
+    def eol
+      _ { lnsp? && nl && :eol }
+    end
+
+    def lnsp?
+      _(/[ \t]*/)
+    end
+
+    def lnsp!
+      _(/[ \t]+/)
+    end
+
+    def sp!
+      _(/\s+/)
+    end
+
+    def sp?
+      _(/\s*/)
+    end
+
+    def ident
+      (w = ident_only) && token_sp? && w
+    end
+
+    # raw token match
+    # if token is ident, then exact match performed with whole next ident
+    # else only string match
+    # and then whitespace is consumed
+    def `(token)
+      @p.match {
+        if _is_ident?(token)
+          _{ ident_only == token } || fail!(pat: token)
+        else
+          @p.match(token)
+        end && token_sp? && token
+      }
+    end
+
+    def _is_ident?(tok)
+      @_is_ident ||= Hash.new { |h, k|
+        h[k] = self.class.parse(:ident_only, k) == k
+      }
+      @_is_ident[tok]
+    end
+
+    def ident_only
+      txt(/[a-zA-Z_]\w*/)
+    end
+
+    def token_sp?
+      _(/\s*/)
+    end
+  end
+end

data/lib/fupeg/parser.rb

@@ -4,6 +4,11 @@ require "strscan"
 
 module FuPeg
   class Parser
+    attr_accessor :debug
+    attr_accessor :file
+    attr_reader :failed
+    attr_reader :str
+
     def initialize(str, pos = 0)
       reset!(str, pos)
     end
@@ -19,40 +24,56 @@ module FuPeg
         @scan.pos = pos
       end
       @failed = nil
+      @debug = false
       @cut = CutPoint.new
     end
 
-    attr_reader :failed
-
     def bytepos
       @scan.pos
     end
 
-    def charpos
-      @str_size - @str.byteslice(@scan.pos..).size
+    def charpos(pos = @scan.pos)
+      @str_size - @str.byteslice(pos..).size
     end
 
-    Fail = Struct.new(:stack, :pos, :bytepos)
+    Fail = Struct.new(:stack, :bytepos, :pattern)
 
-    def fail!(skip = 2)
-      if !@failed || bytepos > @failed.bytepos
+    def fail!(*, pat: nil, skip: 2)
+      if debug || !@failed || bytepos > @failed.bytepos
         stack = caller_locations(skip)
         stack.delete_if do |loc|
-          if loc.path == __FILE__
-            loc.label =~ /\b(_bt|each|block)\b/
+          path = loc.path
+          if path == __FILE__
+            true
+          elsif path.start_with?(__dir__)
+            loc.label =~ /\b(backtrack|each|block)\b/
           end
         end
-        pos = position_for_charpos(charpos)
-        @failed = Fail.new(stack, pos, bytepos)
+        @failed = Fail.new(stack, bytepos, pat)
+        report_failed($stderr) if debug
       end
       nil
     end
 
+    def failed_position
+      position(bytepos: @failed.bytepos)
+    end
+
     def report_failed(out)
-      pos = @failed.pos
-      out << "Failed at #{pos.lineno}:#{pos.colno} :\n"
-      out << pos.line + "\n"
-      out << (" " * (pos.colno - 1) + "^\n")
+      pos = position(bytepos: @failed.bytepos)
+      out << if @failed.pattern
+        "Failed #{failed.pattern.inspect} at #{pos.lineno}:#{pos.colno}"
+      else
+        "Failed at #{pos.lineno}:#{pos.colno}"
+      end
+      if @file
+        out << " of #{@file}"
+      end
+      out << ":\n"
+      out << pos.line.chomp + "\n"
+      curpos = pos.line[...pos.colno].gsub("\t", " " * 8).size
+      curpos = 1 if curpos == 0 && @failed.bytepos == @str.bytesize
+      out << (" " * (curpos - 1) + "^\n")
       out << "Call stack:\n"
       @failed.stack.each do |loc|
         out << "#{loc.path}:#{loc.lineno} in #{loc.label}\n"
@@ -60,75 +81,11 @@ module FuPeg
       out
     end
 
-    def dot
-      @scan.scan(/./m) || fail!
-    end
-
-    begin
-      StringScanner.new("x").skip("x")
-      def lit(reg_or_str)
-        @scan.scan(reg_or_str) || fail!
-      end
-    rescue
-      def lit(reg_or_str)
-        if String === reg_or_str
-          @__match_lit_cache ||= Hash.new { |h, s| h[s] = Regexp.new(Regexp.escape(s)) }
-          reg_or_str = @__match_lit_cache[reg_or_str]
-        end
-        @scan.scan(reg_or_str) || fail!
-      end
-    end
-
-    def seq(*args, &block)
-      _bt(&block)
-    end
-
-    def opt(&block)
-      _rewind(nil, @failed, _bt(&block) || true)
-    end
-
-    def rep(range = 0.., &block)
-      range = range..range if Integer === range
-      range = 0..range.max if range.begin.nil?
-      unless Integer === range.min && (range.end.nil? || Integer === range.max)
-        raise "Range malformed #{range}"
-      end
-      _bt do
-        max = range.end && range.max
-        ar = []
-        (1..max).each do
-          res = _bt(&block)
-          break unless res
-          ar << res
-        end
-        (ar.size >= range.min) ? ar : fail!
-      end
-    end
-
-    def text(&block)
-      pos = @scan.pos
-      _bt(&block) && @str.byteslice(pos, @scan.pos - pos)
-    end
-
-    def will?(&block)
-      _rewind(@scan.pos, false, _bt(&block))
-    end
-
-    def wont!(&block)
-      _rewind(@scan.pos, @failed, !_bt(&block)) || fail!
-    end
-
-    # cut point handling
-    #   cut do
-    #     seq { lit("{") && cut! && lit("}") } ||
-    #     !cut? && seq { lit("[") && cut! && lit("]") } ||
-    #     !cut? && dot
-    #   end
     class CutPoint
       attr_accessor :next
 
       def initialize
-        @cut = false
+        @cut = nil
         @next = nil
       end
 
@@ -137,13 +94,13 @@ module FuPeg
         @cut = true
       end
 
-      def cut?
-        @cut
+      def can_continue?
+        @cut ? nil : true
       end
     end
 
-    # for use with cut! and cut?
-    def cut
+    # for use with cut! and cont?
+    def with_cut_point
       prev_cut = @cut
       @cut = CutPoint.new
       prev_cut.next = @cut
@@ -153,51 +110,98 @@ module FuPeg
       @cut = prev_cut
     end
 
-    def cut!
-      @cut.cut!
-    end
-
-    def cut?
-      @cut.cut?
+    def current_cutpoint
+      @cut
     end
 
     # Position handling for failures
 
     Position = Struct.new(:lineno, :colno, :line, :charpos)
 
-    private
-
     def init_line_ends
       @line_ends = [-1]
-      pos = 0
-      while (pos = @str.index("\n", pos))
-        @line_ends << @pos
-        pos += 1
+      scan = StringScanner.new(@str)
+      while scan.skip_until(/\n|\r\n?/)
+        @line_ends << scan.pos - 1
       end
-      @line_ends << @str.size
+      @line_ends << @str.bytesize
     end
 
-    public
-
-    def position_for_charpos(charpos)
-      lineno = @line_ends.bsearch_index { |x| x >= charpos }
+    def position(bytepos: @scan.pos)
+      lineno = @line_ends.bsearch_index { |x| x >= bytepos }
       case lineno
       when nil
-        raise "Position #{charpos} is larger than string size #{@str.size}"
+        raise "Position #{bytepos} is larger than string byte size #{@str.bytesize}"
       else
         prev_end = @line_ends[lineno - 1]
         line_start = prev_end + 1
-        column = charpos - prev_end
+        column = @str.byteslice(line_start, bytepos - prev_end).size
       end
-      line = @str[line_start..@line_ends[lineno]]
-      Position.new(lineno, column, line, charpos)
+      if bytepos == @str.bytesize
+        if @str[-1] == "\n"
+          lineno, column = lineno + 1, 1
+        else
+          column += 1
+        end
+      end
+      line = @str.byteslice(line_start..@line_ends[lineno])
+      Position.new(lineno, column, line, charpos(bytepos))
     end
 
     # helper methods
 
-    private
+    begin
+      StringScanner.new("x").skip("x")
+      def match(lit = nil, &block)
+        block ? backtrack(&block) : (!lit || @scan.skip(lit) && true || fail!(pat: lit))
+      end
+    rescue
+      def match(lit = nil, &block)
+        if String === lit
+          @_lit_cache ||= {}
+          lit = @_lit_cache[lit] ||= Regexp.new(Regexp.escape(lit))
+        end
+        block ? backtrack(&block) : (!lit || @scan.skip(lit) && true || fail!(pat: lit))
+      end
+    end
+
+    def text(lit = nil, &block)
+      pos = @scan.pos
+      match(lit, &block) && @str.byteslice(pos, @scan.pos - pos)
+    end
 
-    def _bt
+    def bounds(lit = nil, &block)
+      pos = @scan.pos
+      match(lit, &block) && pos...@scan.pos
+    end
+
+    def repetition(range = 0.., lit = nil, &block)
+      range = range..range if Integer === range
+      range = 0..range.max if range.begin.nil?
+      unless Integer === range.min && (range.end.nil? || Integer === range.max)
+        raise "Range malformed #{range}"
+      end
+      backtrack do
+        max = range.end && range.max
+        ar = []
+        (1..max).each do |i|
+          res = backtrack { yield i == 1 }
+          break unless res
+          ar << res
+        end
+        (ar.size >= range.min) ? ar : fail!
+      end
+    end
+
+    def dot
+      match(/./m)
+    end
+
+    def eof?
+      @scan.eos?
+    end
+
+    def backtrack
       pos = @scan.pos
       res = yield
       if res
@@ -212,10 +216,25 @@ module FuPeg
       raise
     end
 
-    def _rewind(pos, failed, val)
-      @scan.pos = pos if pos
-      @failed = failed if failed != false
-      val
+    def look_ahead(positive, lit = nil, &block)
+      if block
+        p, f = @scan.pos, @failed
+        r = yield
+        @scan.pos = p
+        if positive ? r : !r
+          @failed = f
+          true
+        else
+          fail!
+        end
+      else
+        m = @scan.match?(lit)
+        if positive ? m : !m
+          true
+        else
+          fail!(pat: lit)
+        end
+      end
     end
   end
 end

data/lib/fupeg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FuPeg
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

data/lib/fupeg.rb

@@ -2,10 +2,9 @@
 
 require_relative "fupeg/version"
 require_relative "fupeg/parser"
+require_relative "fupeg/grammar"
 
 module FuPeg
-  VERSION = "0.1.0"
-
   class Error < StandardError; end
   # Your code goes here...
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fupeg
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Yura Sokolov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-14 00:00:00.000000000 Z
+date: 2023-08-28 00:00:00.000000000 Z
 dependencies: []
 description: "\n    Simple backtracing parser, using ruby logical operators for primitive
   sequence/choice\n    and slim wrappers for other PEG style operators and backtrace.\n
@@ -26,7 +26,9 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/calc.rb
 - lib/fupeg.rb
+- lib/fupeg/grammar.rb
 - lib/fupeg/parser.rb
 - lib/fupeg/version.rb
 - sig/fupeg.rbs