neg 1.0.0 → 1.1.0

data/CHANGELOG.md

@@ -2,6 +2,13 @@
 # neg - CHANGELOG.md
 
 
+## neg - 1.1.0    released 2013-02-28
+
+- giving the "right" error on unconsumed input
+- get rid of UnconsumedInputError
+- translator: introduce n.flattened_results
+
+
 ## neg - 1.0.0    released 2013-01-16
 
 - initial release

data/README.md

@@ -1,4 +1,3 @@
-
 # neg
 
 A neg narser.
@@ -30,11 +29,11 @@ Here is the classical arithmetic example:
 
     expression  == operation
 
-    operator    == `+` | `-` | `*` | `/`
-    operation   == value + (operator + value) * 0
-    value       == parenthese | number
-    parenthese  == `(` + expression + `)`
-    number      == `-` * -1 + _('0-9') * 1
+    operator     == `+` | `-` | `*` | `/`
+    operation    == value + (operator + value) * 0
+    value        == parentheses | number
+    parentheses  == `(` + expression + `)`
+    number       == `-` * -1 + _('0-9') * 1
   end
 
   tree = ArithParser.parse("1+(2*12)")
@@ -116,7 +115,7 @@ It's a nested assemblage of result nodes.
 [ :bfalse, [ 0, 1, 1 ], true, 'false', [] ]
 ```
 
-In case of successful parsing, the succes? == false also get all pruned. In case of failed parsing, they are left in the output parse tree.
+In case of successful parsing, the success? == false also get all pruned. In case of failed parsing, they are left in the output parse tree.
 
 A translator turns a raw parse tree into some final result. Look below and at the JSON parser sample in the specs for more information. If the parse failed and a translator is present, a ParseError is raised.
 
@@ -132,11 +131,11 @@ class CompactArithParser < Neg::Parser
 
     expression  == operation
 
-    operator    == `+` | `-` | `*` | `/`
-    operation   == value + (operator + value) * 0
-    value       == parenthese | number
-    parenthese  == `(` + expression + `)`
-    number      == `-` * -1 + _('0-9') * 1
+    operator     == `+` | `-` | `*` | `/`
+    operation    == value + (operator + value) * 0
+    value        == parentheses | number
+    parentheses  == `(` + expression + `)`
+    number       == `-` * -1 + _('0-9') * 1
   end
 
   translator do
@@ -146,7 +145,7 @@ class CompactArithParser < Neg::Parser
     on(:value)     { |n| n.results.first }
 
     on(:expression) { |n|
-      results = n.results.flatten(2)
+      results = n.flattened_results
       results.size == 1 ? results.first : results
     }
   end
@@ -156,8 +155,12 @@ CompactArithParser.parse("1+2+3")
   # => [ 1, '+', 2, '+', 3 ]
 ```
 
+The original of this parser lies in [spec/sample_arith_spec.rb](spec/sample_arith_spec.rb). Please note that it's very dumb (like everything in neg) and, for example, avoids carefully dealing with operator precedence for its target language.
+
 As said above, when a translator is present and the parsing fails (before the translator kicks in), a ParseError is raised, with fancy methods to navigate the failed parse tree.
 
+See also the [sample JSON parser](spec/sample_json_parser_spec.rb) and a [tiny toy scheme parser](spec/sample_scheme_spec.rb).
+
 
 ## presentations
 

data/TODO.txt

@@ -1,17 +1,10 @@
 
-[o] unconsumed input!
-[o] ^ 0, 1, [ min, max ]
-[o] switch from ^ to * (how * is related to +)
-[o] _ (any)
-[o] chars
-[o] lookahead present/absent
-  ~ x --> present
-  ! x --> absent
-
-[ ] blankslate
-[ ] drop UnconsumedInput, replace with regular [ false, ... ] output
-[ ] x * '?' / x * '+' / x * '*' as shortcuts
-[ ] memoization (only at non-terminal level?)
-
-[ ] "xxx" instead of `xxx` (trick on the right side)
+[o] investigate translator rule returning [] (json empty array for instance)
+    does it get discarded like the others?
+[o] include the on(nil) rule by default?
+[o] raise error (with #error_tree) when translator hits unparsable input
+[o] document ...['name'] in the README
+
+[o] verify that the Parslet UnconsumedInput problem doesn't appear in neg
+[o] second look at the blankslate
 

data/lib/neg/errors.rb

@@ -27,19 +27,20 @@ module Neg
 
   class NegError < StandardError; end
 
-  class UnconsumedInputError < NegError; end
   class ParserError < NegError; end
 
   class ParseError < NegError
 
-    attr_reader :tree
+    attr_reader :tree, :position
 
     def initialize(tree)
 
       @tree = tree
       @nodes = list_nodes(tree)
 
-      super(deepest_error[3])
+      d = deepest_error
+      @position = d[1]
+      super(d[3])
     end
 
     def errors
@@ -49,15 +50,30 @@ module Neg
 
     def deepest_error
 
-      errors.inject { |e, n| e[1][0] < n[1][0] ? n : e }
+      # let's keep the tree depth (e[5]) for later
+
+      errors.inject do |eold, enew|
+        if eold[1][0] <= enew[1][0]
+          enew
+        else
+          eold
+        end
+      end
     end
 
+    def offset;  @position[0]; end
+    def line;    @position[1]; end
+    def column;  @position[2]; end
+
     protected
 
-    def list_nodes(start, accumulator=[])
+    def list_nodes(start, depth=0, accumulator=[])
+
+      start = start.dup
+      start << depth
 
       accumulator << start
-      start[4].each { |n| list_nodes(n, accumulator) }
+      start[4].each { |n| list_nodes(n, depth + 1, accumulator) }
 
       accumulator
     end

data/lib/neg/parser.rb

@@ -58,15 +58,26 @@ module Neg
       pa
     end
 
+    def self.reduce(result)
+
+      if result[0] && result[2] && result[3]
+        result[4] =
+          []
+      else
+        result[4] =
+          result[4].each_with_object([]) { |cr, a| a << reduce(cr) if cr[2] }
+      end
+
+      result
+    end
+
     def self.parse(s, opts={})
 
       i = Neg::Input(s)
 
       result = __send__(@root).parse(i, opts)
 
-      raise UnconsumedInputError.new(
-        "remaining: #{i.remains.inspect}"
-      ) if result[2] && ( ! i.eoi?)
+      result[2] = false if result[2] && ( ! i.eoi?)
 
       if @translator && opts[:translate] != false
         if result[2]
@@ -74,8 +85,10 @@ module Neg
         else
           raise ParseError.new(result)
         end
-      else
+      elsif result[2] == false || opts[:noreduce]
         result
+      else
+        reduce(result)
       end
     end
 
@@ -123,10 +136,6 @@ module Neg
 
         input.rewind(start) unless success
 
-        #if success && children.size == 1 && children.first[1] == start
-        #  return children.first
-        #end
-
         [ nil, start, success, result, children ]
       end
     end
@@ -144,13 +153,13 @@ module Neg
         @child = pa
       end
 
-      def reduce(children_results)
+      def refine(children_results)
 
         children_results.collect { |cr|
           if cr[0] && cr[0].to_s.match(/^_/).nil?
             false
           elsif cr[2]
-            cr[3] ? cr[3] : reduce(cr[4])
+            cr[3] ? cr[3] : refine(cr[4])
           else
             nil
           end
@@ -166,11 +175,10 @@ module Neg
         return r if r[0] == false
         return r if r[1].is_a?(String)
 
-        report = reduce(r[2])
+        report = refine(r[2])
+        report = report.include?(false) ? nil : report.join
 
-        return r if report.include?(false)
-
-        [ true, report.join, opts[:noreduce] ? r[2] : [] ]
+        [ true, report, r[2] ]
       end
 
       def parse(input_or_string, opts)
@@ -214,14 +222,28 @@ module Neg
         rs = []
 
         loop do
-          r = @child.parse(i, opts)
-          break if ! r[2] && rs.size >= @min && (@max.nil? || rs.size <= @max)
-          rs << r
-          break if ! r[2]
-          break if rs.size == @max
+          rs << @child.parse(i, opts)
+          break if rs.last[2] == false
         end
 
-        success = (rs.empty? || rs.last[2]) && (rs.size >= @min)
+        rs.size.downto(1) do |i|
+
+          r = rs[i - 1]
+
+          next if r[2] == false
+
+          if @max && i > @max
+            r[2] = false
+            r[3] = "did not expect #{r[3].inspect} (min #{@min} max #{@max})"
+          end
+        end
+
+        trs = rs.take_while { |r| r[2] == true }
+        rs = trs + [ rs.find { |r| r[2] == false } ]
+
+        success = trs.size >= @min && (@max.nil? || trs.size <= @max)
+
+        i.rewind(rs.last[1]) if success && rs.last[2] == false
 
         [ success, nil, rs ]
       end

data/lib/neg/translator.rb

@@ -40,8 +40,8 @@ module Neg
     def self.translate(parse_tree)
 
       results =
-        parse_tree[4].each_with_object([]) { |node, a|
-          catch(nil) { a << translate(node) }
+        parse_tree[4].each_with_object([]) { |tree, a|
+          catch(nil) { a << translate(tree) } if tree[2]
         }
 
       apply(parse_tree, results)
@@ -70,6 +70,20 @@ module Neg
       def column;    @parse_tree[1][2]; end
 
       def result;    @parse_tree[3]; end
+
+      # Useful when the grammar has something of the form:
+      #
+      #   array == `[` + (value + (`,` + value) * 0) * 0 + `]`
+      #
+      # It flattens the value array.
+      #
+      # Look at the spec/sample_* files to see it in action.
+      #
+      def flattened_results
+
+        f2 = results.flatten(2)
+        f2.any? ? [ f2.shift ] + f2.flatten(2) : []
+      end
     end
   end
 end

data/lib/neg/version.rb

@@ -25,6 +25,6 @@
 
 module Neg
 
-  VERSION = '1.0.0'
+  VERSION = '1.1.0'
 end
 

data/neg.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.platform = Gem::Platform::RUBY
   s.authors = [ 'John Mettraux' ]
   s.email = [ 'jmettraux@gmail.com' ]
-  s.homepage = 'https://github.com/jmettraux/leg'
+  s.homepage = 'https://github.com/jmettraux/neg'
   s.rubyforge_project = 'ruote'
   s.summary = 'a neg narser'
 

data/spec/greedy_spec.rb

@@ -0,0 +1,115 @@
+
+require 'spec_helper'
+
+
+describe 'neg and errors' do
+
+  class BlockParser < Neg::Parser
+
+    parser do
+
+      blocks == nl? + block + (nl + block) * 0 + nl?
+
+      block == sp? + `begin` + sp + _('a-z') + nl + body + sp? + `end`
+      body == ((line | block) + nl) * 0
+      line == sp + `line`
+
+      nl == _("\s\n") * 1
+      nl? == _("\s\n") * 0
+      sp == _(" ") * 1
+      sp? == _(" ") * 0
+    end
+
+    translator do
+    end
+  end
+
+  it 'parses a single empty block' do
+
+    BlockParser.parse(%{
+      begin a
+      end
+    })
+  end
+
+  it 'parses nested blocks' do
+
+    BlockParser.parse(%{
+      begin a
+        begin b
+        end
+      end
+    })
+  end
+
+  it 'parses successive blocks' do
+
+    BlockParser.parse(%{
+      begin a
+      end
+      begin b
+      end
+    })
+  end
+
+  it 'fails gracefully on a missing end (0)' do
+
+    err = nil
+
+    begin
+      BlockParser.parse(%{
+        begin a
+          begin b
+        end
+      })
+    rescue => err
+    end
+
+    err.class.should == Neg::ParseError
+    err.position.should == [ 53, 5, 6 ]
+    err.message.should == 'expected "end", got ""'
+  end
+
+  it 'fails gracefully on a missing end (1)' do
+
+    err = nil
+
+    begin
+      BlockParser.parse(%{
+        begin a
+        end
+        begin b
+          begin c
+        end
+      })
+    rescue => err
+    end
+
+    err.class.should == Neg::ParseError
+    err.position.should == [ 81, 7, 6 ]
+    err.message.should == 'expected "end", got ""'
+  end
+
+  it 'fails gracefully on a li too much (2)' do
+
+    err = nil
+
+    begin
+      BlockParser.parse(%{
+        begin a
+        end
+        begin b
+          begin c
+            li
+          end
+        end
+      })
+    rescue => err
+    end
+
+    err.class.should == Neg::ParseError
+    err.position.should == [ 75, 6, 12 ]
+    err.message.should == 'expected "end", got "li\n"'
+  end
+end
+

data/spec/parser_repetition_spec.rb

@@ -18,14 +18,15 @@ describe Neg::Parser::RepetitionParser do
         [ :text, [ 0, 1, 1 ], true, '', [] ]
     end
 
-    it 'fails gracefully' do
-
-      lambda {
-        parser.parse('xx')
-      }.should raise_error(
-        Neg::UnconsumedInputError,
-        'remaining: "x"')
-    end
+    #it 'fails gracefully' do
+    #  lambda {
+    #    parser.parse('xx')
+    #  }.should raise_error(
+    #    Neg::UnconsumedInputError,
+    #    'remaining: "x"')
+    #end
+      #
+      # UnconsumedInputError is gone.
 
     it 'is rendered correctly via #to_s' do
 
@@ -57,14 +58,15 @@ describe Neg::Parser::RepetitionParser do
         [ :text, [ 0, 1, 1 ], true, 'xxx', [] ]
     end
 
-    it 'fails gracefully' do
-
-      lambda {
-        parser.parse('a')
-      }.should raise_error(
-        Neg::UnconsumedInputError,
-        'remaining: "a"')
-    end
+    #it 'fails gracefully' do
+    #  lambda {
+    #    parser.parse('a')
+    #  }.should raise_error(
+    #    Neg::UnconsumedInputError,
+    #    'remaining: "a"')
+    #end
+      #
+      # UnconsumedInputError is gone.
   end
 
   context '`x` * 2 (at least 2)' do
@@ -113,12 +115,13 @@ describe Neg::Parser::RepetitionParser do
           [ nil, [ 2, 1, 3 ], false, 'expected "x", got ""', [] ] ] ]
     end
 
-    it 'fails gracefully (unconsumed input)' do
-
-      lambda {
-        parser.parse('xxxx')
-      }.should raise_error(Neg::UnconsumedInputError, 'remaining: "x"')
-    end
+    #it 'fails gracefully (unconsumed input)' do
+    #  lambda {
+    #    parser.parse('xxxx')
+    #  }.should raise_error(Neg::UnconsumedInputError, 'remaining: "x"')
+    #end
+      #
+      # UnconsumedInputError is gone.
 
     it 'is rendered correctly via #to_s' do
 

data/spec/parser_spec.rb

@@ -33,30 +33,52 @@ describe 'Neg::Parser' do
   #
   # not worth the pain for now
 
-  describe '.parse' do
+#  describe '.parse' do
+#
+#    let(:parser) {
+#      Class.new(Neg::Parser) do
+#        text == `x`
+#      end
+#    }
+#
+#    it 'raises on unconsumed input' do
+#
+#      lambda {
+#        parser.parse('xy')
+#      }.should raise_error(
+#        Neg::UnconsumedInputError,
+#        'remaining: "y"')
+#    end
+#
+#    it 'raises on unconsumed input (...)' do
+#
+#      lambda {
+#        parser.parse('xyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy')
+#      }.should raise_error(
+#        Neg::UnconsumedInputError,
+#        'remaining: "yyyyyyy..."')
+#    end
+#  end
+  #
+  # UnconsumedInputError is gone.
+
+  context 'unconsumed input' do
 
     let(:parser) {
       Class.new(Neg::Parser) do
-        text == `x`
+        text == `x` * -1
       end
     }
 
-    it 'raises on unconsumed input' do
-
-      lambda {
-        parser.parse('xy')
-      }.should raise_error(
-        Neg::UnconsumedInputError,
-        'remaining: "y"')
-    end
-
-    it 'raises on unconsumed input (...)' do
+    it 'fails gracefully' do
 
-      lambda {
-        parser.parse('xyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy')
-      }.should raise_error(
-        Neg::UnconsumedInputError,
-        'remaining: "yyyyyyy..."')
+      parser.parse('xx').should ==
+        [ :text,
+          [ 0, 1, 1 ],
+           false,
+            'x',
+             [ [ nil, [0, 1, 1], true, 'x', [] ],
+               [ nil, [1, 1, 2], false, 'did not expect "x" (min 0 max 1)', [] ] ] ]
     end
   end
 end

data/spec/sample_arith_spec.rb

@@ -8,11 +8,11 @@ describe 'sample math parser' do
 
     expression  == operation
 
-    operator    == `+` | `-` | `*` | `/`
-    operation   == value + (operator + value) * 0
-    value       == parenthese | number
-    parenthese  == `(` + expression + `)`
-    number      == `-` * -1 + _('0-9') * 1
+    operator     == `+` | `-` | `*` | `/`
+    operation    == value + (operator + value) * 0
+    value        == parentheses | number
+    parentheses  == `(` + expression + `)`
+    number       == `-` * -1 + _('0-9') * 1
   end
 
   class ArithTranslator < Neg::Translator
@@ -22,7 +22,7 @@ describe 'sample math parser' do
     on(:value)     { |n| n.results.first }
 
     on(:expression) { |n|
-      results = n.results.flatten(2)
+      results = n.flattened_results
       results.size == 1 ? results.first : results
     }
   end

data/spec/sample_compact_spec.rb

@@ -10,11 +10,11 @@ describe 'sample compact arith parser' do
 
       expression  == operation
 
-      operator    == `+` | `-` | `*` | `/`
-      operation   == value + (operator + value) * 0
-      value       == parenthese | number
-      parenthese  == `(` + expression + `)`
-      number      == `-` * -1 + _('0-9') * 1
+      operator     == `+` | `-` | `*` | `/`
+      operation    == value + (operator + value) * 0
+      value        == parentheses | number
+      parentheses  == `(` + expression + `)`
+      number       == `-` * -1 + _('0-9') * 1
     end
 
     translator do

data/spec/sample_json_parser_spec.rb

@@ -40,14 +40,8 @@ describe 'sample JSON parser' do
       on(:value) { |n| n.results.first.first }
       on(:spaces?) { throw nil }
 
-      on(:object) { |n|
-        f2 = n.results.flatten(2)
-        Hash[f2.any? ? [ f2.shift ] + f2.flatten(2) : []]
-      }
-      on(:array) { |n|
-        f2 = n.results.flatten(2)
-        f2.any? ? [ f2.shift ] + f2.flatten(2) : []
-      }
+      on(:object) { |n| Hash[n.flattened_results] }
+      on(:array) { |n| n.flattened_results }
 
       on(:string) { |n| eval(n.result) }
 
@@ -198,8 +192,8 @@ describe 'sample JSON parser' do
   it 'raises a ParseError on incorrect input' do
 
     lambda do
-      JsonParser.parse("x")
-    end.should raise_error(Neg::ParseError, 'expected "{", got "x"')
+      JsonParser.parse('x')
+    end.should raise_error(Neg::ParseError, 'expected "null", got "x"')
   end
 end
 

data/spec/sample_scheme_spec.rb

@@ -0,0 +1,39 @@
+
+require 'spec_helper'
+
+
+describe 'sample scheme parser' do
+
+  class SchemeParser < Neg::Parser
+
+    parser do
+
+      expression == list | atom
+
+      list == `(` + (expression + (` ` + expression) * 0) * 0 + `)`
+      atom == _("^() ") * 1
+    end
+
+    translator do
+
+      on(:expression) { |n| n.results.first }
+      on(:atom) { |n| n.result }
+
+      on(:list) { |n| n.flattened_results }
+    end
+  end
+
+  it 'parses numbers' do
+
+    SchemeParser.parse("13").should == "13"
+    SchemeParser.parse("-13").should == "-13"
+  end
+
+  it 'parses lists' do
+
+    SchemeParser.parse("()").should == []
+    SchemeParser.parse("(a b c)").should == [ 'a', 'b', 'c' ]
+    SchemeParser.parse("(a (b c))").should == [ 'a', [ 'b', 'c' ] ]
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: neg
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-16 00:00:00.000000000 Z
+date: 2013-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -51,31 +51,33 @@ extensions: []
 extra_rdoc_files: []
 files:
 - Rakefile
-- lib/neg/errors.rb
 - lib/neg/input.rb
+- lib/neg/errors.rb
+- lib/neg/version.rb
 - lib/neg/parser.rb
 - lib/neg/translator.rb
-- lib/neg/version.rb
 - lib/neg.rb
-- spec/input_spec.rb
-- spec/parser_alternative_spec.rb
-- spec/parser_character_spec.rb
-- spec/parser_lookahead_parser_spec.rb
+- spec/sample_arith_spec.rb
+- spec/spec_helper.rb
 - spec/parser_non_terminal_spec.rb
-- spec/parser_repetition_spec.rb
+- spec/greedy_spec.rb
+- spec/parser_character_spec.rb
+- spec/parser_string_spec.rb
+- spec/sample_scheme_spec.rb
 - spec/parser_sequence_spec.rb
+- spec/sample_json_parser_spec.rb
+- spec/parser_alternative_spec.rb
 - spec/parser_spec.rb
-- spec/parser_string_spec.rb
-- spec/sample_arith_spec.rb
 - spec/sample_compact_spec.rb
-- spec/sample_json_parser_spec.rb
-- spec/spec_helper.rb
+- spec/input_spec.rb
+- spec/parser_lookahead_parser_spec.rb
+- spec/parser_repetition_spec.rb
 - neg.gemspec
-- LICENSE.txt
 - TODO.txt
-- CHANGELOG.md
+- LICENSE.txt
 - README.md
-homepage: https://github.com/jmettraux/leg
+- CHANGELOG.md
+homepage: https://github.com/jmettraux/neg
 licenses: []
 post_install_message: 
 rdoc_options: []
@@ -89,7 +91,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2623637621873378549
+      hash: 4523696272151903388
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -98,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2623637621873378549
+      hash: 4523696272151903388
 requirements: []
 rubyforge_project: ruote
 rubygems_version: 1.8.24