rparsec-ruby19 1.0

data/rparsec/token.rb

@@ -0,0 +1,43 @@
+require 'rparsec/misc'
+
+module RParsec
+
+#
+# Represents a token during lexical analysis.
+#
+class Token
+  extend DefHelper
+  
+  def_ctor :kind, :text, :index
+  
+  #
+  # The type of the token
+  #
+  attr_reader :kind
+  
+  #
+  # The text of the matched range
+  #
+  attr_reader :text
+  
+  #
+  # The starting index of the matched range
+  #
+  attr_reader :index
+  
+  #
+  # The length of the token.
+  #
+  def length
+    @text.length
+  end
+  
+  #
+  # String representation of the token.
+  # 
+  def to_s
+    "#{@kind}: #{@text}"
+  end
+end
+
+end # module

data/test/src/expression_test.rb

@@ -0,0 +1,124 @@
+require 'import'
+import :parsers, :functors, :expressions
+require 'parser_test'
+
+class ExpressionParserTest < ParserTestCase
+  include Functors
+  def setup
+    @use_around = true
+  end
+  def around(p)
+    return p unless @use_around
+    delim = whitespace.many_
+    p << delim;
+  end
+  def negate
+    around(char('-')) >> value(Neg)
+  end
+  def positive
+    around(char('+')) >> value(Id)
+  end
+  def increment
+    around(string('++') >> value(Inc))
+  end
+  def decrement
+    around(string('--')) >> value(Dec)
+  end
+  def plus
+    around(char('+')) >> value(Plus)
+  end
+  def minus
+    around(char('-')) >> value(Minus)
+  end
+  def mul
+    around(char('*')) >> value(Mul)
+  end
+  def space_mul
+    not_among('+-*/%').peek >> value(Mul)
+  end
+  def div
+    around(char('/')) >> value(Div)
+  end
+  def rdiv
+    around(str('//')) >> value(Div)
+  end
+  def mod
+    around(char('%')) >> value(Mod)
+  end
+  def lparen
+    around(char('('))
+  end
+  def rparen
+    around(char(')'))
+  end
+  def int
+    around(integer).map(&To_i)
+  end
+  def testPrefix
+    parser = whitespace.many_ >> int.prefix(negate) << eof
+    assertParser(' - -3 ', 3, parser)
+    assertParser(' 3', 3, parser)
+    assertParser('3 ', 3, parser)
+    assertParser('-3', -3, parser)
+    assertError(' - -', 'integer expected', parser, 4)
+    assertError(' - -5 3', 'EOF expected', parser, 6)
+  end
+  def testPostfix
+    parser = whitespace.many_ >> int.postfix(increment) << eof
+    assertParser(' 3++ ++', 5, parser)
+    assertParser('3++++ ', 5, parser)
+    assertParser(' 3', 3, parser)
+    assertError(' ++', 'integer expected', parser, 1)
+    assertError('5++ 3', 'EOF expected', parser, 4)
+  end
+  def testInfixn
+    parser = whitespace.many_ >> int.infixn(plus) << eof
+    assertParser(' 1 + 2 ', 3, parser)
+    assertParser('1 + 2 ', 3, parser)
+    assertError('1+2 +3', 'EOF expected', parser, 4)
+  end
+  def testInfixl
+    parser = whitespace.many_ >> int.infixl(minus) << eof
+    assertParser(' 1-2 -3 ', -4, parser)
+    assertParser('1 - 2 ', -1, parser)
+    assertParser('1  ', 1, parser)
+    assertError('1-2-3-', 'integer expected', parser, 6)
+  end
+  def testInfixr
+    parser = whitespace.many_ >> int.infixr(minus) << eof
+    assertParser(' 1-2 -3 ', 2, parser)
+    assertParser(' 1-2 -3-4 ', -2, parser)
+    assertParser('1 - 2 ', -1, parser)
+    assertParser('1  ', 1, parser)
+    assertError('1-2-3-', 'integer expected', parser, 6)
+  end
+  def testExpression
+    @use_around = false
+    ops = OperatorTable.new do |tbl|
+      tbl.infixl(plus, 20)
+      tbl.infixl(minus, 20)
+      tbl.infixl(mul, 40)
+      tbl.infixl(space_mul, 40)
+      tbl.infixl(div, 40)
+      tbl.prefix(negate, 60)
+      tbl.prefix(positive, 60)
+      tbl.postfix(increment, 50)
+      tbl.postfix(decrement, 50)
+      tbl.infixr(rdiv, 40)
+    end
+    expr = nil
+    term = int | char(?() >> lazy{expr} << char(?))
+    delim = whitespace.many_
+    expr = delim >> Expressions.build(term, ops, delim)
+    
+    assertParser('1', 1, expr)
+    assertParser('1+2', 3, expr)
+    assertParser('(1-2)', -1, expr)
+    assertParser('2-3* 2', -4, expr)
+    assertParser("\n ((2-3 )*-+2--) ", 3, expr)
+    assertParser('((2-3 )*-+2--/2//2) ', 3, expr)
+    assertParser('((2-3 )*-+2--/2//2--) ', 1, expr)
+    assertParser('((2-3 )*-+2--//4//2) ', 2, expr)
+    assertParser('((2-3 )*-+2--/2/2) ', 0, expr)
+  end
+end

data/test/src/full_parser_test.rb

@@ -0,0 +1,95 @@
+require 'import'
+import :parsers, :keywords, :operators, :functors, :expressions
+require 'parser_test'
+
+class FullParserTest < ParserTestCase
+  def calculate_simple_cases(val, cases, default)
+    cases.each do |cond, consequence|
+      # cond, consequence = *cs
+      return consequence if cond == val
+    end
+    default
+  end
+  def calculate_full_cases(cases, default)
+    cases.each do |cond, consequence|
+      return consequence if cond
+    end
+    default
+  end
+  def parser
+    keywords = Keywords.case_sensitive(%w{case when else end and or not true false})
+    ops = Operators.new(%w{+ - * / % ++ -- == > < >= <= != : ( )})
+    lexer = integer.token(:int)|keywords.lexer|ops.lexer
+    delim = whitespaces |comment_line('#')
+    lexeme = lexer.lexeme(delim) << eof
+    expr = nil
+    lazy_expr = lazy{expr}
+    compare = ops['>'] >> Gt | ops['<'] >> Lt | ops['>='] >> Ge | ops['<='] >> Le |
+      ops['=='] >> Eq | ops['!='] >> Ne
+    comparison = sequence(lazy_expr, compare, lazy_expr) {|e1,f,e2|f.call(e1,e2)}
+    bool = nil
+    lazy_bool = lazy{bool}
+    bool_term = keywords[:true] >> true | keywords[:false] >> false |
+      comparison | ops['('] >> lazy_bool << ops[')']
+    bool_table = OperatorTable.new.
+      infixl(keywords[:or] >> Or, 20).
+      infixl(keywords[:and] >> And, 30).
+      infixl(keywords[:not] >> Not, 30)
+    
+    bool = Expressions.build(bool_term, bool_table)
+    simple_case = sequence(keywords[:when], lazy_expr, ops[':'], lazy_expr) do |w,cond,t,val|
+      [cond, val]
+    end
+    full_case = sequence(keywords[:when], bool, ops[':'], lazy_expr) do |w,cond,t,val|
+      [cond, val]
+    end
+    default_case = (keywords[:else] >> lazy_expr).optional
+    simple_when_then = sequence(lazy_expr, simple_case.many, default_case, 
+      keywords[:end]) do |val, cases, default|
+      calculate_simple_cases(val, cases, default)
+    end
+    full_when_then = sequence(full_case.many, default_case, keywords[:end]) do |cases, default|
+      calculate_full_cases(cases, default)
+    end
+    case_expr = keywords[:case] >> (simple_when_then | full_when_then)
+    
+    term = token(:int, &To_i) | (ops['('] >> lazy_expr << ops[')']) | case_expr
+    table = OperatorTable.new.
+      infixl(ops['+'] >> Plus, 20).
+      infixl(ops['-'] >> Minus, 20).
+      infixl(ops['*'] >> Mul, 30).
+      infixl(ops['/'] >> Div, 30).
+      postfix(ops['++'] >> Inc, 40).
+      postfix(ops['--'] >> Dec, 40).
+      prefix(ops['-'] >> Neg, 50)
+    expr = Expressions.build(term, table)
+    lexeme.nested(expr << eof)
+  end
+  def verify(code, expected=eval(code))
+    assertParser(code, expected, self.parser)
+  end
+  def testNumber
+    verify(' 1')
+  end
+  def testSimpleCalc
+    verify('1 - 2')
+  end
+  def testComplexCalculationWithComment
+    verify('2*15/-(5- -2) #this is test')
+  end
+  def testSimpleCaseWhen
+    verify('case 1 when 1: 0 else 1 end')
+  end
+  def testSimpleCaseWhenWithRegularCalc
+    verify('case 1 when 1*1: (1-2) when 3:4 end+1')
+  end
+  def testFullCaseWhen
+      assertParser('3*case when 1==0 and 1==1: 1 when 1==1 : 2 end', 6, parser)
+    begin
+      parser.parse('3*case when (1==0 and 1==1): 1 when 1==1 then 2 end')
+      fail('should have failed')
+      rescue ParserException => e
+        assert(e.message.include?(': expected, then at line 1, col 42'))
+    end
+  end
+end

data/test/src/functor_test.rb

@@ -0,0 +1,66 @@
+require 'import'
+require 'runit/testcase'
+import :parsers, :functors
+
+include RParsec
+class Proc
+  include FunctorMixin
+end
+class Method
+  include FunctorMixin
+end
+    
+class FunctorTestCase < RUNIT::TestCase
+  include Functors
+  def verify(expected, f, *args)
+    assert_equal(expected, f.call(*args))
+  end
+  def testConst
+    verify(1, const(1), 'a')
+  end
+  def testMethod
+    verify(true, 1.method(:kind_of?), Fixnum)
+  end
+  def testFlip
+    verify(1, Minus.flip, 1, 2)
+  end
+  def testCompose
+    verify(-3, Neg.compose(Plus), 1, 2)
+    verify(-3, Neg << Plus, 1, 2)
+    verify(-3, Plus >> Neg, 1, 2)
+    verify(3, Neg << Neg << Plus, 1, 2)
+  end
+  def testCurry
+    assert_equal(3, Plus.curry.call(1).call(2))
+    assert_equal(-1, Minus.curry.call(1).call(2))
+  end
+  def testReverse_curry
+    assert_equal(1, Minus.reverse_curry.call(1).call(2))
+  end
+  def testUncurry
+    verify(-1, Minus.curry.uncurry, 1, 2)
+  end
+  def testReverse_uncurry
+    verify(-1, Minus.reverse_curry.reverse_uncurry, 1, 2)
+    verify(1, Minus.reverse_curry.uncurry, 1, 2)
+    verify(1, Minus.curry.reverse_uncurry, 1, 2)
+  end
+  def testRepeat
+    cnt=0
+    inc = proc {cnt+=1}
+    n = 10
+    verify(n, (inc*n))
+    assert_equal(10, cnt)
+  end
+  def testPower
+    double = Mul.curry.call(2)
+    verify(8, double ** 3, 1)
+    verify(nil, double ** 0, 1)
+  end
+  def testNth
+    verify(2, nth(1), 1, 2, 3)
+  end
+  def testMethodIsMixedIn
+    verify(false, 1.method(:kind_of?).compose(Id), String)
+  end
+end

data/test/src/import.rb

@@ -0,0 +1,5 @@
+# $: << "#{File.dirname(__FILE__)}/../.."
+
+def import *names
+  names.each {|lib|require "rparsec/#{lib}"}
+end

data/test/src/keyword_test.rb

@@ -0,0 +1,28 @@
+require 'import'
+import :parsers, :keywords
+require 'parser_test'
+
+class KeywordTestCase < ParserTestCase
+  Insensitive = Keywords.case_insensitive(%w{select from where group by order having}){|x|x.downcase}
+  Sensitive = Keywords.case_sensitive(%w{new delete if else then void int}){|x|x}
+  def verifySensitiveKeyword(code, keyword)
+    assertParser(code, keyword, Sensitive.lexer.lexeme.nested(Sensitive.parser(keyword)))
+    assertParser(code, keyword, Sensitive.lexer.lexeme.nested(Sensitive.parser(keyword.to_sym)))
+  end
+  def verifyInsensitiveKeyword(code, keyword)
+    assertParser(code, keyword.downcase, 
+      Insensitive.lexer.lexeme.nested(Insensitive.parser(keyword)))
+    assertParser(code, keyword.downcase,Insensitive.lexer.lexeme.nested(Insensitive.parser(keyword.downcase.to_sym)))
+  end
+  def testCaseSensitiveKeywords
+    verifySensitiveKeyword('new int void', 'new')
+  end
+  def testCaseInsensitiveKeywords
+    verifyInsensitiveKeyword('select from where', 'SeleCt')
+  end
+  def testBasics
+    assert(!Insensitive.case_sensitive?)
+    assert(Sensitive.case_sensitive?)
+    assert(:keyword, Sensitive.keyword_symbol)
+  end
+end

data/test/src/operator_test.rb

@@ -0,0 +1,21 @@
+require 'import'
+import :parsers, :operators, :functors
+require 'parser_test'
+
+class OperatorTestCase < ParserTestCase
+  Ops = Operators.new(%w{++ + - -- * / ~}, &Id)
+  def verifyToken(src, op)
+    verifyParser(src, op, Ops[op])
+  end
+  def verifyParser(src, expected, parser)
+    assertParser(src, expected, Ops.lexer.lexeme.nested(parser))
+  end
+  def testAll
+    verifyToken('++ -', '++')
+    verifyParser('++ + -- ++ - +', '-', 
+      (Ops['++']|Ops['--']|Ops['+']).many_ >> Ops['-'])
+  end
+  def testSort
+    assert_equal(%w{+++ ++- ++ + --- -- -}, Operators.sort(%w{++ - + -- +++ ++- ---}))
+  end
+end

data/test/src/parser_test.rb

@@ -0,0 +1,53 @@
+require 'import'
+require 'rubyunit'
+import :parsers, :functors
+
+include RParsec
+
+class ParserTestCase < RUNIT::TestCase
+  include Functors
+  include Parsers
+  def assertParser(code, expected, parser)
+    assert_equal(expected, parser.parse(code))
+  end
+  def assertError(code, expected, parser, index=0, line=1, col=1+index)
+    begin
+      parser.parse(code)
+      assert_fail("error should have happened")
+      rescue ParserException => e
+        assert_equal(index, e.index)
+        msg = expected
+        msg = add_encountered(msg, current(code,index)) << " at line #{line}, col #{col}." unless expected.include? 'at line'
+        assert_equal(msg, e.message)
+    end
+  end
+  def assertGrammar(code, expected, lexer, grammar)
+    assertParser(code, expected, lexer.nested(grammar))
+  end
+  def assertGrammarError(code, expected, token_name, lexer, grammar, index=0, line=1, col=1+index)
+    parser = lexer.nested(grammar)
+    begin
+      parser.parse(code)
+      assert_fail("error should have happened")
+      rescue ParserException => e
+        assert_equal(index, e.index)
+        msg = expected
+        msg = "#{msg}, #{token_name}" << " at line #{line}, col #{col}." unless expected.include? 'at line'
+        assert_equal(msg, e.message)
+    end
+  end
+  def current(code, index)
+    return "EOF" if code.length <= index
+    c = code[index]
+    if c.kind_of? Fixnum
+      "'"<<c<<"'"
+    else
+      c.to_s
+    end
+  end
+  def add_encountered(msg, encountered)
+    result = msg.dup
+    result << ', ' unless msg.strip.length == 0 || msg =~ /.*(\.|,)\s*$/
+    "#{result}#{encountered}"
+  end
+end

data/test/src/perf_benchmark.rb

@@ -0,0 +1,25 @@
+require 'import'
+import :rparsec
+require 'benchmark'
+require 'rubyunit'
+class PerfTestCase < RUNIT::TestCase
+  include Parsers
+  include Functors
+  def test1
+    code = "1+#{'1+' * 6000}1"
+    ops = OperatorTable.new do |tbl|
+      tbl.infixl(char(?+) >> Plus, 20)
+      tbl.infixl(char(?-) >> Minus, 20)
+      tbl.infixl(char(?*) >> Mul, 40)
+      tbl.infixl(char(?/) >> Div, 40)
+      tbl.prefix(char(?-) >> Neg, 60)
+    end
+    expr = nil
+    term = integer.map(&To_i) | char(?() >> lazy{expr} << char(?))
+    delim = whitespace.many_
+    expr = delim >> Expressions.build(term, ops, delim)
+    Benchmark.bm do |x|
+        x.report("parsing") {puts(expr.parse(code))}
+    end
+  end
+end