pegex 0.0.2 → 0.0.3

data/lib/pegex/tree.rb

@@ -2,7 +2,7 @@ require 'pegex/receiver'
 
 class Pegex::Tree < Pegex::Receiver
   def gotrule got=nil
-    return $pegex_nil if got.nil?
+    return Pegex::Constant::Null if got.nil?
 
     if self.parser.parent['-wrap']
       return {self.parser.rule => (got || [])}

data/lib/pegex/tree/wrap.rb

@@ -2,8 +2,8 @@ require 'pegex/tree'
 
 class Pegex::Tree::Wrap < Pegex::Tree
   def gotrule got
-    return got if @parser.parent['-pass']
-    return $pegex_nil unless got
+    return got || Pegex::Constant::Null if @parser.parent['-pass']
+    return Pegex::Constant::Null unless got
     return @parser.rule => got
   end
 

data/test/export-api.rb

@@ -0,0 +1,38 @@
+require 'test/unit'
+
+class Test::Unit::TestCase
+  def test
+    require 'pegex'
+    assert method('pegex'),
+      'pegex is exported'
+
+    parser1 = pegex "foo: <bar>\n"
+
+    assert parser1.kind_of?(Pegex::Parser),
+      'pegex returns a Pegex::Parser object'
+
+    assert_equal parser1.grammar.tree['+toprule'], 'foo',
+      'pegex() contains a grammar with a compiled tree'
+
+    parser2 = pegex(<<'...');
+number: /<DIGIT>+/
+...
+
+    begin
+      parser2.parse '123'
+      assert true, 'parser2.parse worked'
+    rescue
+      assert false, "parser2.parse failed: #{$!.message}"
+    end
+
+    assert parser2.kind_of?(Pegex::Parser),
+      'grammar property is Pegex::Parser object'
+
+    tree2 = parser2.grammar.tree
+    assert tree2, 'Grammar object has tree';
+    assert tree2.kind_of?(Hash), 'Grammar object is compiled to a tree'
+
+    assert_equal tree2['+toprule'], 'number',
+      '_FIRST_RULE is set correctly'
+  end
+end

data/test/flatten.rb

@@ -0,0 +1,30 @@
+require 'test/unit'
+require 'pegex'
+require 'pegex/receiver'
+
+class TestFlatten < Test::Unit::TestCase
+  def test_flatten
+    grammar = <<'...'
+a: (((b)))+
+b: (c | d)
+c: /(x)/
+d: /y/
+...
+    parser = pegex(grammar, R)
+    got = parser.parse('xxx')
+
+    assert_equal got.join(''), 'xxx', 'Array was flattened'
+  end
+end
+
+class R < Pegex::Receiver
+  def got_a(got)
+    got.flatten
+  end
+  def got_b(got)
+    [got]
+  end
+  def got_c(got)
+    [got]
+  end
+end

data/test/grammar-api.rb

@@ -1,6 +1,25 @@
-require './test/lib/test_pegex'
+require 'testml'
+require 'testml/compiler/lite'
+require 'testml/util'; include TestML::Util
 require 'pegex/grammar'
 
+TestML.new(
+  compiler: TestML::Compiler::Lite,
+).testml = <<'...'
+%TestML 0.1.0
+Plan = 1
+# Label = 'MyGrammar1 compiled a tree from its text'
+grammar_api() == 'foo'
+...
+
+class TestML::Bridge
+  def grammar_api
+    grammar = MyGrammar1.new
+    grammar.make_tree
+    native grammar.tree['+toprule']
+  end
+end
+
 class MyGrammar1 < Pegex::Grammar
   def initialize
     @text = <<'...'
@@ -12,10 +31,3 @@ baz: /def/
 ...
   end
 end
-
-TestML.run do |t|
-  g1 = MyGrammar1.new
-  g1.make_tree
-  t.assert_equal g1.tree['+toprule'], 'foo',
-    'MyGrammar1 compiled a tree from its text'
-end

data/test/lib/xxx.rb

@@ -1,13 +1,13 @@
 def XXX object
-  require 'psych'
-  puts Psych.dump object
+  require 'yaml'
+  puts YAML.dump object
   puts 'XXX from: ' + caller.first
   exit
 end
 
 def YYY object, show_caller=true
-  require 'psych'
-  puts Psych.dump object
+  require 'yaml'
+  puts YAML.dump object
   puts 'YYY from: ' + caller.first if show_caller
   return object
 end

data/test/mice.pgx

@@ -0,0 +1,7 @@
+phrase: !one number ~ things EOL
+
+one: /1/
+
+number: DIGIT
+
+things: /blind<SPACE>mice/

data/test/parse.rb

@@ -0,0 +1,18 @@
+require 'test/unit'
+require 'pegex'
+require 'pegex/input'
+
+class TestParse < Test::Unit::TestCase
+  def test_parse
+    grammar_file = 'test/mice.pgx';
+    # XXX need to add file support to Pegex::Input
+    # input =  Pegex::Input.new(file: grammar_file)
+    input = File.read(grammar_file)
+    begin
+      pegex(input).parse("3 blind mice\n")
+      assert true, "!<rule> works"
+    rescue
+      assert false, "!<rule> fails"
+    end
+  end
+end

data/test/repeat.rb

@@ -0,0 +1,10 @@
+require 'test/unit'
+require 'pegex'
+
+class TestRepeat < Test::Unit::TestCase
+  def test_repeat
+    parser = pegex('a: /<ANY>*?(x+)<ANY>*/')
+    assert_equal parser.parse('xxxx')['a'], 'xxxx', 'First parse works'
+    assert_equal parser.parse('xxxx')['a'], 'xxxx', 'Second parse works'
+  end
+end

data/test/sample.rb

@@ -0,0 +1,72 @@
+require 'test/unit'
+
+class TestSample < Test::Unit::TestCase
+  def test_sample
+    grammar_text = <<'...'
+contact:
+    name_section
+    phone_section
+    address_section
+
+name_section:
+    / Name <COLON> <BLANK>+ /
+    name
+    EOL
+
+name: /(<WORD>+)<BLANK>(<WORD>+)/
+
+phone_section: /Phone<COLON><BLANK>+/ <phone_number> <EOL>
+phone_number: term
+
+address_section:
+    /Address<COLON><EOL>/
+    street_line
+    city_line
+    country_line?
+
+street_line: indent street EOL
+street: /<NS><ANY>*/
+city_line: indent city EOL
+city: term
+country_line: indent country EOL
+country: term
+
+term: /(
+    <NS>            # NS is "non-space"
+    <ANY>*
+)/
+
+indent: /<BLANK>{2}/
+...
+
+    input = <<'...'
+Name: Ingy Net
+Phone: 919-876-5432
+Address:
+  1234 Main St
+  Niceville
+  OK
+...
+
+    want = <<'...'
+...
+
+    require 'pegex/grammar'
+    require 'pegex/receiver'
+    require 'pegex/compiler'
+    grammar = Pegex::Grammar.new do |i|
+      i.tree = Pegex::Compiler.new.compile(grammar_text).tree
+    end
+    parser = Pegex::Parser.new do |i|
+      i.grammar = grammar
+      i.receiver = Pegex::Receiver.new
+    end
+    ast1 = parser.parse(input)
+
+    assert true, 'parsed'
+# TODO
+#     got = YAML.dump(ast1)
+#     assert_equal got, want, 'It works'
+  end
+end
+

data/test/testml.rb

@@ -0,0 +1,25 @@
+require 'test/unit'
+require 'testml'
+require 'testml/compiler/lite'
+$:.unshift "#{Dir.getwd}/test"
+$:.unshift "#{Dir.getwd}/test/lib"
+require 'testml_bridge'
+
+class TestMLTestCase < Test::Unit::TestCase
+  def run_testml_file(file)
+    TestML.new(
+      testml: file,
+      bridge: TestMLBridge,
+      compiler: TestML::Compiler::Lite,
+    ).run(self)
+  end
+
+  (Dir.glob('test/testml/*.tml')
+    .collect {|f| f.sub(/^test\//, '')}
+  ).each do |file|
+    method_name = 'test_' + file.gsub(/\W+/, '_').sub(/_tml$/, '')
+    define_method(method_name.to_sym) do
+      run_testml_file(file)
+    end
+  end
+end

data/test/testml.yaml

@@ -0,0 +1,3 @@
+source_testml_dir: ../../pegex-tml
+local_testml_dir: ./testml
+exclude_testml_files: []

data/test/{compiler-checks.tml → testml/compiler-checks.tml}

@@ -1,12 +1,10 @@
-#!/usr/bin/env testml
+%TestML 0.1.0
 
-%TestML 1.0
+Plan = 48
 
-Plan = 50;
+*grammar.bootstrap_compile.yaml.clean == *yaml
+*grammar.compile.yaml.clean == *yaml
 
-*grammar.compile.yaml.clean == *yaml;
-
-testml_data <<'...'
 === Empty Grammar
 --- grammar
 --- yaml
@@ -256,10 +254,10 @@ a:
 
 === Meta Lines
 --- grammar
-%grammar        foo
-%version    1.1.1
-%extends bar bar  
-%include   bazzy 
+\%grammar        foo
+\%version    1.1.1
+\%extends bar bar  
+\%include   bazzy 
 a: /b/
 --- yaml
 +extends: bar bar
@@ -268,4 +266,3 @@ a: /b/
 +version: 1.1.1
 a:
   .rgx: b
-...

data/test/{compiler-equivalence.rb → testml/compiler-equivalence.tml}

@@ -1,12 +1,7 @@
-require './test/lib/test_pegex'
+%TestML 0.1.0
 
-TestML.require_or_skip 'psych'
+*grammar1.compile.yaml == *grammar2.compile.yaml
 
-TestML.run do |t|
-  t.eval '*grammar1.compile.yaml == *grammar2.compile.yaml'
-end
-
-TestML.data <<'...'
 === Simple Test Case
 --- grammar1
 a: /x/
@@ -76,4 +71,3 @@ a b: /O HAI/
 --- grammar2
 a: /O HAI/
 b: /O HAI/
-...

data/test/{compiler.tml → testml/compiler.tml}

@@ -1,3 +1,16 @@
+%TestML 0.1.0
+
+Plan = 63
+
+Label = '$BlockLabel - Compiler output matches bootstrap?'
+*grammar.compile.yaml == *grammar.bootstrap_compile.yaml
+
+Label = '$BlockLabel - Compressed grammar compiles the same?'
+*grammar.compress.compile.yaml == *grammar.compress.compile.yaml
+
+Label = '$BlockLabel - Compressed grammar matches uncompressed?'
+*grammar.compress.compile.yaml == *grammar.compile.yaml
+
 === Single Regex
 --- grammar
 a: /x/

data/test/{error.rb → testml/error.tml}

@@ -1,17 +1,7 @@
-require './test/lib/test_pegex'
+%TestML 0.1.0
 
-TestML.run do |t|
-  t.eval 'parse_input(*grammar, *input).Catch ~~ *error'
-end
+parse_input(*grammar, *input).Catch ~~ *error
 
-class TestPegex
-  def parse_input grammar, input
-    parser = pegex grammar
-    return parser.parse input
-  end
-end
-
-TestML.data <<'...'
 === Error fails at furthest match
 # XXX This one not testing much.
 --- grammar
@@ -158,4 +148,3 @@ a: /a+/ %%~%%^%% ~
 aaa
 --- error: Rule ending syntax error
 # --- error: Illegal characters in separator indicator
-...

data/test/testml/optimize.tml

@@ -0,0 +1,14 @@
+%TestML 0.1.0
+
+# XXX Skipping this test for now. Might need a %Skip in TestML
+'1' == '1'
+# *grammar.compile.optimize.yaml.clean == *yaml
+
+
+=== Question Mark Expansion
+--- SKIP
+--- grammar
+a: /(:foo)/
+--- yaml
+a:
+  .rgx: /(?:foo)/

data/test/{tree-pegex.tml → testml/tree-pegex.tml}

@@ -1,5 +1,15 @@
+%TestML 0.1.0
+
+Plan = 2
+
+Label = '$BlockLabel - Pegex::Tree'
+parse_to_tree(*grammar, *input).yaml.clean == *tree
+Label = '$BlockLabel - Pegex::Tree::Wrap'
+parse_to_tree_wrap(*grammar, *input).yaml.clean == *wrap
+Label = '$BlockLabel - TestAST'
+parse_to_tree_test(*grammar, *input).yaml.clean == *ast
+
 === Part of Pegex Grammar
---- SKIP
 --- grammar
 \# This is the Pegex grammar for Pegex grammars!
 grammar: ( <comment>* <rule_definition> )+ <comment>*

data/test/{tree.tml → testml/tree.tml}

@@ -1,3 +1,14 @@
+%TestML 0.1.0
+
+Plan = 56
+
+Label = '$BlockLabel - Pegex::Tree'
+parse_to_tree(*grammar, *input).yaml.clean == *tree
+Label = '$BlockLabel - Pegex::Tree::Wrap'
+parse_to_tree_wrap(*grammar, *input).yaml.clean == *wrap
+Label = '$BlockLabel - t::TestAST'
+parse_to_tree_test(*grammar, *input).yaml.clean == *ast
+
 === Single Regex - Single Capture
 --- grammar
 a: /x*(y*)z*<EOL>/
@@ -57,13 +68,12 @@ d: /d/
 --- tree
 - []
 - []
---- wrapSKIP
+--- wrap
 a:
 - []
 - []
 
 === A subrule
---- SKIP
 --- grammar
 a: <b> /(y+)/ <EOL>
 b: /(x+)/
@@ -109,7 +119,6 @@ a:
   - zzz
 
 === + Modifier
---- SKIP
 --- grammar
 a: ( b c )+ <EOL>
 b: /(x*)/
@@ -186,7 +195,6 @@ a:
   c: ccc
 
 === Assertion not captured
---- SKIP
 --- grammar
 a: =x x y EOL
 x: /(x+)/
@@ -202,7 +210,6 @@ a:
 - y: yyyy
 
 === Empty regex group plus rule
---- SKIP
 --- grammar
 a: <b>* <c> <EOL>
 b: /xxx/
@@ -218,7 +225,6 @@ a:
 - c: yyy
 
 === Rule to Rule to Rule
---- SKIP
 --- grammar
 a: <b>
 b: <c>*
@@ -262,7 +268,6 @@ a:
   - c: c
 
 === Rule with Separator
---- SKIP
 --- grammar
 a: <c>* % <d>
 c: /(c+)/
@@ -388,7 +393,6 @@ a:
 - b: b
 
 === Quantifier on the Separator
---- SKIP
 --- grammar
 a: <b>2-4 %% <c>*
 b: /(b)/
@@ -422,7 +426,6 @@ a:
 - - b: b
   - b: b
 
-# TODO - deal with psych null output
 === False Values
 --- grammar
 a: <zero> <empty> <undef>
@@ -447,3 +450,14 @@ a:
 - b: bb
 - c: cc
 - d: dd
+
+=== 2 + 1
+--- SKIP
+--- grammar
+a: <b>2 b
+b: /(b)/
+--- input: bbb
+--- ast
+- b
+- b
+- b