pegex 0.0.1

data/test/compiler.tml

@@ -0,0 +1,111 @@
+=== Single Regex
+--- grammar
+a: /x/
+
+=== Single Reference
+--- grammar
+a: <b>
+
+=== Single Error
+--- grammar
+a: `b`
+
+=== Simple All Group
+--- grammar
+a: /b/ <c>
+
+=== Simple Any Group
+--- grammar
+a: <b> | <c>
+
+=== Bracketed All Group
+--- grammar
+a: ( <b> /c/ )
+
+=== Bracketed Any Group
+--- grammar
+a: ( <b> | /c/ | `d` )
+
+=== Bracketed Group in Unbracketed Group
+--- grammar
+a: <b> ( <c> | <d> )
+
+=== And over Or Precedence
+--- grammar
+a: <b> <c> | <d> <e> | <f> % <g>
+
+=== Multiple Rules
+--- grammar
+a: <b>
+b: <c>
+
+=== Simple Grammar
+--- grammar
+a: ( <b> <c>* )
+b: /x/
+c: /y+/
+
+=== Semicolons OK
+--- grammar
+a: <b>;
+b: <c>
+c: /d/;
+
+=== Unbracketed
+--- grammar
+a: <b> <c> <d>
+b: <c> | <d>
+
+=== Not Rule
+--- grammar
+a: !<b> <c>
+
+=== Multiline
+--- grammar
+a: <b>
+   <c>
+b:
+    /c/ <d>
+    <e>;
+c:
+    <d> |
+    ( /e/ <f> )
+    | `g`
+
+=== Various Groups
+--- grammar
+a: <b> ( <c> | <d> )
+b: ( <c> | <d> ) <e>
+c: <d> | ( <e> <f>) | <g>
+d: <e> | (<f> <g>) | <h> | ( `i` )
+e: ( <f> )
+
+=== Modifiers
+--- grammar
+a: !<a> =<b>
+b: ( /c/ <d> )+
+c: ( /c/ <d> )+
+
+=== Any Group Plus Rule
+--- grammar
+a: /w/ ( <x>+ | <y>* ) <z>?
+
+=== Equivalent
+--- grammar
+a: <b>
+c: !<d>
+
+=== Regex and Rule
+--- grammar
+a_b: /c/ <d>
+
+=== Quantified group
+--- grammar
+a: <b> ( <c>* | <d>+ )+
+e: ( <f> !<g> )?
+
+=== Failures to test later
+--- SKIP
+--- grammar
+b: ( /x/ )+
+

data/test/error.rb

@@ -0,0 +1,161 @@
+require './test/lib/test_pegex'
+
+TestML.run do |t|
+  t.eval 'parse_input(*grammar, *input).Catch ~~ *error'
+end
+
+class TestPegex
+  def parse_input grammar, input
+    parser = pegex grammar
+    return parser.parse input
+  end
+end
+
+TestML.data <<'...'
+=== Error fails at furthest match
+# XXX This one not testing much.
+--- grammar
+a: b+ c
+b: /b/
+c: /c/
+--- input
+bbbbddddd
+--- error: ddddd\n
+
+### TODO ###
+# === Pegex: Illegal meta rule
+# --- grammar
+# %grammar Test
+# %foobar Quark
+# a: /a+/
+# --- input
+# aaa
+# --- error: Illegal meta rule
+
+=== Pegex: Rule header syntax error
+--- grammar
+a|: /a+/
+--- input
+aaa
+--- error: Rule header syntax error
+
+=== Pegex: Rule ending syntax error
+--- grammar
+a: /a+/ |
+--- input
+aaa
+--- error: Rule ending syntax error
+
+=== Pegex: Illegal rule modifier
+--- grammar
+a: /a+/
+b: ^<a>1-2
+--- input
+aaa
+--- error: Illegal rule modifier
+
+=== Pegex: Missing > in rule reference
+--- grammar
+a: /a+/
+b: !<a1-2
+--- input
+aaa
+--- error: Missing > in rule reference
+
+=== Pegex: Missing < in rule reference
+--- grammar
+a: /a+/
+b: !a>1-2
+--- input
+aaa
+--- error: Rule ending syntax error
+# --- error: Missing < in rule reference
+
+=== Pegex: Illegal character in rule quantifier
+--- grammar
+a: /a+/
+b: !a^1-2
+--- input
+aaa
+--- error: Rule ending syntax error
+# --- error: Illegal character in rule quantifier
+
+=== Pegex: Unprotected rule name with numeric quantifier
+--- grammar
+a: /a+/
+b: !a1-2
+--- input
+aaa
+--- error: Rule ending syntax error
+# --- error: Unprotected rule name with numeric quantifier
+
+=== Pegex: Runaway regular expression
+--- grammar
+a: /a+
+--- input
+aaa
+--- error: Runaway regular expression
+
+=== Pegex: Illegal group rule modifier
+--- grammar
+a: /a+/
+b: !(a =<a>)1-2
+--- input
+aaa
+--- error: Illegal group rule modifier
+
+=== Pegex: Runaway rule group
+--- grammar
+a: /a+/
+b: .(a =<a>1-2
+--- input
+aaa
+--- error: Runaway rule group
+
+=== Pegex: Illegal character in group rule quantifier
+--- grammar
+a: /a+/
+b: .(a =<a>)^2
+--- input
+aaa
+--- error: Rule ending syntax error
+# --- error: Illegal character in group rule quantifier
+
+=== Pegex: Multi-line error messages not allowed
+--- grammar
+a: /a+/
+b: `This is legal`
+c: `This is
+ 
+illegal`
+--- input
+aaa
+--- error: Multi-line error messages not allowed
+
+=== Pegex: Runaway error message
+--- grammar
+a: /a+/
+b: `This is legal`
+c: `This is
+
+illegal
+--- input
+aaa
+--- error: Runaway error message
+
+=== Pegex: Leading separator form (BOK) no longer supported
+--- grammar
+a: /a+/ %%% ~
+--- input
+aaa
+--- error: Rule ending syntax error
+# --- error: Leading separator form (BOK) no longer supported
+
+=== Pegex: Illegal characters in separator indicator
+--- grammar
+a: /a+/ %%~%%^%% ~
+--- input
+aaa
+--- error: Rule ending syntax error
+# --- error: Illegal characters in separator indicator
+...

data/test/export_ok.rb

@@ -0,0 +1,36 @@
+require './test/lib/test_pegex'
+
+TestML.run do |t|
+  require 'pegex'
+  t.assert method('pegex'),
+    'pegex is exported'
+
+  parser1 = pegex "foo: <bar>\n"
+
+  t.assert parser1.kind_of?(Pegex::Parser),
+    'pegex returns a Pegex::Parser object'
+
+  t.assert_equal parser1.grammar.tree['+toprule'], 'foo',
+    'pegex() contains a grammar with a compiled tree'
+
+  parser2 = pegex(<<'...');
+number: /<DIGIT>+/
+...
+
+  begin
+    parser2.parse '123'
+    t.assert true, 'parser2.parse worked'
+  rescue
+    t.assert false, "parser2.parse failed: #{$!.message}"
+  end
+
+  t.assert parser2.kind_of?(Pegex::Parser),
+    'grammar property is Pegex::Parser object'
+
+  tree2 = parser2.grammar.tree
+  t.assert tree2, 'Grammar object has tree';
+  t.assert tree2.kind_of?(Hash), 'Grammar object is compiled to a tree'
+
+  t.assert_equal tree2['+toprule'], 'number',
+    '_FIRST_RULE is set correctly'
+end

data/test/grammar-api.rb

@@ -0,0 +1,21 @@
+require './test/lib/test_pegex'
+require 'pegex/grammar'
+
+class MyGrammar1 < Pegex::Grammar
+  def initialize
+    @text = <<'...'
+foo: /xyz/ <bar>
+bar:
+    /abc/ |
+    <baz>
+baz: /def/
+...
+  end
+end
+
+TestML.run do |t|
+  g1 = MyGrammar1.new
+  g1.make_tree
+  t.assert_equal g1.tree['+toprule'], 'foo',
+    'MyGrammar1 compiled a tree from its text'
+end

data/test/lib/recursive_sort.rb

@@ -0,0 +1,17 @@
+class Object
+  def recursive_sort
+    self
+  end
+end
+
+class Array
+  def recursive_sort
+    map &:recursive_sort
+  end
+end
+
+class Hash
+  def recursive_sort
+    Hash[keys.sort.map {|k| [k, self[k].recursive_sort]}]
+  end
+end

data/test/lib/test_pegex.rb

@@ -0,0 +1,33 @@
+require 'testml/lite'
+require 'pegex/compiler'
+require 'recursive_sort'
+
+class TestPegex < TestML::Lite
+  include TestML::Lite::TestCases
+
+  def compile grammar_text
+    $grammar_text = grammar_text
+    tree = Pegex::Compiler.new.parse(grammar_text).combinate.tree
+    tree.delete '+toprule'
+    return tree
+  end
+
+  def yaml object
+    require 'psych'
+    Psych.dump object.recursive_sort
+  end
+
+  def clean yaml
+    yaml.sub! /^\.\.\.\n/, ''
+    yaml.sub! /\A---\s/, ''
+    yaml.gsub! /'(\d+)'/, '\1'
+    yaml.gsub! /\+eok: true/, '+eok: 1'
+    return yaml
+  end
+
+  def on_fail
+    puts "Parsing this Pegex grammar:"
+    puts $grammar_text
+    puts
+  end
+end

data/test/lib/testast.rb

@@ -0,0 +1,15 @@
+require 'pegex/tree'
+
+class TestAST < Pegex::Tree
+  def got_zero got
+    return 0
+  end
+
+  def got_empty got
+    return ''
+  end
+
+  def got_undef got
+    return nil
+  end
+end

data/test/lib/xxx.rb

@@ -0,0 +1,13 @@
+def XXX object
+  require 'psych'
+  puts Psych.dump object
+  puts 'XXX from: ' + caller.first
+  exit
+end
+
+def YYY object, show_caller=true
+  require 'psych'
+  puts Psych.dump object
+  puts 'YYY from: ' + caller.first if show_caller
+  return object
+end

data/test/tree-pegex.tml

@@ -0,0 +1,35 @@
+=== Part of Pegex Grammar
+--- SKIP
+--- grammar
+\# This is the Pegex grammar for Pegex grammars!
+grammar: ( <comment>* <rule_definition> )+ <comment>*
+rule_definition: /<WS>*/ <rule_name> /<COLON><WS>*/ <rule_line>
+rule_name: /(<ALPHA><WORD>*)/
+comment: /<HASH><line><EOL>/
+line: /<ANY>*/
+rule_line: /(<line>)<EOL>/
+
+--- input
+\# This is the Pegex grammar for Pegex grammars!
+grammar: ( <comment>* <rule_definition> )+ <comment>*
+rule_definition: /<WS>*/ <rule_name> /<COLON><WS>*/ <rule_line>
+--- tree
+- - - []
+    - - grammar
+      - ( <comment>* <rule_definition> )+ <comment>*
+  - - []
+    - - rule_definition
+      - /<WS>*/ <rule_name> /<COLON><WS>*/ <rule_line>
+- []
+--- wrap
+grammar:
+- - - []
+    - rule_definition:
+      - rule_name: grammar
+      - rule_line: ( <comment>* <rule_definition> )+ <comment>*
+  - - []
+    - rule_definition:
+      - rule_name: rule_definition
+      - rule_line: /<WS>*/ <rule_name> /<COLON><WS>*/ <rule_line>
+- []
+

data/test/tree.rb

@@ -0,0 +1,47 @@
+require './test/lib/test_pegex'
+
+TestML.require_or_skip 'psych'
+
+TestML.run do |t|
+  files = [
+    'test/tree.tml',
+    'test/tree-pegex.tml',
+  ]
+
+  files.each do |f|
+    t.data f
+    t.eval '*grammar', t.method('run_tree_tests')
+  end
+end
+
+class TestPegex
+  require 'pegex/tree'
+  require 'pegex/tree/wrap'
+  require 'testast'
+  def run_tree_tests block, expr=nil
+    label '$BlockLabel - Pegex::Tree'
+    run_test(
+      block,
+      "parse_to_tree('Pegex::Tree', *grammar, *input).yaml.clean == *tree",
+    )
+
+    label('$BlockLabel - Pegex::Tree::Wrap');
+    run_test(
+      block,
+      "parse_to_tree('Pegex::Tree::Wrap', *grammar, *input).yaml.clean == *wrap",
+    )
+
+    label('$BlockLabel - t::TestAST');
+    run_test(
+      block,
+      "parse_to_tree('TestAST', *grammar, *input).yaml.clean == *ast",
+    )
+  end
+
+  require 'pegex'
+  def parse_to_tree(receiver, grammar, input)
+    require receiver.downcase.gsub /::/, '/'
+    parser = pegex(grammar, Kernel.eval(receiver))
+    return parser.parse(input)
+  end
+end