rley 0.3.08 → 0.3.09

data/examples/parsers/parsing_right_recursive.rb

@@ -1,70 +0,0 @@
-# Purpose: to demonstrate how to build and render a parse tree
-
-require 'pp' # TODO remove this dependency
-require 'rley'  # Load the gem
-
-# Steps to render a parse tree (of a valid parsed input):
-# 1. Define a grammar
-# 2. Create a tokenizer for the language
-# 3. Create a parser for that grammar
-# 4. Tokenize the input
-# 5. Let the parser process the input
-# 6. Generate a parse tree from the parse result
-# 7. Render the parse tree (in JSON)
-
-########################################
-# Step 1. Define a problematic grammar
-# Grammar Z: A grammar with hidden left recursion and a cycle
-# (based on example in book of D. Grune, C JH. Jacobs,
-# "Parsing Techniques: A Practical Guide"
-# Springer, ISBN: 978-1-4419-1901-4, (2010) p. 224
-# Let's create the grammar step-by-step with the grammar builder:
-builder = Rley::Syntax::GrammarBuilder.new
-builder.add_terminals('a')
-builder.add_production('S' => %w(a S))
-builder.add_production('S' => []) # Empty RHS
-
-# And now build the grammar...
-right_recursive_gram = builder.grammar
-
-
-########################################
-# 2. Create a tokenizer for the language
-# The tokenizer transforms the input into an array of tokens
-def tokenizer(aText, aGrammar)
-  tokens = aText.chars.map do |lexeme|
-    case lexeme
-      when 'a'
-        terminal = aGrammar.name2symbol[lexeme]
-      else
-        msg = "Unknown input text '#{lexeme}'"
-        raise StandardError, msg
-    end
-    Rley::Parser::Token.new(lexeme, terminal)
-  end
-
-  return tokens
-end
-
-########################################
-# Step 3. Create a parser for that grammar
-parser = Rley::Parser::EarleyParser.new(right_recursive_gram)
-
-########################################
-# Step 3. Tokenize the input
-valid_input = 'aaaa'
-tokens = tokenizer(valid_input, right_recursive_gram)
-
-########################################
-# Step 5. Let the parser process the input
-result = parser.parse(tokens)
-puts "Parsing success? #{result.success?}"
-#pp result
-
-result.chart.state_sets.each_with_index do |aStateSet, index|
-  puts "State[#{index}]"
-  puts "========"
-  aStateSet.states.each { |aState| puts aState.to_s }
-end
-
-# End of file

data/examples/parsers/parsing_tricky.rb

@@ -1,91 +0,0 @@
-# Purpose: to demonstrate how to build and render a parse tree
-
-require 'pp' # TODO remove this dependency
-require 'rley'  # Load the gem
-
-# Steps to render a parse tree (of a valid parsed input):
-# 1. Define a grammar
-# 2. Create a tokenizer for the language
-# 3. Create a parser for that grammar
-# 4. Tokenize the input
-# 5. Let the parser process the input
-# 6. Generate a parse tree from the parse result
-# 7. Render the parse tree (in JSON)
-
-########################################
-# Step 1. Define a problematic grammar
-# Grammar Z: A grammar with hidden left recursion and a cycle
-# (based on example in article of Elizabeth Scott, "SPPF-Style Parsing From Earley Recognisers"
-# Electronic Notes in Theoretical Computer Science 203 (2008) 53–67
-# Let's create the grammar step-by-step with the grammar builder:
-builder = Rley::Syntax::GrammarBuilder.new
-builder.add_terminals('a', 'b')
-builder.add_production('S' => %w(A T))
-builder.add_production('S' => %w(a T))
-builder.add_production('A' => 'a')
-builder.add_production('A' => %w(B A))
-builder.add_production('B' => []) # Empty RHS
-builder.add_production('T' => %w(b b b))
-
-# And now build the grammar...
-grammar_tricky = builder.grammar
-
-
-########################################
-# 2. Create a tokenizer for the language
-# The tokenizer transforms the input into an array of tokens
-def tokenizer(aText, aGrammar)
-  tokens = aText.chars.map do |lexeme|
-    case lexeme
-      when 'a', 'b'
-        terminal = aGrammar.name2symbol[lexeme]
-      else
-        msg = "Unknown input text '#{lexeme}'"
-        raise StandardError, msg
-    end
-    Rley::Parser::Token.new(lexeme, terminal)
-  end
-
-  return tokens
-end
-
-########################################
-# Step 3. Create a parser for that grammar
-parser = Rley::Parser::EarleyParser.new(grammar_tricky)
-
-########################################
-# Step 3. Tokenize the input
-valid_input = 'abbb'
-tokens = tokenizer(valid_input, grammar_tricky)
-
-########################################
-# Step 5. Let the parser process the input
-result = parser.parse(tokens)
-puts "Parsing success? #{result.success?}"
-#pp result
-
-result.chart.state_sets.each_with_index do |aStateSet, index|
-  puts "State[#{index}]"
-  puts "========"
-  aStateSet.states.each { |aState| puts aState.to_s }
-end
-
-=begin
-########################################
-# Step 6. Generate a parse tree from the parse result
-ptree = result.parse_tree
-pp ptree
-#=begin
-########################################
-# Step 7. Render the parse tree (in JSON)
-# Let's create a parse tree visitor
-visitor = Rley::ParseTreeVisitor.new(ptree)
-
-#Here we create a renderer object...
-renderer = Rley::Formatter::Json.new(STDOUT)
-
-# Now emit the parse tree as JSON on the console output
-puts "JSON rendering of the parse tree for '#{valid_input}' input:"
-renderer.render(visitor)
-=end
-# End of file

data/examples/parsers/tracing_parser.rb

@@ -1,54 +0,0 @@
-# Purpose: to use a grammar that causes some Earley parsers to fail.
-# See: http://stackoverflow.com/questions/22311323/earley-parser-recursion
-require 'rley'  # Load the gem
-
-# Steps to parse some valid input:
-# 1. Define a grammar
-# 2. Create a tokenizer for the language
-# 3. Create a parser for that grammar
-# 4. Tokenize the input
-# 5. Let the parser process the input & trace its progress
-
-
-########################################
-# Step 1. Define a grammar that might cause infinite recursion
-# Let's create the grammar step-by-step with the grammar builder:
-builder = Rley::Syntax::GrammarBuilder.new
-builder.add_terminals('ident')
-builder.add_production('S' => 'E')
-builder.add_production('E' => ['E', 'E'] )
-builder.add_production('E' => 'ident')
-
-# And now build the grammar...
-grammar_tricky = builder.grammar
-
-
-########################################
-# 2. Create a tokenizer for the language
-# The tokenizer transforms the input into an array of tokens
-def tokenizer(aText, aGrammar)
-  terminal = aGrammar.name2symbol['ident']
-  
-  tokens = aText.chars.map do |ch|
-    Rley::Parser::Token.new(ch, terminal)
-  end
-  
-  return tokens
-end
-
-########################################
-# Step 3. Create a parser for that grammar
-parser = Rley::Parser::EarleyParser.new(grammar_tricky)
-
-########################################
-# Step 3. Tokenize the input
-valid_input = 'abcdefg'
-tokens = tokenizer(valid_input, grammar_tricky)
-
-########################################
-# Step 5. Let the parser process the input
-# Force the parser to trace its parsing progress.
-result = parser.parse(tokens, 1)
-puts "Parsing success? #{result.success?}"
-
-# End of file

data/examples/recognizers/recognizer_abc.rb

@@ -1,71 +0,0 @@
-# Purpose: to demonstrate how to build a recognizer
-# A recognizer is a kind of parser that indicates whether the input
-# complies to the grammar or not.
-
-require 'rley'  # Load the gem
-
-# Steps to build a recognizer:
-# 1. Define a grammar
-# 2. Create a parser for that grammar
-# 3. Build the input
-# 4. Let the parser process the input
-# 5. Check the parser's result to see whether the input was valid (=recognized)
-
-########################################
-# Step 1. Define a grammar for a very simple language
-# It recognizes/generates strings like 'b', 'abc', 'aabcc', 'aaabccc',...
-# (based on example in N. Wirth's book "Compiler Construction", p. 6)
-# Let's create the grammar step-by-step with the grammar builder:
-builder = Rley::Syntax::GrammarBuilder.new
-builder.add_terminals('a', 'b', 'c')
-builder.add_production('S' => 'A')
-builder.add_production('A' => %w(a A c))
-builder.add_production('A' => 'b')
-
-# And now build the grammar...
-grammar_abc = builder.grammar
-
-# Keep track of the terminal symbols of the grammar:
-term_a = grammar_abc.name2symbol['a']
-term_b = grammar_abc.name2symbol['b']
-term_c = grammar_abc.name2symbol['c']
-
-########################################
-# Step 2. Create a parser for that grammar
-parser = Rley::Parser::EarleyParser.new(grammar_abc)
-
-########################################
-# Step 3. Build the input
-# Mimicking the output of a tokenizer
-valid_input = [
-  Rley::Parser::Token.new('a', term_a),
-  Rley::Parser::Token.new('a', term_a),
-  Rley::Parser::Token.new('b', term_b),
-  Rley::Parser::Token.new('c', term_c),
-  Rley::Parser::Token.new('c', term_c)
-]
-
-########################################
-# Step 4. Let the parser process the input
-result = parser.parse(valid_input)
-
-
-########################################
-# Step 5. Check the parser's result to see whether the input was valid
-puts "Successful parse of 'aabcc'? #{result.success?}"
-# Output: Successful parse of 'aabcc'? true
-
-
-
-# Let's redo steps 3, 4, 5 again with an invalid input.
-invalid_input = [
-  Rley::Parser::Token.new('a', term_a),
-  Rley::Parser::Token.new('a', term_a),
-  Rley::Parser::Token.new('b', term_b),
-  Rley::Parser::Token.new('c', term_c)
-]
-result = parser.parse(invalid_input)
-puts "Successful parse of 'aabc'? #{result.success?}"
-# Output: Successful parse of 'aabc'? false
-
-# End of file