rley 0.3.08 → 0.3.09

data/examples/grammars/grammar_abc.rb

@@ -1,26 +0,0 @@
-# Purpose: to demonstrate how to build a very simple grammar
-require 'rley'  # Load the gem
-
-# A very simple language
-# It recognizes/generates strings like 'b', 'abc', 'aabcc', 'aaabccc',...
-# (based on example in N. Wirth's book "Compiler Construction", p. 6)
-# S ::= A.
-# A ::= "a" A "c".
-# A ::= "b".
-
-
-# Let's create the grammar step-by-step with the grammar builder:
-builder = Rley::Syntax::GrammarBuilder.new
-builder.add_terminals('a', 'b', 'c')
-builder.add_production('S' => 'A')
-builder.add_production('A' => %w(a A c))
-builder.add_production('A' => 'b')
-
-# And now build the grammar...
-grammar_abc = builder.grammar
-
-# Prove that it is a grammar
-puts grammar_abc.class.name
-
-# End of file
-

data/examples/parsers/demo-JSON/JSON_grammar.rb

@@ -1,31 +0,0 @@
-# Purpose: to demonstrate how to build and render a parse tree for JSON
-# language
-require 'rley'  # Load the gem
-
-
-########################################
-# Define a grammar for JSON
-builder = Rley::Syntax::GrammarBuilder.new
-builder.add_terminals('KEYWORD') # For true, false, null keywords
-builder.add_terminals('JSON_STRING', 'JSON_NUMBER')
-builder.add_terminals('LACCOL', 'RACCOL') # For '{', '}' delimiters
-builder.add_terminals('LBRACKET', 'RBRACKET') # For '[', ']' delimiters
-builder.add_terminals('COLON', 'COMMA') # For ':', ',' separators
-builder.add_production('json_text' => 'json_value')
-builder.add_production('json_value' => 'json_object')
-builder.add_production('json_value' => 'json_array')
-builder.add_production('json_value' => 'JSON_STRING')
-builder.add_production('json_value' => 'JSON_NUMBER')
-builder.add_production('json_value' => 'KEYWORD')
-builder.add_production('json_object' => %w[LACCOL json_pairs RACCOL])
-builder.add_production('json_object' => ['LACCOL', 'RACCOL'])
-builder.add_production('json_pairs' => %w[json_pairs COMMA single_pair])
-builder.add_production('json_pairs' => 'single_pair')
-builder.add_production('single_pair' => %w[JSON_STRING COLON json_value])
-builder.add_production('json_array' => %w[LBRACKET array_items RBRACKET])
-builder.add_production('json_array' => ['RBRACKET', 'RBRACKET'])
-builder.add_production('array_items' => %w[array_items COMMA json_value])
-builder.add_production('array_items' => %w[json_value])
-
-# And now build the grammar...
-GrammarJSON = builder.grammar

data/examples/parsers/demo-JSON/JSON_lexer.rb

@@ -1,114 +0,0 @@
-# File: JSON_lexer.rb
-# Lexer for the JSON data format
-require 'rley'  # Load the gem
-require 'strscan'
-
-# Lexer for JSON.
-class JSONLexer
-  attr_reader(:scanner)
-  attr_reader(:lineno)
-  attr_reader(:line_start)
-  attr_reader(:name2symbol)
-
-  @@lexeme2name = {
-    '{' => 'LACCOL',
-    '}' => 'RACCOL',
-    '[' => 'LBRACKET',
-    ']' => 'RBRACKET',
-    ',' => 'COMMA',
-    ':' => 'COLON'
-  }
-
-  class ScanError < StandardError ; end
-
-public
-  def initialize(source, aGrammar)
-    @scanner = StringScanner.new(source)
-    @name2symbol = aGrammar.name2symbol
-    @lineno =  1
-  end
-
-  def tokens()
-    tok_sequence = []
-    until @scanner.eos? do
-      token = _next_token
-      tok_sequence << token unless token.nil?
-    end
-
-    return tok_sequence
-  end
-
-private
-  def _next_token()
-    token = nil
-    skip_whitespaces
-    curr_ch = scanner.getch # curr_ch is at start of token or eof reached...
-
-    begin
-      break if curr_ch.nil?
-
-      case curr_ch
-        when '{', '}', '[', ']', ',', ':'
-          type_name = @@lexeme2name[curr_ch]
-          token_type = name2symbol[type_name]
-          token = Rley::Parser::Token.new(curr_ch, token_type)
-
-        # LITERALS
-        when '"'  # Start string delimiter found
-          value = scanner.scan(/([^"\\]|\\.)*/)
-          end_delimiter = scanner.getch()
-          raise ScanError.new('No closing quotes (") found') if end_delimiter.nil?
-          token_type = name2symbol['JSON_STRING']
-          token = Rley::Parser::Token.new(value, token_type)
-
-        when /[ftn]/  # First letter of keywords
-          @scanner.pos = scanner.pos - 1 # Simulate putback
-          keyw = scanner.scan(/false|true|null/)
-          if keyw.nil?
-            invalid_keyw = scanner.scan(/\w+/)
-            raise ScanError.new("Invalid keyword: #{invalid_keyw}")
-          else
-            token_type = name2symbol['KEYWORD']
-            token = Rley::Parser::Token.new(keyw, token_type)
-          end
-
-
-        when /[-0-9]/ # Start character of number literal found
-          @scanner.pos = scanner.pos - 1 # Simulate putback
-          value = scanner.scan(/-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9])?/)
-          token_type = name2symbol['JSON_NUMBER']
-          token = Rley::Parser::Token.new(value, token_type)
-
-
-        else # Unknown token
-          erroneous = curr_ch.nil? ? '' : curr_ch
-          sequel = scanner.scan(/.{1,20}/)
-          erroneous += sequel unless sequel.nil?
-          raise ScanError.new("Unknown token #{erroneous}")
-      end #case
-
-
-    end while (token.nil? && curr_ch = scanner.getch())
-
-    return token
-  end
-
-
-  def skip_whitespaces()
-    matched = scanner.scan(/[ \t\f\n\r]+/)
-    return if matched.nil?
-
-    newline_count = 0
-    matched.scan(/\n\r?|\r/) { |_| newline_count += 1 }
-    newline_detected(newline_count)
-  end
-
-
-  def newline_detected(count)
-    @lineno += count
-    @line_start = scanner.pos()
-  end
-
-end # class
-
-

data/examples/parsers/demo-JSON/JSON_parser.rb

@@ -1,89 +0,0 @@
-# Purpose: to demonstrate how to build and render a parse tree for JSON
-# language
-require 'pp'
-require 'rley'  # Load the gem
-require_relative 'json_lexer'
-
-# Steps to render a parse tree (of a valid parsed input):
-# 1. Define a grammar
-# 2. Create a parser for that grammar
-# 3. Tokenize the input
-# 4. Let the parser process the input
-# 5. Generate a parse tree from the parse result
-# 6. Render the parse tree (in JSON)
-
-########################################
-# Step 1. Load a grammar for JSON
-require_relative 'JSON_grammar'
-
-# A JSON parser derived from our general Earley parser.
-class JSONParser < Rley::Parser::EarleyParser
-  attr_reader(:source_file)
-
-  # Constructor
-  def initialize()
-    # Builder the Earley parser with the JSON grammar
-    super(GrammarJSON)
-  end
-  
-  def parse_file(aFilename)
-    tokens = tokenize_file(aFilename)
-    result = parse(tokens)
-
-    return result
-  end
-  
-  private
-  
-  def tokenize_file(aFilename)
-    input_source = nil
-    File.open(aFilename, 'r') { |f| input_source = f.read }
-
-    lexer = JSONLexer.new(input_source, GrammarJSON)
-    return lexer.tokens
-  end
-end # class
-
-=begin
-########################################
-# Step 3. Create a parser for that grammar
-# parser = Rley::Parser::EarleyParser.new(GrammarJSON)
-parser = JSONParser.new
-
-
-########################################
-# Step 4. Tokenize the input file
-file_name = 'sample02.json'
-=begin
-input_source = nil
-File.open(file_name, 'r') { |f| input_source = f.read }
-
-lexer = JSONLexer.new(input_source, GrammarJSON)
-tokens =  lexer.tokens
-#=end
-
-########################################
-# Step 5. Let the parser process the input
-result = parser.parse_file(file_name) # parser.parse(tokens)
-unless result.success?
-  puts "Parsing of '#{file_name}' failed"
-  exit(1)
-end
-
-########################################
-# Step 6. Generate a parse tree from the parse result
-ptree = result.parse_tree
-
-########################################
-# Step 7. Render the parse tree (in JSON)
-# Let's create a parse tree visitor
-visitor = Rley::ParseTreeVisitor.new(ptree)
-
-#Here we create a renderer object...
-renderer = Rley::Formatter::Json.new(STDOUT)
-
-# Now emit the parse tree as JSON on the console output
-puts "JSON rendering of the parse tree for '#{file_name}' input:"
-renderer.render(visitor)
-=end
-# End of file

data/examples/parsers/demo-JSON/demo_json.rb

@@ -1,42 +0,0 @@
-require_relative 'JSON_parser'
-
-# Create a JSON parser object
-parser = JSONParser.new 
-
-# Parse the input file with name given in command-line
-if ARGV.empty?
-  msg = <<-END_MSG
-Command-line symtax:
-  ruby #{__FILE__} filename
-  where:
-    filename is the name of a JSON file
-    
-  Example:
-  ruby #{__FILE__} sample01.json
-END_MSG
-  puts msg
-  exit(1)
-end
-file_name = ARGV[0]
-result = parser.parse_file(file_name) # result object contains parse details
-
-unless result.success?
-  # Stop if the parse failed...
-  puts "Parsing of '#{file_name}' failed"
-  exit(1)
-end
-
-# Generate a parse tree from the parse result
-ptree = result.parse_tree
-
-# Do something with the parse tree: render it on the output console.
-# Step a: Let's create a parse tree visitor
-visitor = Rley::ParseTreeVisitor.new(ptree)
-
-# Step b: Select the rendition format to be JSON
-renderer = Rley::Formatter::Json.new(STDOUT)
-
-# Step c: Now emit the parse tree as JSON on the console output
-puts "JSON rendering of the parse tree for '#{file_name}' input:"
-renderer.render(visitor)
-# End of file

data/examples/parsers/parsing_L0.rb

@@ -1,124 +0,0 @@
-# Purpose: to demonstrate how to build and render a parse tree for the L0
-# language
-require 'pp'
-require 'rley'  # Load the gem
-
-# Steps to render a parse tree (of a valid parsed input):
-# 1. Define a grammar
-# 2. Create a tokenizer for the language
-# 3. Create a parser for that grammar
-# 4. Tokenize the input
-# 5. Let the parser process the input
-# 6. Generate a parse tree from the parse result
-# 7. Render the parse tree (in JSON)
-
-########################################
-# Step 1. Define a grammar for a micro English-like language
-# based on Jurafky & Martin L0 language (chapter 12 of the book).
-# It defines the syntax of a sentence in a language with a 
-# very limited syntax and lexicon in the context of airline reservation.
-builder = Rley::Syntax::GrammarBuilder.new
-builder.add_terminals('Noun', 'Verb', 'Pronoun', 'Proper-Noun')
-builder.add_terminals('Determiner', 'Preposition', )
-builder.add_production('S' => %w[NP VP])
-builder.add_production('NP' => 'Pronoun')
-builder.add_production('NP' => 'Proper-Noun')
-builder.add_production('NP' => %w[Determiner Nominal])
-builder.add_production('Nominal' => %w[Nominal Noun])
-builder.add_production('Nominal' => 'Noun')
-builder.add_production('VP' => 'Verb')
-builder.add_production('VP' => %w[Verb NP])
-builder.add_production('VP' => %w[Verb NP PP])
-builder.add_production('VP' => %w[Verb PP])
-builder.add_production('PP' => %w[Preposition PP])
-
-# And now build the grammar...
-grammar_l0 = builder.grammar
-
-
-########################################
-# 2. Create a tokenizer for the language
-# The tokenizer transforms the input into an array of tokens
-# This is a very simplistic implementation for demo purposes.
-
-# The lexicon is just a Hash with pairs of the form:
-# word =>terminal symbol name
-L0_lexicon = {
-  'flight' => 'Noun',
-  'breeze' => 'Noun',
-  'trip' => 'Noun',
-  'morning' => 'Noun',
-  'is' => 'Verb',
-  'prefer' => 'Verb',
-  'like' => 'Verb',
-  'need' => 'Verb',
-  'want' => 'Verb',
-  'fly' => 'Verb',
-  'me' => 'Pronoun',
-  'I' => 'Pronoun',
-  'you' => 'Pronoun',
-  'it' => 'Pronoun',
-  'Alaska' => 'Proper-Noun',
-  'Baltimore' => 'Proper-Noun',
-  'Chicago' => 'Proper-Noun',
-  'United' => 'Proper-Noun',
-  'American' => 'Proper-Noun',
-  'the' => 'Determiner',
-  'a' => 'Determiner',
-  'an' => 'Determiner',
-  'this' => 'Determiner',
-  'these' => 'Determiner',
-  'that' => 'Determiner',
-  'from' => 'Preposition',
-  'to' => 'Preposition',
-  'on' => 'Preposition',
-  'near' => 'Preposition'
-}
-
-# Highly simplified tokenizer implementation.
-def tokenizer(aText, aGrammar)
-  tokens = aText.scan(/\S+/).map do |word|
-    term_name = L0_lexicon[word]
-    if term_name.nil?
-      raise StandardError, "Word '#{word}' not found in lexicon"
-    end
-    terminal = aGrammar.name2symbol[term_name]
-    Rley::Parser::Token.new(word, terminal)
-  end
-  
-  return tokens
-end
-
-########################################
-# Step 3. Create a parser for that grammar
-parser = Rley::Parser::EarleyParser.new(grammar_l0)
-
-########################################
-# Step 3. Tokenize the input
-valid_input = 'I prefer a morning flight'
-# Another sentence: it is a flight from Chicago
-tokens = tokenizer(valid_input, grammar_l0)
-
-########################################
-# Step 5. Let the parser process the input
-result = parser.parse(tokens)
-
-puts "Parsing success? #{result.success?}"
-
-
-########################################
-# Step 6. Generate a parse tree from the parse result
-ptree = result.parse_tree
-
-########################################
-# Step 7. Render the parse tree (in JSON)
-# Let's create a parse tree visitor
-visitor = Rley::ParseTreeVisitor.new(ptree)
-
-#Here we create a renderer object...
-renderer = Rley::Formatter::Json.new(STDOUT)
-
-# Now emit the parse tree as JSON on the console output
-puts "JSON rendering of the parse tree for '#{valid_input}' input:"
-renderer.render(visitor)
-# End of file

data/examples/parsers/parsing_L1.rb

@@ -1,137 +0,0 @@
-# Purpose: to demonstrate how to build and render a parse tree for the L1
-# language
-require 'pp'
-require 'rley'  # Load the gem
-
-# Steps to render a parse tree (of a valid parsed input):
-# 1. Define a grammar
-# 2. Create a tokenizer for the language
-# 3. Create a parser for that grammar
-# 4. Tokenize the input
-# 5. Let the parser process the input
-# 6. Generate a parse tree from the parse result
-# 7. Render the parse tree (in JSON)
-
-########################################
-# Step 1. Define a grammar for a miniature English-like language
-# based on Jurafky & Martin L1 language (chapter 13).
-# It defines the syntax of a sentence in a language with a 
-# very limited syntax and lexicon in the context of airline reservation.
-builder = Rley::Syntax::GrammarBuilder.new
-builder.add_terminals('Noun', 'Verb', 'Pronoun', 'Proper-Noun')
-builder.add_terminals('Aux', 'Determiner', 'Preposition')
-builder.add_production('Sentence' => 'S')
-builder.add_production('S' => %w[Aux NP VP])
-builder.add_production('S' => %w[NP VP])
-builder.add_production('S' => %w[VP])
-builder.add_production('NP' => 'Pronoun')
-builder.add_production('NP' => 'Proper-Noun')
-builder.add_production('NP' => %w[Determiner Nominal])
-builder.add_production('Nominal' => %w[Nominal Noun])
-builder.add_production('Nominal' => %w[Nominal PP])
-builder.add_production('Nominal' => 'Noun')
-builder.add_production('VP' => 'Verb')
-builder.add_production('VP' => %w[Verb NP])
-builder.add_production('VP' => %w[Verb NP PP])
-builder.add_production('VP' => %w[Verb PP])
-builder.add_production('VP' => %w[VP PP])
-builder.add_production('PP' => %w[Preposition NP])
-
-# And now build the grammar...
-grammar_l1 = builder.grammar
-
-
-########################################
-# 2. Create a tokenizer for the language
-# The tokenizer transforms the input into an array of tokens
-# This is a very simplistic implementation for demo purposes.
-
-# The lexicon is just a Hash with pairs of the form:
-# word =>terminal symbol name
-L1_lexicon = {
-  'does' => 'Aux',
-  'flight' => 'Noun',
-  'trip' => 'Noun',
-  'meal' => 'Noun',
-  'money' => 'Noun',
-  'morning' => 'Noun',
-  'is' => 'Verb',
-  'book' => 'Verb',
-  'prefer' => 'Verb',
-  'like' => 'Verb',
-  'need' => 'Verb',
-  'want' => 'Verb',
-  'fly' => 'Verb',
-  'show' => 'Verb',
-  'me' => 'Pronoun',
-  'I' => 'Pronoun',
-  'she' => 'Pronoun',
-  'you' => 'Pronoun',
-  'it' => 'Pronoun',
-  'Alaska' => 'Proper-Noun',
-  'Baltimore' => 'Proper-Noun',
-  'Chicago' => 'Proper-Noun',
-  'Houston' => 'Proper-Noun',
-  'NWA' => 'Proper-Noun',
-  'United' => 'Proper-Noun',
-  'American' => 'Proper-Noun',
-  'the' => 'Determiner',
-  'a' => 'Determiner',
-  'an' => 'Determiner',
-  'this' => 'Determiner',
-  'these' => 'Determiner',
-  'that' => 'Determiner',
-  'from' => 'Preposition',
-  'to' => 'Preposition',
-  'on' => 'Preposition',
-  'near' => 'Preposition',
-  'through' => 'Preposition'
-}
-
-# Highly simplified tokenizer implementation.
-def tokenizer(aText, aGrammar)
-  tokens = aText.scan(/\S+/).map do |word|
-    term_name = L1_lexicon[word]
-    if term_name.nil?
-      raise StandardError, "Word '#{word}' not found in lexicon"
-    end
-    terminal = aGrammar.name2symbol[term_name]
-    Rley::Parser::Token.new(word, terminal)
-  end
-  
-  return tokens
-end
-
-########################################
-# Step 3. Tokenize the input
-valid_input = 'I want the flight from Alaska through Chicago to Houston'
-
-# Another sentence: it is a flight from Chicago
-tokens = tokenizer(valid_input, grammar_l1)
-
-########################################
-# Step 4. Create a parser for that grammar
-parser = Rley::Parser::EarleyParser.new(grammar_l1)
-
-########################################
-# Step 5. Let the parser process the input
-result = parser.parse(tokens)
-puts "Parsing success? #{result.success?}"
-
-
-########################################
-# Step 6. Generate a parse tree from the parse result
-ptree = result.parse_tree
-
-########################################
-# Step 7. Render the parse tree (in JSON)
-# Let's create a parse tree visitor
-visitor = Rley::ParseTreeVisitor.new(ptree)
-
-#Here we create a renderer object...
-renderer = Rley::Formatter::Json.new(STDOUT)
-
-# Now emit the parse tree as JSON on the console output
-puts "JSON rendering of the parse tree for '#{valid_input}' input:"
-renderer.render(visitor)
-# End of file