rley 0.5.14 → 0.6.00

data/examples/data_formats/JSON/json_ast_builder.rb

@@ -7,7 +7,7 @@ require_relative 'json_ast_nodes'
 # The Builder pattern creates a complex object
 # (say, a parse tree) from simpler objects (terminal and non-terminal
 # nodes) and using a step by step approach.
-class JSONASTBuilder < Rley::Parser::ParseTreeBuilder 
+class JSONASTBuilder < Rley::ParseRep::ParseTreeBuilder 
   Terminal2NodeClass = {
                          'false' => JSONBooleanNode,
                          'true' => JSONBooleanNode,
@@ -18,77 +18,18 @@ class JSONASTBuilder < Rley::Parser::ParseTreeBuilder
 
   protected
   
-  def return_first_child(_range, _tokens, theChildren)
-    return theChildren[0]
+  def terminal2node()
+    Terminal2NodeClass
   end
-
-  def return_second_child(_range, _tokens, theChildren)
-    return theChildren[1]
-  end      
-
-  # Overriding method.
-  # Create a parse tree object with given
-  # node as root node.
-  def create_tree(aRootNode)
-    return Rley::PTree::ParseTree.new(aRootNode)
-  end
-
-  # Overriding method.
-  # Factory method for creating a node object for the given
-  # input token.
-  # @param terminal [Terminal] Terminal symbol associated with the token
-  # @param aTokenPosition [Integer] Position of token in the input stream
-  # @param aToken [Token] The input token
-  def new_leaf_node(_production, terminal, aTokenPosition, aToken)
-    klass = Terminal2NodeClass.fetch(terminal.name, JSONTerminalNode)
-    return klass.new(aToken, aTokenPosition)
+  
+  # Default class for representing terminal nodes.
+  # @return [Class]
+  def terminalnode_class()
+    JSONTerminalNode
   end
-
-  # Method to override.
-  # Factory method for creating a parent node object.
-  # @param aProduction [Production] Production rule
-  # @param aRange [Range] Range of tokens matched by the rule
-  # @param theTokens [Array] The input tokens
-  # @param theChildren [Array] Children nodes (one per rhs symbol)
-  def new_parent_node(aProduction, aRange, theTokens, theChildren)
-    node = case aProduction.name
-      when 'JSON-text_0' # rule 'JSON-text' => 'value'
-        return_first_child(aRange, theTokens, theChildren)
-
-      when /value_\d/
-        return_first_child(aRange, theTokens, theChildren)
-
-      when 'object_0'
-        reduce_object_0(aProduction, aRange, theTokens, theChildren)
-
-      when 'object_1'
-        reduce_object_1(aRange, theTokens, theChildren)
-
-      when 'member-list_0'
-        reduce_member_list_0(aRange, theTokens, theChildren)
-
-      when 'member-list_1'
-        reduce_member_list_1(aProduction, aRange, theTokens, theChildren)
-
-      when 'member_0'
-        reduce_member_0(aProduction, aRange, theTokens, theChildren)
-
-      when 'array_0'
-        reduce_array_0(aProduction, aRange, theTokens, theChildren)
-
-      when 'array_1'
-        reduce_array_1(aRange, theTokens, theChildren)
-
-      when 'array-items_0'
-        reduce_array_items_0(aRange, theTokens, theChildren)
-
-      when 'array-items_1'
-        reduce_array_items_1(aProduction, aRange, theTokens, theChildren)
-      else
-        raise StandardError, "Don't know production #{aProduction.name}"
-    end
-
-    return node
+  
+  def reduce_JSON_text_0(_aProd, _range, _tokens, theChildren)
+    return_first_child(_range, _tokens, theChildren)
   end
   
   # rule 'object' => %w[begin-object member-list end-object]

data/examples/data_formats/JSON/json_demo.rb

@@ -1,8 +1,9 @@
 require_relative 'cli_options'
-require_relative 'json_parser'
+require_relative 'json_lexer'
 require_relative 'json_minifier'
 require_relative 'json_ast_builder'
 
+
 prog_name = 'json_demo'
 prog_version = '0.3.0'
 
@@ -14,16 +15,6 @@ if ARGV.empty?
 end
 
 file_name = ARGV[0]
-# Create a JSON parser object
-parser = JSONParser.new 
-result = parser.parse_file(file_name) # result object contains parse details
-
-unless result.success?
-  # Stop if parse failed...
-  puts "Parsing of '#{file_name}' failed"
-  puts result.failure_reason.message
-  exit(1)
-end
 
 tree_rep = cli_options[:rep]
 renderer = nil
@@ -43,14 +34,41 @@ case cli_options[:format]
     raise StandardError, msg if tree_rep == :cst
 end
 
-tree_builder = tree_rep == :ast ? JSONASTBuilder : nil
+
+# Create a Rley facade object
+# If necessary, select AST representation
+engine = Rley::Engine.new do |cfg|
+  builder = tree_rep == :ast ? JSONASTBuilder : nil
+  cfg.repr_builder = builder
+end
+
+########################################
+# Step 1. Load a grammar for JSON
+require_relative 'json_grammar'
+engine.use_grammar(GrammarJSON)
+
+
+input_source = nil
+File.open(file_name, 'r') { |f| input_source = f.read }
+lexer = JSONLexer.new(input_source)
+
+result = engine.parse(lexer.tokens)
+
+unless result.success?
+  # Stop if parse failed...
+  puts "Parsing of '#{file_name}' failed"
+  puts result.failure_reason.message
+  exit(1)
+end
+
+
 
 # Generate a parse tree from the parse result
-ptree = result.parse_tree(tree_builder)
+ptree = engine.convert(result)
 
 if renderer
   # Let's create a parse tree visitor
-  visitor = Rley::ParseTreeVisitor.new(ptree)
+  visitor = engine.ptree_visitor(ptree)
 
   # Now output formatted parse tree
   renderer.render(visitor)

data/examples/data_formats/JSON/json_grammar.rb

@@ -13,7 +13,7 @@ builder = Rley::Syntax::GrammarBuilder.new do
   add_terminals('begin-object', 'end-object') # For '{', '}' delimiters
   add_terminals('begin-array', 'end-array') # For '[', ']' delimiters
   add_terminals('name-separator', 'value-separator') # For ':', ',' separators
-  rule 'JSON-text' => 'value'
+  rule 'JSON_text' => 'value'
   rule 'value' => 'false'
   rule 'value' => 'null'
   rule 'value' => 'true'   

data/examples/data_formats/JSON/json_lexer.rb

@@ -8,7 +8,6 @@ class JSONLexer
   attr_reader(:scanner)
   attr_reader(:lineno)
   attr_reader(:line_start)
-  attr_reader(:name2symbol)
 
   @@lexeme2name = {
     '{' => 'begin-object',
@@ -21,9 +20,8 @@ class JSONLexer
 
   class ScanError < StandardError; end
 
-  def initialize(source, aGrammar)
+  def initialize(source)
     @scanner = StringScanner.new(source)
-    @name2symbol = aGrammar.name2symbol
     @lineno = 1
   end
 
@@ -49,8 +47,7 @@ class JSONLexer
 
       case curr_ch
         when '{', '}', '[', ']', ',', ':'
-          type_name = @@lexeme2name[curr_ch]
-          token_type = name2symbol[type_name]
+          token_type = @@lexeme2name[curr_ch]
           token = Rley::Lexical::Token.new(curr_ch, token_type)
 
         when /[ftn]/ # First letter of keywords
@@ -60,8 +57,7 @@ class JSONLexer
             invalid_keyw = scanner.scan(/\w+/)
             raise ScanError.new("Invalid keyword: #{invalid_keyw}")
           else
-            token_type = name2symbol[keyw]
-            token = Rley::Lexical::Token.new(keyw, token_type)
+            token = Rley::Lexical::Token.new(keyw, keyw)
           end
 
         # LITERALS
@@ -70,14 +66,12 @@ class JSONLexer
           end_delimiter = scanner.getch
           err_msg = 'No closing quotes (") found'
           raise ScanError.new(err_msg) if end_delimiter.nil?
-          token_type = name2symbol['string']
-          token = Rley::Lexical::Token.new(value, token_type)
+          token = Rley::Lexical::Token.new(value, 'string')
 
         when /[-0-9]/ # Start character of number literal found
           @scanner.pos = scanner.pos - 1 # Simulate putback
           value = scanner.scan(/-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9])?/)
-          token_type = name2symbol['number']
-          token = Rley::Lexical::Token.new(value, token_type)
+          token = Rley::Lexical::Token.new(value, 'number')
 
         else # Unknown token
           erroneous = curr_ch.nil? ? '' : curr_ch

data/examples/general/SRL/lib/ast_builder.rb

@@ -1,5 +1,4 @@
 require 'stringio'
-require_relative 'ast_building'
 require_relative 'regex_repr'
 
 # The purpose of a ASTBuilder is to build piece by piece an AST
@@ -9,14 +8,17 @@ require_relative 'regex_repr'
 # The Builder pattern creates a complex object
 # (say, a parse tree) from simpler objects (terminal and non-terminal
 # nodes) and using a step by step approach.
-class ASTBuilder < Rley::Parser::ParseTreeBuilder
-  include ASTBuilding
+class ASTBuilder < Rley::ParseRep::ASTBaseBuilder
 
   Terminal2NodeClass = { }.freeze
 
   attr_reader :options
 
   protected
+  
+  def terminal2node()
+    Terminal2NodeClass
+  end
 
   # Overriding method.
   # Factory method for creating a node object for the given
@@ -30,31 +32,6 @@ class ASTBuilder < Rley::Parser::ParseTreeBuilder
     return node
   end
 
-  # Method to override.
-  # Factory method for creating a parent node object.
-  # @param aProduction [Production] Production rule
-  # @param aRange [Range] Range of tokens matched by the rule
-  # @param theTokens [Array] The input tokens
-  # @param theChildren [Array] Children nodes (one per rhs symbol)
-  def new_parent_node(aProduction, aRange, theTokens, theChildren)
-    short_name = aProduction.name
-    method_name = 'reduce_' + short_name
-    if self.respond_to?(method_name, true)
-      node = send(method_name, aProduction, aRange, theTokens, theChildren)
-    else
-      # Default action...
-      node = case aProduction.rhs.size
-               when 0
-                 nil
-               when 1
-                 return_first_child(aRange, theTokens, theChildren)
-               else
-                raise StandardError, "Don't know production '#{aProduction.name}'"
-             end            
-    end
-    return node
-  end
-
   def multiplicity(lowerBound, upperBound)
     return SRL::Regex::Multiplicity.new(lowerBound, upperBound, :greedy)
   end

data/examples/general/SRL/lib/tokenizer.rb

@@ -15,7 +15,6 @@ module SRL
     attr_reader(:scanner)
     attr_reader(:lineno)
     attr_reader(:line_start)
-    attr_reader(:name2symbol)
 
     @@lexeme2name = {
       '(' => 'LPAREN',
@@ -78,9 +77,8 @@ module SRL
 
     class ScanError < StandardError; end
 
-    def initialize(source, aGrammar)
+    def initialize(source)
       @scanner = StringScanner.new(source)
-      @name2symbol = aGrammar.name2symbol
       @lineno = 1
     end
 
@@ -132,9 +130,8 @@ module SRL
     end
 
     def build_token(aSymbolName, aLexeme)
-      token_type = name2symbol[aSymbolName]
       begin
-        token = Rley::Lexical::Token.new(aLexeme, token_type)
+        token = Rley::Lexical::Token.new(aLexeme, aSymbolName)
       rescue Exception => ex
         puts "Failing with '#{aSymbolName}' and '#{aLexeme}'"
         raise ex

data/examples/general/SRL/spec/integration_spec.rb

@@ -1,19 +1,26 @@
 require_relative 'spec_helper' # Use the RSpec framework
-require_relative '../lib/parser'
+require_relative '../lib/tokenizer'
+require_relative '../lib/grammar'
 require_relative '../lib/ast_builder'
 
 describe 'Integration tests:' do
   def parse(someSRL)
-    parser = SRL::Parser.new
-    result = parser.parse_SRL(someSRL)
+    tokenizer = SRL::Tokenizer.new(someSRL)
+    @engine.parse(tokenizer.tokens)
   end
 
   def regexp_repr(aResult)
     # Generate an abstract syntax parse tree from the parse result
-    regexp_expr_builder = ASTBuilder
-    tree = aResult.parse_tree(regexp_expr_builder)
+    tree = @engine.convert(aResult)
     regexp = tree.root
   end
+  
+  before(:each) do
+    @engine = Rley::Engine.new do |config|
+      config.repr_builder = ASTBuilder
+    end
+    @engine.use_grammar(SRL::Grammar)
+  end
 
   context 'Parsing character ranges:' do
     it "should parse 'letter from ... to ...' syntax" do

data/examples/general/SRL/spec/tokenizer_spec.rb

@@ -1,5 +1,4 @@
 require_relative 'spec_helper' # Use the RSpec framework
-require_relative '../lib/grammar'
 require_relative '../lib/tokenizer' # Load the class under test
 
 
@@ -8,18 +7,18 @@ module SRL
     def match_expectations(aTokenizer, theExpectations)
       aTokenizer.tokens.each_with_index do |token, i|
         terminal, lexeme = theExpectations[i]
-        expect(token.terminal.name).to eq(terminal)
+        expect(token.terminal).to eq(terminal)
         expect(token.lexeme).to eq(lexeme)
       end
     end
   
   
-    subject { Tokenizer.new('', SRL::Grammar) }    
+    subject { Tokenizer.new('') }    
     
     context 'Initialization:' do
 
       it 'should be initialized with a text to tokenize and a grammar' do
-        expect { Tokenizer.new('anything', SRL::Grammar) }.not_to raise_error
+        expect { Tokenizer.new('anything') }.not_to raise_error
       end
 
       it 'should have its scanner initialized' do
@@ -28,20 +27,20 @@ module SRL
     end # context
 
     context 'Single token recognition:' do
-      # it 'should tokenize delimiters and separators' do
-        # subject.scanner.string = ','
-        # token = subject.tokens.first
-        # expect(token).to be_kind_of(Rley::Lexical::Token)
-        # expect(token.terminal.name).to eq('COMMA')
-        # expect(token.lexeme).to eq(',')
-      # end
+      it 'should tokenize delimiters and separators' do
+        subject.scanner.string = ','
+        token = subject.tokens.first
+        expect(token).to be_kind_of(Rley::Lexical::Token)
+        expect(token.terminal).to eq('COMMA')
+        expect(token.lexeme).to eq(',')
+      end
       
       it 'should tokenize keywords' do
         sample = 'between Exactly oncE optional TWICE'
         subject.scanner.string = sample
         subject.tokens.each do |tok|
           expect(tok).to be_kind_of(Rley::Lexical::Token)
-          expect(tok.terminal.name).to eq(tok.lexeme.upcase)
+          expect(tok.terminal).to eq(tok.lexeme.upcase)
         end
       end
       
@@ -49,7 +48,7 @@ module SRL
         subject.scanner.string = ' 123 '
         token = subject.tokens.first
         expect(token).to be_kind_of(Rley::Lexical::Token)
-        expect(token.terminal.name).to eq('INTEGER')
+        expect(token.terminal).to eq('INTEGER')
         expect(token.lexeme).to eq('123')
       end 
 
@@ -57,7 +56,7 @@ module SRL
         subject.scanner.string = ' 1 '
         token = subject.tokens.first
         expect(token).to be_kind_of(Rley::Lexical::Token)
-        expect(token.terminal.name).to eq('DIGIT_LIT')
+        expect(token.terminal).to eq('DIGIT_LIT')
         expect(token.lexeme).to eq('1')
       end       
     end # context

data/examples/general/SRL/srl_demo.rb

@@ -1,4 +1,5 @@
-require_relative './lib/parser'
+require_relative './lib/tokenizer'
+require_relative './lib/grammar'
 require_relative './lib/ast_builder'
 
 def print_title(aTitle)
@@ -17,9 +18,6 @@ def print_tree(aTitle, aParseTree)
   puts ''
 end
 
-# Create a calculator parser object
-parser = SRL::Parser.new
-
 # Parse the input expression in command-line
 if ARGV.empty?
   my_name = File.basename(__FILE__)
@@ -42,7 +40,16 @@ END_MSG
   exit(1)
 end
 puts ARGV[0]
-result = parser.parse_SRL(ARGV[0])
+
+# Create a Rley facade object
+engine = Rley::Engine.new
+
+########################################
+# Step 1. Load a grammar for calculator
+engine.use_grammar(SRL::Grammar)
+
+lexer = SRL::Tokenizer.new(ARGV[0])
+result = engine.parse(lexer.tokens)
 
 unless result.success?
   # Stop if the parse failed...
@@ -53,12 +60,12 @@ end
 
 
 # Generate a concrete syntax parse tree from the parse result
-cst_ptree = result.parse_tree
+cst_ptree = engine.convert(result)
 print_tree('Concrete Syntax Tree (CST)', cst_ptree)
 
-# Generate a regexp literal representation from the parse result
-tree_builder = ASTBuilder
-ast_ptree = result.parse_tree(tree_builder)
+# Generate an abstract syntax tree (AST) from the parse result
+engine.configuration.repr_builder = ASTBuilder
+ast_ptree = engine.convert(result)
 
 # Now output the regexp literal
 root = ast_ptree.root