rley 0.5.07 → 0.5.08

data/examples/general/SRL/lib/parser.rb

@@ -0,0 +1,26 @@
+# Purpose: to demonstrate how to build and render a parse tree for JSON
+# language
+require_relative 'tokenizer'
+require_relative 'grammar'
+module SRL
+  # A parser for a subset of Simple Regex Language
+  class Parser < Rley::Parser::GFGEarleyParser
+    attr_reader(:source_file)
+
+    # Constructor
+    def initialize()
+      # Builder the Earley parser with the calculator grammar
+      super(Grammar)
+    end
+
+    def parse_SRL(aText)
+      lexer = Tokenizer.new(aText, grammar)
+      tokens = lexer.tokens
+      result = parse(tokens)
+
+      return result
+    end
+  end # class
+end # module
+
+# End of file

data/examples/general/SRL/lib/regex/multiplicity.rb

@@ -0,0 +1,94 @@
+# File: Multiplicity.rb
+
+module SRL
+  module Regex # This module is used as a namespace
+    # The multiplicity specifies by how much a given expression can be repeated.
+    class Multiplicity
+      # The lowest acceptable repetition count
+      attr_reader(:lower_bound)
+      
+      # The highest possible repetition count
+      attr_reader(:upper_bound)
+
+      # An indicator that specifies how to repeat (:greedy, :lazy, :possessive)
+      attr_reader(:policy)
+      
+      # @param aLowerBound [Integer]
+      # @param anUpperBound [Integer, Symbol] integer or :more symbol
+      # @param aPolicy [Symbol] One of: (:greedy, :lazy, :possessive)
+      def initialize(aLowerBound, anUpperBound, aPolicy)
+        @lower_bound = valid_lower_bound(aLowerBound)
+        @upper_bound = valid_upper_bound(anUpperBound)
+        @policy = valid_policy(aPolicy)
+      end
+      
+    public
+      # Purpose: Return the String representation of the multiplicity.
+      def to_str()
+        case upper_bound
+          when :more
+            case lower_bound
+              when 0
+                subresult = '*'
+              when 1
+                subresult = '+'
+              else
+                subresult = "{#{lower_bound},}"
+            end
+            
+          when lower_bound
+            subresult = "{#{lower_bound}}"
+          else
+            if [lower_bound, upper_bound] == [0, 1]
+              subresult = '?'
+            else
+              subresult = "{#{lower_bound},#{upper_bound}}"
+            end
+        end
+        
+        suffix = case policy
+          when :greedy
+            ''
+          when :lazy
+            '?'
+          when :possessive
+            '+'
+        end
+        
+        return subresult + suffix
+      end
+      
+      
+    private
+      # Validation method. Return the validated lower bound value
+      def valid_lower_bound(aLowerBound)
+        err_msg = "Invalid lower bound of repetition count #{aLowerBound}"
+        raise StandardError, err_msg unless aLowerBound.kind_of?(Integer) 
+        return aLowerBound
+      end
+      
+      # Validation method. Return the validated lower bound value
+      def valid_upper_bound(anUpperBound)
+        err_msg = "Invalid upper bound of repetition count #{anUpperBound}"
+        unless anUpperBound.kind_of?(Integer) || (anUpperBound == :more)	
+          raise StandardError, err_msg
+        end
+        
+        return anUpperBound
+      end	
+      
+      # Validation method. Return the validated policy value.
+      def valid_policy(aPolicy)
+        err_msg = "Invalid repetition policy '#{aPolicy}'."
+        valid_policies = [:greedy, :lazy, :possessive]
+        raise StandardError, err_msg unless valid_policies.include? aPolicy
+
+        return aPolicy
+      end
+
+    end # class
+
+  end # module
+end # module
+
+# End of file

data/examples/general/SRL/lib/regex_repr.rb

@@ -0,0 +1 @@
+require_relative './regex/multiplicity'

data/examples/general/SRL/lib/srl_demo.rb

@@ -0,0 +1,67 @@
+require_relative 'parser'
+require_relative 'ast_builder'
+
+def print_title(aTitle)
+  puts aTitle
+  puts '=' * aTitle.size
+end
+
+def print_tree(aTitle, aParseTree)
+  # Let's create a parse tree visitor
+  visitor = Rley::ParseTreeVisitor.new(aParseTree)
+
+  # Now output formatted parse tree
+  print_title(aTitle)
+  renderer = Rley::Formatter::Asciitree.new($stdout)
+  renderer.render(visitor)
+  puts ''
+end
+
+# Create a calculator parser object
+parser = SRL::Parser.new
+
+# Parse the input expression in command-line
+if ARGV.empty?
+  my_name = File.basename(__FILE__)
+  msg = <<-END_MSG
+Demo parser for the SRL, the Simple Regex Language (https://simple-regex.com/).
+Ultimately it will support SRL in full, currently it parses only the 
+SRL quantifiers.
+The utility prints the resulting regular expression.
+
+Command-line syntax:
+  ruby #{my_name} filename
+  where:
+    the file name is a SRL source file.
+
+  Examples:
+  ruby #{my_name} sample01.srl
+END_MSG
+  puts msg
+  exit(1)
+end
+puts ARGV[0]
+result = parser.parse_expression(ARGV[0])
+
+unless result.success?
+  # Stop if the parse failed...
+  puts "Parsing of '#{ARGV[0]}' failed"
+  puts "Reason: #{result.failure_reason.message}"
+  exit(1)
+end
+
+
+# Generate a concrete syntax parse tree from the parse result
+cst_ptree = result.parse_tree
+print_tree('Concrete Syntax Tree (CST)', cst_ptree)
+
+# Generate an abstract syntax parse tree from the parse result
+tree_builder = ASTBuilder
+ast_ptree = result.parse_tree(tree_builder)
+# print_tree('Abstract Syntax Tree (AST)', ast_ptree)
+
+# # Now perform the computation of math expression
+# root = ast_ptree.root
+# print_title('Result:')
+# puts root.interpret.to_s # Output the expression result
+# End of file

data/examples/general/SRL/lib/tokenizer.rb

@@ -0,0 +1,101 @@
+# File: srl_tokenizer.rb
+# Tokenizer for SRL (Simple Regex Language)
+require 'strscan'
+require 'rley' # Load the gem
+
+module SRL
+  # The tokenizer should recognize:
+  # Keywords: as, capture, letter
+  # Integer literals including single digit
+  # String literals (quote delimited)
+  # Single character literal
+  # Delimiters: parentheses '(' and ')'
+  # Separators: comma (optional)
+  class Tokenizer
+    attr_reader(:scanner)
+    attr_reader(:lineno)
+    attr_reader(:line_start)
+    attr_reader(:name2symbol)
+
+    @@lexeme2name = {
+      '(' => 'LPAREN',
+      ')' => 'RPAREN',
+      ',' => 'COMMA'
+    }.freeze
+    
+    # Here are all the SRL keywords (in uppercase)
+    @@keywords = %w[
+      AND
+      AT
+      BETWEEN
+      EXACTLY
+      LEAST
+      MORE
+      NEVER
+      ONCE
+      OPTIONAL
+      OR
+      TIMES
+      TWICE
+    ].map { |x| [x, x] } .to_h
+    
+    class ScanError < StandardError; end
+
+    def initialize(source, aGrammar)
+      @scanner = StringScanner.new(source)
+      @name2symbol = aGrammar.name2symbol
+      @lineno = 1
+    end
+
+    def tokens()
+      tok_sequence = []
+      until @scanner.eos?
+        token = _next_token
+        tok_sequence << token unless token.nil?
+      end
+
+      return tok_sequence
+    end
+
+    private
+
+    def _next_token()
+      skip_whitespaces
+      curr_ch = scanner.peek(1)
+      return nil if curr_ch.nil?
+      
+      token = nil
+
+      if '(),'.include? curr_ch
+        # Single character token
+        token = build_token(@@lexeme2name[curr_ch], scanner.getch)  
+      elsif (lexeme = scanner.scan(/[0-9]{2,}/))
+        token = build_token('INTEGER', lexeme) # An integer has two or more digits
+      elsif (lexeme = scanner.scan(/[0-9]/))
+        token = build_token('DIGIT', lexeme) 
+      elsif (lexeme = scanner.scan(/[a-zA-Z]{2,}/))
+        token = build_token(@@keywords[lexeme.upcase], lexeme)
+        # TODO: handle case unknown identifier
+      elsif (lexeme = scanner.scan(/\w/))
+        puts 'Buff'
+        token = build_token('CHAR', lexeme)      
+      else # Unknown token
+        erroneous = curr_ch.nil? ? '' : curr_ch
+        sequel = scanner.scan(/.{1,20}/)
+        erroneous += sequel unless sequel.nil?
+        raise ScanError.new("Unknown token #{erroneous}")
+      end
+
+      return token
+    end
+    
+    def build_token(aSymbolName, aLexeme)
+      token_type = name2symbol[aSymbolName]
+      return Rley::Lexical::Token.new(aLexeme, token_type)    
+    end
+
+    def skip_whitespaces()
+      scanner.scan(/[ \t\f\n\r]+/)
+    end
+  end # class
+end # module

data/examples/general/SRL/spec/integration_spec.rb

@@ -0,0 +1,103 @@
+require_relative 'spec_helper' # Use the RSpec framework
+require_relative '../lib/parser'
+require_relative '../lib/ast_builder'
+
+describe 'Integration tests:' do
+  def parse(someSRL)
+    parser = SRL::Parser.new
+    result = parser.parse_SRL(someSRL)
+  end
+
+  def regexp_repr(aResult)
+    # Generate an abstract syntax parse tree from the parse result
+    regexp_expr_builder = ASTBuilder
+    tree = aResult.parse_tree(regexp_expr_builder)
+    regexp = tree.root
+  end
+
+  context 'Parsing quantifiers:' do
+    it "should parse 'once' syntax" do
+      result = parse('once')
+      expect(result).to be_success
+
+      regexp = regexp_repr(result)
+      expect(regexp.to_str).to eq('{1}')
+    end
+
+    it "should parse 'twice' syntax" do
+      result = parse('twice')
+      expect(result).to be_success
+      
+      regexp = regexp_repr(result)
+      expect(regexp.to_str).to eq('{2}')      
+    end
+
+    it "should parse 'optional' syntax" do
+      result = parse('optional')
+      expect(result).to be_success
+      
+      regexp = regexp_repr(result)
+      expect(regexp.to_str).to eq('?')      
+    end
+
+    it "should parse 'exactly ... times' syntax" do
+      result = parse('exactly 4 times')
+      expect(result).to be_success
+      
+      regexp = regexp_repr(result)
+      expect(regexp.to_str).to eq('{4}')        
+    end
+
+    it "should parse 'between ... and ... times' syntax" do
+      result = parse('between 2 and 4 times')
+      expect(result).to be_success
+
+      # Dropping 'times' keyword is shorter syntax
+      expect(parse('between 2 and 4')).to be_success
+      
+      regexp = regexp_repr(result)
+      expect(regexp.to_str).to eq('{2, 4}')        
+    end
+
+    it "should parse 'once or more' syntax" do
+      result = parse('once or more')
+      expect(result).to be_success
+    end
+
+    it "should parse 'never or more' syntax" do
+      result = parse('never or more')
+      expect(result).to be_success
+    end
+
+    it "should parse 'at least  ... times' syntax" do
+      result = parse('at least 10 times')
+      expect(result).to be_success
+      
+      regexp = regexp_repr(result)
+      expect(regexp.to_str).to eq('{10,}')      
+    end
+
+  end # context
+
+end # describe
+
+
+=begin
+
+unless result.success?
+  # Stop if the parse failed...
+  puts "Parsing of '#{ARGV[0]}' failed"
+  puts "Reason: #{result.failure_reason.message}"
+  exit(1)
+end
+
+
+# Generate a concrete syntax parse tree from the parse result
+cst_ptree = result.parse_tree
+print_tree('Concrete Syntax Tree (CST)', cst_ptree)
+
+# Generate an abstract syntax parse tree from the parse result
+tree_builder = ASTBuilder
+ast_ptree = result.parse_tree(tree_builder)
+=end
+

data/examples/general/SRL/spec/regex/multiplicity_spec.rb

@@ -0,0 +1,83 @@
+# File: Multiplicity_spec.rb
+
+require 'pp'
+require_relative '../spec_helper'	# Use the RSpec test framework
+require_relative '../../lib/regex/multiplicity'
+
+module SRL
+  # Reopen the module, in order to get rid of fully qualified names
+  module Regex	# This module is used as a namespace
+
+    describe Multiplicity do
+
+      context "Creation & initialisation" do
+        it "should be created with 3 arguments" do
+          # Valid cases: initialized with two integer values and a policy symbol
+          [:greedy, :lazy, :possessive].each do |aPolicy|
+            expect { Multiplicity.new(0, 1, aPolicy) }.not_to raise_error
+          end
+          
+          # Invalid case: initialized with invalid policy value
+          err = "Invalid repetition policy 'wrong'."
+          expect { Multiplicity.new(0, :more, 'wrong') }.to raise_error(StandardError, err)
+        end
+      
+      end
+      
+      context "Provided services" do
+        it 'should know its text representation' do
+          policy2text = { :greedy => '' , :lazy => '?', :possessive => '+' }
+          
+          # Case: zero or one
+          policy2text.keys.each do |aPolicy|
+            multi = Multiplicity.new(0, 1, aPolicy)
+            expect(multi.to_str).to eq("?#{policy2text[aPolicy]}")
+          end
+          
+          # Case: zero or more
+          policy2text.keys.each do |aPolicy|
+            multi = Multiplicity.new(0, :more, aPolicy)
+            expect(multi.to_str).to eq("*#{policy2text[aPolicy]}")
+          end
+          
+          # Case: one or more
+          policy2text.keys.each do |aPolicy|
+            multi = Multiplicity.new(1, :more, aPolicy)
+            expect(multi.to_str).to eq("+#{policy2text[aPolicy]}")
+          end
+          
+          # Case: exactly m times
+          policy2text.keys.each do |aPolicy|
+            samples = [1, 2, 5, 100]
+            samples.each do |aCount|
+              multi = Multiplicity.new(aCount, aCount, aPolicy)
+              expect(multi.to_str).to eq("{#{aCount}}#{policy2text[aPolicy]}")
+            end
+          end
+
+          # Case: m, n times
+          policy2text.keys.each do |aPolicy|
+            samples = [1, 2, 5, 100]
+            samples.each do |aCount|
+              upper = aCount + 1 + rand(20)
+              multi = Multiplicity.new(aCount, upper, aPolicy)
+              expect(multi.to_str).to eq("{#{aCount},#{upper}}#{policy2text[aPolicy]}")
+            end
+          end
+
+          # Case: m or more
+          policy2text.keys.each do |aPolicy|
+            samples = [2, 3, 5, 100]
+            samples.each do |aCount|
+              multi = Multiplicity.new(aCount, :more, aPolicy)
+              expect(multi.to_str).to eq("{#{aCount},}#{policy2text[aPolicy]}")
+            end
+          end			
+        end
+      end	
+
+    end
+
+  end # module
+end # module
+# End of file