lrama 0.1.0

data/lib/lrama/parser.rb

@@ -0,0 +1,321 @@
+require "lrama/report"
+
+module Lrama
+  # Parser for parse.y, generates a grammar
+  class Parser
+    include Lrama::Report::Duration
+
+    T = Lrama::Lexer::Token
+
+    class TokenScanner
+      def initialize(tokens)
+        @tokens = tokens
+        @index = 0
+      end
+
+      def current_token
+        @tokens[@index]
+      end
+
+      def current_type
+        current_token && current_token.type
+      end
+
+      def next
+        token = current_token
+        @index += 1
+        return token
+      end
+
+      def consume(*token_types)
+        if token_types.include?(current_type)
+          token = current_token
+          self.next
+          return token
+        end
+
+        return nil
+      end
+
+      def consume!(*token_types)
+        consume(*token_types) || (raise "#{token_types} is expected but #{current_type}. #{current_token}")
+      end
+
+      def consume_multi(*token_types)
+        a = []
+
+        while token_types.include?(current_type)
+          a << current_token
+          self.next
+        end
+
+        raise "No token is consumed. #{token_types}" if a.empty?
+
+        return a
+      end
+
+      def eots?
+        current_token.nil?
+      end
+    end
+
+    def initialize(text)
+      @text = text
+    end
+
+    def parse
+      report_duration(:parse) do
+        lexer = Lexer.new(@text)
+        grammar = Grammar.new
+        process_prologue(grammar, lexer)
+        parse_bison_declarations(TokenScanner.new(lexer.bison_declarations_tokens), grammar)
+        parse_grammar_rules(TokenScanner.new(lexer.grammar_rules_tokens), grammar)
+        process_epilogue(grammar, lexer)
+        grammar.prepare
+        grammar.compute_nullable
+        grammar.validate!
+
+        grammar
+      end
+    end
+
+    private
+
+    def process_prologue(grammar, lexer)
+      grammar.prologue_first_lineno = lexer.prologue.first[1] if lexer.prologue.first
+      grammar.prologue = lexer.prologue.map(&:first).join
+    end
+
+    def process_epilogue(grammar, lexer)
+      grammar.epilogue_first_lineno = lexer.epilogue.first[1] if lexer.epilogue.first
+      grammar.epilogue = lexer.epilogue.map(&:first).join
+    end
+
+    def parse_bison_declarations(ts, grammar)
+      precedence_number = 0
+
+      while !ts.eots? do
+        case ts.current_type
+        when T::P_expect
+          ts.next
+          grammar.expect = ts.consume!(T::Number).s_value
+        when T::P_define
+          ts.next
+          # Ignore
+          ts.consume_multi(T::Ident)
+        when T::P_printer
+          lineno = ts.current_token.line
+          ts.next
+          code = ts.consume!(T::User_code)
+          code = grammar.build_code(:printer, code)
+          ident_or_tags = ts.consume_multi(T::Ident, T::Tag)
+          grammar.add_printer(ident_or_tags: ident_or_tags, code: code, lineno: lineno)
+        when T::P_lex_param
+          ts.next
+          code = ts.consume!(T::User_code)
+          code = grammar.build_code(:lex_param, code)
+          grammar.lex_param = code.token_code.s_value
+        when T::P_parse_param
+          ts.next
+          code = ts.consume!(T::User_code)
+          code = grammar.build_code(:parse_param, code)
+          grammar.parse_param = code.token_code.s_value
+        when T::P_initial_action
+          ts.next
+          code = ts.consume!(T::User_code)
+          code = grammar.build_code(:initial_action, code)
+          ts.consume(T::Semicolon)
+          grammar.initial_action = code
+        when T::P_union
+          lineno = ts.current_token.line
+          ts.next
+          code = ts.consume!(T::User_code)
+          code = grammar.build_code(:union, code)
+          ts.consume(T::Semicolon)
+          grammar.set_union(code, lineno)
+        when T::P_token
+          # %token tag? (ident number? string?)+
+          #
+          # * ident can be char, e.g. '\\', '\t', '\13'
+          # * number is a token_id for term
+          #
+          # These are valid token declaration (from CRuby parse.y)
+          #
+          # %token END_OF_INPUT   0 "end-of-input"
+          # %token <id> '\\'      "backslash"
+          # %token tSP            "escaped space"
+          # %token tUPLUS         132 "unary+"
+          # %token tCOLON3        ":: at EXPR_BEG"
+          # %token tSTRING_DBEG tSTRING_DVAR tLAMBEG tLABEL_END
+          #
+          #
+          # See: https://www.gnu.org/software/bison/manual/html_node/Symbol-Decls.html
+          ts.next
+          opt_tag = ts.consume(T::Tag)
+
+          while (id = ts.consume(T::Ident, T::Char)) do
+            opt_number = ts.consume(T::Number)
+            opt_string = ts.consume(T::String)
+            # Can replace 0 (EOF)
+            grammar.add_term(
+              id: id,
+              alias_name: opt_string && opt_string.s_value,
+              token_id: opt_number && opt_number.s_value,
+              tag: opt_tag,
+              replace: true,
+            )
+          end
+        when T::P_type
+          # %type tag? (ident|char|string)+
+          #
+          # See: https://www.gnu.org/software/bison/manual/html_node/Symbol-Decls.html
+          ts.next
+          opt_tag = ts.consume(T::Tag)
+
+          while (id = ts.consume(T::Ident, T::Char, T::String)) do
+            grammar.add_type(
+              id: id,
+              tag: opt_tag
+            )
+          end
+        when T::P_nonassoc
+          # %nonassoc (ident|char|string)+
+          ts.next
+          while (id = ts.consume(T::Ident, T::Char, T::String)) do
+            sym = grammar.add_term(id: id)
+            grammar.add_nonassoc(sym, precedence_number)
+          end
+          precedence_number += 1
+        when T::P_left
+          # %left (ident|char|string)+
+          ts.next
+          while (id = ts.consume(T::Ident, T::Char, T::String)) do
+            sym = grammar.add_term(id: id)
+            grammar.add_left(sym, precedence_number)
+          end
+          precedence_number += 1
+        when T::P_right
+          # %right (ident|char|string)+
+          ts.next
+          while (id = ts.consume(T::Ident, T::Char, T::String)) do
+            sym = grammar.add_term(id: id)
+            grammar.add_right(sym, precedence_number)
+          end
+          precedence_number += 1
+        when nil
+          # end of input
+          raise "Reach to end of input within declarations"
+        else
+          raise "Unexpected token: #{ts.current_token}"
+        end
+      end
+    end
+
+    def parse_grammar_rules(ts, grammar)
+      while !ts.eots? do
+        parse_grammar_rule(ts, grammar)
+      end
+    end
+
+    # TODO: Take care of %prec of rule.
+    #       If %prec exists, user code before %prec
+    #       is NOT an action. For example "{ code 3 }" is NOT an action.
+    #
+    #  keyword_class { code 2 } tSTRING '!' keyword_end { code 3 } %prec "="
+    def parse_grammar_rule(ts, grammar)
+      # LHS
+      lhs = ts.consume!(T::Ident_Colon) # class:
+      lhs.type = T::Ident
+
+      rhs = parse_grammar_rule_rhs(ts, grammar)
+
+      grammar.add_rule(lhs: lhs, rhs: rhs, lineno: rhs.first ? rhs.first.line : lhs.line)
+
+      while true do
+        case ts.current_type
+        when T::Bar
+          # |
+          bar_lineno = ts.current_token.line
+          ts.next
+          rhs = parse_grammar_rule_rhs(ts, grammar)
+          grammar.add_rule(lhs: lhs, rhs: rhs, lineno: rhs.first ? rhs.first.line : bar_lineno)
+        when T::Semicolon
+          # ;
+          ts.next
+          break
+        when T::Ident_Colon
+          # Next lhs can be here because ";" is optional.
+          # Do not consume next token.
+          break
+        when nil
+          # end of input can be here when ";" is omitted
+          break
+        else
+          raise "Unexpected token: #{ts.current_token}"
+        end
+      end
+    end
+
+    def parse_grammar_rule_rhs(ts, grammar)
+      a = []
+      prec_seen = false
+      code_after_prec = false
+
+      while true do
+        # TODO: Srting can be here
+        case ts.current_type
+        when T::Ident
+          # keyword_class
+
+          raise "Ident after %prec" if prec_seen
+          a << ts.current_token
+          ts.next
+        when T::Char
+          # '!'
+
+          raise "Char after %prec" if prec_seen
+          a << ts.current_token
+          ts.next
+        when T::P_prec
+          # %prec tPLUS
+          #
+          # See: https://www.gnu.org/software/bison/manual/html_node/Contextual-Precedence.html
+
+          ts.next
+          prec_seen = true
+          precedence_id = ts.consume!(T::Ident, T::String, T::Char)
+          precedence_sym = grammar.find_symbol_by_id!(precedence_id)
+          a << precedence_sym
+        when T::User_code
+          # { code } in the middle of rhs
+
+          if prec_seen
+            raise "Multiple User_code after %prec" if code_after_prec
+            code_after_prec = true
+          end
+
+          code = ts.current_token
+          grammar.build_references(code)
+          a << code
+          ts.next
+        when T::Bar
+          # |
+          break
+        when T::Semicolon
+          # ;
+          break
+        when T::Ident_Colon
+          # Next lhs can be here because ";" is optional.
+          break
+        when nil
+          # end of input can be here when ";" is omitted
+          break
+        else
+          raise "Unexpected token: #{ts.current_token}"
+        end
+      end
+
+      return a
+    end
+  end
+end

data/lib/lrama/report.rb

@@ -0,0 +1,35 @@
+module Lrama
+  class Report
+    module Profile
+      def self.report_profile
+        require "stackprof"
+
+        StackProf.run(mode: :cpu, raw: true, out: 'tmp/stackprof-cpu-myapp.dump') do
+          yield
+        end
+      end
+    end
+
+    module Duration
+      def self.enable
+        @_report_duration_enabled = true
+      end
+
+      def self.enabled?
+        !!@_report_duration_enabled
+      end
+
+      def report_duration(method_name)
+        time1 = Time.now.to_f
+        result = yield
+        time2 = Time.now.to_f
+
+        if Duration.enabled?
+          puts sprintf("%s %10.5f s", method_name, time2 - time1)
+        end
+
+        return result
+      end
+    end
+  end
+end