lrama 0.5.1 → 0.5.3

data/lib/lrama/grammar.rb

@@ -1,283 +1,15 @@
-require "forwardable"
+require "lrama/grammar/code"
+require "lrama/grammar/error_token"
+require "lrama/grammar/precedence"
+require "lrama/grammar/printer"
+require "lrama/grammar/reference"
+require "lrama/grammar/rule"
+require "lrama/grammar/symbol"
+require "lrama/grammar/union"
 require "lrama/lexer"
 
 module Lrama
-  Rule = Struct.new(:id, :lhs, :rhs, :code, :nullable, :precedence_sym, :lineno, keyword_init: true) do
-    # TODO: Change this to display_name
-    def to_s
-      l = lhs.id.s_value
-      r = rhs.empty? ? "ε" : rhs.map {|r| r.id.s_value }.join(", ")
-
-      "#{l} -> #{r}"
-    end
-
-    # Used by #user_actions
-    def as_comment
-      l = lhs.id.s_value
-      r = rhs.empty? ? "%empty" : rhs.map {|r| r.display_name }.join(" ")
-
-      "#{l}: #{r}"
-    end
-
-    def precedence
-      precedence_sym && precedence_sym.precedence
-    end
-
-    def initial_rule?
-      id == 0
-    end
-
-    def translated_code
-      if code
-        code.translated_code
-      else
-        nil
-      end
-    end
-  end
-
-  # Symbol is both of nterm and term
-  # `number` is both for nterm and term
-  # `token_id` is tokentype for term, internal sequence number for nterm
-  #
-  # TODO: Add validation for ASCII code range for Token::Char
-  Symbol = Struct.new(:id, :alias_name, :number, :tag, :term, :token_id, :nullable, :precedence, :printer, keyword_init: true) do
-    attr_writer :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol
-
-    def term?
-      term
-    end
-
-    def nterm?
-      !term
-    end
-
-    def eof_symbol?
-      !!@eof_symbol
-    end
-
-    def error_symbol?
-      !!@error_symbol
-    end
-
-    def undef_symbol?
-      !!@undef_symbol
-    end
-
-    def accept_symbol?
-      !!@accept_symbol
-    end
-
-    def display_name
-      if alias_name
-        alias_name
-      else
-        id.s_value
-      end
-    end
-
-    # name for yysymbol_kind_t
-    #
-    # See: b4_symbol_kind_base
-    def enum_name
-      case
-      when accept_symbol?
-        name = "YYACCEPT"
-      when eof_symbol?
-        name = "YYEOF"
-      when term? && id.type == Token::Char
-        if alias_name
-          name = number.to_s + alias_name
-        else
-          name = number.to_s + id.s_value
-        end
-      when term? && id.type == Token::Ident
-        name = id.s_value
-      when nterm? && (id.s_value.include?("$") || id.s_value.include?("@"))
-        name = number.to_s + id.s_value
-      when nterm?
-        name = id.s_value
-      else
-        raise "Unexpected #{self}"
-      end
-
-      "YYSYMBOL_" + name.gsub(/[^a-zA-Z_0-9]+/, "_")
-    end
-
-    # comment for yysymbol_kind_t
-    def comment
-      case
-      when accept_symbol?
-        # YYSYMBOL_YYACCEPT
-        id.s_value
-      when eof_symbol?
-        # YYEOF
-        alias_name
-      when (term? && 0 < token_id && token_id < 128)
-        # YYSYMBOL_3_backslash_, YYSYMBOL_14_
-        alias_name || id.s_value
-      when id.s_value.include?("$") || id.s_value.include?("@")
-        # YYSYMBOL_21_1
-        id.s_value
-      else
-        # YYSYMBOL_keyword_class, YYSYMBOL_strings_1
-        alias_name || id.s_value
-      end
-    end
-  end
-
   Type = Struct.new(:id, :tag, keyword_init: true)
-
-  Code = Struct.new(:type, :token_code, keyword_init: true) do
-    extend Forwardable
-
-    def_delegators "token_code", :s_value, :line, :column, :references
-
-    # $$, $n, @$, @n is translated to C code
-    def translated_code
-      case type
-      when :user_code
-        translated_user_code
-      when :initial_action
-        translated_initial_action_code
-      end
-    end
-
-    # * ($1) error
-    # * ($$) *yyvaluep
-    # * (@1) error
-    # * (@$) *yylocationp
-    def translated_printer_code(tag)
-      t_code = s_value.dup
-
-      references.reverse.each do |ref|
-        first_column = ref.first_column
-        last_column = ref.last_column
-
-        case
-        when ref.number == "$" && ref.type == :dollar # $$
-          # Omit "<>"
-          member = tag.s_value[1..-2]
-          str = "((*yyvaluep).#{member})"
-        when ref.number == "$" && ref.type == :at # @$
-          str = "(*yylocationp)"
-        when ref.type == :dollar # $n
-          raise "$#{ref.number} can not be used in %printer."
-        when ref.type == :at # @n
-          raise "@#{ref.number} can not be used in %printer."
-        else
-          raise "Unexpected. #{self}, #{ref}"
-        end
-
-        t_code[first_column..last_column] = str
-      end
-
-      return t_code
-    end
-
-
-    private
-
-    # * ($1) yyvsp[i]
-    # * ($$) yyval
-    # * (@1) yylsp[i]
-    # * (@$) yyloc
-    def translated_user_code
-      t_code = s_value.dup
-
-      references.reverse.each do |ref|
-        first_column = ref.first_column
-        last_column = ref.last_column
-
-        case
-        when ref.number == "$" && ref.type == :dollar # $$
-          # Omit "<>"
-          member = ref.tag.s_value[1..-2]
-          str = "(yyval.#{member})"
-        when ref.number == "$" && ref.type == :at # @$
-          str = "(yyloc)"
-        when ref.type == :dollar # $n
-          i = -ref.position_in_rhs + ref.number
-          # Omit "<>"
-          member = ref.tag.s_value[1..-2]
-          str = "(yyvsp[#{i}].#{member})"
-        when ref.type == :at # @n
-          i = -ref.position_in_rhs + ref.number
-          str = "(yylsp[#{i}])"
-        else
-          raise "Unexpected. #{self}, #{ref}"
-        end
-
-        t_code[first_column..last_column] = str
-      end
-
-      return t_code
-    end
-
-    # * ($1) error
-    # * ($$) yylval
-    # * (@1) error
-    # * (@$) yylloc
-    def translated_initial_action_code
-      t_code = s_value.dup
-
-      references.reverse.each do |ref|
-        first_column = ref.first_column
-        last_column = ref.last_column
-
-        case
-        when ref.number == "$" && ref.type == :dollar # $$
-          str = "yylval"
-        when ref.number == "$" && ref.type == :at # @$
-          str = "yylloc"
-        when ref.type == :dollar # $n
-          raise "$#{ref.number} can not be used in initial_action."
-        when ref.type == :at # @n
-          raise "@#{ref.number} can not be used in initial_action."
-        else
-          raise "Unexpected. #{self}, #{ref}"
-        end
-
-        t_code[first_column..last_column] = str
-      end
-
-      return t_code
-    end
-  end
-
-  # type: :dollar or :at
-  # ex_tag: "$<tag>1" (Optional)
-  Reference = Struct.new(:type, :number, :ex_tag, :first_column, :last_column, :referring_symbol, :position_in_rhs, keyword_init: true) do
-    def tag
-      if ex_tag
-        ex_tag
-      else
-        referring_symbol.tag
-      end
-    end
-  end
-
-  Precedence = Struct.new(:type, :precedence, keyword_init: true) do
-    include Comparable
-
-    def <=>(other)
-      self.precedence <=> other.precedence
-    end
-  end
-
-  Printer = Struct.new(:ident_or_tags, :code, :lineno, keyword_init: true) do
-    def translated_code(member)
-      code.translated_printer_code(member)
-    end
-  end
-
-  Union = Struct.new(:code, :lineno, keyword_init: true) do
-    def braces_less_code
-      # Remove braces
-      code.s_value[1..-2]
-    end
-  end
-
   Token = Lrama::Lexer::Token
 
   # Grammar is the result of parsing an input grammar file
@@ -287,7 +19,7 @@ module Lrama
 
     attr_reader :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol, :aux
     attr_accessor :union, :expect,
-                  :printers,
+                  :printers, :error_tokens,
                   :lex_param, :parse_param, :initial_action,
                   :symbols, :types,
                   :rules, :_rules,
@@ -295,6 +27,7 @@ module Lrama
 
     def initialize
       @printers = []
+      @error_tokens = []
       @symbols = []
       @types = []
       @_rules = []
@@ -314,6 +47,10 @@ module Lrama
       @printers << Printer.new(ident_or_tags: ident_or_tags, code: code, lineno: lineno)
     end
 
+    def add_error_token(ident_or_tags:, code:, lineno:)
+      @error_tokens << ErrorToken.new(ident_or_tags, code, lineno)
+    end
+
     def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false)
       if token_id && (sym = @symbols.find {|s| s.token_id == token_id })
         if replace
@@ -382,8 +119,8 @@ module Lrama
     end
 
     def build_references(token_code)
-      token_code.references.map! do |type, number, tag, first_column, last_column|
-        Reference.new(type: type, number: number, ex_tag: tag, first_column: first_column, last_column: last_column)
+      token_code.references.map! do |type, value, tag, first_column, last_column|
+        Reference.new(type: type, value: value, ex_tag: tag, first_column: first_column, last_column: last_column)
       end
 
       token_code
@@ -419,12 +156,14 @@ module Lrama
       fill_sym_to_rules
       fill_nterm_type
       fill_symbol_printer
+      fill_symbol_error_token
       @symbols.sort_by!(&:number)
     end
 
     # TODO: More validation methods
     def validate!
       validate_symbol_number_uniqueness!
+      validate_no_declared_type_reference!
     end
 
     def compute_nullable
@@ -627,15 +366,23 @@ module Lrama
               ref.position_in_rhs = i - 1
               next if ref.type == :at
               # $$, $n, @$, @n can be used in any actions
-              number = ref.number
 
-              if number == "$"
+              if ref.value == "$"
                 # TODO: Should be postponed after middle actions are extracted?
                 ref.referring_symbol = lhs
-              else
-                raise "Can not refer following component. #{number} >= #{i}. #{token}" if number >= i
-                rhs1[number - 1].referred = true
-                ref.referring_symbol = rhs1[number - 1]
+              elsif ref.value.is_a?(Integer)
+                raise "Can not refer following component. #{ref.value} >= #{i}. #{token}" if ref.value >= i
+                rhs1[ref.value - 1].referred = true
+                ref.referring_symbol = rhs1[ref.value - 1]
+              elsif ref.value.is_a?(String)
+                target_tokens = ([lhs] + rhs1 + [code]).compact.first(i)
+                referring_symbol_candidate = target_tokens.filter {|token| token.referred_by?(ref.value) }
+                raise "Referring symbol `#{ref.value}` is duplicated. #{token}" if referring_symbol_candidate.size >= 2
+                raise "Referring symbol `#{ref.value}` is not found. #{token}" if referring_symbol_candidate.count == 0
+
+                referring_symbol = referring_symbol_candidate.first
+                referring_symbol.referred = true
+                ref.referring_symbol = referring_symbol
               end
             end
           end
@@ -837,6 +584,23 @@ module Lrama
       end
     end
 
+    def fill_symbol_error_token
+      @symbols.each do |sym|
+        @error_tokens.each do |error_token|
+          error_token.ident_or_tags.each do |ident_or_tag|
+            case ident_or_tag.type
+            when Token::Ident
+              sym.error_token = error_token if sym.id == ident_or_tag
+            when Token::Tag
+              sym.error_token = error_token if sym.tag == ident_or_tag
+            else
+              raise "Unknown token type. #{error_token}"
+            end
+          end
+        end
+      end
+    end
+
     def validate_symbol_number_uniqueness!
       invalid = @symbols.group_by(&:number).select do |number, syms|
         syms.count > 1
@@ -846,5 +610,23 @@ module Lrama
 
       raise "Symbol number is duplicated. #{invalid}"
     end
+
+    def validate_no_declared_type_reference!
+      errors = []
+
+      rules.each do |rule|
+        next unless rule.code
+
+        rule.code.references.select do |ref|
+          ref.type == :dollar && !ref.tag
+        end.each do |ref|
+          errors << "$#{ref.value} of '#{rule.lhs.id.s_value}' has no declared type"
+        end
+      end
+
+      return if errors.empty?
+
+      raise errors.join("\n")
+    end
   end
 end

data/lib/lrama/lexer/token.rb

@@ -0,0 +1,76 @@
+module Lrama
+  class Lexer
+    class Token < Struct.new(:type, :s_value, :alias, keyword_init: true)
+      Type = Struct.new(:id, :name, keyword_init: true)
+
+      attr_accessor :line, :column, :referred
+      # For User_code
+      attr_accessor :references
+
+      def to_s
+        "#{super} line: #{line}, column: #{column}"
+      end
+
+      def referred_by?(string)
+        [self.s_value, self.alias].include?(string)
+      end
+
+      def ==(other)
+        self.class == other.class && self.type == other.type && self.s_value == other.s_value
+      end
+
+      def numberize_references(lhs, rhs)
+        self.references.map! {|ref|
+          ref_name = ref[1]
+          if ref_name.is_a?(::String) && ref_name != '$'
+            value =
+              if lhs.referred_by?(ref_name)
+                '$'
+              else
+                rhs.find_index {|token| token.referred_by?(ref_name) } + 1
+              end
+            [ref[0], value, ref[2], ref[3], ref[4]]
+          else
+            ref
+          end
+        }
+      end
+
+      @i = 0
+      @types = []
+
+      def self.define_type(name)
+        type = Type.new(id: @i, name: name.to_s)
+        const_set(name, type)
+        @types << type
+        @i += 1
+      end
+
+      # Token types
+      define_type(:P_expect)         # %expect
+      define_type(:P_define)         # %define
+      define_type(:P_printer)        # %printer
+      define_type(:P_error_token)    # %error-token
+      define_type(:P_lex_param)      # %lex-param
+      define_type(:P_parse_param)    # %parse-param
+      define_type(:P_initial_action) # %initial-action
+      define_type(:P_union)          # %union
+      define_type(:P_token)          # %token
+      define_type(:P_type)           # %type
+      define_type(:P_nonassoc)       # %nonassoc
+      define_type(:P_left)           # %left
+      define_type(:P_right)          # %right
+      define_type(:P_prec)           # %prec
+      define_type(:User_code)        # { ... }
+      define_type(:Tag)              # <int>
+      define_type(:Number)           # 0
+      define_type(:Ident_Colon)      # k_if:, k_if  : (spaces can be there)
+      define_type(:Ident)            # api.pure, tNUMBER
+      define_type(:Named_Ref)        # [foo]
+      define_type(:Semicolon)        # ;
+      define_type(:Bar)              # |
+      define_type(:String)           # "str"
+      define_type(:Char)             # '+'
+    end
+  end
+end

data/lib/lrama/lexer.rb

@@ -1,58 +1,12 @@
 require "strscan"
-require "lrama/report"
+require "lrama/report/duration"
+require "lrama/lexer/token"
 
 module Lrama
   # Lexer for parse.y
   class Lexer
     include Lrama::Report::Duration
 
-    # s_value is semantic value
-    Token = Struct.new(:type, :s_value, keyword_init: true) do
-      Type = Struct.new(:id, :name, keyword_init: true)
-
-      attr_accessor :line, :column, :referred
-      # For User_code
-      attr_accessor :references
-
-      def to_s
-        "#{super} line: #{line}, column: #{column}"
-      end
-
-      @i = 0
-      @types = []
-
-      def self.define_type(name)
-        type = Type.new(id: @i, name: name.to_s)
-        const_set(name, type)
-        @types << type
-        @i += 1
-      end
-
-      # Token types
-      define_type(:P_expect)         # %expect
-      define_type(:P_define)         # %define
-      define_type(:P_printer)        # %printer
-      define_type(:P_lex_param)      # %lex-param
-      define_type(:P_parse_param)    # %parse-param
-      define_type(:P_initial_action) # %initial-action
-      define_type(:P_union)          # %union
-      define_type(:P_token)          # %token
-      define_type(:P_type)           # %type
-      define_type(:P_nonassoc)       # %nonassoc
-      define_type(:P_left)           # %left
-      define_type(:P_right)          # %right
-      define_type(:P_prec)           # %prec
-      define_type(:User_code)        # { ... }
-      define_type(:Tag)              # <int>
-      define_type(:Number)           # 0
-      define_type(:Ident_Colon)      # k_if:, k_if  : (spaces can be there)
-      define_type(:Ident)            # api.pure, tNUMBER
-      define_type(:Semicolon)        # ;
-      define_type(:Bar)              # |
-      define_type(:String)           # "str"
-      define_type(:Char)             # '+'
-    end
-
     # States
     #
     # See: https://www.gnu.org/software/bison/manual/html_node/Grammar-Outline.html
@@ -166,16 +120,23 @@ module Lrama
           tokens << create_token(Token::Number, Integer(ss[0]), line, ss.pos - column)
         when ss.scan(/(<[a-zA-Z0-9_]+>)/)
           tokens << create_token(Token::Tag, ss[0], line, ss.pos - column)
+        when ss.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]\s*:/)
+          tokens << create_token(Token::Ident_Colon, ss[1], line, ss.pos - column)
+          tokens << create_token(Token::Named_Ref, ss[2], line, ss.pos - column)
         when ss.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)\s*:/)
           tokens << create_token(Token::Ident_Colon, ss[1], line, ss.pos - column)
         when ss.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)/)
           tokens << create_token(Token::Ident, ss[0], line, ss.pos - column)
+        when ss.scan(/\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/)
+          tokens << create_token(Token::Named_Ref, ss[1], line, ss.pos - column)
         when ss.scan(/%expect/)
           tokens << create_token(Token::P_expect, ss[0], line, ss.pos - column)
         when ss.scan(/%define/)
           tokens << create_token(Token::P_define, ss[0], line, ss.pos - column)
         when ss.scan(/%printer/)
           tokens << create_token(Token::P_printer, ss[0], line, ss.pos - column)
+        when ss.scan(/%error-token/)
+          tokens << create_token(Token::P_error_token, ss[0], line, ss.pos - column)
         when ss.scan(/%lex-param/)
           tokens << create_token(Token::P_lex_param, ss[0], line, ss.pos - column)
         when ss.scan(/%parse-param/)
@@ -257,6 +218,9 @@ module Lrama
         when ss.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
           tag = ss[1] ? create_token(Token::Tag, ss[1], line, str.length) : nil
           references << [:dollar, Integer(ss[2]), tag, str.length, str.length + ss[0].length - 1]
+        when ss.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_.][-a-zA-Z0-9_.]*)/) # $foo, $expr, $<long>program
+          tag = ss[1] ? create_token(Token::Tag, ss[1], line, str.length) : nil
+          references << [:dollar, ss[2], tag, str.length, str.length + ss[0].length - 1]
         when ss.scan(/@\$/) # @$
           references << [:at, "$", nil, str.length, str.length + ss[0].length - 1]
         when ss.scan(/@(\d)+/) # @1

data/lib/lrama/output.rb

@@ -1,20 +1,24 @@
 require "erb"
 require "forwardable"
-require "lrama/report"
+require "lrama/report/duration"
 
 module Lrama
   class Output
     extend Forwardable
     include Report::Duration
 
-    attr_reader :grammar_file_path, :context, :grammar
+    attr_reader :grammar_file_path, :context, :grammar, :error_recovery
 
     def_delegators "@context", :yyfinal, :yylast, :yyntokens, :yynnts, :yynrules, :yynstates,
                                :yymaxutok, :yypact_ninf, :yytable_ninf
 
     def_delegators "@grammar", :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol
 
-    def initialize(out:, output_file_path:, template_name:, grammar_file_path:, header_out: nil, header_file_path: nil, context:, grammar:)
+    def initialize(
+      out:, output_file_path:, template_name:, grammar_file_path:,
+      header_out: nil, header_file_path: nil,
+      context:, grammar:, error_recovery: false
+    )
       @out = out
       @output_file_path = output_file_path
       @template_name = template_name
@@ -23,6 +27,7 @@ module Lrama
       @header_file_path = header_file_path
       @context = context
       @grammar = grammar
+      @error_recovery = error_recovery
     end
 
     if ERB.instance_method(:initialize).parameters.last.first == :key
@@ -98,6 +103,10 @@ module Lrama
       int_array_to_string(@context.yytranslate)
     end
 
+    def yytranslate_inverted
+      int_array_to_string(@context.yytranslate_inverted)
+    end
+
     def yyrline
       int_array_to_string(@context.yyrline)
     end
@@ -155,6 +164,25 @@ module Lrama
       STR
     end
 
+    def symbol_actions_for_error_token
+      str = ""
+
+      @grammar.symbols.each do |sym|
+        next unless sym.error_token
+
+        str << <<-STR
+    case #{sym.enum_name}: /* #{sym.comment}  */
+#line #{sym.error_token.lineno} "#{@grammar_file_path}"
+         #{sym.error_token.translated_code(sym.tag)}
+#line [@oline@] [@ofile@]
+        break;
+
+        STR
+      end
+
+      str
+    end
+
     # b4_user_actions
     def user_actions
       str = ""

data/lib/lrama/parser/token_scanner.rb

@@ -14,6 +14,10 @@ module Lrama
         current_token && current_token.type
       end
 
+      def previous_token
+        @tokens[@index - 1]
+      end
+
       def next
         token = current_token
         @index += 1