lrama 0.5.2 → 0.5.4

data/lib/lrama/grammar.rb

@@ -1,293 +1,23 @@
-require "forwardable"
+require "lrama/grammar/auxiliary"
+require "lrama/grammar/code"
+require "lrama/grammar/error_token"
+require "lrama/grammar/precedence"
+require "lrama/grammar/printer"
+require "lrama/grammar/reference"
+require "lrama/grammar/rule"
+require "lrama/grammar/symbol"
+require "lrama/grammar/union"
 require "lrama/lexer"
+require "lrama/type"
 
 module Lrama
-  Rule = Struct.new(:id, :lhs, :rhs, :code, :nullable, :precedence_sym, :lineno, keyword_init: true) do
-    # TODO: Change this to display_name
-    def to_s
-      l = lhs.id.s_value
-      r = rhs.empty? ? "ε" : rhs.map {|r| r.id.s_value }.join(", ")
-
-      "#{l} -> #{r}"
-    end
-
-    # Used by #user_actions
-    def as_comment
-      l = lhs.id.s_value
-      r = rhs.empty? ? "%empty" : rhs.map {|r| r.display_name }.join(" ")
-
-      "#{l}: #{r}"
-    end
-
-    def precedence
-      precedence_sym && precedence_sym.precedence
-    end
-
-    def initial_rule?
-      id == 0
-    end
-
-    def translated_code
-      if code
-        code.translated_code
-      else
-        nil
-      end
-    end
-  end
-
-  # Symbol is both of nterm and term
-  # `number` is both for nterm and term
-  # `token_id` is tokentype for term, internal sequence number for nterm
-  #
-  # TODO: Add validation for ASCII code range for Token::Char
-  Symbol = Struct.new(:id, :alias_name, :number, :tag, :term, :token_id, :nullable, :precedence, :printer, keyword_init: true) do
-    attr_writer :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol
-
-    def term?
-      term
-    end
-
-    def nterm?
-      !term
-    end
-
-    def eof_symbol?
-      !!@eof_symbol
-    end
-
-    def error_symbol?
-      !!@error_symbol
-    end
-
-    def undef_symbol?
-      !!@undef_symbol
-    end
-
-    def accept_symbol?
-      !!@accept_symbol
-    end
-
-    def display_name
-      if alias_name
-        alias_name
-      else
-        id.s_value
-      end
-    end
-
-    # name for yysymbol_kind_t
-    #
-    # See: b4_symbol_kind_base
-    def enum_name
-      case
-      when accept_symbol?
-        name = "YYACCEPT"
-      when eof_symbol?
-        name = "YYEOF"
-      when term? && id.type == Token::Char
-        if alias_name
-          name = number.to_s + alias_name
-        else
-          name = number.to_s + id.s_value
-        end
-      when term? && id.type == Token::Ident
-        name = id.s_value
-      when nterm? && (id.s_value.include?("$") || id.s_value.include?("@"))
-        name = number.to_s + id.s_value
-      when nterm?
-        name = id.s_value
-      else
-        raise "Unexpected #{self}"
-      end
-
-      "YYSYMBOL_" + name.gsub(/[^a-zA-Z_0-9]+/, "_")
-    end
-
-    # comment for yysymbol_kind_t
-    def comment
-      case
-      when accept_symbol?
-        # YYSYMBOL_YYACCEPT
-        id.s_value
-      when eof_symbol?
-        # YYEOF
-        alias_name
-      when (term? && 0 < token_id && token_id < 128)
-        # YYSYMBOL_3_backslash_, YYSYMBOL_14_
-        alias_name || id.s_value
-      when id.s_value.include?("$") || id.s_value.include?("@")
-        # YYSYMBOL_21_1
-        id.s_value
-      else
-        # YYSYMBOL_keyword_class, YYSYMBOL_strings_1
-        alias_name || id.s_value
-      end
-    end
-  end
-
-  Type = Struct.new(:id, :tag, keyword_init: true)
-
-  Code = Struct.new(:type, :token_code, keyword_init: true) do
-    extend Forwardable
-
-    def_delegators "token_code", :s_value, :line, :column, :references
-
-    # $$, $n, @$, @n is translated to C code
-    def translated_code
-      case type
-      when :user_code
-        translated_user_code
-      when :initial_action
-        translated_initial_action_code
-      end
-    end
-
-    # * ($1) error
-    # * ($$) *yyvaluep
-    # * (@1) error
-    # * (@$) *yylocationp
-    def translated_printer_code(tag)
-      t_code = s_value.dup
-
-      references.reverse.each do |ref|
-        first_column = ref.first_column
-        last_column = ref.last_column
-
-        case
-        when ref.value == "$" && ref.type == :dollar # $$
-          # Omit "<>"
-          member = tag.s_value[1..-2]
-          str = "((*yyvaluep).#{member})"
-        when ref.value == "$" && ref.type == :at # @$
-          str = "(*yylocationp)"
-        when ref.type == :dollar # $n
-          raise "$#{ref.value} can not be used in %printer."
-        when ref.type == :at # @n
-          raise "@#{ref.value} can not be used in %printer."
-        else
-          raise "Unexpected. #{self}, #{ref}"
-        end
-
-        t_code[first_column..last_column] = str
-      end
-
-      return t_code
-    end
-
-
-    private
-
-    # * ($1) yyvsp[i]
-    # * ($$) yyval
-    # * (@1) yylsp[i]
-    # * (@$) yyloc
-    def translated_user_code
-      t_code = s_value.dup
-
-      references.reverse.each do |ref|
-        first_column = ref.first_column
-        last_column = ref.last_column
-
-        case
-        when ref.value == "$" && ref.type == :dollar # $$
-          # Omit "<>"
-          member = ref.tag.s_value[1..-2]
-          str = "(yyval.#{member})"
-        when ref.value == "$" && ref.type == :at # @$
-          str = "(yyloc)"
-        when ref.type == :dollar # $n
-          i = -ref.position_in_rhs + ref.value
-          # Omit "<>"
-          member = ref.tag.s_value[1..-2]
-          str = "(yyvsp[#{i}].#{member})"
-        when ref.type == :at # @n
-          i = -ref.position_in_rhs + ref.value
-          str = "(yylsp[#{i}])"
-        else
-          raise "Unexpected. #{self}, #{ref}"
-        end
-
-        t_code[first_column..last_column] = str
-      end
-
-      return t_code
-    end
-
-    # * ($1) error
-    # * ($$) yylval
-    # * (@1) error
-    # * (@$) yylloc
-    def translated_initial_action_code
-      t_code = s_value.dup
-
-      references.reverse.each do |ref|
-        first_column = ref.first_column
-        last_column = ref.last_column
-
-        case
-        when ref.value == "$" && ref.type == :dollar # $$
-          str = "yylval"
-        when ref.value == "$" && ref.type == :at # @$
-          str = "yylloc"
-        when ref.type == :dollar # $n
-          raise "$#{ref.value} can not be used in initial_action."
-        when ref.type == :at # @n
-          raise "@#{ref.value} can not be used in initial_action."
-        else
-          raise "Unexpected. #{self}, #{ref}"
-        end
-
-        t_code[first_column..last_column] = str
-      end
-
-      return t_code
-    end
-  end
-
-  # type: :dollar or :at
-  # ex_tag: "$<tag>1" (Optional)
-  Reference = Struct.new(:type, :value, :ex_tag, :first_column, :last_column, :referring_symbol, :position_in_rhs, keyword_init: true) do
-    def tag
-      if ex_tag
-        ex_tag
-      else
-        referring_symbol.tag
-      end
-    end
-  end
-
-  Precedence = Struct.new(:type, :precedence, keyword_init: true) do
-    include Comparable
-
-    def <=>(other)
-      self.precedence <=> other.precedence
-    end
-  end
-
-  Printer = Struct.new(:ident_or_tags, :code, :lineno, keyword_init: true) do
-    def translated_code(member)
-      code.translated_printer_code(member)
-    end
-  end
-
-  Union = Struct.new(:code, :lineno, keyword_init: true) do
-    def braces_less_code
-      # Remove braces
-      code.s_value[1..-2]
-    end
-  end
-
   Token = Lrama::Lexer::Token
 
   # Grammar is the result of parsing an input grammar file
   class Grammar
-    # Grammar file information not used by States but by Output
-    Aux = Struct.new(:prologue_first_lineno, :prologue, :epilogue_first_lineno, :epilogue, keyword_init: true)
-
     attr_reader :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol, :aux
     attr_accessor :union, :expect,
-                  :printers,
+                  :printers, :error_tokens,
                   :lex_param, :parse_param, :initial_action,
                   :symbols, :types,
                   :rules, :_rules,
@@ -295,6 +25,7 @@ module Lrama
 
     def initialize
       @printers = []
+      @error_tokens = []
       @symbols = []
       @types = []
       @_rules = []
@@ -305,7 +36,7 @@ module Lrama
       @error_symbol = nil
       @undef_symbol = nil
       @accept_symbol = nil
-      @aux = Aux.new
+      @aux = Auxiliary.new
 
       append_special_symbols
     end
@@ -314,6 +45,10 @@ module Lrama
       @printers << Printer.new(ident_or_tags: ident_or_tags, code: code, lineno: lineno)
     end
 
+    def add_error_token(ident_or_tags:, code:, lineno:)
+      @error_tokens << ErrorToken.new(ident_or_tags: ident_or_tags, code: code, lineno: lineno)
+    end
+
     def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false)
       if token_id && (sym = @symbols.find {|s| s.token_id == token_id })
         if replace
@@ -419,12 +154,14 @@ module Lrama
       fill_sym_to_rules
       fill_nterm_type
       fill_symbol_printer
+      fill_symbol_error_token
       @symbols.sort_by!(&:number)
     end
 
     # TODO: More validation methods
     def validate!
       validate_symbol_number_uniqueness!
+      validate_no_declared_type_reference!
     end
 
     def compute_nullable
@@ -476,6 +213,41 @@ module Lrama
       end
     end
 
+    def compute_first_set
+      terms.each do |term|
+        term.first_set = Set.new([term]).freeze
+        term.first_set_bitmap = Lrama::Bitmap.from_array([term.number])
+      end
+
+      nterms.each do |nterm|
+        nterm.first_set = Set.new([]).freeze
+        nterm.first_set_bitmap = Lrama::Bitmap.from_array([])
+      end
+
+      while true do
+        changed = false
+
+        @rules.each do |rule|
+          rule.rhs.each do |r|
+            if rule.lhs.first_set_bitmap | r.first_set_bitmap != rule.lhs.first_set_bitmap
+              changed = true
+              rule.lhs.first_set_bitmap = rule.lhs.first_set_bitmap | r.first_set_bitmap
+            end
+
+            break unless r.nullable
+          end
+        end
+
+        break unless changed
+      end
+
+      nterms.each do |nterm|
+        nterm.first_set = Lrama::Bitmap.to_array(nterm.first_set_bitmap).map do |number|
+          find_symbol_by_number!(number)
+        end.to_set
+      end
+    end
+
     def find_symbol_by_s_value(s_value)
       @symbols.find do |sym|
         sym.id.s_value == s_value
@@ -845,6 +617,23 @@ module Lrama
       end
     end
 
+    def fill_symbol_error_token
+      @symbols.each do |sym|
+        @error_tokens.each do |error_token|
+          error_token.ident_or_tags.each do |ident_or_tag|
+            case ident_or_tag.type
+            when Token::Ident
+              sym.error_token = error_token if sym.id == ident_or_tag
+            when Token::Tag
+              sym.error_token = error_token if sym.tag == ident_or_tag
+            else
+              raise "Unknown token type. #{error_token}"
+            end
+          end
+        end
+      end
+    end
+
     def validate_symbol_number_uniqueness!
       invalid = @symbols.group_by(&:number).select do |number, syms|
         syms.count > 1
@@ -854,5 +643,23 @@ module Lrama
 
       raise "Symbol number is duplicated. #{invalid}"
     end
+
+    def validate_no_declared_type_reference!
+      errors = []
+
+      rules.each do |rule|
+        next unless rule.code
+
+        rule.code.references.select do |ref|
+          ref.type == :dollar && !ref.tag
+        end.each do |ref|
+          errors << "$#{ref.value} of '#{rule.lhs.id.s_value}' has no declared type"
+        end
+      end
+
+      return if errors.empty?
+
+      raise errors.join("\n")
+    end
   end
 end

data/lib/lrama/lexer/token/type.rb

@@ -0,0 +1,8 @@
+module Lrama
+  class Lexer
+    class Token < Struct.new(:type, :s_value, :alias, keyword_init: true)
+      class Type < Struct.new(:id, :name, keyword_init: true)
+      end
+    end
+  end
+end

data/lib/lrama/lexer/token.rb

@@ -0,0 +1,77 @@
+require 'lrama/lexer/token/type'
+
+module Lrama
+  class Lexer
+    class Token
+
+      attr_accessor :line, :column, :referred
+      # For User_code
+      attr_accessor :references
+
+      def to_s
+        "#{super} line: #{line}, column: #{column}"
+      end
+
+      def referred_by?(string)
+        [self.s_value, self.alias].include?(string)
+      end
+
+      def ==(other)
+        self.class == other.class && self.type == other.type && self.s_value == other.s_value
+      end
+
+      def numberize_references(lhs, rhs)
+        self.references.map! {|ref|
+          ref_name = ref[1]
+          if ref_name.is_a?(::String) && ref_name != '$'
+            value =
+              if lhs.referred_by?(ref_name)
+                '$'
+              else
+                rhs.find_index {|token| token.referred_by?(ref_name) } + 1
+              end
+            [ref[0], value, ref[2], ref[3], ref[4]]
+          else
+            ref
+          end
+        }
+      end
+
+      @i = 0
+      @types = []
+
+      def self.define_type(name)
+        type = Type.new(id: @i, name: name.to_s)
+        const_set(name, type)
+        @types << type
+        @i += 1
+      end
+
+      # Token types
+      define_type(:P_expect)         # %expect
+      define_type(:P_define)         # %define
+      define_type(:P_printer)        # %printer
+      define_type(:P_error_token)    # %error-token
+      define_type(:P_lex_param)      # %lex-param
+      define_type(:P_parse_param)    # %parse-param
+      define_type(:P_initial_action) # %initial-action
+      define_type(:P_union)          # %union
+      define_type(:P_token)          # %token
+      define_type(:P_type)           # %type
+      define_type(:P_nonassoc)       # %nonassoc
+      define_type(:P_left)           # %left
+      define_type(:P_right)          # %right
+      define_type(:P_prec)           # %prec
+      define_type(:User_code)        # { ... }
+      define_type(:Tag)              # <int>
+      define_type(:Number)           # 0
+      define_type(:Ident_Colon)      # k_if:, k_if  : (spaces can be there)
+      define_type(:Ident)            # api.pure, tNUMBER
+      define_type(:Named_Ref)        # [foo]
+      define_type(:Semicolon)        # ;
+      define_type(:Bar)              # |
+      define_type(:String)           # "str"
+      define_type(:Char)             # '+'
+    end
+  end
+end

data/lib/lrama/lexer.rb

@@ -1,84 +1,12 @@
 require "strscan"
-require "lrama/report"
+require "lrama/report/duration"
+require "lrama/lexer/token"
 
 module Lrama
   # Lexer for parse.y
   class Lexer
     include Lrama::Report::Duration
 
-    # s_value is semantic value
-    Token = Struct.new(:type, :s_value, :alias, keyword_init: true) do
-      Type = Struct.new(:id, :name, keyword_init: true)
-
-      attr_accessor :line, :column, :referred
-      # For User_code
-      attr_accessor :references
-
-      def to_s
-        "#{super} line: #{line}, column: #{column}"
-      end
-
-      def referred_by?(string)
-        [self.s_value, self.alias].include?(string)
-      end
-
-      def ==(other)
-        self.class == other.class && self.type == other.type && self.s_value == other.s_value
-      end
-
-      def numberize_references(lhs, rhs)
-        self.references.map! {|ref|
-          ref_name = ref[1]
-          if ref_name.is_a?(String) && ref_name != '$'
-            value =
-              if lhs.referred_by?(ref_name)
-                '$'
-              else
-                rhs.find_index {|token| token.referred_by?(ref_name) } + 1
-              end
-            [ref[0], value, ref[2], ref[3], ref[4]]
-          else
-            ref
-          end
-        }
-      end
-
-      @i = 0
-      @types = []
-
-      def self.define_type(name)
-        type = Type.new(id: @i, name: name.to_s)
-        const_set(name, type)
-        @types << type
-        @i += 1
-      end
-
-      # Token types
-      define_type(:P_expect)         # %expect
-      define_type(:P_define)         # %define
-      define_type(:P_printer)        # %printer
-      define_type(:P_lex_param)      # %lex-param
-      define_type(:P_parse_param)    # %parse-param
-      define_type(:P_initial_action) # %initial-action
-      define_type(:P_union)          # %union
-      define_type(:P_token)          # %token
-      define_type(:P_type)           # %type
-      define_type(:P_nonassoc)       # %nonassoc
-      define_type(:P_left)           # %left
-      define_type(:P_right)          # %right
-      define_type(:P_prec)           # %prec
-      define_type(:User_code)        # { ... }
-      define_type(:Tag)              # <int>
-      define_type(:Number)           # 0
-      define_type(:Ident_Colon)      # k_if:, k_if  : (spaces can be there)
-      define_type(:Ident)            # api.pure, tNUMBER
-      define_type(:Named_Ref)        # [foo]
-      define_type(:Semicolon)        # ;
-      define_type(:Bar)              # |
-      define_type(:String)           # "str"
-      define_type(:Char)             # '+'
-    end
-
     # States
     #
     # See: https://www.gnu.org/software/bison/manual/html_node/Grammar-Outline.html
@@ -207,6 +135,8 @@ module Lrama
           tokens << create_token(Token::P_define, ss[0], line, ss.pos - column)
         when ss.scan(/%printer/)
           tokens << create_token(Token::P_printer, ss[0], line, ss.pos - column)
+        when ss.scan(/%error-token/)
+          tokens << create_token(Token::P_error_token, ss[0], line, ss.pos - column)
         when ss.scan(/%lex-param/)
           tokens << create_token(Token::P_lex_param, ss[0], line, ss.pos - column)
         when ss.scan(/%parse-param/)

data/lib/lrama/output.rb

@@ -1,20 +1,24 @@
 require "erb"
 require "forwardable"
-require "lrama/report"
+require "lrama/report/duration"
 
 module Lrama
   class Output
     extend Forwardable
     include Report::Duration
 
-    attr_reader :grammar_file_path, :context, :grammar
+    attr_reader :grammar_file_path, :context, :grammar, :error_recovery
 
     def_delegators "@context", :yyfinal, :yylast, :yyntokens, :yynnts, :yynrules, :yynstates,
                                :yymaxutok, :yypact_ninf, :yytable_ninf
 
     def_delegators "@grammar", :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol
 
-    def initialize(out:, output_file_path:, template_name:, grammar_file_path:, header_out: nil, header_file_path: nil, context:, grammar:)
+    def initialize(
+      out:, output_file_path:, template_name:, grammar_file_path:,
+      header_out: nil, header_file_path: nil,
+      context:, grammar:, error_recovery: false
+    )
       @out = out
       @output_file_path = output_file_path
       @template_name = template_name
@@ -23,6 +27,7 @@ module Lrama
       @header_file_path = header_file_path
       @context = context
       @grammar = grammar
+      @error_recovery = error_recovery
     end
 
     if ERB.instance_method(:initialize).parameters.last.first == :key
@@ -98,6 +103,10 @@ module Lrama
       int_array_to_string(@context.yytranslate)
     end
 
+    def yytranslate_inverted
+      int_array_to_string(@context.yytranslate_inverted)
+    end
+
     def yyrline
       int_array_to_string(@context.yyrline)
     end
@@ -155,6 +164,25 @@ module Lrama
       STR
     end
 
+    def symbol_actions_for_error_token
+      str = ""
+
+      @grammar.symbols.each do |sym|
+        next unless sym.error_token
+
+        str << <<-STR
+    case #{sym.enum_name}: /* #{sym.comment}  */
+#line #{sym.error_token.lineno} "#{@grammar_file_path}"
+         #{sym.error_token.translated_code(sym.tag)}
+#line [@oline@] [@ofile@]
+        break;
+
+        STR
+      end
+
+      str
+    end
+
     # b4_user_actions
     def user_actions
       str = ""
@@ -224,7 +252,7 @@ module Lrama
     end
 
     def extract_param_name(param)
-      /\A(.)+([a-zA-Z0-9_]+)\z/.match(param)[2]
+      /\A(\W*)([a-zA-Z0-9_]+)\z/.match(param.split.last)[2]
     end
 
     def parse_param_name