lrama 0.5.9 → 0.5.10

data/lib/lrama/report/profile.rb

@@ -1,18 +1,7 @@
 module Lrama
   class Report
     module Profile
-      # 1. Wrap target method with Profile.report_profile like below:
-      #
-      #   Lrama::Report::Profile.report_profile { method }
-      #
-      # 2. Run lrama command, for example
-      #
-      #   $ ./exe/lrama --trace=time spec/fixtures/integration/ruby_3_2_0/parse.tmp.y
-      #
-      # 3. Generate html file
-      #
-      #   $ stackprof --d3-flamegraph tmp/stackprof-cpu-myapp.dump > tmp/flamegraph.html
-      #
+      # See "Profiling Lrama" in README.md for how to use.
       def self.report_profile
         require "stackprof"
 

data/lib/lrama/version.rb

@@ -1,3 +1,3 @@
 module Lrama
-  VERSION = "0.5.9".freeze
+  VERSION = "0.5.10".freeze
 end

data/parser.y

@@ -1,5 +1,5 @@
 class Lrama::Parser
-  expect 7
+  expect 1
 
   token C_DECLARATION CHARACTER IDENT_COLON IDENTIFIER INTEGER STRING TAG
 
@@ -36,15 +36,13 @@ rule
                    | "%lex-param" params
                        {
                          val[1].each {|token|
-                           token.references = []
-                           @grammar.lex_param = @grammar.build_code(:lex_param, token).token_code.s_value
+                           @grammar.lex_param = Grammar::Code::NoReferenceCode.new(type: :lex_param, token_code: token).token_code.s_value
                          }
                        }
                    | "%parse-param" params
                        {
                          val[1].each {|token|
-                           token.references = []
-                           @grammar.parse_param = @grammar.build_code(:parse_param, token).token_code.s_value
+                           @grammar.parse_param = Grammar::Code::NoReferenceCode.new(type: :parse_param, token_code: token).token_code.s_value
                          }
                        }
                    | "%code" IDENTIFIER "{"
@@ -69,7 +67,7 @@ rule
                        }
                      "}"
                        {
-                         @grammar.initial_action = @grammar.build_code(:initial_action, val[3])
+                         @grammar.initial_action = Grammar::Code::InitialActionCode.new(type: :initial_action, token_code: val[3])
                        }
                    | ";"
 
@@ -83,7 +81,10 @@ rule
                          }
                        "}"
                          {
-                           @grammar.set_union(@grammar.build_code(:union, val[3]), val[3].line)
+                           @grammar.set_union(
+                             Grammar::Code::NoReferenceCode.new(type: :union, token_code: val[3]),
+                             val[3].line
+                           )
                          }
                      | symbol_declaration
                      | "%destructor" "{"
@@ -105,7 +106,11 @@ rule
                          }
                        "}" generic_symlist
                          {
-                           @grammar.add_printer(ident_or_tags: val[6], code: @grammar.build_code(:printer, val[3]), lineno: val[3].line)
+                           @grammar.add_printer(
+                             ident_or_tags: val[6],
+                             token_code: val[3],
+                             lineno: val[3].line
+                           )
                          }
                      | "%error-token" "{"
                          {
@@ -117,7 +122,11 @@ rule
                          }
                        "}" generic_symlist
                          {
-                           @grammar.add_error_token(ident_or_tags: val[6], code: @grammar.build_code(:error_token, val[3]), lineno: val[3].line)
+                           @grammar.add_error_token(
+                             ident_or_tags: val[6],
+                             token_code: val[3],
+                             lineno: val[3].line
+                           )
                          }
 
   symbol_declaration: "%token" token_declarations
@@ -252,9 +261,9 @@ rule
                                        {
                                          result = [{tag: val[0], tokens: val[1]}]
                                        }
-                                   | token_declarations_for_precedence token_declaration_list_for_precedence
+                                   | token_declarations_for_precedence TAG token_declaration_list_for_precedence
                                        {
-                                         result = val[0].append({tag: nil, tokens: val[1]})
+                                         result = val[0].append({tag: val[1], tokens: val[2]})
                                        }
 
   token_declaration_list_for_precedence: token_declaration_for_precedence { result = [val[0]] }
@@ -262,8 +271,8 @@ rule
 
   token_declaration_for_precedence: id
 
-  id: IDENTIFIER { raise "Ident after %prec" if @prec_seen }
-    | CHARACTER { raise "Char after %prec" if @prec_seen }
+  id: IDENTIFIER { on_action_error("ident after %prec", val[0]) if @prec_seen }
+    | CHARACTER { on_action_error("char after %prec", val[0]) if @prec_seen }
 
   grammar: rules_or_grammar_declaration
          | grammar rules_or_grammar_declaration
@@ -275,56 +284,74 @@ rule
            {
              lhs = val[0]
              lhs.alias_name = val[1]
-             val[3].each {|hash|
-               @grammar.add_rule(lhs: lhs, rhs: hash[:rhs], lineno: hash[:lineno])
-             }
+             val[3].each do |builder|
+               builder.lhs = lhs
+               builder.complete_input
+               @grammar.add_rule_builder(builder)
+             end
            }
 
   rhs_list: rhs
               {
-                result = [{rhs: val[0], lineno: val[0].first&.line || @lexer.line - 1}]
+                builder = val[0]
+                if !builder.line
+                  builder.line = @lexer.line - 1
+                end
+                result = [builder]
               }
           | rhs_list "|" rhs
               {
-                result = val[0].append({rhs: val[2], lineno: val[2].first&.line || @lexer.line - 1})
+                builder = val[2]
+                if !builder.line
+                  builder.line = @lexer.line - 1
+                end
+                result = val[0].append(builder)
               }
           | rhs_list ";"
 
   rhs: /* empty */
          {
            reset_precs
-           result = []
+           result = Grammar::RuleBuilder.new(@rule_counter, @midrule_action_counter)
          }
      | "%empty"
          {
            reset_precs
-           result = []
+           result = Grammar::RuleBuilder.new(@rule_counter, @midrule_action_counter)
          }
      | rhs symbol named_ref_opt
          {
            token = val[1]
            token.alias_name = val[2]
-           result = val[0].append(token)
+           builder = val[0]
+           builder.add_rhs(token)
+           result = builder
          }
-     | rhs "?"
-          {
-            token = Lrama::Lexer::Token::Parameterizing.new(s_value: val[1])
-            result = val[0].append(token)
-          }
-     | rhs "+"
-          {
-            token = Lrama::Lexer::Token::Parameterizing.new(s_value: val[1])
-            result = val[0].append(token)
-          }
-     | rhs "*"
-          {
-             token = Lrama::Lexer::Token::Parameterizing.new(s_value: val[1])
-             result = val[0].append(token)
-          }
+     | rhs IDENTIFIER parameterizing_suffix
+         {
+           token = Lrama::Lexer::Token::Parameterizing.new(s_value: val[2], location: @lexer.location, args: [val[1]])
+           builder = val[0]
+           builder.add_rhs(token)
+           result = builder
+         }
+     | rhs IDENTIFIER "(" symbol ")"
+         {
+           token = Lrama::Lexer::Token::Parameterizing.new(s_value: val[1].s_value, location: @lexer.location, args: [val[3]])
+           builder = val[0]
+           builder.add_rhs(token)
+           result = builder
+         }
+     | rhs IDENTIFIER "(" symbol "," symbol ")"
+        {
+          token = Lrama::Lexer::Token::Parameterizing.new(s_value: val[1].s_value, location: @lexer.location, args: [val[3], val[5]])
+          builder = val[0]
+          builder.add_rhs(token)
+          result = builder
+        }
      | rhs "{"
          {
            if @prec_seen
-             raise "Multiple User_code after %prec" if @code_after_prec
+             on_action_error("multiple User_code after %prec", val[0])  if @code_after_prec
              @code_after_prec = true
            end
            begin_c_declaration("}")
@@ -337,15 +364,23 @@ rule
          {
            token = val[3]
            token.alias_name = val[6]
-           result = val[0].append(token)
+           builder = val[0]
+           builder.user_code = token
+           result = builder
          }
      | rhs "%prec" symbol
          {
            sym = @grammar.find_symbol_by_id!(val[2])
-           result = val[0].append(sym)
            @prec_seen = true
+           builder = val[0]
+           builder.precedence_sym = sym
+           result = builder
          }
 
+  parameterizing_suffix: "?"
+                       | "+"
+                       | "*"
+
   named_ref_opt: # empty
                | '[' IDENTIFIER ']' { result = val[1].s_value }
 
@@ -387,18 +422,18 @@ def initialize(text, path, debug = false)
   @text = text
   @path = path
   @yydebug = debug
+  @rule_counter = Lrama::Grammar::Counter.new(0)
+  @midrule_action_counter = Lrama::Grammar::Counter.new(1)
 end
 
 def parse
   report_duration(:parse) do
     @lexer = Lrama::Lexer.new(@text)
-    @grammar = Lrama::Grammar.new
+    @grammar = Lrama::Grammar.new(@rule_counter)
     @precedence_number = 0
     reset_precs
     do_parse
     @grammar.prepare
-    @grammar.compute_nullable
-    @grammar.compute_first_set
     @grammar.validate!
     @grammar
   end
@@ -409,18 +444,40 @@ def next_token
 end
 
 def on_error(error_token_id, error_value, value_stack)
-  if error_value.respond_to?(:line) && error_value.respond_to?(:column)
-    line = error_value.line
-    first_column = error_value.column
+  if error_value.is_a?(Lrama::Lexer::Token)
+    line = error_value.first_line
+    first_column = error_value.first_column
+    last_column = error_value.last_column
+    value = "'#{error_value.s_value}'"
+  else
+    line = @lexer.line
+    first_column = @lexer.head_column
+    last_column = @lexer.column
+    value = error_value.inspect
+  end
+
+  raise ParseError, <<~ERROR
+    #{@path}:#{line}:#{first_column}: parse error on value #{value} (#{token_to_str(error_token_id) || '?'})
+    #{@text.split("\n")[line - 1]}
+    #{carrets(first_column, last_column)}
+  ERROR
+end
+
+def on_action_error(error_message, error_value)
+  if error_value.is_a?(Lrama::Lexer::Token)
+    line = error_value.first_line
+    first_column = error_value.first_column
+    last_column = error_value.last_column
   else
     line = @lexer.line
     first_column = @lexer.head_column
+    last_column = @lexer.column
   end
 
   raise ParseError, <<~ERROR
-    #{@path}:#{line}:#{first_column}: parse error on value #{error_value.inspect} (#{token_to_str(error_token_id) || '?'})
+    #{@path}:#{line}: #{error_message}
     #{@text.split("\n")[line - 1]}
-    #{carrets(first_column)}
+    #{carrets(first_column, last_column)}
   ERROR
 end
 
@@ -441,6 +498,6 @@ def end_c_declaration
   @lexer.end_symbol = nil
 end
 
-def carrets(first_column)
-  ' ' * (first_column + 1) + '^' * (@lexer.column - first_column)
+def carrets(first_column, last_column)
+  ' ' * (first_column + 1) + '^' * (last_column - first_column)
 end

data/rbs_collection.lock.yaml

@@ -2,7 +2,7 @@
 sources:
 - type: git
   name: ruby/gem_rbs_collection
-  revision: 2de2d4535caba275f3b8533684aab110d921f553
+  revision: 25286c51a19927f28623aee3cd36655f902399ba
   remote: https://github.com/ruby/gem_rbs_collection.git
   repo_dir: gems
 path: ".gem_rbs_collection"
@@ -15,6 +15,10 @@ gems:
   version: '0'
   source:
     type: stdlib
+- name: forwardable
+  version: '0'
+  source:
+    type: stdlib
 - name: rake
   version: '13.0'
   source:

data/rbs_collection.yaml

@@ -20,3 +20,4 @@ gems:
   # It's unnecessary if you don't use rbs as a library.
   - name: rbs
     ignore: true
+  - name: forwardable

data/sig/lrama/grammar/code/printer_code.rbs

@@ -0,0 +1,15 @@
+module Lrama
+  class Grammar
+    class Code
+      class PrinterCode < Code
+        @tag: untyped
+        def initialize: (?type: untyped, ?token_code: untyped, ?tag: untyped) -> void
+
+        private
+
+        # ref: Lrama::Grammar::Code.token_code.references
+        def reference_to_c: (untyped ref) -> untyped
+      end
+    end
+  end
+end

data/sig/lrama/grammar/code.rbs

@@ -0,0 +1,24 @@
+module Lrama
+  class Grammar
+    class Code
+      extend Forwardable
+
+      attr_accessor type: untyped
+      attr_accessor token_code: untyped
+
+      # delegated
+      def s_value: -> String
+      def line: -> Integer
+      def column: -> untyped
+      def references: -> untyped
+
+      def initialize: (?type: untyped, ?token_code: untyped) -> void
+
+      def translated_code: () -> String
+
+      private
+
+      def reference_to_c: (untyped ref) -> untyped
+    end
+  end
+end

data/sig/lrama/grammar/counter.rbs

@@ -0,0 +1,11 @@
+module Lrama
+  class Grammar
+    class Counter
+      @number: Integer
+
+      def initialize: (Integer number) -> void
+
+      def increment: () -> Integer
+    end
+  end
+end

data/sig/lrama/grammar/parameterizing_rules/builder.rbs

@@ -0,0 +1,10 @@
+module Lrama
+  class Grammar
+    class ParameterizingRules
+      class Builder
+        def initialize: (Lexer::Token token, Counter rule_counter, Lexer::Token lhs, Lexer::Token::UserCode? user_code, Lexer::Token? precedence_sym, Integer? line) -> void
+        def build: () -> Array[Rule]
+      end
+    end
+  end
+end

data/sig/lrama/grammar/precedence.rbs

@@ -0,0 +1,11 @@
+module Lrama
+  class Grammar
+    class Precedence
+      include Comparable
+      attr_accessor type: Symbol
+      attr_accessor precedence: Integer
+
+      def <=>: (Precedence other) -> Integer
+    end
+  end
+end

data/sig/lrama/grammar/printer.rbs

@@ -0,0 +1,11 @@
+module Lrama
+  class Grammar
+    class Printer
+      attr_accessor ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag]
+      attr_accessor token_code: Grammar::Code
+      attr_accessor lineno: Integer
+
+      def translated_code: (Lexer::Token member) -> String
+    end
+  end
+end

data/sig/lrama/grammar/reference.rbs

@@ -1,21 +1,21 @@
 module Lrama
   class Grammar
     class Reference
-      # TODO: Replace untyped referring_symbol with (Grammar::Symbol|Lexer::Token)
       attr_accessor type: Symbol
-      attr_accessor value: (String|Integer)
+      attr_accessor name: String
+      attr_accessor index: Integer
       attr_accessor ex_tag: Lexer::Token?
       attr_accessor first_column: Integer
       attr_accessor last_column: Integer
-      attr_accessor referring_symbol: untyped
       attr_accessor position_in_rhs: Integer?
 
       def initialize: (
-        type: Symbol, value: (String|Integer), ex_tag: Lexer::Token?,
+        type: Symbol, ?name: String, ?index: Integer, ?ex_tag: Lexer::Token?,
         first_column: Integer, last_column: Integer,
-        referring_symbol: untyped, position_in_rhs: Integer?
+        ?position_in_rhs: Integer?
       ) -> void
 
+      def value: () -> (String|Integer)
       def tag: () -> untyped
     end
   end

data/sig/lrama/grammar/rule.rbs

@@ -0,0 +1,13 @@
+module Lrama
+  class Grammar
+    class Rule
+      attr_accessor original_rule: Rule
+
+      def initialize: (
+        ?id: untyped, ?_lhs: untyped, ?lhs: untyped, ?_rhs: untyped, ?rhs: untyped,
+        ?token_code: untyped, ?position_in_original_rule_rhs: untyped, ?nullable: untyped,
+        ?precedence_sym: untyped, ?lineno: untyped
+      ) -> void
+    end
+  end
+end

data/sig/lrama/grammar/rule_builder.rbs

@@ -0,0 +1,41 @@
+module Lrama
+  class Grammar
+    class RuleBuilder
+      attr_accessor lhs: Lexer::Token
+      attr_accessor line: Integer?
+      attr_reader rhs: Array[Lexer::Token]
+      attr_reader user_code: Lexer::Token::UserCode?
+      attr_reader precedence_sym: Lexer::Token?
+
+      @rule_counter: Counter
+      @midrule_action_counter: Counter
+      @position_in_original_rule_rhs: Integer?
+      @skip_preprocess_references: bool
+      @user_code: Lexer::Token::UserCode?
+      @rule_builders_for_derived_rules: Array[RuleBuilder]
+      @rules: Array[Rule]
+      @replaced_rhs: Array[Lexer::Token]
+      @parameterizing_rules: Array[Rule]
+      @midrule_action_rules: Array[Rule]
+
+      def initialize: (Counter rule_counter, Counter midrule_action_counter, ?Integer position_in_original_rule_rhs, ?skip_preprocess_references: bool) -> void
+      def add_rhs: (Lexer::Token rhs) -> void
+      def user_code=: (Lexer::Token::UserCode user_code) -> void
+      def precedence_sym=: (Lexer::Token user_code) -> void
+      def complete_input: () -> void
+      def setup_rules: () -> void
+      def parameterizing_rules: () -> Array[Rule]
+      def midrule_action_rules: () -> Array[Rule]
+      def rules: () -> Array[Rule]
+
+      private
+
+      def freeze_rhs: () -> void
+      def preprocess_references: () -> void
+      def build_rules: () -> void
+      def process_rhs: () -> void
+      def numberize_references: () -> void
+      def flush_user_code: () -> void
+    end
+  end
+end

data/sig/lrama/lexer/location.rbs

@@ -0,0 +1,14 @@
+module Lrama
+  class Lexer
+    class Location
+      attr_reader first_line: Integer
+      attr_reader first_column: Integer
+      attr_reader last_line: Integer
+      attr_reader last_column: Integer
+
+      def initialize: (first_line: Integer, first_column: Integer, last_line: Integer, last_column: Integer) -> void
+
+      def ==: (Location other) -> bool
+    end
+  end
+end

data/sig/lrama/lexer/token/parameterizing.rbs

@@ -2,6 +2,13 @@ module Lrama
   class Lexer
     class Token
       class Parameterizing < Token
+        attr_accessor args: Array[Lrama::Lexer::Token]
+
+        def option?: () -> bool
+        def nonempty_list?: () -> bool
+        def list?: () -> bool
+        def separated_nonempty_list?: () -> bool
+        def separated_list?: () -> bool
       end
     end
   end

data/sig/lrama/lexer/token/tag.rbs

@@ -2,6 +2,7 @@ module Lrama
   class Lexer
     class Token
       class Tag < Token
+        def member: () -> String
       end
     end
   end

data/sig/lrama/lexer/token/user_code.rbs

@@ -2,7 +2,14 @@ module Lrama
   class Lexer
     class Token
       class UserCode < Token
-        attr_accessor references: Array[[Symbol, (String|Integer), Token?, Integer, Integer]]
+        @references: Array[Lrama::Grammar::Reference]
+
+        def references: () -> Array[Lrama::Grammar::Reference]
+
+        private
+
+        def _references: () -> Array[Lrama::Grammar::Reference]
+        def scan_reference: (StringScanner scanner) -> Lrama::Grammar::Reference?
       end
     end
   end

data/sig/lrama/lexer/token.rbs

@@ -3,15 +3,20 @@ module Lrama
     class Token
       attr_accessor s_value: String
       attr_accessor alias_name: String
-      attr_accessor line: Integer
-      attr_accessor column: Integer
+      attr_accessor location: Location
       attr_accessor referred: bool
 
-      def initialize: (?s_value: String, ?alias_name: String) -> void
+      def initialize: (?s_value: String, ?alias_name: String, ?location: Location) -> void
 
       def to_s: () -> String
       def referred_by?: (String string) -> bool
       def ==: (Token other) -> bool
+      def first_line: () -> Integer
+      def first_column: () -> Integer
+      def last_line: () -> Integer
+      def last_column: () -> Integer
+      alias line first_line
+      alias column first_column
     end
   end
 end

data/sig/stdlib/strscan/string_scanner.rbs

@@ -0,0 +1,5 @@
+class StringScanner
+  # TODO: Is it better to define `StringScanner#fetch` whose type '(Integer) -> String' ?
+  def []: (Integer) -> String
+        | ...
+end

data/template/bison/yacc.c

@@ -69,11 +69,13 @@
 
 
 <%# b4_user_pre_prologue -%>
+<%- if output.aux.prologue -%>
 /* First part of user prologue.  */
 #line <%= output.aux.prologue_first_lineno %> "<%= output.grammar_file_path %>"
 
 <%= output.aux.prologue %>
 #line [@oline@] [@ofile@]
+<%- end -%>
 
 <%# b4_cast_define -%>
 # ifndef YY_CAST
@@ -1483,6 +1485,7 @@ YYLTYPE yylloc = yyloc_default;
 <%# b4_declare_parser_state_variables -%>
     /* Number of syntax errors so far.  */
     int yynerrs = 0;
+    YY_USE (yynerrs); /* Silence compiler warning.  */
 
     yy_state_fast_t yystate = 0;
     /* Number of tokens to shift before error messages enabled.  */
@@ -2043,7 +2046,9 @@ yyreturnlab:
 }
 
 <%# b4_percent_code_get([[epilogue]]) -%>
+<%- if output.aux.epilogue -%>
 #line <%= output.aux.epilogue_first_lineno - 1 %> "<%= output.grammar_file_path %>"
 
 <%= output.aux.epilogue -%>
+<%- end -%>