lrama 0.5.8 → 0.5.10

data/lib/lrama/report/profile.rb

@@ -1,18 +1,7 @@
 module Lrama
   class Report
     module Profile
-      # 1. Wrap target method with Profile.report_profile like below:
-      #
-      #   Lrama::Report::Profile.report_profile { method }
-      #
-      # 2. Run lrama command, for example
-      #
-      #   $ ./exe/lrama --trace=time spec/fixtures/integration/ruby_3_2_0/parse.tmp.y
-      #
-      # 3. Generate html file
-      #
-      #   $ stackprof --d3-flamegraph tmp/stackprof-cpu-myapp.dump > tmp/flamegraph.html
-      #
+      # See "Profiling Lrama" in README.md for how to use.
       def self.report_profile
         require "stackprof"
 

data/lib/lrama/version.rb

@@ -1,3 +1,3 @@
 module Lrama
-  VERSION = "0.5.8".freeze
+  VERSION = "0.5.10".freeze
 end

data/parser.y

@@ -1,6 +1,10 @@
 class Lrama::Parser
+  expect 1
+
   token C_DECLARATION CHARACTER IDENT_COLON IDENTIFIER INTEGER STRING TAG
+
 rule
+
   input: prologue_declarations bison_declarations "%%" grammar epilogue_opt
 
   prologue_declarations: # empty
@@ -8,14 +12,12 @@ rule
 
   prologue_declaration: "%{"
                           {
-                            @lexer.status = :c_declaration
-                            @lexer.end_symbol = '%}'
+                            begin_c_declaration("%}")
                             @grammar.prologue_first_lineno = @lexer.line
                           }
                         C_DECLARATION
                           {
-                            @lexer.status = :initial
-                            @lexer.end_symbol = nil
+                            end_c_declaration
                           }
                         "%}"
                           {
@@ -34,86 +36,97 @@ rule
                    | "%lex-param" params
                        {
                          val[1].each {|token|
-                           token.references = []
-                           @grammar.lex_param = @grammar.build_code(:lex_param, token).token_code.s_value
+                           @grammar.lex_param = Grammar::Code::NoReferenceCode.new(type: :lex_param, token_code: token).token_code.s_value
                          }
                        }
                    | "%parse-param" params
                        {
                          val[1].each {|token|
-                           token.references = []
-                           @grammar.parse_param = @grammar.build_code(:parse_param, token).token_code.s_value
+                           @grammar.parse_param = Grammar::Code::NoReferenceCode.new(type: :parse_param, token_code: token).token_code.s_value
                          }
                        }
+                   | "%code" IDENTIFIER "{"
+                       {
+                         begin_c_declaration("}")
+                       }
+                     C_DECLARATION
+                       {
+                         end_c_declaration
+                       }
+                     "}"
+                       {
+                         @grammar.add_percent_code(id: val[1], code: val[4])
+                       }
                    | "%initial-action" "{"
                        {
-                         @lexer.status = :c_declaration
-                         @lexer.end_symbol = '}'
+                         begin_c_declaration("}")
                        }
                      C_DECLARATION
                        {
-                         @lexer.status = :initial
-                         @lexer.end_symbol = nil
+                         end_c_declaration
                        }
                      "}"
                        {
-                         @grammar.initial_action = @grammar.build_code(:initial_action, val[3])
+                         @grammar.initial_action = Grammar::Code::InitialActionCode.new(type: :initial_action, token_code: val[3])
                        }
                    | ";"
 
   grammar_declaration: "%union" "{"
                          {
-                           @lexer.status = :c_declaration
-                           @lexer.end_symbol = '}'
+                           begin_c_declaration("}")
                          }
                        C_DECLARATION
                          {
-                           @lexer.status = :initial
-                           @lexer.end_symbol = nil
+                           end_c_declaration
                          }
                        "}"
                          {
-                           @grammar.set_union(@grammar.build_code(:union, val[3]), val[3].line)
+                           @grammar.set_union(
+                             Grammar::Code::NoReferenceCode.new(type: :union, token_code: val[3]),
+                             val[3].line
+                           )
                          }
                      | symbol_declaration
                      | "%destructor" "{"
                          {
-                           @lexer.status = :c_declaration
-                           @lexer.end_symbol = '}'
+                           begin_c_declaration("}")
                          }
                        C_DECLARATION
                          {
-                           @lexer.status = :initial
-                           @lexer.end_symbol = nil
+                           end_c_declaration
                          }
-                         "}" generic_symlist
+                       "}" generic_symlist
                      | "%printer" "{"
                          {
-                           @lexer.status = :c_declaration
-                           @lexer.end_symbol = '}'
+                           begin_c_declaration("}")
                          }
                        C_DECLARATION
                          {
-                           @lexer.status = :initial
-                           @lexer.end_symbol = nil
+                           end_c_declaration
                          }
                        "}" generic_symlist
                          {
-                           @grammar.add_printer(ident_or_tags: val[6], code: @grammar.build_code(:printer, val[3]), lineno: val[3].line)
+                           @grammar.add_printer(
+                             ident_or_tags: val[6],
+                             token_code: val[3],
+                             lineno: val[3].line
+                           )
                          }
                      | "%error-token" "{"
                          {
-                           @lexer.status = :c_declaration
-                           @lexer.end_symbol = '}'
+                           begin_c_declaration("}")
                          }
                        C_DECLARATION
                          {
-                           @lexer.status = :initial
-                           @lexer.end_symbol = nil
+                           end_c_declaration
                          }
                        "}" generic_symlist
                          {
-                           @grammar.add_error_token(ident_or_tags: val[6], code: @grammar.build_code(:error_token, val[3]), lineno: val[3].line)
+                           @grammar.add_error_token(
+                             ident_or_tags: val[6],
+                             token_code: val[3],
+                             lineno: val[3].line
+                           )
                          }
 
   symbol_declaration: "%token" token_declarations
@@ -217,13 +230,11 @@ rule
 
   params: params "{"
             {
-              @lexer.status = :c_declaration
-              @lexer.end_symbol = '}'
+              begin_c_declaration("}")
             }
           C_DECLARATION
             {
-              @lexer.status = :initial
-              @lexer.end_symbol = nil
+              end_c_declaration
             }
           "}"
             {
@@ -231,13 +242,11 @@ rule
             }
         | "{"
             {
-              @lexer.status = :c_declaration
-              @lexer.end_symbol = '}'
+              begin_c_declaration("}")
             }
           C_DECLARATION
             {
-              @lexer.status = :initial
-              @lexer.end_symbol = nil
+              end_c_declaration
             }
           "}"
             {
@@ -252,9 +261,9 @@ rule
                                        {
                                          result = [{tag: val[0], tokens: val[1]}]
                                        }
-                                   | token_declarations_for_precedence token_declaration_list_for_precedence
+                                   | token_declarations_for_precedence TAG token_declaration_list_for_precedence
                                        {
-                                         result = val[0].append({tag: nil, tokens: val[1]})
+                                         result = val[0].append({tag: val[1], tokens: val[2]})
                                        }
 
   token_declaration_list_for_precedence: token_declaration_for_precedence { result = [val[0]] }
@@ -262,8 +271,8 @@ rule
 
   token_declaration_for_precedence: id
 
-  id: IDENTIFIER { raise "Ident after %prec" if @prec_seen }
-    | CHARACTER { raise "Char after %prec" if @prec_seen }
+  id: IDENTIFIER { on_action_error("ident after %prec", val[0]) if @prec_seen }
+    | CHARACTER { on_action_error("char after %prec", val[0]) if @prec_seen }
 
   grammar: rules_or_grammar_declaration
          | grammar rules_or_grammar_declaration
@@ -274,81 +283,104 @@ rule
   rules: id_colon named_ref_opt ":" rhs_list
            {
              lhs = val[0]
-             lhs.alias = val[1]
-             val[3].each {|hash|
-               @grammar.add_rule(lhs: lhs, rhs: hash[:rhs], lineno: hash[:lineno])
-             }
+             lhs.alias_name = val[1]
+             val[3].each do |builder|
+               builder.lhs = lhs
+               builder.complete_input
+               @grammar.add_rule_builder(builder)
+             end
            }
 
   rhs_list: rhs
               {
-                result = [{rhs: val[0], lineno: val[0].first&.line || @lexer.line - 1}]
+                builder = val[0]
+                if !builder.line
+                  builder.line = @lexer.line - 1
+                end
+                result = [builder]
               }
           | rhs_list "|" rhs
               {
-                result = val[0].append({rhs: val[2], lineno: val[2].first&.line || @lexer.line - 1})
+                builder = val[2]
+                if !builder.line
+                  builder.line = @lexer.line - 1
+                end
+                result = val[0].append(builder)
               }
           | rhs_list ";"
 
   rhs: /* empty */
          {
-           result = []
-           @prec_seen = false
-           @code_after_prec = false
+           reset_precs
+           result = Grammar::RuleBuilder.new(@rule_counter, @midrule_action_counter)
+         }
+     | "%empty"
+         {
+           reset_precs
+           result = Grammar::RuleBuilder.new(@rule_counter, @midrule_action_counter)
          }
      | rhs symbol named_ref_opt
          {
            token = val[1]
-           token.alias = val[2]
-           result = val[0].append(token)
+           token.alias_name = val[2]
+           builder = val[0]
+           builder.add_rhs(token)
+           result = builder
          }
-     | rhs "{"
+     | rhs IDENTIFIER parameterizing_suffix
          {
-           if @prec_seen
-             raise "Multiple User_code after %prec" if @code_after_prec
-             @code_after_prec = true
-           end
-           @lexer.status = :c_declaration
-           @lexer.end_symbol = '}'
+           token = Lrama::Lexer::Token::Parameterizing.new(s_value: val[2], location: @lexer.location, args: [val[1]])
+           builder = val[0]
+           builder.add_rhs(token)
+           result = builder
          }
-       C_DECLARATION
+     | rhs IDENTIFIER "(" symbol ")"
          {
-           @lexer.status = :initial
-           @lexer.end_symbol = nil
+           token = Lrama::Lexer::Token::Parameterizing.new(s_value: val[1].s_value, location: @lexer.location, args: [val[3]])
+           builder = val[0]
+           builder.add_rhs(token)
+           result = builder
          }
-       "}" named_ref_opt
-         {
-           token = val[3]
-           token.alias = val[6]
-           result = val[0].append(token)
-         }
-     | "{"
+     | rhs IDENTIFIER "(" symbol "," symbol ")"
+        {
+          token = Lrama::Lexer::Token::Parameterizing.new(s_value: val[1].s_value, location: @lexer.location, args: [val[3], val[5]])
+          builder = val[0]
+          builder.add_rhs(token)
+          result = builder
+        }
+     | rhs "{"
          {
            if @prec_seen
-             raise "Multiple User_code after %prec" if @code_after_prec
+             on_action_error("multiple User_code after %prec", val[0])  if @code_after_prec
              @code_after_prec = true
            end
-           @lexer.status = :c_declaration
-           @lexer.end_symbol = '}'
+           begin_c_declaration("}")
          }
        C_DECLARATION
          {
-           @lexer.status = :initial
-           @lexer.end_symbol = nil
+           end_c_declaration
          }
        "}" named_ref_opt
          {
-           token = val[2]
-           token.alias = val[5]
-           result = [token]
+           token = val[3]
+           token.alias_name = val[6]
+           builder = val[0]
+           builder.user_code = token
+           result = builder
          }
      | rhs "%prec" symbol
          {
            sym = @grammar.find_symbol_by_id!(val[2])
-           result = val[0].append(sym)
            @prec_seen = true
+           builder = val[0]
+           builder.precedence_sym = sym
+           result = builder
          }
 
+  parameterizing_suffix: "?"
+                       | "+"
+                       | "*"
+
   named_ref_opt: # empty
                | '[' IDENTIFIER ']' { result = val[1].s_value }
 
@@ -357,14 +389,12 @@ rule
   epilogue_opt: # empty
               | "%%"
                   {
-                    @lexer.status = :c_declaration
-                    @lexer.end_symbol = '\Z'
+                    begin_c_declaration('\Z')
                     @grammar.epilogue_first_lineno = @lexer.line + 1
                   }
                 C_DECLARATION
                   {
-                    @lexer.status = :initial
-                    @lexer.end_symbol = nil
+                    end_c_declaration
                     @grammar.epilogue = val[2].s_value
                   }
 
@@ -381,28 +411,29 @@ rule
   generic_symlist_item: symbol
                       | TAG
 
-  string_as_id: STRING { result = Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Ident, s_value: val[0]) }
+  string_as_id: STRING { result = Lrama::Lexer::Token::Ident.new(s_value: val[0]) }
 end
 
 ---- inner
 
 include Lrama::Report::Duration
 
-def initialize(text, path)
+def initialize(text, path, debug = false)
   @text = text
   @path = path
+  @yydebug = debug
+  @rule_counter = Lrama::Grammar::Counter.new(0)
+  @midrule_action_counter = Lrama::Grammar::Counter.new(1)
 end
 
 def parse
   report_duration(:parse) do
     @lexer = Lrama::Lexer.new(@text)
-    @grammar = Lrama::Grammar.new
+    @grammar = Lrama::Grammar.new(@rule_counter)
     @precedence_number = 0
+    reset_precs
     do_parse
-    @grammar.extract_references
     @grammar.prepare
-    @grammar.compute_nullable
-    @grammar.compute_first_set
     @grammar.validate!
     @grammar
   end
@@ -413,10 +444,60 @@ def next_token
 end
 
 def on_error(error_token_id, error_value, value_stack)
-  source = @text.split("\n")[error_value.line - 1]
+  if error_value.is_a?(Lrama::Lexer::Token)
+    line = error_value.first_line
+    first_column = error_value.first_column
+    last_column = error_value.last_column
+    value = "'#{error_value.s_value}'"
+  else
+    line = @lexer.line
+    first_column = @lexer.head_column
+    last_column = @lexer.column
+    value = error_value.inspect
+  end
+
   raise ParseError, <<~ERROR
-    #{@path}:#{@lexer.line}:#{@lexer.column}: parse error on value #{error_value.inspect} (#{token_to_str(error_token_id) || '?'})
-    #{source}
-    #{' ' * @lexer.column}^
+    #{@path}:#{line}:#{first_column}: parse error on value #{value} (#{token_to_str(error_token_id) || '?'})
+    #{@text.split("\n")[line - 1]}
+    #{carrets(first_column, last_column)}
   ERROR
 end
+
+def on_action_error(error_message, error_value)
+  if error_value.is_a?(Lrama::Lexer::Token)
+    line = error_value.first_line
+    first_column = error_value.first_column
+    last_column = error_value.last_column
+  else
+    line = @lexer.line
+    first_column = @lexer.head_column
+    last_column = @lexer.column
+  end
+
+  raise ParseError, <<~ERROR
+    #{@path}:#{line}: #{error_message}
+    #{@text.split("\n")[line - 1]}
+    #{carrets(first_column, last_column)}
+  ERROR
+end
+
+private
+
+def reset_precs
+  @prec_seen = false
+  @code_after_prec = false
+end
+
+def begin_c_declaration(end_symbol)
+  @lexer.status = :c_declaration
+  @lexer.end_symbol = end_symbol
+end
+
+def end_c_declaration
+  @lexer.status = :initial
+  @lexer.end_symbol = nil
+end
+
+def carrets(first_column, last_column)
+  ' ' * (first_column + 1) + '^' * (last_column - first_column)
+end

data/rbs_collection.lock.yaml

@@ -2,7 +2,7 @@
 sources:
 - type: git
   name: ruby/gem_rbs_collection
-  revision: 95ad664324500c9eec78569b45da98c65a27a511
+  revision: 25286c51a19927f28623aee3cd36655f902399ba
   remote: https://github.com/ruby/gem_rbs_collection.git
   repo_dir: gems
 path: ".gem_rbs_collection"
@@ -11,6 +11,22 @@ gems:
   version: '0'
   source:
     type: stdlib
+- name: fileutils
+  version: '0'
+  source:
+    type: stdlib
+- name: forwardable
+  version: '0'
+  source:
+    type: stdlib
+- name: rake
+  version: '13.0'
+  source:
+    type: git
+    name: ruby/gem_rbs_collection
+    revision: 2de2d4535caba275f3b8533684aab110d921f553
+    remote: https://github.com/ruby/gem_rbs_collection.git
+    repo_dir: gems
 - name: stackprof
   version: '0.2'
   source:

data/rbs_collection.yaml

@@ -20,3 +20,4 @@ gems:
   # It's unnecessary if you don't use rbs as a library.
   - name: rbs
     ignore: true
+  - name: forwardable

data/sample/calc.y

@@ -15,10 +15,12 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <ctype.h>
+%}
 
+%code provides {
 static int yylex(YYSTYPE *val, YYLTYPE *loc);
 static int yyerror(YYLTYPE *loc, const char *str);
-%}
+}
 
 %union {
     int val;

data/sample/parse.y

@@ -4,10 +4,14 @@
 
 %{
 // Prologue
+%}
+
+%code provides {
+
 static enum yytokentype yylex(YYSTYPE *lval, YYLTYPE *yylloc);
 static void yyerror(YYLTYPE *yylloc, const char *msg);
 
-%}
+}
 
 %expect 0
 %define api.pure

data/sig/lrama/grammar/code/printer_code.rbs

@@ -0,0 +1,15 @@
+module Lrama
+  class Grammar
+    class Code
+      class PrinterCode < Code
+        @tag: untyped
+        def initialize: (?type: untyped, ?token_code: untyped, ?tag: untyped) -> void
+
+        private
+
+        # ref: Lrama::Grammar::Code.token_code.references
+        def reference_to_c: (untyped ref) -> untyped
+      end
+    end
+  end
+end

data/sig/lrama/grammar/code.rbs

@@ -0,0 +1,24 @@
+module Lrama
+  class Grammar
+    class Code
+      extend Forwardable
+
+      attr_accessor type: untyped
+      attr_accessor token_code: untyped
+
+      # delegated
+      def s_value: -> String
+      def line: -> Integer
+      def column: -> untyped
+      def references: -> untyped
+
+      def initialize: (?type: untyped, ?token_code: untyped) -> void
+
+      def translated_code: () -> String
+
+      private
+
+      def reference_to_c: (untyped ref) -> untyped
+    end
+  end
+end

data/sig/lrama/grammar/counter.rbs

@@ -0,0 +1,11 @@
+module Lrama
+  class Grammar
+    class Counter
+      @number: Integer
+
+      def initialize: (Integer number) -> void
+
+      def increment: () -> Integer
+    end
+  end
+end

data/sig/lrama/grammar/parameterizing_rules/builder.rbs

@@ -0,0 +1,10 @@
+module Lrama
+  class Grammar
+    class ParameterizingRules
+      class Builder
+        def initialize: (Lexer::Token token, Counter rule_counter, Lexer::Token lhs, Lexer::Token::UserCode? user_code, Lexer::Token? precedence_sym, Integer? line) -> void
+        def build: () -> Array[Rule]
+      end
+    end
+  end
+end

data/sig/lrama/grammar/percent_code.rbs

@@ -0,0 +1,10 @@
+module Lrama
+  class Grammar
+    class PercentCode
+      attr_reader id: Lexer::Token::Ident
+      attr_reader code: Lexer::Token::UserCode
+
+      def initialize: (Lexer::Token::Ident id, Lexer::Token::UserCode code) -> void
+    end
+  end
+end

data/sig/lrama/grammar/precedence.rbs

@@ -0,0 +1,11 @@
+module Lrama
+  class Grammar
+    class Precedence
+      include Comparable
+      attr_accessor type: Symbol
+      attr_accessor precedence: Integer
+
+      def <=>: (Precedence other) -> Integer
+    end
+  end
+end

data/sig/lrama/grammar/printer.rbs

@@ -0,0 +1,11 @@
+module Lrama
+  class Grammar
+    class Printer
+      attr_accessor ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag]
+      attr_accessor token_code: Grammar::Code
+      attr_accessor lineno: Integer
+
+      def translated_code: (Lexer::Token member) -> String
+    end
+  end
+end