lrama 0.5.7 → 0.5.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c786e8955350ca51e6226aacd493073f7c31714ef7d5911dfb16259aa9415513
-  data.tar.gz: d167b1a2df9dcbc8f31912e8842388aa7d6a788a8d3eeff0278c6d3ed5ecbac6
+  metadata.gz: 5369d73367ea2fee299dbb73336a7801c98faa51bb3f68e66327c12f5a2d6716
+  data.tar.gz: b79943902e78f921d3361800cd1c371cebc594f527e940b83be3d26fcab190a9
 SHA512:
-  metadata.gz: c89fe932ef32b5f441b87df33c2431c8631a069c293306d0a39c858e462731e11d753117ab4e7c48b4e4e41015405328a8d8f5590fbce3544e67babdaeddd8a6
-  data.tar.gz: 619d1affd9f09c5c1b7748f712005b60073638be709e8bae986e39801173cf9de32555cf05837dbb7450360989989f5277e76a496eec702ace75c3b4af944471
+  metadata.gz: 93fd0bb99180b74f7a98b1186273b9324ac8855f599add59a4eb86b7b38b7930975c32463568ae93ff5d829d7c605dbd8a264c8127e40d92f7f7b0bd46637b0a
+  data.tar.gz: 06a43beddb6b78b4a1866164c63ebff7309c134b74dc32b5272e181f88ba4b2ff2ea30d756a16016e194e6c0babf0e6ee1536984e90835798eaa57be8eb09c9f

data/.github/workflows/test.yaml

@@ -57,7 +57,7 @@ jobs:
 
       - name: Check for parser.rb is up to date
         run: |
-          bundle exec rake build:racc_parser
+          bundle exec rake build:parser
           git diff --color --no-ext-diff --ignore-submodules --exit-code lib/lrama/parser.rb
   steep-check:
     runs-on: ubuntu-20.04

data/Gemfile

@@ -6,7 +6,7 @@ gem "rspec"
 gem "pry"
 # stackprof doesn't support Windows
 gem "stackprof", platforms: [:ruby]
-gem "racc"
+gem "racc", "1.7.3"
 gem "rake"
 gem "rbs", require: false
 gem "steep", require: false

data/README.md

@@ -65,13 +65,13 @@ This also requires Lrama to be able to run with only default gems because BASERU
 
 ## Development
 
-### How to generate new_parser.rb
+### How to generate parser.rb
 
 ```shell
-$ rake build:racc_parser
+$ rake build:parser
 ```
 
-`new_parser.rb` is generated from `parser.y` by Racc.
+`parser.rb` is generated from `parser.y` by Racc.
 Run the rake command when you update `parser.y` then commit changes of both files.
 
 ### Test

data/Rakefile

@@ -1,13 +1,8 @@
 require "bundler/gem_tasks"
 
 namespace "build" do
-  desc "build parser from parser.y by using Racc"
-  task :racc_parser do
-    `bundle exec racc parser.y --embedded -o lib/lrama/parser.rb`
-  end
-
-  desc "build parser for debugging"
-  task :racc_verbose_parser do
-    `bundle exec racc parser.y --embedded -o lib/lrama/parser.rb -t --log-file=parser.output`
+  desc "build parser from parser.y"
+  task :parser do
+    sh "bundle exec racc parser.y --embedded -o lib/lrama/parser.rb -t --log-file=parser.output"
   end
 end

data/Steepfile

@@ -6,8 +6,16 @@ target :lib do
 
   check "lib/lrama/bitmap.rb"
   check "lib/lrama/digraph.rb"
+  check "lib/lrama/grammar/percent_code.rb"
+  # TODO: Include this file once Lrama::Grammar::Symbol type is defined
+  # check "lib/lrama/grammar/reference.rb"
+  check "lib/lrama/lexer/token.rb"
+  check "lib/lrama/lexer/token/char.rb"
+  check "lib/lrama/lexer/token/ident.rb"
+  check "lib/lrama/lexer/token/parameterizing.rb"
+  check "lib/lrama/lexer/token/tag.rb"
+  check "lib/lrama/lexer/token/user_code.rb"
   check "lib/lrama/report/duration.rb"
   check "lib/lrama/report/profile.rb"
-  check "lib/lrama/token/type.rb"
   check "lib/lrama/warning.rb"
 end

data/lib/lrama/command.rb

@@ -8,7 +8,7 @@ module Lrama
       warning = Lrama::Warning.new
       text = options.y.read
       options.y.close if options.y != STDIN
-      grammar = Lrama::Parser.new(text).parse
+      grammar = Lrama::Parser.new(text, options.grammar_file, options.debug).parse
       states = Lrama::States.new(grammar, warning, trace_state: (options.trace_opts[:automaton] || options.trace_opts[:closure]))
       states.compute
       context = Lrama::Context.new(states)
@@ -20,6 +20,11 @@ module Lrama
         end
       end
 
+      if options.trace_opts && options.trace_opts[:rules]
+        puts "Grammar rules:"
+        puts grammar.rules
+      end
+
       File.open(options.outfile, "w+") do |f|
         Lrama::Output.new(
           out: f,

data/lib/lrama/grammar/percent_code.rb

@@ -0,0 +1,12 @@
+module Lrama
+  class Grammar
+    class PercentCode
+      attr_reader :id, :code
+
+      def initialize(id, code)
+        @id = id
+        @code = code
+      end
+    end
+  end
+end

data/lib/lrama/grammar/symbol.rb

@@ -47,9 +47,9 @@ module Lrama
           name = "YYACCEPT"
         when eof_symbol?
           name = "YYEOF"
-        when term? && id.type == Token::Char
+        when term? && id.is_a?(Lrama::Lexer::Token::Char)
           name = number.to_s + display_name
-        when term? && id.type == Token::Ident
+        when term? && id.is_a?(Lrama::Lexer::Token::Ident)
           name = id.s_value
         when nterm? && (id.s_value.include?("$") || id.s_value.include?("@"))
           name = number.to_s + id.s_value

data/lib/lrama/grammar.rb

@@ -3,6 +3,7 @@ require "strscan"
 require "lrama/grammar/auxiliary"
 require "lrama/grammar/code"
 require "lrama/grammar/error_token"
+require "lrama/grammar/percent_code"
 require "lrama/grammar/precedence"
 require "lrama/grammar/printer"
 require "lrama/grammar/reference"
@@ -13,11 +14,9 @@ require "lrama/lexer"
 require "lrama/type"
 
 module Lrama
-  Token = Lrama::Lexer::Token
-
   # Grammar is the result of parsing an input grammar file
   class Grammar
-    attr_reader :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol, :aux
+    attr_reader :percent_codes, :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol, :aux
     attr_accessor :union, :expect,
                   :printers, :error_tokens,
                   :lex_param, :parse_param, :initial_action,
@@ -26,6 +25,8 @@ module Lrama
                   :sym_to_rules
 
     def initialize
+      # Code defined by "%code"
+      @percent_codes = []
       @printers = []
       @error_tokens = []
       @symbols = []
@@ -43,6 +44,10 @@ module Lrama
       append_special_symbols
     end
 
+    def add_percent_code(id:, code:)
+      @percent_codes << PercentCode.new(id, code)
+    end
+
     def add_printer(ident_or_tags:, code:, lineno:)
       @printers << Printer.new(ident_or_tags: ident_or_tags, code: code, lineno: lineno)
     end
@@ -122,16 +127,7 @@ module Lrama
       @_rules << [lhs, rhs, lineno]
     end
 
-    def build_references(token_code)
-      token_code.references.map! do |type, value, tag, first_column, last_column|
-        Reference.new(type: type, value: value, ex_tag: tag, first_column: first_column, last_column: last_column)
-      end
-
-      token_code
-    end
-
     def build_code(type, token_code)
-      build_references(token_code)
       Code.new(type: type, token_code: token_code)
     end
 
@@ -152,6 +148,7 @@ module Lrama
     end
 
     def prepare
+      extract_references
       normalize_rules
       collect_symbols
       replace_token_with_symbol
@@ -308,46 +305,53 @@ module Lrama
       @nterms ||= @symbols.select(&:nterm?)
     end
 
+    def scan_reference(scanner)
+      start = scanner.pos
+      case
+      # $ references
+      # It need to wrap an identifier with brackets to use ".-" for identifiers
+      when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
+        tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil
+        return Reference.new(type: :dollar, value: "$", ex_tag: tag, first_column: start, last_column: scanner.pos - 1)
+      when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
+        tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil
+        return Reference.new(type: :dollar, value: Integer(scanner[2]), ex_tag: tag, first_column: start, last_column: scanner.pos - 1)
+      when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
+        tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil
+        return Reference.new(type: :dollar, value: scanner[2], ex_tag: tag, first_column: start, last_column: scanner.pos - 1)
+      when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
+        tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil
+        return Reference.new(type: :dollar, value: scanner[2], ex_tag: tag, first_column: start, last_column: scanner.pos - 1)
+
+      # @ references
+      # It need to wrap an identifier with brackets to use ".-" for identifiers
+      when scanner.scan(/@\$/) # @$
+        return Reference.new(type: :at, value: "$", first_column: start, last_column: scanner.pos - 1)
+      when scanner.scan(/@(\d+)/) # @1
+        return Reference.new(type: :at, value: Integer(scanner[1]), first_column: start, last_column: scanner.pos - 1)
+      when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
+        return Reference.new(type: :at, value: scanner[1], first_column: start, last_column: scanner.pos - 1)
+      when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
+        return Reference.new(type: :at, value: scanner[1], first_column: start, last_column: scanner.pos - 1)
+      end
+    end
+
+    private
+
     def extract_references
       unless initial_action.nil?
         scanner = StringScanner.new(initial_action.s_value)
         references = []
 
         while !scanner.eos? do
-          start = scanner.pos
-          case
-          # $ references
-          # It need to wrap an identifier with brackets to use ".-" for identifiers
-          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
-            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-            references << [:dollar, "$", tag, start, scanner.pos - 1]
-          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
-            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-            references << [:dollar, Integer(scanner[2]), tag, start, scanner.pos - 1]
-          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
-            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
-          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
-            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
-
-          # @ references
-          # It need to wrap an identifier with brackets to use ".-" for identifiers
-          when scanner.scan(/@\$/) # @$
-            references << [:at, "$", nil, start, scanner.pos - 1]
-          when scanner.scan(/@(\d+)/) # @1
-            references << [:at, Integer(scanner[1]), nil, start, scanner.pos - 1]
-          when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
-            references << [:at, scanner[1], nil, start, scanner.pos - 1]
-          when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
-            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          if reference = scan_reference(scanner)
+            references << reference
           else
             scanner.getch
           end
         end
 
         initial_action.token_code.references = references
-        build_references(initial_action.token_code)
       end
 
       @printers.each do |printer|
@@ -355,40 +359,14 @@ module Lrama
         references = []
 
         while !scanner.eos? do
-          start = scanner.pos
-          case
-          # $ references
-          # It need to wrap an identifier with brackets to use ".-" for identifiers
-          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
-            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-            references << [:dollar, "$", tag, start, scanner.pos - 1]
-          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
-            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-            references << [:dollar, Integer(scanner[2]), tag, start, scanner.pos - 1]
-          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
-            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
-          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
-            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
-
-          # @ references
-          # It need to wrap an identifier with brackets to use ".-" for identifiers
-          when scanner.scan(/@\$/) # @$
-            references << [:at, "$", nil, start, scanner.pos - 1]
-          when scanner.scan(/@(\d+)/) # @1
-            references << [:at, Integer(scanner[1]), nil, start, scanner.pos - 1]
-          when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
-            references << [:at, scanner[1], nil, start, scanner.pos - 1]
-          when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
-            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          if reference = scan_reference(scanner)
+            references << reference
           else
             scanner.getch
           end
         end
 
         printer.code.token_code.references = references
-        build_references(printer.code.token_code)
       end
 
       @error_tokens.each do |error_token|
@@ -396,78 +374,27 @@ module Lrama
         references = []
 
         while !scanner.eos? do
-          start = scanner.pos
-          case
-          # $ references
-          # It need to wrap an identifier with brackets to use ".-" for identifiers
-          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
-            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-            references << [:dollar, "$", tag, start, scanner.pos - 1]
-          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
-            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-            references << [:dollar, Integer(scanner[2]), tag, start, scanner.pos - 1]
-          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
-            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
-          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
-            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
-
-          # @ references
-          # It need to wrap an identifier with brackets to use ".-" for identifiers
-          when scanner.scan(/@\$/) # @$
-            references << [:at, "$", nil, start, scanner.pos - 1]
-          when scanner.scan(/@(\d+)/) # @1
-            references << [:at, Integer(scanner[1]), nil, start, scanner.pos - 1]
-          when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
-            references << [:at, scanner[1], nil, start, scanner.pos - 1]
-          when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
-            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          if reference = scan_reference(scanner)
+            references << reference
           else
             scanner.getch
           end
         end
 
         error_token.code.token_code.references = references
-        build_references(error_token.code.token_code)
       end
 
       @_rules.each do |lhs, rhs, _|
         rhs.each_with_index do |token, index|
-          next if token.class == Lrama::Grammar::Symbol || token.type != Lrama::Lexer::Token::User_code
+          next unless token.class == Lrama::Lexer::Token::UserCode
 
           scanner = StringScanner.new(token.s_value)
           references = []
 
           while !scanner.eos? do
-            start = scanner.pos
             case
-            # $ references
-            # It need to wrap an identifier with brackets to use ".-" for identifiers
-            when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
-              tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-              references << [:dollar, "$", tag, start, scanner.pos - 1]
-            when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
-              tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-              references << [:dollar, Integer(scanner[2]), tag, start, scanner.pos - 1]
-            when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
-              tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-              references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
-            when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
-              tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
-              references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
-
-            # @ references
-            # It need to wrap an identifier with brackets to use ".-" for identifiers
-            when scanner.scan(/@\$/) # @$
-              references << [:at, "$", nil, start, scanner.pos - 1]
-            when scanner.scan(/@(\d+)/) # @1
-              references << [:at, Integer(scanner[1]), nil, start, scanner.pos - 1]
-            when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
-              references << [:at, scanner[1], nil, start, scanner.pos - 1]
-            when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
-              references << [:at, scanner[1], nil, start, scanner.pos - 1]
-
+            when reference = scan_reference(scanner)
+              references << reference
             when scanner.scan(/\/\*/)
               scanner.scan_until(/\*\//)
             else
@@ -476,22 +403,11 @@ module Lrama
           end
 
           token.references = references
-          token.numberize_references(lhs, rhs)
-          build_references(token)
+          numberize_references(lhs, rhs, token.references)
         end
       end
     end
 
-    def create_token(type, s_value, line, column)
-      t = Token.new(type: type, s_value: s_value)
-      t.line = line
-      t.column = column
-
-      return t
-    end
-
-    private
-
     def find_nterm_by_id!(id)
       nterms.find do |nterm|
         nterm.id == id
@@ -505,29 +421,54 @@ module Lrama
       # @empty_symbol = term
 
       # YYEOF
-      term = add_term(id: Token.new(type: Token::Ident, s_value: "YYEOF"), alias_name: "\"end of file\"", token_id: 0)
+      term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYEOF"), alias_name: "\"end of file\"", token_id: 0)
       term.number = 0
       term.eof_symbol = true
       @eof_symbol = term
 
       # YYerror
-      term = add_term(id: Token.new(type: Token::Ident, s_value: "YYerror"), alias_name: "error")
+      term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYerror"), alias_name: "error")
       term.number = 1
       term.error_symbol = true
       @error_symbol = term
 
       # YYUNDEF
-      term = add_term(id: Token.new(type: Token::Ident, s_value: "YYUNDEF"), alias_name: "\"invalid token\"")
+      term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYUNDEF"), alias_name: "\"invalid token\"")
       term.number = 2
       term.undef_symbol = true
       @undef_symbol = term
 
       # $accept
-      term = add_nterm(id: Token.new(type: Token::Ident, s_value: "$accept"))
+      term = add_nterm(id: Lrama::Lexer::Token::Ident.new(s_value: "$accept"))
       term.accept_symbol = true
       @accept_symbol = term
     end
 
+    def numberize_references(lhs, rhs, references)
+      references.map! {|ref|
+        ref_name = ref.value
+        if ref_name.is_a?(::String) && ref_name != '$'
+          value =
+            if lhs.referred_by?(ref_name)
+              '$'
+            else
+              index = rhs.find_index {|token| token.referred_by?(ref_name) }
+
+              if index
+                index + 1
+              else
+                raise "'#{ref_name}' is invalid name."
+              end
+            end
+
+          ref.value = value
+          ref
+        else
+          ref
+        end
+      }
+    end
+
     # 1. Add $accept rule to the top of rules
     # 2. Extract precedence and last action
     # 3. Extract action in the middle of RHS into new Empty rule
@@ -570,7 +511,7 @@ module Lrama
           case
           when r.is_a?(Symbol) # precedence_sym
             precedence_sym = r
-          when (r.type == Token::User_code) && precedence_sym.nil? && code.nil? && rhs1.empty?
+          when r.is_a?(Lrama::Lexer::Token::UserCode) && precedence_sym.nil? && code.nil? && rhs1.empty?
             code = r
           else
             rhs1 << r
@@ -580,7 +521,7 @@ module Lrama
 
         # Bison n'th component is 1-origin
         (rhs1 + [code]).compact.each.with_index(1) do |token, i|
-          if token.type == Token::User_code
+          if token.is_a?(Lrama::Lexer::Token::UserCode)
             token.references.each do |ref|
               # Need to keep position_in_rhs for actions in the middle of RHS
               ref.position_in_rhs = i - 1
@@ -609,9 +550,9 @@ module Lrama
         end
 
         rhs2 = rhs1.map do |token|
-          if token.type == Token::User_code
+          if token.is_a?(Lrama::Lexer::Token::UserCode)
             prefix = token.referred ? "@" : "$@"
-            new_token = Token.new(type: Token::Ident, s_value: prefix + extracted_action_number.to_s)
+            new_token = Lrama::Lexer::Token::Ident.new(s_value: prefix + extracted_action_number.to_s)
             extracted_action_number += 1
             a << [new_token, token]
             new_token
@@ -627,8 +568,12 @@ module Lrama
         end
 
         c = code ? Code.new(type: :user_code, token_code: code) : nil
-        @rules << Rule.new(id: @rules.count, lhs: lhs, rhs: rhs2, code: c, precedence_sym: precedence_sym, lineno: lineno)
-
+        # Expand Parameterizing rules
+        if rhs2.any? {|r| r.is_a?(Lrama::Lexer::Token::Parameterizing) }
+          expand_parameterizing_rules(lhs, rhs2, c, precedence_sym, lineno)
+        else
+          @rules << Rule.new(id: @rules.count, lhs: lhs, rhs: rhs2, code: c, precedence_sym: precedence_sym, lineno: lineno)
+        end
         add_nterm(id: lhs)
         a.each do |new_token, _|
           add_nterm(id: new_token)
@@ -636,14 +581,37 @@ module Lrama
       end
     end
 
+    def expand_parameterizing_rules(lhs, rhs, code, precedence_sym, lineno)
+      token = Lrama::Lexer::Token::Ident.new(s_value: rhs[0].s_value)
+      if rhs.any? {|r| r.is_a?(Lrama::Lexer::Token::Parameterizing) && r.option? }
+        option_token = Lrama::Lexer::Token::Ident.new(s_value: "option_#{rhs[0].s_value}")
+        add_term(id: option_token)
+        @rules << Rule.new(id: @rules.count, lhs: lhs, rhs: [option_token], code: code, precedence_sym: precedence_sym, lineno: lineno)
+        @rules << Rule.new(id: @rules.count, lhs: option_token, rhs: [], code: code, precedence_sym: precedence_sym, lineno: lineno)
+        @rules << Rule.new(id: @rules.count, lhs: option_token, rhs: [token], code: code, precedence_sym: precedence_sym, lineno: lineno)
+      elsif rhs.any? {|r| r.is_a?(Lrama::Lexer::Token::Parameterizing) && r.nonempty_list? }
+        nonempty_list_token = Lrama::Lexer::Token::Ident.new(s_value: "nonempty_list_#{rhs[0].s_value}")
+        add_term(id: nonempty_list_token)
+        @rules << Rule.new(id: @rules.count, lhs: lhs, rhs: [nonempty_list_token], code: code, precedence_sym: precedence_sym, lineno: lineno)
+        @rules << Rule.new(id: @rules.count, lhs: nonempty_list_token, rhs: [token], code: code, precedence_sym: precedence_sym, lineno: lineno)
+        @rules << Rule.new(id: @rules.count, lhs: nonempty_list_token, rhs: [nonempty_list_token, token], code: code, precedence_sym: precedence_sym, lineno: lineno)
+      elsif rhs.any? {|r| r.is_a?(Lrama::Lexer::Token::Parameterizing) && r.list? }
+        list_token = Lrama::Lexer::Token::Ident.new(s_value: "list_#{rhs[0].s_value}")
+        add_term(id: list_token)
+        @rules << Rule.new(id: @rules.count, lhs: lhs, rhs: [list_token], code: code, precedence_sym: precedence_sym, lineno: lineno)
+        @rules << Rule.new(id: @rules.count, lhs: list_token, rhs: [], code: code, precedence_sym: precedence_sym, lineno: lineno)
+        @rules << Rule.new(id: @rules.count, lhs: list_token, rhs: [list_token, token], code: code, precedence_sym: precedence_sym, lineno: lineno)
+      end
+    end
+
     # Collect symbols from rules
     def collect_symbols
       @rules.flat_map(&:rhs).each do |s|
         case s
-        when Token
-          if s.type == Token::Char
-            add_term(id: s)
-          end
+        when Lrama::Lexer::Token::Char
+          add_term(id: s)
+        when Lrama::Lexer::Token
+          # skip
         when Symbol
           # skip
         else
@@ -684,7 +652,7 @@ module Lrama
 
         # If id is Token::Char, it uses ASCII code
         if sym.term? && sym.token_id.nil?
-          if sym.id.type == Token::Char
+          if sym.id.is_a?(Lrama::Lexer::Token::Char)
             # Ignore ' on the both sides
             case sym.id.s_value[1..-2]
             when "\\b"
@@ -737,7 +705,7 @@ module Lrama
           rule.code.references.each do |ref|
             next if ref.type == :at
 
-            if ref.referring_symbol.type != Token::User_code
+            if !ref.referring_symbol.is_a?(Lrama::Lexer::Token::UserCode)
               ref.referring_symbol = token_to_symbol(ref.referring_symbol)
             end
           end
@@ -747,7 +715,7 @@ module Lrama
 
     def token_to_symbol(token)
       case token
-      when Token
+      when Lrama::Lexer::Token
         find_symbol_by_id!(token)
       when Symbol
         token
@@ -793,10 +761,10 @@ module Lrama
       @symbols.each do |sym|
         @printers.each do |printer|
           printer.ident_or_tags.each do |ident_or_tag|
-            case ident_or_tag.type
-            when Token::Ident
+            case ident_or_tag
+            when Lrama::Lexer::Token::Ident
               sym.printer = printer if sym.id == ident_or_tag
-            when Token::Tag
+            when Lrama::Lexer::Token::Tag
               sym.printer = printer if sym.tag == ident_or_tag
             else
               raise "Unknown token type. #{printer}"
@@ -810,10 +778,10 @@ module Lrama
       @symbols.each do |sym|
         @error_tokens.each do |error_token|
           error_token.ident_or_tags.each do |ident_or_tag|
-            case ident_or_tag.type
-            when Token::Ident
+            case ident_or_tag
+            when Lrama::Lexer::Token::Ident
               sym.error_token = error_token if sym.id == ident_or_tag
-            when Token::Tag
+            when Lrama::Lexer::Token::Tag
               sym.error_token = error_token if sym.tag == ident_or_tag
             else
               raise "Unknown token type. #{error_token}"

data/lib/lrama/lexer/token/char.rb

@@ -0,0 +1,8 @@
+module Lrama
+  class Lexer
+    class Token
+      class Char < Token
+      end
+    end
+  end
+end

data/lib/lrama/lexer/token/ident.rb

@@ -0,0 +1,8 @@
+module Lrama
+  class Lexer
+    class Token
+      class Ident < Token
+      end
+    end
+  end
+end

data/lib/lrama/lexer/token/parameterizing.rb

@@ -0,0 +1,19 @@
+module Lrama
+  class Lexer
+    class Token
+      class Parameterizing < Token
+        def option?
+          self.s_value == "?"
+        end
+
+        def nonempty_list?
+          self.s_value == "+"
+        end
+
+        def list?
+          self.s_value == "*"
+        end
+      end
+    end
+  end
+end