lrama 0.5.5 → 0.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d79363afacc07dac12ab5c1a861123e099626591104079c962e02f3df74a24f3
-  data.tar.gz: d8ddf78087e27510ab9da808301505a10ec0151ef55d1df3a59cc80ffde81b53
+  metadata.gz: c786e8955350ca51e6226aacd493073f7c31714ef7d5911dfb16259aa9415513
+  data.tar.gz: d167b1a2df9dcbc8f31912e8842388aa7d6a788a8d3eeff0278c6d3ed5ecbac6
 SHA512:
-  metadata.gz: acc15bb56862ea03c6b195253ba881786acf498719f930b8a8435a60f86b44d4383778cc215e7497971833c4be596a9c41a6c5134f77703a9f5cd0a6b5714ad2
-  data.tar.gz: d45b2ec3a22ce2e29beb6a5ec0f4d7b142cebca3b6ca76786fa19b841c257d3841d5a7a43d7d564459b8f9c5b05faa7229bd9b2e2178592ec3988b64f77b535c
+  metadata.gz: c89fe932ef32b5f441b87df33c2431c8631a069c293306d0a39c858e462731e11d753117ab4e7c48b4e4e41015405328a8d8f5590fbce3544e67babdaeddd8a6
+  data.tar.gz: 619d1affd9f09c5c1b7748f712005b60073638be709e8bae986e39801173cf9de32555cf05837dbb7450360989989f5277e76a496eec702ace75c3b4af944471

data/.github/workflows/codespell.yaml

@@ -0,0 +1,16 @@
+name: CodeSpell
+on:
+  - pull_request
+jobs:
+  codespell:
+    name: CodeSpell
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: CodeSpell
+        uses: codespell-project/actions-codespell@master
+        with:
+          check_filenames: true
+          check_hidden: true
+          ignore_words_file: .codespellignore
+          exclude_file: lib/lrama/parser.rb

data/.github/workflows/test.yaml

@@ -15,7 +15,7 @@ jobs:
       matrix:
         ruby: ['head', '3.2', '3.1', '3.0', '2.7', '2.6', '2.5']
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
@@ -29,7 +29,7 @@ jobs:
       matrix:
         ruby: ['head']
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
@@ -38,13 +38,27 @@ jobs:
       - run: bundle exec rspec
   check-misc:
     runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        ruby: ['head']
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - run: bundle install
+
       # Copy from https://github.com/ruby/ruby/blob/089227e94823542acfdafa68541d330eee42ffea/.github/workflows/check_misc.yml#L27
       - name: Check for trailing spaces
         run: |
-          git grep -I -n '[	 ]$' -- '*.rb' '*.[chy]' '*.rs' && exit 1 || :
+          git grep -I -n '[	 ]$' -- '*.rb' '*.[chy]' '*.rs' ':!spec/' && exit 1 || :
           git grep -n '^[	 ][	 ]*$' -- '*.md' && exit 1 || :
+
+      - name: Check for parser.rb is up to date
+        run: |
+          bundle exec rake build:racc_parser
+          git diff --color --no-ext-diff --ignore-submodules --exit-code lib/lrama/parser.rb
   steep-check:
     runs-on: ubuntu-20.04
     strategy:
@@ -52,7 +66,7 @@ jobs:
       matrix:
         ruby: ['head']
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
@@ -65,13 +79,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        baseruby: ['3.0']
+        # '3.0' is the oldest living ruby version
+        # '2.5' is for BASERUBY
+        baseruby: ['head', '3.0', '2.5']
         ruby_branch: ['master']
     defaults:
       run:
         working-directory: ../ruby/build
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.baseruby }}

data/.gitignore

@@ -4,3 +4,4 @@
 /Gemfile.lock
 /pkg/
 coverage/
+/parser.output

data/Gemfile

@@ -6,6 +6,7 @@ gem "rspec"
 gem "pry"
 # stackprof doesn't support Windows
 gem "stackprof", platforms: [:ruby]
+gem "racc"
 gem "rake"
 gem "rbs", require: false
 gem "steep", require: false

data/README.md

@@ -61,12 +61,78 @@ This branch generates "parse.c" compatible with Bison 3.8.2 for ruby 3.0, 3.1, 3
 
 Lrama is executed with BASERUBY when building ruby from source code. Therefore Lrama needs to support BASERUBY, currently 2.5, or later version.
 
-This also requires Lrama to be able to run with only default gems and bundled gems.
+This also requires Lrama to be able to run with only default gems because BASERUBY runs with `--disable=gems` option.
 
-## Build Ruby
+## Development
+
+### How to generate new_parser.rb
+
+```shell
+$ rake build:racc_parser
+```
+
+`new_parser.rb` is generated from `parser.y` by Racc.
+Run the rake command when you update `parser.y` then commit changes of both files.
+
+### Test
+
+Running tests:
+
+```shell
+$ bundle install
+$ bundle exec rspec
+```
+
+Running type check:
+
+```shell
+$ bundle install
+$ bundle exec rbs collection install
+$ bundle exec steep check
+```
+
+### Profiling Lrama
+
+#### 1. Create parse.tmp.y in ruby/ruby
+
+```shell
+$ ruby tool/id2token.rb parse.y > parse.tmp.y
+$ cp parse.tmp.y dir/lrama/tmp
+```
+
+#### 2. Enable Profiler
+
+```diff
+diff --git a/exe/lrama b/exe/lrama
+index ba5fb06..2497178 100755
+--- a/exe/lrama
++++ b/exe/lrama
+@@ -3,4 +3,6 @@
+ $LOAD_PATH << File.join(__dir__, "../lib")
+ require "lrama"
+
+-Lrama::Command.new.run(ARGV.dup)
++Lrama::Report::Profile.report_profile do
++  Lrama::Command.new.run(ARGV.dup)
++end
+```
+
+#### 3. Run Lrama
+
+```shell
+$ exe/lrama -o parse.tmp.c --header=parse.tmp.h tmp/parse.tmp.y
+```
+
+#### 4. Generate Flamegraph
+
+```shell
+$ stackprof --d3-flamegraph tmp/stackprof-cpu-myapp.dump > tmp/flamegraph.html
+```
+
+### Build Ruby
 
 1. Install Lrama
-2. Run `make YACC=lrama`
+2. Run `make main`
 
 ## Release flow
 

data/Rakefile

@@ -1 +1,13 @@
 require "bundler/gem_tasks"
+
+namespace "build" do
+  desc "build parser from parser.y by using Racc"
+  task :racc_parser do
+    `bundle exec racc parser.y --embedded -o lib/lrama/parser.rb`
+  end
+
+  desc "build parser for debugging"
+  task :racc_verbose_parser do
+    `bundle exec racc parser.y --embedded -o lib/lrama/parser.rb -t --log-file=parser.output`
+  end
+end

data/Steepfile

@@ -1,10 +1,13 @@
 # D = Steep::Diagnostic
 #
 target :lib do
+  repo_path '.gem_rbs_collection/'
   signature "sig"
 
   check "lib/lrama/bitmap.rb"
+  check "lib/lrama/digraph.rb"
   check "lib/lrama/report/duration.rb"
   check "lib/lrama/report/profile.rb"
+  check "lib/lrama/token/type.rb"
   check "lib/lrama/warning.rb"
 end

data/exe/lrama

@@ -3,4 +3,4 @@
 $LOAD_PATH << File.join(__dir__, "../lib")
 require "lrama"
 
-Lrama::Command.new(ARGV.dup).run
+Lrama::Command.new.run(ARGV.dup)

data/lib/lrama/command.rb

@@ -1,53 +1,35 @@
-require 'optparse'
-
 module Lrama
   class Command
-    def initialize(argv)
-      @argv = argv
-
-      @skeleton = "bison/yacc.c"
-      @header = false
-      @header_file = nil
-      @report = []
-      @report_file = nil
-      @outfile = "y.tab.c"
-      @trace = []
-      @error_recovery = false
-      @grammar_file = nil
-      @report_file = nil
-      @trace_opts = nil
-      @report_opts = nil
-    end
+    def run(argv)
+      options = OptionParser.new.parse(argv)
 
-    def run
-      parse_option
-
-      Report::Duration.enable if @trace_opts[:time]
+      Report::Duration.enable if options.trace_opts[:time]
 
       warning = Lrama::Warning.new
-      grammar = Lrama::Parser.new(@y.read).parse
-      @y.close if @y != STDIN
-      states = Lrama::States.new(grammar, warning, trace_state: (@trace_opts[:automaton] || @trace_opts[:closure]))
+      text = options.y.read
+      options.y.close if options.y != STDIN
+      grammar = Lrama::Parser.new(text).parse
+      states = Lrama::States.new(grammar, warning, trace_state: (options.trace_opts[:automaton] || options.trace_opts[:closure]))
       states.compute
       context = Lrama::Context.new(states)
 
-      if @report_file
+      if options.report_file
         reporter = Lrama::StatesReporter.new(states)
-        File.open(@report_file, "w+") do |f|
-          reporter.report(f, **@report_opts)
+        File.open(options.report_file, "w+") do |f|
+          reporter.report(f, **options.report_opts)
         end
       end
 
-      File.open(@outfile, "w+") do |f|
+      File.open(options.outfile, "w+") do |f|
         Lrama::Output.new(
           out: f,
-          output_file_path: @outfile,
-          template_name: @skeleton,
-          grammar_file_path: @grammar_file,
-          header_file_path: @header_file,
+          output_file_path: options.outfile,
+          template_name: options.skeleton,
+          grammar_file_path: options.grammar_file,
+          header_file_path: options.header_file,
           context: context,
           grammar: grammar,
-          error_recovery: @error_recovery,
+          error_recovery: options.error_recovery,
         ).render
       end
 
@@ -55,108 +37,5 @@ module Lrama
         exit 1
       end
     end
-
-    private
-
-    def validate_report(report)
-      bison_list = %w[states itemsets lookaheads solved counterexamples cex all none]
-      others = %w[verbose]
-      list = bison_list + others
-      not_supported = %w[cex none]
-      h = { grammar: true }
-
-      report.each do |r|
-        if list.include?(r) && !not_supported.include?(r)
-          h[r.to_sym] = true
-        else
-          raise "Invalid report option \"#{r}\"."
-        end
-      end
-
-      if h[:all]
-        (bison_list - not_supported).each do |r|
-          h[r.to_sym] = true
-        end
-
-        h.delete(:all)
-      end
-
-      return h
-    end
-
-    def validate_trace(trace)
-      list = %w[
-        none locations scan parse automaton bitsets
-        closure grammar resource sets muscles tools
-        m4-early m4 skeleton time ielr cex all
-      ]
-      h = {}
-
-      trace.each do |t|
-        if list.include?(t)
-          h[t.to_sym] = true
-        else
-          raise "Invalid trace option \"#{t}\"."
-        end
-      end
-
-      return h
-    end
-
-    def parse_option
-      opt = OptionParser.new
-
-      # opt.on('-h') {|v| p v }
-      opt.on('-V', '--version') {|v| puts "lrama #{Lrama::VERSION}"; exit 0 }
-
-      # Tuning the Parser
-      opt.on('-S', '--skeleton=FILE') {|v| @skeleton = v }
-      opt.on('-t') {  } # Do nothing
-
-      # Output Files:
-      opt.on('-h', '--header=[FILE]') {|v| @header = true; @header_file = v }
-      opt.on('-d') { @header = true }
-      opt.on('-r', '--report=THINGS', Array) {|v| @report = v }
-      opt.on('--report-file=FILE')    {|v| @report_file = v }
-      opt.on('-v') {  } # Do nothing
-      opt.on('-o', '--output=FILE')   {|v| @outfile = v }
-
-      # Hidden
-      opt.on('--trace=THINGS', Array) {|v| @trace = v }
-
-      # Error Recovery
-      opt.on('-e') {|v| @error_recovery = true }
-
-      opt.parse!(@argv)
-
-      @trace_opts = validate_trace(@trace)
-      @report_opts = validate_report(@report)
-
-      @grammar_file = @argv.shift
-
-      if !@grammar_file
-        abort "File should be specified\n"
-      end
-
-      if @grammar_file == '-'
-        @grammar_file = @argv.shift or abort "File name for STDIN should be specified\n"
-        @y = STDIN
-      else
-        @y = File.open(@grammar_file, 'r')
-      end
-
-      if !@report.empty? && @report_file.nil? && @grammar_file
-        @report_file = File.dirname(@grammar_file) + "/" + File.basename(@grammar_file, ".*") + ".output"
-      end
-
-      if !@header_file && @header
-        case
-        when @outfile
-          @header_file = File.dirname(@outfile) + "/" + File.basename(@outfile, ".*") + ".h"
-        when @grammar_file
-          @header_file = File.dirname(@grammar_file) + "/" + File.basename(@grammar_file, ".*") + ".h"
-        end
-      end
-    end
   end
 end

data/lib/lrama/context.rb

@@ -170,7 +170,7 @@ module Lrama
       return a
     end
 
-    # Mapping from rule number to lenght of RHS.
+    # Mapping from rule number to length of RHS.
     # Dummy rule is appended as the first element whose value is 0
     # because 0 means error in yydefact.
     def yyr2
@@ -214,7 +214,7 @@ module Lrama
       (rule_id + 1) * -1
     end
 
-    # Symbol number is assinged to term first then nterm.
+    # Symbol number is assigned to term first then nterm.
     # This method calculates sequence_number for nterm.
     def nterm_number_to_sequence_number(nterm_number)
       nterm_number - @states.terms.count
@@ -259,7 +259,7 @@ module Lrama
           actions[conflict.symbol.number] = ErrorActionNumber
         end
 
-        # If default_reduction_rule, replase default_reduction_rule in
+        # If default_reduction_rule, replace default_reduction_rule in
         # actions with zero.
         if state.default_reduction_rule
           actions.map! do |e|
@@ -272,7 +272,7 @@ module Lrama
         end
 
         # If no default_reduction_rule, default behavior is an
-        # error then replase ErrorActionNumber with zero.
+        # error then replace ErrorActionNumber with zero.
         if !state.default_reduction_rule
           actions.map! do |e|
             if e == ErrorActionNumber

data/lib/lrama/digraph.rb

@@ -40,8 +40,7 @@ module Lrama
       end
 
       if @h[x] == d
-        while true do
-          z = @stack.pop
+        while (z = @stack.pop) do
           @h[z] = Float::INFINITY
           break if z == x
           @result[z] = @result[x] # F (Top of S) = F x

data/lib/lrama/grammar/union.rb

@@ -2,8 +2,8 @@ module Lrama
   class Grammar
     class Union < Struct.new(:code, :lineno, keyword_init: true)
       def braces_less_code
-        # Remove braces
-        code.s_value[1..-2]
+        # Braces is already removed by lexer
+        code.s_value
       end
     end
   end

data/lib/lrama/grammar.rb

@@ -1,3 +1,5 @@
+require "strscan"
+
 require "lrama/grammar/auxiliary"
 require "lrama/grammar/code"
 require "lrama/grammar/error_token"
@@ -306,6 +308,188 @@ module Lrama
       @nterms ||= @symbols.select(&:nterm?)
     end
 
+    def extract_references
+      unless initial_action.nil?
+        scanner = StringScanner.new(initial_action.s_value)
+        references = []
+
+        while !scanner.eos? do
+          start = scanner.pos
+          case
+          # $ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, "$", tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, Integer(scanner[2]), tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+
+          # @ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/@\$/) # @$
+            references << [:at, "$", nil, start, scanner.pos - 1]
+          when scanner.scan(/@(\d+)/) # @1
+            references << [:at, Integer(scanner[1]), nil, start, scanner.pos - 1]
+          when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
+            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
+            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          else
+            scanner.getch
+          end
+        end
+
+        initial_action.token_code.references = references
+        build_references(initial_action.token_code)
+      end
+
+      @printers.each do |printer|
+        scanner = StringScanner.new(printer.code.s_value)
+        references = []
+
+        while !scanner.eos? do
+          start = scanner.pos
+          case
+          # $ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, "$", tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, Integer(scanner[2]), tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+
+          # @ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/@\$/) # @$
+            references << [:at, "$", nil, start, scanner.pos - 1]
+          when scanner.scan(/@(\d+)/) # @1
+            references << [:at, Integer(scanner[1]), nil, start, scanner.pos - 1]
+          when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
+            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
+            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          else
+            scanner.getch
+          end
+        end
+
+        printer.code.token_code.references = references
+        build_references(printer.code.token_code)
+      end
+
+      @error_tokens.each do |error_token|
+        scanner = StringScanner.new(error_token.code.s_value)
+        references = []
+
+        while !scanner.eos? do
+          start = scanner.pos
+          case
+          # $ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, "$", tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, Integer(scanner[2]), tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+
+          # @ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/@\$/) # @$
+            references << [:at, "$", nil, start, scanner.pos - 1]
+          when scanner.scan(/@(\d+)/) # @1
+            references << [:at, Integer(scanner[1]), nil, start, scanner.pos - 1]
+          when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
+            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
+            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          else
+            scanner.getch
+          end
+        end
+
+        error_token.code.token_code.references = references
+        build_references(error_token.code.token_code)
+      end
+
+      @_rules.each do |lhs, rhs, _|
+        rhs.each_with_index do |token, index|
+          next if token.class == Lrama::Grammar::Symbol || token.type != Lrama::Lexer::Token::User_code
+
+          scanner = StringScanner.new(token.s_value)
+          references = []
+
+          while !scanner.eos? do
+            start = scanner.pos
+            case
+            # $ references
+            # It need to wrap an identifier with brackets to use ".-" for identifiers
+            when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
+              tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+              references << [:dollar, "$", tag, start, scanner.pos - 1]
+            when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
+              tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+              references << [:dollar, Integer(scanner[2]), tag, start, scanner.pos - 1]
+            when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
+              tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+              references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+            when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
+              tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+              references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+
+            # @ references
+            # It need to wrap an identifier with brackets to use ".-" for identifiers
+            when scanner.scan(/@\$/) # @$
+              references << [:at, "$", nil, start, scanner.pos - 1]
+            when scanner.scan(/@(\d+)/) # @1
+              references << [:at, Integer(scanner[1]), nil, start, scanner.pos - 1]
+            when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
+              references << [:at, scanner[1], nil, start, scanner.pos - 1]
+            when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
+              references << [:at, scanner[1], nil, start, scanner.pos - 1]
+
+            when scanner.scan(/\/\*/)
+              scanner.scan_until(/\*\//)
+            else
+              scanner.getch
+            end
+          end
+
+          token.references = references
+          token.numberize_references(lhs, rhs)
+          build_references(token)
+        end
+      end
+    end
+
+    def create_token(type, s_value, line, column)
+      t = Token.new(type: type, s_value: s_value)
+      t.line = line
+      t.column = column
+
+      return t
+    end
+
     private
 
     def find_nterm_by_id!(id)
@@ -470,7 +654,9 @@ module Lrama
 
     # Fill #number and #token_id
     def fill_symbol_number
-      # TODO: why start from 256
+      # Character literal in grammar file has
+      # token id corresponding to ASCII code by default,
+      # so start token_id from 256.
       token_id = 256
 
       # YYEMPTY = -2