lrama 0.5.6 → 0.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e54c51af6f1d3632293cbd7f68762bf7cff63758d3ce633113805a629b92072b
-  data.tar.gz: 5d053543f0e00c9fb20c40f5ca8236e499292f35fa8cf6fb8e85cac33b930814
+  metadata.gz: c786e8955350ca51e6226aacd493073f7c31714ef7d5911dfb16259aa9415513
+  data.tar.gz: d167b1a2df9dcbc8f31912e8842388aa7d6a788a8d3eeff0278c6d3ed5ecbac6
 SHA512:
-  metadata.gz: 87bafe9650720b154855e055e53d700cfba67d31489dbc855c716bc150dff35e92d0fc974c14cb81a07ae68febbd69118c1720a628e2819fec1ebfa304acad55
-  data.tar.gz: 19b0c51748cef053d205bbf13e8ff238bda2fdbb101d304f6bfe9633b4f88fe2c4a1f452627528fd2c9e78bc97955a2a5f7543212aa581c3a9f7ae96dd3d70f8
+  metadata.gz: c89fe932ef32b5f441b87df33c2431c8631a069c293306d0a39c858e462731e11d753117ab4e7c48b4e4e41015405328a8d8f5590fbce3544e67babdaeddd8a6
+  data.tar.gz: 619d1affd9f09c5c1b7748f712005b60073638be709e8bae986e39801173cf9de32555cf05837dbb7450360989989f5277e76a496eec702ace75c3b4af944471

data/.github/workflows/codespell.yaml

@@ -0,0 +1,16 @@
+name: CodeSpell
+on:
+  - pull_request
+jobs:
+  codespell:
+    name: CodeSpell
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: CodeSpell
+        uses: codespell-project/actions-codespell@master
+        with:
+          check_filenames: true
+          check_hidden: true
+          ignore_words_file: .codespellignore
+          exclude_file: lib/lrama/parser.rb

data/.github/workflows/test.yaml

@@ -38,13 +38,27 @@ jobs:
       - run: bundle exec rspec
   check-misc:
     runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        ruby: ['head']
     steps:
       - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - run: bundle install
+
       # Copy from https://github.com/ruby/ruby/blob/089227e94823542acfdafa68541d330eee42ffea/.github/workflows/check_misc.yml#L27
       - name: Check for trailing spaces
         run: |
-          git grep -I -n '[	 ]$' -- '*.rb' '*.[chy]' '*.rs' && exit 1 || :
+          git grep -I -n '[	 ]$' -- '*.rb' '*.[chy]' '*.rs' ':!spec/' && exit 1 || :
           git grep -n '^[	 ][	 ]*$' -- '*.md' && exit 1 || :
+
+      - name: Check for parser.rb is up to date
+        run: |
+          bundle exec rake build:racc_parser
+          git diff --color --no-ext-diff --ignore-submodules --exit-code lib/lrama/parser.rb
   steep-check:
     runs-on: ubuntu-20.04
     strategy:
@@ -65,7 +79,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        baseruby: ['3.0']
+        # '3.0' is the oldest living ruby version
+        # '2.5' is for BASERUBY
+        baseruby: ['head', '3.0', '2.5']
         ruby_branch: ['master']
     defaults:
       run:

data/.gitignore

@@ -4,3 +4,4 @@
 /Gemfile.lock
 /pkg/
 coverage/
+/parser.output

data/Gemfile

@@ -6,6 +6,7 @@ gem "rspec"
 gem "pry"
 # stackprof doesn't support Windows
 gem "stackprof", platforms: [:ruby]
+gem "racc"
 gem "rake"
 gem "rbs", require: false
 gem "steep", require: false

data/README.md

@@ -61,12 +61,78 @@ This branch generates "parse.c" compatible with Bison 3.8.2 for ruby 3.0, 3.1, 3
 
 Lrama is executed with BASERUBY when building ruby from source code. Therefore Lrama needs to support BASERUBY, currently 2.5, or later version.
 
-This also requires Lrama to be able to run with only default gems and bundled gems.
+This also requires Lrama to be able to run with only default gems because BASERUBY runs with `--disable=gems` option.
 
-## Build Ruby
+## Development
+
+### How to generate new_parser.rb
+
+```shell
+$ rake build:racc_parser
+```
+
+`new_parser.rb` is generated from `parser.y` by Racc.
+Run the rake command when you update `parser.y` then commit changes of both files.
+
+### Test
+
+Running tests:
+
+```shell
+$ bundle install
+$ bundle exec rspec
+```
+
+Running type check:
+
+```shell
+$ bundle install
+$ bundle exec rbs collection install
+$ bundle exec steep check
+```
+
+### Profiling Lrama
+
+#### 1. Create parse.tmp.y in ruby/ruby
+
+```shell
+$ ruby tool/id2token.rb parse.y > parse.tmp.y
+$ cp parse.tmp.y dir/lrama/tmp
+```
+
+#### 2. Enable Profiler
+
+```diff
+diff --git a/exe/lrama b/exe/lrama
+index ba5fb06..2497178 100755
+--- a/exe/lrama
++++ b/exe/lrama
+@@ -3,4 +3,6 @@
+ $LOAD_PATH << File.join(__dir__, "../lib")
+ require "lrama"
+
+-Lrama::Command.new.run(ARGV.dup)
++Lrama::Report::Profile.report_profile do
++  Lrama::Command.new.run(ARGV.dup)
++end
+```
+
+#### 3. Run Lrama
+
+```shell
+$ exe/lrama -o parse.tmp.c --header=parse.tmp.h tmp/parse.tmp.y
+```
+
+#### 4. Generate Flamegraph
+
+```shell
+$ stackprof --d3-flamegraph tmp/stackprof-cpu-myapp.dump > tmp/flamegraph.html
+```
+
+### Build Ruby
 
 1. Install Lrama
-2. Run `make YACC=lrama`
+2. Run `make main`
 
 ## Release flow
 

data/Rakefile

@@ -1 +1,13 @@
 require "bundler/gem_tasks"
+
+namespace "build" do
+  desc "build parser from parser.y by using Racc"
+  task :racc_parser do
+    `bundle exec racc parser.y --embedded -o lib/lrama/parser.rb`
+  end
+
+  desc "build parser for debugging"
+  task :racc_verbose_parser do
+    `bundle exec racc parser.y --embedded -o lib/lrama/parser.rb -t --log-file=parser.output`
+  end
+end

data/Steepfile

@@ -1,10 +1,13 @@
 # D = Steep::Diagnostic
 #
 target :lib do
+  repo_path '.gem_rbs_collection/'
   signature "sig"
 
   check "lib/lrama/bitmap.rb"
+  check "lib/lrama/digraph.rb"
   check "lib/lrama/report/duration.rb"
   check "lib/lrama/report/profile.rb"
+  check "lib/lrama/token/type.rb"
   check "lib/lrama/warning.rb"
 end

data/lib/lrama/command.rb

@@ -6,8 +6,9 @@ module Lrama
       Report::Duration.enable if options.trace_opts[:time]
 
       warning = Lrama::Warning.new
-      grammar = Lrama::Parser.new(options.y.read).parse
+      text = options.y.read
       options.y.close if options.y != STDIN
+      grammar = Lrama::Parser.new(text).parse
       states = Lrama::States.new(grammar, warning, trace_state: (options.trace_opts[:automaton] || options.trace_opts[:closure]))
       states.compute
       context = Lrama::Context.new(states)

data/lib/lrama/context.rb

@@ -170,7 +170,7 @@ module Lrama
       return a
     end
 
-    # Mapping from rule number to lenght of RHS.
+    # Mapping from rule number to length of RHS.
     # Dummy rule is appended as the first element whose value is 0
     # because 0 means error in yydefact.
     def yyr2
@@ -214,7 +214,7 @@ module Lrama
       (rule_id + 1) * -1
     end
 
-    # Symbol number is assinged to term first then nterm.
+    # Symbol number is assigned to term first then nterm.
     # This method calculates sequence_number for nterm.
     def nterm_number_to_sequence_number(nterm_number)
       nterm_number - @states.terms.count
@@ -259,7 +259,7 @@ module Lrama
           actions[conflict.symbol.number] = ErrorActionNumber
         end
 
-        # If default_reduction_rule, replase default_reduction_rule in
+        # If default_reduction_rule, replace default_reduction_rule in
         # actions with zero.
         if state.default_reduction_rule
           actions.map! do |e|
@@ -272,7 +272,7 @@ module Lrama
         end
 
         # If no default_reduction_rule, default behavior is an
-        # error then replase ErrorActionNumber with zero.
+        # error then replace ErrorActionNumber with zero.
         if !state.default_reduction_rule
           actions.map! do |e|
             if e == ErrorActionNumber

data/lib/lrama/digraph.rb

@@ -40,8 +40,7 @@ module Lrama
       end
 
       if @h[x] == d
-        while true do
-          z = @stack.pop
+        while (z = @stack.pop) do
           @h[z] = Float::INFINITY
           break if z == x
           @result[z] = @result[x] # F (Top of S) = F x

data/lib/lrama/grammar/union.rb

@@ -2,8 +2,8 @@ module Lrama
   class Grammar
     class Union < Struct.new(:code, :lineno, keyword_init: true)
       def braces_less_code
-        # Remove braces
-        code.s_value[1..-2]
+        # Braces is already removed by lexer
+        code.s_value
       end
     end
   end

data/lib/lrama/grammar.rb

@@ -1,3 +1,5 @@
+require "strscan"
+
 require "lrama/grammar/auxiliary"
 require "lrama/grammar/code"
 require "lrama/grammar/error_token"
@@ -306,6 +308,188 @@ module Lrama
       @nterms ||= @symbols.select(&:nterm?)
     end
 
+    def extract_references
+      unless initial_action.nil?
+        scanner = StringScanner.new(initial_action.s_value)
+        references = []
+
+        while !scanner.eos? do
+          start = scanner.pos
+          case
+          # $ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, "$", tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, Integer(scanner[2]), tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+
+          # @ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/@\$/) # @$
+            references << [:at, "$", nil, start, scanner.pos - 1]
+          when scanner.scan(/@(\d+)/) # @1
+            references << [:at, Integer(scanner[1]), nil, start, scanner.pos - 1]
+          when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
+            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
+            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          else
+            scanner.getch
+          end
+        end
+
+        initial_action.token_code.references = references
+        build_references(initial_action.token_code)
+      end
+
+      @printers.each do |printer|
+        scanner = StringScanner.new(printer.code.s_value)
+        references = []
+
+        while !scanner.eos? do
+          start = scanner.pos
+          case
+          # $ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, "$", tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, Integer(scanner[2]), tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+
+          # @ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/@\$/) # @$
+            references << [:at, "$", nil, start, scanner.pos - 1]
+          when scanner.scan(/@(\d+)/) # @1
+            references << [:at, Integer(scanner[1]), nil, start, scanner.pos - 1]
+          when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
+            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
+            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          else
+            scanner.getch
+          end
+        end
+
+        printer.code.token_code.references = references
+        build_references(printer.code.token_code)
+      end
+
+      @error_tokens.each do |error_token|
+        scanner = StringScanner.new(error_token.code.s_value)
+        references = []
+
+        while !scanner.eos? do
+          start = scanner.pos
+          case
+          # $ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, "$", tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, Integer(scanner[2]), tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+            references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+
+          # @ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/@\$/) # @$
+            references << [:at, "$", nil, start, scanner.pos - 1]
+          when scanner.scan(/@(\d+)/) # @1
+            references << [:at, Integer(scanner[1]), nil, start, scanner.pos - 1]
+          when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
+            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
+            references << [:at, scanner[1], nil, start, scanner.pos - 1]
+          else
+            scanner.getch
+          end
+        end
+
+        error_token.code.token_code.references = references
+        build_references(error_token.code.token_code)
+      end
+
+      @_rules.each do |lhs, rhs, _|
+        rhs.each_with_index do |token, index|
+          next if token.class == Lrama::Grammar::Symbol || token.type != Lrama::Lexer::Token::User_code
+
+          scanner = StringScanner.new(token.s_value)
+          references = []
+
+          while !scanner.eos? do
+            start = scanner.pos
+            case
+            # $ references
+            # It need to wrap an identifier with brackets to use ".-" for identifiers
+            when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
+              tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+              references << [:dollar, "$", tag, start, scanner.pos - 1]
+            when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
+              tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+              references << [:dollar, Integer(scanner[2]), tag, start, scanner.pos - 1]
+            when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
+              tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+              references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+            when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
+              tag = scanner[1] ? Lrama::Lexer::Token.new(type: Lrama::Lexer::Token::Tag, s_value: scanner[1]) : nil
+              references << [:dollar, scanner[2], tag, start, scanner.pos - 1]
+
+            # @ references
+            # It need to wrap an identifier with brackets to use ".-" for identifiers
+            when scanner.scan(/@\$/) # @$
+              references << [:at, "$", nil, start, scanner.pos - 1]
+            when scanner.scan(/@(\d+)/) # @1
+              references << [:at, Integer(scanner[1]), nil, start, scanner.pos - 1]
+            when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
+              references << [:at, scanner[1], nil, start, scanner.pos - 1]
+            when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
+              references << [:at, scanner[1], nil, start, scanner.pos - 1]
+
+            when scanner.scan(/\/\*/)
+              scanner.scan_until(/\*\//)
+            else
+              scanner.getch
+            end
+          end
+
+          token.references = references
+          token.numberize_references(lhs, rhs)
+          build_references(token)
+        end
+      end
+    end
+
+    def create_token(type, s_value, line, column)
+      t = Token.new(type: type, s_value: s_value)
+      t.line = line
+      t.column = column
+
+      return t
+    end
+
     private
 
     def find_nterm_by_id!(id)
@@ -470,7 +654,9 @@ module Lrama
 
     # Fill #number and #token_id
     def fill_symbol_number
-      # TODO: why start from 256
+      # Character literal in grammar file has
+      # token id corresponding to ASCII code by default,
+      # so start token_id from 256.
       token_id = 256
 
       # YYEMPTY = -2