lrama 0.5.2 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 26bcef726eebf61de01200b02250b2aef91ca957df43db5d72e99ae30e320f4e
-  data.tar.gz: 751b62d184806c0cff319bffc4990ad0de9dd143accafdf6bdf1652eb02512b7
+  metadata.gz: f2e675399217ba6b1c8cc57aaa36bbf863ed8b22dc3e22777c88d0d0aaf1cb26
+  data.tar.gz: 940b0da60c6b25edb1c10ed80539b17617e033d7ccc3a5c9c6036959c356ae37
 SHA512:
-  metadata.gz: 02001b5b137d023c4f5ce48c1d9284ce90c5aad2ec269d7ef7a17e6ffa65be9e553e213400fc63d8256c0209908fc4b314de261993442af4a25772449453fbeb
-  data.tar.gz: 76a9f6bac19c0dcbdc6f9e2834d3c437f9fc5eb999d1b45d20362d52a94c9dc74bca3411b681993e28f388374df7f9b8ea7d0f534e53399efb8cc24112f75ea4
+  metadata.gz: 26eb48911cf5ba12b3087390bb7f7053165ff092a8919f7969e31b3e784a0085e876140d419cf4b463426cbfc4ae5f3625504a6c1433f9f9842ee9c32490082f
+  data.tar.gz: 8bce42121ffb5d45076dc3a1664e61209fec0f1a06a08d50d3a4279eb5158911b35165e21f8de0ac87c19e7cab1d2b7b0cac16bcaba0606ec61ea3b1c6195e0b

data/.github/workflows/test.yaml

@@ -22,6 +22,15 @@ jobs:
           bundler-cache: true
       - run: bundle install
       - run: bundle exec rspec
+  check-misc:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      # Copy from https://github.com/ruby/ruby/blob/089227e94823542acfdafa68541d330eee42ffea/.github/workflows/check_misc.yml#L27
+      - name: Check for trailing spaces
+        run: |
+          git grep -I -n '[	 ]$' -- '*.rb' '*.[chy]' '*.rs' && exit 1 || :
+          git grep -n '^[	 ][	 ]*$' -- '*.md' && exit 1 || :
   steep-check:
     runs-on: ubuntu-20.04
     strategy:
@@ -58,7 +67,7 @@ jobs:
       - run: mkdir -p tool/lrama
         working-directory: ../ruby
       - name: Copy Lrama to ruby/tool
-        run: cp -r exe lib template ../ruby/tool/lrama
+        run: cp -r LEGAL.md MIT exe lib template ../ruby/tool/lrama
         working-directory:
       - run: tree tool/lrama
         working-directory: ../ruby

data/.gitignore

@@ -3,3 +3,4 @@
 .irbrc
 /Gemfile.lock
 /pkg/
+coverage/

data/Gemfile

@@ -8,3 +8,4 @@ gem "stackprof"
 gem "rake"
 gem "rbs", require: false
 gem "steep", require: false
+gem "simplecov", require: false

data/LEGAL.md

@@ -5,22 +5,7 @@ mentioned below.
 
 ## GNU General Public License version 3
 
-These files are licensed under the GNU General Public License version 3. See these files for more information.
+These files are licensed under the GNU General Public License version 3 or later. See these files for more information.
 
 * template/bison/yacc.c
 * template/bison/yacc.h
-
-## Same with Ruby
-
-These files are licensed same with Ruby. See https://github.com/ruby/ruby/blob/master/COPYING for more information.
-
-* spec/fixtures/integration/ruby_3_0_5/parse.tmp.y
-* spec/fixtures/integration/ruby_3_0_5/y.tab.c
-* spec/fixtures/integration/ruby_3_0_5/y.tab.h
-* spec/fixtures/integration/ruby_3_1_0/parse.tmp.y
-* spec/fixtures/integration/ruby_3_1_0/y.tab.c
-* spec/fixtures/integration/ruby_3_1_0/y.tab.h
-* spec/fixtures/integration/ruby_3_2_0/parse.tmp.y
-* spec/fixtures/integration/ruby_3_2_0/y.tab.c
-* spec/fixtures/integration/ruby_3_2_0/y.tab.h
-

data/README.md

@@ -49,10 +49,20 @@ Enter the formula:
 
 ## Versions and Branches
 
+### v0_5 (`master` branch)
+
+This branch is for Ruby 3.3. `lrama_0_5` branch is created from this branch, once Ruby 3.3 is released.
+
 ### v0_4 (`lrama_0_4` branch)
 
 This branch generates "parse.c" compatible with Bison 3.8.2 for ruby 3.0, 3.1, 3.2. The first version migrated to ruby is ["0.4.0"](https://github.com/ruby/ruby/pull/7798) therefore keep this branch for Bison compatible branch.
 
+## Supported Ruby version
+
+Lrama is executed with BASERUBY when building ruby from source code. Therefore Lrama needs to support BASERUBY, currently 2.5, or later version.
+
+This also requires Lrama to be able to run with only default gems and bundled gems.
+
 ## Build Ruby
 
 1. Install Lrama
@@ -62,7 +72,7 @@ This branch generates "parse.c" compatible with Bison 3.8.2 for ruby 3.0, 3.1, 3
 
 1. Update `Lrama::VERSION`
 2. Release as a gem by `rake release`
-3. Update Lrama in ruby/ruby by `cp -r LEGAL.md MIT exe lib ruby/tool/lrama`
+3. Update Lrama in ruby/ruby by `cp -r LEGAL.md MIT exe lib template ruby/tool/lrama`
 4. Create new release on [GitHub](https://github.com/ruby/lrama/releases)
 
 ## License

data/Steepfile

@@ -4,6 +4,7 @@ target :lib do
   signature "sig"
 
   check "lib/lrama/bitmap.rb"
-  check "lib/lrama/report.rb"
+  check "lib/lrama/report/duration.rb"
+  check "lib/lrama/report/profile.rb"
   check "lib/lrama/warning.rb"
 end

data/doc/TODO.md

@@ -1,7 +1,6 @@
 # TODO
 
 * command
-  * [ ] Add "--bison" option
 * lexer
   * [x] Basic functionalities
 * parser
@@ -27,7 +26,7 @@
   * [x] Table compaction
   * [x] -d option
   * yacc.c
-    * [ ] %lex-param
+    * [x] %lex-param
     * [x] %parse-param
     * [x] %printer
     * [x] Replace $, @ in user codes
@@ -45,8 +44,14 @@
 * Reporting
   * [ ] Bison style
     * [ ] Wrap not selected reduce with "[]". See basic.output file generated by Bison.
+* Counterexamples
+  * [x] Nonunifying Counterexamples
+  * [ ] Unifying Counterexamples
+  * [ ] Performance improvement using reverse_transitions and reverse_productions
 * Error Tolerance
-  * [x] Subset of Corchuelo et al.
+  * [x] Corchuelo et al. algorithm with N = 1 (this means the next token when error is raised)
+  * [x] Add new decl for error token semantic value initialization (%error-token)
+  * [x] Use YYMALLOC & YYFREE
 * Lex state
 * CI
   * [x] Setup CI

data/exe/lrama

@@ -4,4 +4,4 @@
 $LOAD_PATH << File.join(__dir__, "../lib")
 require "lrama"
 
-Lrama::Command.new.run(ARGV.dup)
+Lrama::Command.new(ARGV.dup).run

data/lib/lrama/command.rb

@@ -2,94 +2,57 @@ require 'optparse'
 
 module Lrama
   class Command
-    def run(argv)
-      opt = OptionParser.new
-
-      # opt.on('-h') {|v| p v }
-      opt.on('-V', '--version') {|v| puts Lrama::VERSION ; exit 0 }
-
-      # Tuning the Parser
-      skeleton = "bison/yacc.c"
-
-      opt.on('-S', '--skeleton=FILE') {|v| skeleton = v }
-      opt.on('-t') {  } # Do nothing
-
-      # Output Files:
-      header = false
-      header_file = nil
-      report = []
-      report_file = nil
-      outfile = "y.tab.c"
-
-      opt.on('-h', '--header=[FILE]') {|v| header = true; header_file = v }
-      opt.on('-d') { header = true }
-      opt.on('-r', '--report=THINGS') {|v| report = v.split(',') }
-      opt.on('--report-file=FILE')    {|v| report_file = v }
-      opt.on('-v') {  } # Do nothing
-      opt.on('-o', '--output=FILE')   {|v| outfile = v }
-
-      # Hidden
-      trace = []
-      opt.on('--trace=THINGS') {|v| trace = v.split(',') }
-
-      # Error Recovery
-      error_recovery = false
-      opt.on('-e') {|v| error_recovery = true }
-
-      opt.parse!(argv)
-
-      trace_opts = validate_trace(trace)
-      report_opts = validate_report(report)
-
-      grammar_file = argv.shift
-
-      if !report.empty? && report_file.nil? && grammar_file
-        report_file = File.dirname(grammar_file) + "/" + File.basename(grammar_file, ".*") + ".output"
-      end
+    def initialize(argv)
+      @argv = argv
+
+      @version = nil
+      @skeleton = "bison/yacc.c"
+      @header = false
+      @header_file = nil
+      @report = []
+      @report_file = nil
+      @outfile = "y.tab.c"
+      @trace = []
+      @error_recovery = false
+      @grammar_file = nil
+      @report_file = nil
+      @trace_opts = nil
+      @report_opts = nil
+    end
 
-      if !header_file && header
-        case
-        when outfile
-          header_file = File.dirname(outfile) + "/" + File.basename(outfile, ".*") + ".h"
-        when grammar_file
-          header_file = File.dirname(grammar_file) + "/" + File.basename(grammar_file, ".*") + ".h"
-        end
-      end
+    def run
+      parse_option
 
-      if !grammar_file
-        abort "File should be specified\n"
+      if @version
+        puts Lrama::VERSION
+        exit 0
       end
 
-      Report::Duration.enable if trace_opts[:time]
+      Report::Duration.enable if @trace_opts[:time]
 
       warning = Lrama::Warning.new
-      if grammar_file == '-'
-        grammar_file = argv.shift or abort "File name for STDIN should be specified\n"
-        y = STDIN.read
-      else
-        y = File.read(grammar_file)
-      end
-      grammar = Lrama::Parser.new(y).parse
-      states = Lrama::States.new(grammar, warning, trace_state: (trace_opts[:automaton] || trace_opts[:closure]))
+      grammar = Lrama::Parser.new(@y.read).parse
+      states = Lrama::States.new(grammar, warning, trace_state: (@trace_opts[:automaton] || @trace_opts[:closure]))
       states.compute
       context = Lrama::Context.new(states)
 
-      if report_file
+      if @report_file
         reporter = Lrama::StatesReporter.new(states)
-        File.open(report_file, "w+") do |f|
-          reporter.report(f, **report_opts)
+        File.open(@report_file, "w+") do |f|
+          reporter.report(f, **@report_opts)
         end
       end
 
-      File.open(outfile, "w+") do |f|
+      File.open(@outfile, "w+") do |f|
         Lrama::Output.new(
           out: f,
-          output_file_path: outfile,
-          template_name: skeleton,
-          grammar_file_path: grammar_file,
-          header_file_path: header_file,
+          output_file_path: @outfile,
+          template_name: @skeleton,
+          grammar_file_path: @grammar_file,
+          header_file_path: @header_file,
           context: context,
           grammar: grammar,
+          error_recovery: @error_recovery,
         ).render
       end
 
@@ -104,7 +67,7 @@ module Lrama
       bison_list = %w[states itemsets lookaheads solved counterexamples cex all none]
       others = %w[verbose]
       list = bison_list + others
-      not_supported = %w[counterexamples cex none]
+      not_supported = %w[cex none]
       h = { grammar: true }
 
       report.each do |r|
@@ -144,5 +107,61 @@ module Lrama
 
       return h
     end
+
+    def parse_option
+      opt = OptionParser.new
+
+      # opt.on('-h') {|v| p v }
+      opt.on('-V', '--version') {|v| @version = true }
+
+      # Tuning the Parser
+      opt.on('-S', '--skeleton=FILE') {|v| @skeleton = v }
+      opt.on('-t') {  } # Do nothing
+
+      # Output Files:
+      opt.on('-h', '--header=[FILE]') {|v| @header = true; @header_file = v }
+      opt.on('-d') { @header = true }
+      opt.on('-r', '--report=THINGS', Array) {|v| @report = v }
+      opt.on('--report-file=FILE')    {|v| @report_file = v }
+      opt.on('-v') {  } # Do nothing
+      opt.on('-o', '--output=FILE')   {|v| @outfile = v }
+
+      # Hidden
+      opt.on('--trace=THINGS', Array) {|v| @trace = v }
+
+      # Error Recovery
+      opt.on('-e') {|v| @error_recovery = true }
+
+      opt.parse!(@argv)
+
+      @trace_opts = validate_trace(@trace)
+      @report_opts = validate_report(@report)
+
+      @grammar_file = @argv.shift
+
+      if !@grammar_file
+        abort "File should be specified\n"
+      end
+
+      if @grammar_file == '-'
+        @grammar_file = @argv.shift or abort "File name for STDIN should be specified\n"
+        @y = STDIN
+      else
+        @y = File.open(@grammar_file, 'r')
+      end
+
+      if !@report.empty? && @report_file.nil? && @grammar_file
+        @report_file = File.dirname(@grammar_file) + "/" + File.basename(@grammar_file, ".*") + ".output"
+      end
+
+      if !@header_file && @header
+        case
+        when @outfile
+          @header_file = File.dirname(@outfile) + "/" + File.basename(@outfile, ".*") + ".h"
+        when @grammar_file
+          @header_file = File.dirname(@grammar_file) + "/" + File.basename(@grammar_file, ".*") + ".h"
+        end
+      end
+    end
   end
 end

data/lib/lrama/context.rb

@@ -1,4 +1,4 @@
-require "lrama/report"
+require "lrama/report/duration"
 
 module Lrama
   # This is passed to a template
@@ -89,6 +89,16 @@ module Lrama
       return a
     end
 
+    def yytranslate_inverted
+      a = Array.new(@states.symbols.count, @states.undef_symbol.token_id)
+
+      @states.terms.each do |term|
+        a[term.number] = term.token_id
+      end
+
+      return a
+    end
+
     # Mapping from rule number to line number of the rule is defined.
     # Dummy rule is appended as the first element whose value is 0
     # because 0 means error in yydefact.

data/lib/lrama/counterexamples/derivation.rb

@@ -0,0 +1,63 @@
+module Lrama
+  class Counterexamples
+    class Derivation
+      attr_reader :item, :left, :right
+      attr_writer :right
+
+      def initialize(item, left, right = nil)
+        @item = item
+        @left = left
+        @right = right
+      end
+
+      def to_s
+        "#<Derivation(#{item.display_name})>"
+      end
+      alias :inspect :to_s
+
+      def render_strings_for_report
+        result = []
+        _render_for_report(self, 0, result, 0)
+        result.map(&:rstrip)
+      end
+
+      def render_for_report
+        render_strings_for_report.join("\n")
+      end
+
+      private
+
+      def _render_for_report(derivation, offset, strings, index)
+        item = derivation.item
+        if strings[index]
+          strings[index] << " " * (offset - strings[index].length)
+        else
+          strings[index] = " " * offset
+        end
+        str = strings[index]
+        str << "#{item.rule_id}: #{item.symbols_before_dot.map(&:display_name).join(" ")} "
+
+        if derivation.left
+          len = str.length
+          str << "#{item.next_sym.display_name}"
+          length = _render_for_report(derivation.left, len, strings, index + 1)
+          # I want String#ljust!
+          str << " " * (length - str.length)
+        else
+          str << " • #{item.symbols_after_dot.map(&:display_name).join(" ")} "
+          return str.length
+        end
+
+        if derivation.right&.left
+          length = _render_for_report(derivation.right.left, str.length, strings, index + 1)
+          str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} "
+          str << " " * (length - str.length) if length > str.length
+        elsif item.next_next_sym
+          str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} "
+        end
+
+        return str.length
+      end
+    end
+  end
+end

data/lib/lrama/counterexamples/example.rb

@@ -0,0 +1,124 @@
+module Lrama
+  class Counterexamples
+    class Example
+      attr_reader :path1, :path2, :conflict, :conflict_symbol
+
+      # path1 is shift conflict when S/R conflict
+      # path2 is always reduce conflict
+      def initialize(path1, path2, conflict, conflict_symbol, counterexamples)
+        @path1 = path1
+        @path2 = path2
+        @conflict = conflict
+        @conflict_symbol = conflict_symbol
+        @counterexamples = counterexamples
+      end
+
+      def type
+        @conflict.type
+      end
+
+      def path1_item
+        @path1.last.to.item
+      end
+
+      def path2_item
+        @path2.last.to.item
+      end
+
+      def derivations1
+        @derivations1 ||= _derivations(path1)
+      end
+
+      def derivations2
+        @derivations2 ||= _derivations(path2)
+      end
+
+      private
+
+      def _derivations(paths)
+        derivation = nil
+        current = :production
+        lookahead_sym = paths.last.to.item.end_of_rule? ? @conflict_symbol : nil
+
+        paths.reverse.each do |path|
+          item = path.to.item
+
+          case current
+          when :production
+            case path
+            when StartPath
+              derivation = Derivation.new(item, derivation)
+              current = :start
+            when TransitionPath
+              derivation = Derivation.new(item, derivation)
+              current = :transition
+            when ProductionPath
+              derivation = Derivation.new(item, derivation)
+              current = :production
+            end
+
+            if lookahead_sym && item.next_next_sym && item.next_next_sym.first_set.include?(lookahead_sym)
+              state_item = @counterexamples.transitions[[path.to, item.next_sym]]
+              derivation2 = find_derivation_for_symbol(state_item, lookahead_sym)
+              derivation.right = derivation2
+              lookahead_sym = nil
+            end
+
+          when :transition
+            case path
+            when StartPath
+              derivation = Derivation.new(item, derivation)
+              current = :start
+            when TransitionPath
+              # ignore
+              current = :transition
+            when ProductionPath
+              # ignore
+              current = :production
+            end
+          else
+            raise "BUG: Unknown #{current}"
+          end
+
+          break if current == :start
+        end
+
+        derivation
+      end
+
+      def find_derivation_for_symbol(state_item, sym)
+        queue = []
+        queue << [state_item]
+
+        while (sis = queue.shift)
+          si = sis.last
+          next_sym = si.item.next_sym
+
+          if next_sym == sym
+            derivation = nil
+
+            sis.reverse.each do |si|
+              derivation = Derivation.new(si.item, derivation)
+            end
+
+            return derivation
+          end
+
+          if next_sym.nterm? && next_sym.first_set.include?(sym)
+            @counterexamples.productions[si].each do |next_item|
+              next if next_item.empty_rule?
+              next_si = StateItem.new(si.state, next_item)
+              next if sis.include?(next_si)
+              queue << (sis + [next_si])
+            end
+
+            if next_sym.nullable
+              next_si = @counterexamples.transitions[[si, next_sym]]
+              queue << (sis + [next_si])
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lrama/counterexamples/path.rb

@@ -0,0 +1,69 @@
+module Lrama
+  class Counterexamples
+    class Path
+      def initialize(from_state_item, to_state_item)
+        @from_state_item = from_state_item
+        @to_state_item = to_state_item
+      end
+
+      def from
+        @from_state_item
+      end
+
+      def to
+        @to_state_item
+      end
+
+      def to_s
+        "#<Path(#{type})>"
+      end
+      alias :inspect :to_s
+    end
+
+    class StartPath < Path
+      def initialize(to_state_item)
+        super nil, to_state_item
+      end
+
+      def type
+        :start
+      end
+
+      def transition?
+        false
+      end
+
+      def production?
+        false
+      end
+    end
+
+    class TransitionPath < Path
+      def type
+        :transition
+      end
+
+      def transition?
+        true
+      end
+
+      def production?
+        false
+      end
+    end
+
+    class ProductionPath < Path
+      def type
+        :production
+      end
+
+      def transition?
+        false
+      end
+
+      def production?
+        true
+      end
+    end
+  end
+end

data/lib/lrama/counterexamples/state_item.rb

@@ -0,0 +1,6 @@
+module Lrama
+  class Counterexamples
+    class StateItem < Struct.new(:state, :item)
+    end
+  end
+end

data/lib/lrama/counterexamples/triple.rb

@@ -0,0 +1,21 @@
+module Lrama
+  class Counterexamples
+    # s: state
+    # itm: item within s
+    # l: precise lookahead set
+    class Triple < Struct.new(:s, :itm, :l)
+      alias :state :s
+      alias :item :itm
+      alias :precise_lookahead_set :l
+
+      def state_item
+        StateItem.new(state, item)
+      end
+
+      def inspect
+        "#{state.inspect}. #{item.display_name}. #{l.map(&:id).map(&:s_value)}"
+      end
+      alias :to_s :inspect
+    end
+  end
+end