lrama 0.7.0 → 0.7.1

data/Rakefile

@@ -5,7 +5,7 @@ require "bundler/gem_tasks"
 namespace "build" do
   desc "build parser from parser.y"
   task :parser do
-    sh "bundle exec racc parser.y --embedded -o lib/lrama/parser.rb -t --log-file=parser.output"
+    sh "bundle exec racc parser.y --embedded --frozen -o lib/lrama/parser.rb -t --log-file=parser.output"
   end
 end
 
@@ -26,10 +26,11 @@ desc "steep check"
 task :steep do
   sh "bundle exec steep check"
 end
+task :steep => :rbs_inline
 
 desc "Run rbs-inline"
 task :rbs_inline do
   sh "bundle exec rbs-inline --output lib/"
 end
 
-task default: %i[spec rbs_inline steep]
+task default: %i[spec steep]

data/Steepfile

@@ -6,19 +6,25 @@ target :lib do
   repo_path '.gem_rbs_collection/'
   signature "sig"
 
-  check "lib/lrama/grammar"
-
   check "lib/lrama/counterexamples"
+  check "lib/lrama/grammar"
   check "lib/lrama/lexer"
-  check "lib/lrama/report"
+  check "lib/lrama/reporter"
   check "lib/lrama/state"
   check "lib/lrama/states"
+  check "lib/lrama/tracer"
+  check "lib/lrama/warnings"
   check "lib/lrama/bitmap.rb"
   check "lib/lrama/counterexamples.rb"
   check "lib/lrama/digraph.rb"
+  check "lib/lrama/erb.rb"
   check "lib/lrama/grammar.rb"
+  check "lib/lrama/lexer.rb"
   check "lib/lrama/logger.rb"
+  check "lib/lrama/option_parser.rb"
   check "lib/lrama/options.rb"
-  check "lib/lrama/trace_reporter.rb"
-  check "lib/lrama/warning.rb"
+  check "lib/lrama/reporter.rb"
+  check "lib/lrama/tracer.rb"
+  check "lib/lrama/version.rb"
+  check "lib/lrama/warnings.rb"
 end

data/doc/Index.md

@@ -49,7 +49,7 @@ Enter the formula:
 
 ## Supported Ruby version
 
-Lrama is executed with BASERUBY when building ruby from source code. Therefore Lrama needs to support BASERUBY, currently 2.5, or later version.
+Lrama is executed with BASERUBY when building ruby from source code. Therefore Lrama needs to support BASERUBY, currently 3.1, or later version.
 
 This also requires Lrama to be able to run with only default gems because BASERUBY runs with `--disable=gems` option.
 

data/doc/development/compressed_state_table/parser.rb

@@ -1,3 +1,4 @@
+# :nodoc: all
 class Parser
   YYNTOKENS = 9
   YYLAST = 13
@@ -253,6 +254,7 @@ class Parser
   end
 end
 
+# :nodoc: all
 class Lexer
   def initialize(tokens)
     @tokens = tokens

data/doc/development/profiling.md

@@ -0,0 +1,44 @@
+# Profiling Lrama
+
+To help improve Lrama's processing speed and reduce its memory usage, Lrama supports two profiling options:
+
+* Call-stack profiling using the stackprof gem
+* Memory profiling using the memory_profiler gem
+
+## Call-stack Profiling Lrama
+
+### 1. Create parse.tmp.y in ruby/ruby
+
+```shell
+$ ruby tool/id2token.rb parse.y > parse.tmp.y
+$ cp parse.tmp.y dir/lrama/tmp
+```
+
+### 2. Run Lrama
+
+```shell
+$ exe/lrama -o parse.tmp.c --header=parse.tmp.h --profile=call-stack tmp/parse.tmp.y
+```
+
+### 3. Generate Flamegraph
+
+```shell
+$ stackprof --d3-flamegraph tmp/stackprof-cpu-myapp.dump > tmp/flamegraph.html
+```
+
+## Memory Profiling Lrama
+
+### 1. Create parse.tmp.y in ruby/ruby
+
+```shell
+$ ruby tool/id2token.rb parse.y > parse.tmp.y
+$ cp parse.tmp.y dir/lrama/tmp
+```
+
+### 2. Run Lrama
+
+```shell
+$ exe/lrama -o parse.tmp.c --header=parse.tmp.h --profile=memory tmp/parse.tmp.y
+```
+
+Then "tmp/memory_profiler.txt" is generated.

data/exe/lrama

@@ -4,4 +4,4 @@
 $LOAD_PATH << File.join(__dir__, "../lib")
 require "lrama"
 
-Lrama::Command.new.run(ARGV.dup)
+Lrama::Command.new(ARGV.dup).run

data/lib/lrama/bitmap.rb

@@ -3,7 +3,10 @@
 
 module Lrama
   module Bitmap
-    # @rbs (Array[Integer] ary) -> Integer
+    # @rbs!
+    #   type bitmap = Integer
+
+    # @rbs (Array[Integer] ary) -> bitmap
     def self.from_array(ary)
       bit = 0
 
@@ -14,21 +17,31 @@ module Lrama
       bit
     end
 
-    # @rbs (Integer int) -> Array[Integer]
+    # @rbs (Integer int) -> bitmap
+    def self.from_integer(int)
+      1 << int
+    end
+
+    # @rbs (bitmap int) -> Array[Integer]
     def self.to_array(int)
       a = [] #: Array[Integer]
       i = 0
 
-      while int > 0 do
-        if int & 1 == 1
+      len = int.bit_length
+      while i < len do
+        if int[i] == 1
           a << i
         end
 
         i += 1
-        int >>= 1
       end
 
       a
     end
+
+    # @rbs (bitmap int, Integer size) -> Array[bool]
+    def self.to_bool_array(int, size)
+      Array.new(size) { |i| int[i] == 1 }
+    end
   end
 end

data/lib/lrama/command.rb

@@ -5,64 +5,116 @@ module Lrama
     LRAMA_LIB = File.realpath(File.join(File.dirname(__FILE__)))
     STDLIB_FILE_PATH = File.join(LRAMA_LIB, 'grammar', 'stdlib.y')
 
-    def run(argv)
-      begin
-        options = OptionParser.new.parse(argv)
-      rescue => e
-        message = e.message
-        message = message.gsub(/.+/, "\e[1m\\&\e[m") if Exception.to_tty?
-        abort message
-      end
-
-      Report::Duration.enable if options.trace_opts[:time]
+    def initialize(argv)
+      @logger = Lrama::Logger.new
+      @options = OptionParser.parse(argv)
+      @tracer = Tracer.new(STDERR, **@options.trace_opts)
+      @reporter = Reporter.new(**@options.report_opts)
+      @warnings = Warnings.new(@logger, @options.warnings)
+    rescue => e
+      abort format_error_message(e.message)
+    end
 
-      text = options.y.read
-      options.y.close if options.y != STDIN
-      begin
-        grammar = Lrama::Parser.new(text, options.grammar_file, options.debug, options.define).parse
-        unless grammar.no_stdlib
-          stdlib_grammar = Lrama::Parser.new(File.read(STDLIB_FILE_PATH), STDLIB_FILE_PATH, options.debug).parse
-          grammar.insert_before_parameterizing_rules(stdlib_grammar.parameterizing_rules)
+    def run
+      Lrama::Reporter::Profile::CallStack.report(@options.profile_opts[:call_stack]) do
+        Lrama::Reporter::Profile::Memory.report(@options.profile_opts[:memory]) do
+          execute_command_workflow
         end
-        grammar.prepare
-        grammar.validate!
-      rescue => e
-        raise e if options.debug
-        message = e.message
-        message = message.gsub(/.+/, "\e[1m\\&\e[m") if Exception.to_tty?
-        abort message
       end
-      states = Lrama::States.new(grammar, trace_state: (options.trace_opts[:automaton] || options.trace_opts[:closure]))
+    end
+
+    private
+
+    def execute_command_workflow
+      @tracer.enable_duration
+      text = read_input
+      grammar = build_grammar(text)
+      states, context = compute_status(grammar)
+      render_reports(states) if @options.report_file
+      @tracer.trace(grammar)
+      render_diagram(grammar)
+      render_output(context, grammar)
+      states.validate!(@logger)
+      @warnings.warn(grammar, states)
+    end
+
+    def read_input
+      text = @options.y.read
+      @options.y.close unless @options.y == STDIN
+      text
+    end
+
+    def build_grammar(text)
+      grammar =
+        Lrama::Parser.new(text, @options.grammar_file, @options.debug, @options.locations, @options.define).parse
+      merge_stdlib(grammar)
+      prepare_grammar(grammar)
+      grammar
+    rescue => e
+      raise e if @options.debug
+      abort format_error_message(e.message)
+    end
+
+    def format_error_message(message)
+      return message unless Exception.to_tty?
+
+      message.gsub(/.+/, "\e[1m\\&\e[m")
+    end
+
+    def merge_stdlib(grammar)
+      return if grammar.no_stdlib
+
+      stdlib_text = File.read(STDLIB_FILE_PATH)
+      stdlib_grammar = Lrama::Parser.new(
+        stdlib_text,
+        STDLIB_FILE_PATH,
+        @options.debug,
+        @options.locations,
+        @options.define,
+      ).parse
+
+      grammar.prepend_parameterized_rules(stdlib_grammar.parameterized_rules)
+    end
+
+    def prepare_grammar(grammar)
+      grammar.prepare
+      grammar.validate!
+    end
+
+    def compute_status(grammar)
+      states = Lrama::States.new(grammar, @tracer)
       states.compute
       states.compute_ielr if grammar.ielr_defined?
-      context = Lrama::Context.new(states)
+      [states, Lrama::Context.new(states)]
+    end
 
-      if options.report_file
-        reporter = Lrama::StatesReporter.new(states)
-        File.open(options.report_file, "w+") do |f|
-          reporter.report(f, **options.report_opts)
-        end
+    def render_reports(states)
+      File.open(@options.report_file, "w+") do |f|
+        @reporter.report(f, states)
       end
+    end
 
-      reporter = Lrama::TraceReporter.new(grammar)
-      reporter.report(**options.trace_opts)
+    def render_diagram(grammar)
+      return unless @options.diagram
 
-      File.open(options.outfile, "w+") do |f|
+      File.open(@options.diagram_file, "w+") do |f|
+        Lrama::Diagram.render(out: f, grammar: grammar)
+      end
+    end
+
+    def render_output(context, grammar)
+      File.open(@options.outfile, "w+") do |f|
         Lrama::Output.new(
           out: f,
-          output_file_path: options.outfile,
-          template_name: options.skeleton,
-          grammar_file_path: options.grammar_file,
-          header_file_path: options.header_file,
+          output_file_path: @options.outfile,
+          template_name: @options.skeleton,
+          grammar_file_path: @options.grammar_file,
+          header_file_path: @options.header_file,
           context: context,
           grammar: grammar,
-          error_recovery: options.error_recovery,
+          error_recovery: @options.error_recovery,
         ).render
       end
-
-      logger = Lrama::Logger.new
-      exit false unless Lrama::GrammarValidator.new(grammar, states, logger).valid?
-      Lrama::Diagnostics.new(grammar, states, logger).run(options.diagnostic)
     end
   end
 end

data/lib/lrama/context.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require_relative "report/duration"
+require_relative "tracer/duration"
 
 module Lrama
   # This is passed to a template
   class Context
-    include Report::Duration
+    include Tracer::Duration
 
     ErrorActionNumber = -Float::INFINITY
     BaseMin = -Float::INFINITY
@@ -231,8 +231,8 @@ module Lrama
         end
 
         # Shift is selected when S/R conflict exists.
-        state.selected_term_transitions.each do |shift, next_state|
-          actions[shift.next_sym.number] = next_state.id
+        state.selected_term_transitions.each do |shift|
+          actions[shift.next_sym.number] = shift.to_state.id
         end
 
         state.resolved_conflicts.select do |conflict|
@@ -292,18 +292,18 @@ module Lrama
       # of a default nterm transition destination.
       @yydefgoto = Array.new(@states.nterms.count, 0)
       # Mapping from nterm to next_states
-      nterm_to_next_states = {}
+      nterm_to_to_states = {}
 
       @states.states.each do |state|
-        state.nterm_transitions.each do |shift, next_state|
-          key = shift.next_sym
-          nterm_to_next_states[key] ||= []
-          nterm_to_next_states[key] << [state, next_state] # [from_state, to_state]
+        state.nterm_transitions.each do |goto|
+          key = goto.next_sym
+          nterm_to_to_states[key] ||= []
+          nterm_to_to_states[key] << [state, goto.to_state] # [from_state, to_state]
         end
       end
 
       @states.nterms.each do |nterm|
-        if (states = nterm_to_next_states[nterm])
+        if (states = nterm_to_to_states[nterm])
           default_state = states.map(&:last).group_by {|s| s }.max_by {|_, v| v.count }.first
           default_goto = default_state.id
           not_default_gotos = []
@@ -417,27 +417,25 @@ module Lrama
 
         res = lowzero - froms_and_tos.first[0]
 
+        # Find the smallest `res` such that `@table[res + from]` is empty for all `from` in `froms_and_tos`
         while true do
-          ok = true
+          advanced = false
 
-          froms_and_tos.each do |from, to|
-            loc = res + from
-
-            if @table[loc]
-              # If the cell of table is set, can not use the cell.
-              ok = false
-              break
-            end
+          while used_res[res]
+            res += 1
+            advanced = true
           end
 
-          if ok && used_res[res]
-            ok = false
+          froms_and_tos.each do |from, to|
+            while @table[res + from]
+              res += 1
+              advanced = true
+            end
           end
 
-          if ok
+          unless advanced
+            # no advance means that the current `res` satisfies the condition
             break
-          else
-            res += 1
           end
         end
 

data/lib/lrama/counterexamples/derivation.rb

@@ -1,34 +1,44 @@
+# rbs_inline: enabled
 # frozen_string_literal: true
 
 module Lrama
   class Counterexamples
     class Derivation
-      attr_reader :item, :left, :right
-      attr_writer :right
+      # @rbs!
+      #   @item: State::Item
+      #   @left: Derivation?
 
-      def initialize(item, left, right = nil)
+      attr_reader :item  #: State::Item
+      attr_reader :left  #: Derivation?
+      attr_accessor :right #: Derivation?
+
+      # @rbs (State::Item item, Derivation? left) -> void
+      def initialize(item, left)
         @item = item
         @left = left
-        @right = right
       end
 
+      # @rbs () -> ::String
       def to_s
         "#<Derivation(#{item.display_name})>"
       end
       alias :inspect :to_s
 
+      # @rbs () -> Array[String]
       def render_strings_for_report
         result = [] #: Array[String]
         _render_for_report(self, 0, result, 0)
         result.map(&:rstrip)
       end
 
+      # @rbs () -> String
       def render_for_report
         render_strings_for_report.join("\n")
       end
 
       private
 
+      # @rbs (Derivation derivation, Integer offset, Array[String] strings, Integer index) -> Integer
       def _render_for_report(derivation, offset, strings, index)
         item = derivation.item
         if strings[index]

data/lib/lrama/counterexamples/example.rb

@@ -1,12 +1,31 @@
+# rbs_inline: enabled
 # frozen_string_literal: true
 
 module Lrama
   class Counterexamples
     class Example
-      attr_reader :path1, :path2, :conflict, :conflict_symbol
+      # TODO: rbs-inline 0.11.0 doesn't support instance variables.
+      #       Move these type declarations above instance variable definitions, once it's supported.
+      #       see: https://github.com/soutaro/rbs-inline/pull/149
+      #
+      # @rbs!
+      #   @path1: ::Array[StateItem]
+      #   @path2: ::Array[StateItem]
+      #   @conflict: State::conflict
+      #   @conflict_symbol: Grammar::Symbol
+      #   @counterexamples: Counterexamples
+      #   @derivations1: Derivation
+      #   @derivations2: Derivation
+
+      attr_reader :path1 #: ::Array[StateItem]
+      attr_reader :path2 #: ::Array[StateItem]
+      attr_reader :conflict #: State::conflict
+      attr_reader :conflict_symbol #: Grammar::Symbol
 
       # path1 is shift conflict when S/R conflict
       # path2 is always reduce conflict
+      #
+      # @rbs (Array[StateItem]? path1, Array[StateItem]? path2, State::conflict conflict, Grammar::Symbol conflict_symbol, Counterexamples counterexamples) -> void
       def initialize(path1, path2, conflict, conflict_symbol, counterexamples)
         @path1 = path1
         @path2 = path2
@@ -15,69 +34,75 @@ module Lrama
         @counterexamples = counterexamples
       end
 
+      # @rbs () -> (:shift_reduce | :reduce_reduce)
       def type
         @conflict.type
       end
 
+      # @rbs () -> State::Item
       def path1_item
-        @path1.last.to.item
+        @path1.last.item
       end
 
+      # @rbs () -> State::Item
       def path2_item
-        @path2.last.to.item
+        @path2.last.item
       end
 
+      # @rbs () -> Derivation
       def derivations1
         @derivations1 ||= _derivations(path1)
       end
 
+      # @rbs () -> Derivation
       def derivations2
         @derivations2 ||= _derivations(path2)
       end
 
       private
 
-      def _derivations(paths)
+      # @rbs (Array[StateItem] state_items) -> Derivation
+      def _derivations(state_items)
         derivation = nil #: Derivation
         current = :production
-        last_path = paths.last #: Path
-        lookahead_sym = last_path.to.item.end_of_rule? ? @conflict_symbol : nil
+        last_state_item = state_items.last #: StateItem
+        lookahead_sym = last_state_item.item.end_of_rule? ? @conflict_symbol : nil
 
-        paths.reverse_each do |path|
-          item = path.to.item
+        state_items.reverse_each do |si|
+          item = si.item
 
           case current
           when :production
-            case path
-            when StartPath
+            case si.type
+            when :start
               derivation = Derivation.new(item, derivation)
               current = :start
-            when TransitionPath
+            when :transition
               derivation = Derivation.new(item, derivation)
               current = :transition
-            when ProductionPath
+            when :production
               derivation = Derivation.new(item, derivation)
               current = :production
             else
-              raise "Unexpected. #{path}"
+              raise "Unexpected. #{si}"
             end
 
             if lookahead_sym && item.next_next_sym && item.next_next_sym.first_set.include?(lookahead_sym)
-              state_item = @counterexamples.transitions[[path.to, item.next_sym]]
-              derivation2 = find_derivation_for_symbol(state_item, lookahead_sym)
+              si2 = @counterexamples.transitions[[si, item.next_sym]]
+              derivation2 = find_derivation_for_symbol(si2, lookahead_sym)
               derivation.right = derivation2 # steep:ignore
               lookahead_sym = nil
             end
 
           when :transition
-            case path
-            when StartPath
+            case si.type
+            when :start
               derivation = Derivation.new(item, derivation)
               current = :start
-            when TransitionPath
+            when :transition
               # ignore
               current = :transition
-            when ProductionPath
+            when :production
               # ignore
               current = :production
             end
@@ -91,6 +116,7 @@ module Lrama
         derivation
       end
 
+      # @rbs (StateItem state_item, Grammar::Symbol sym) -> Derivation?
       def find_derivation_for_symbol(state_item, sym)
         queue = [] #: Array[Array[StateItem]]
         queue << [state_item]
@@ -110,9 +136,8 @@ module Lrama
           end
 
           if next_sym.nterm? && next_sym.first_set.include?(sym)
-            @counterexamples.productions[si].each do |next_item|
-              next if next_item.empty_rule?
-              next_si = StateItem.new(si.state, next_item)
+            @counterexamples.productions[si].each do |next_si|
+              next if next_si.item.empty_rule?
               next if sis.include?(next_si)
               queue << (sis + [next_si])
             end

data/lib/lrama/counterexamples/node.rb

@@ -0,0 +1,30 @@
+# rbs_inline: enabled
+# frozen_string_literal: true
+
+module Lrama
+  class Counterexamples
+    # @rbs generic E < Object -- Type of an element
+    class Node
+      attr_reader :elem #: E
+      attr_reader :next_node #: Node[E]?
+
+      # @rbs [E < Object] (Node[E] node) -> Array[E]
+      def self.to_a(node)
+        a = [] # steep:ignore UnannotatedEmptyCollection
+
+        while (node)
+          a << node.elem
+          node = node.next_node
+        end
+
+        a
+      end
+
+      # @rbs (E elem, Node[E]? next_node) -> void
+      def initialize(elem, next_node)
+        @elem = elem
+        @next_node = next_node
+      end
+    end
+  end
+end