enclave 0.1.0

data/ext/enclave/mruby/tools/lrama/lib/lrama/grammar.rb

@@ -0,0 +1,407 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require_relative "grammar/auxiliary"
+require_relative "grammar/binding"
+require_relative "grammar/code"
+require_relative "grammar/counter"
+require_relative "grammar/destructor"
+require_relative "grammar/error_token"
+require_relative "grammar/parameterizing_rule"
+require_relative "grammar/percent_code"
+require_relative "grammar/precedence"
+require_relative "grammar/printer"
+require_relative "grammar/reference"
+require_relative "grammar/rule"
+require_relative "grammar/rule_builder"
+require_relative "grammar/symbol"
+require_relative "grammar/symbols"
+require_relative "grammar/type"
+require_relative "grammar/union"
+require_relative "lexer"
+
+module Lrama
+  # Grammar is the result of parsing an input grammar file
+  class Grammar
+    extend Forwardable
+
+    attr_reader :percent_codes, :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol, :aux, :parameterizing_rule_resolver
+    attr_accessor :union, :expect, :printers, :error_tokens, :lex_param, :parse_param, :initial_action,
+                  :after_shift, :before_reduce, :after_reduce, :after_shift_error_token, :after_pop_stack,
+                  :symbols_resolver, :types, :rules, :rule_builders, :sym_to_rules, :no_stdlib, :locations, :define
+
+    def_delegators "@symbols_resolver", :symbols, :nterms, :terms, :add_nterm, :add_term, :find_term_by_s_value,
+                                        :find_symbol_by_number!, :find_symbol_by_id!, :token_to_symbol,
+                                        :find_symbol_by_s_value!, :fill_symbol_number, :fill_nterm_type,
+                                        :fill_printer, :fill_destructor, :fill_error_token, :sort_by_number!
+
+    def initialize(rule_counter, define = {})
+      @rule_counter = rule_counter
+
+      # Code defined by "%code"
+      @percent_codes = []
+      @printers = []
+      @destructors = []
+      @error_tokens = []
+      @symbols_resolver = Grammar::Symbols::Resolver.new
+      @types = []
+      @rule_builders = []
+      @rules = []
+      @sym_to_rules = {}
+      @parameterizing_rule_resolver = ParameterizingRule::Resolver.new
+      @empty_symbol = nil
+      @eof_symbol = nil
+      @error_symbol = nil
+      @undef_symbol = nil
+      @accept_symbol = nil
+      @aux = Auxiliary.new
+      @no_stdlib = false
+      @locations = false
+      @define = define.map {|d| d.split('=') }.to_h
+
+      append_special_symbols
+    end
+
+    def create_rule_builder(rule_counter, midrule_action_counter)
+      RuleBuilder.new(rule_counter, midrule_action_counter, @parameterizing_rule_resolver)
+    end
+
+    def add_percent_code(id:, code:)
+      @percent_codes << PercentCode.new(id.s_value, code.s_value)
+    end
+
+    def add_destructor(ident_or_tags:, token_code:, lineno:)
+      @destructors << Destructor.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno)
+    end
+
+    def add_printer(ident_or_tags:, token_code:, lineno:)
+      @printers << Printer.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno)
+    end
+
+    def add_error_token(ident_or_tags:, token_code:, lineno:)
+      @error_tokens << ErrorToken.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno)
+    end
+
+    def add_type(id:, tag:)
+      @types << Type.new(id: id, tag: tag)
+    end
+
+    def add_nonassoc(sym, precedence)
+      set_precedence(sym, Precedence.new(type: :nonassoc, precedence: precedence))
+    end
+
+    def add_left(sym, precedence)
+      set_precedence(sym, Precedence.new(type: :left, precedence: precedence))
+    end
+
+    def add_right(sym, precedence)
+      set_precedence(sym, Precedence.new(type: :right, precedence: precedence))
+    end
+
+    def add_precedence(sym, precedence)
+      set_precedence(sym, Precedence.new(type: :precedence, precedence: precedence))
+    end
+
+    def set_precedence(sym, precedence)
+      raise "" if sym.nterm?
+      sym.precedence = precedence
+    end
+
+    def set_union(code, lineno)
+      @union = Union.new(code: code, lineno: lineno)
+    end
+
+    def add_rule_builder(builder)
+      @rule_builders << builder
+    end
+
+    def add_parameterizing_rule(rule)
+      @parameterizing_rule_resolver.add_parameterizing_rule(rule)
+    end
+
+    def parameterizing_rules
+      @parameterizing_rule_resolver.rules
+    end
+
+    def insert_before_parameterizing_rules(rules)
+      @parameterizing_rule_resolver.rules = rules + @parameterizing_rule_resolver.rules
+    end
+
+    def prologue_first_lineno=(prologue_first_lineno)
+      @aux.prologue_first_lineno = prologue_first_lineno
+    end
+
+    def prologue=(prologue)
+      @aux.prologue = prologue
+    end
+
+    def epilogue_first_lineno=(epilogue_first_lineno)
+      @aux.epilogue_first_lineno = epilogue_first_lineno
+    end
+
+    def epilogue=(epilogue)
+      @aux.epilogue = epilogue
+    end
+
+    def prepare
+      resolve_inline_rules
+      normalize_rules
+      collect_symbols
+      set_lhs_and_rhs
+      fill_default_precedence
+      fill_symbols
+      fill_sym_to_rules
+      compute_nullable
+      compute_first_set
+      set_locations
+    end
+
+    # TODO: More validation methods
+    #
+    # * Validation for no_declared_type_reference
+    def validate!
+      @symbols_resolver.validate!
+      validate_rule_lhs_is_nterm!
+    end
+
+    def find_rules_by_symbol!(sym)
+      find_rules_by_symbol(sym) || (raise "Rules for #{sym} not found")
+    end
+
+    def find_rules_by_symbol(sym)
+      @sym_to_rules[sym.number]
+    end
+
+    def ielr_defined?
+      @define.key?('lr.type') && @define['lr.type'] == 'ielr'
+    end
+
+    private
+
+    def compute_nullable
+      @rules.each do |rule|
+        case
+        when rule.empty_rule?
+          rule.nullable = true
+        when rule.rhs.any?(&:term)
+          rule.nullable = false
+        else
+          # noop
+        end
+      end
+
+      while true do
+        rs  = @rules.select {|e| e.nullable.nil? }
+        nts = nterms.select {|e| e.nullable.nil? }
+        rule_count_1  = rs.count
+        nterm_count_1 = nts.count
+
+        rs.each do |rule|
+          if rule.rhs.all?(&:nullable)
+            rule.nullable = true
+          end
+        end
+
+        nts.each do |nterm|
+          find_rules_by_symbol!(nterm).each do |rule|
+            if rule.nullable
+              nterm.nullable = true
+            end
+          end
+        end
+
+        rule_count_2  = @rules.count {|e| e.nullable.nil? }
+        nterm_count_2 = nterms.count {|e| e.nullable.nil? }
+
+        if (rule_count_1 == rule_count_2) && (nterm_count_1 == nterm_count_2)
+          break
+        end
+      end
+
+      rules.select {|r| r.nullable.nil? }.each do |rule|
+        rule.nullable = false
+      end
+
+      nterms.select {|e| e.nullable.nil? }.each do |nterm|
+        nterm.nullable = false
+      end
+    end
+
+    def compute_first_set
+      terms.each do |term|
+        term.first_set = Set.new([term]).freeze
+        term.first_set_bitmap = Lrama::Bitmap.from_array([term.number])
+      end
+
+      nterms.each do |nterm|
+        nterm.first_set = Set.new([]).freeze
+        nterm.first_set_bitmap = Lrama::Bitmap.from_array([])
+      end
+
+      while true do
+        changed = false
+
+        @rules.each do |rule|
+          rule.rhs.each do |r|
+            if rule.lhs.first_set_bitmap | r.first_set_bitmap != rule.lhs.first_set_bitmap
+              changed = true
+              rule.lhs.first_set_bitmap = rule.lhs.first_set_bitmap | r.first_set_bitmap
+            end
+
+            break unless r.nullable
+          end
+        end
+
+        break unless changed
+      end
+
+      nterms.each do |nterm|
+        nterm.first_set = Lrama::Bitmap.to_array(nterm.first_set_bitmap).map do |number|
+          find_symbol_by_number!(number)
+        end.to_set
+      end
+    end
+
+    def setup_rules
+      @rule_builders.each do |builder|
+        builder.setup_rules
+      end
+    end
+
+    def append_special_symbols
+      # YYEMPTY (token_id: -2, number: -2) is added when a template is evaluated
+      # term = add_term(id: Token.new(Token::Ident, "YYEMPTY"), token_id: -2)
+      # term.number = -2
+      # @empty_symbol = term
+
+      # YYEOF
+      term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYEOF"), alias_name: "\"end of file\"", token_id: 0)
+      term.number = 0
+      term.eof_symbol = true
+      @eof_symbol = term
+
+      # YYerror
+      term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYerror"), alias_name: "error")
+      term.number = 1
+      term.error_symbol = true
+      @error_symbol = term
+
+      # YYUNDEF
+      term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYUNDEF"), alias_name: "\"invalid token\"")
+      term.number = 2
+      term.undef_symbol = true
+      @undef_symbol = term
+
+      # $accept
+      term = add_nterm(id: Lrama::Lexer::Token::Ident.new(s_value: "$accept"))
+      term.accept_symbol = true
+      @accept_symbol = term
+    end
+
+    def resolve_inline_rules
+      while @rule_builders.any?(&:has_inline_rules?) do
+        @rule_builders = @rule_builders.flat_map do |builder|
+          if builder.has_inline_rules?
+            builder.resolve_inline_rules
+          else
+            builder
+          end
+        end
+      end
+    end
+
+    def normalize_rules
+      # Add $accept rule to the top of rules
+      rule_builder = @rule_builders.first # : RuleBuilder
+      lineno = rule_builder ? rule_builder.line : 0
+      @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [rule_builder.lhs, @eof_symbol.id], token_code: nil, lineno: lineno)
+
+      setup_rules
+
+      @rule_builders.each do |builder|
+        builder.rules.each do |rule|
+          add_nterm(id: rule._lhs, tag: rule.lhs_tag)
+          @rules << rule
+        end
+      end
+
+      @rules.sort_by!(&:id)
+    end
+
+    # Collect symbols from rules
+    def collect_symbols
+      @rules.flat_map(&:_rhs).each do |s|
+        case s
+        when Lrama::Lexer::Token::Char
+          add_term(id: s)
+        when Lrama::Lexer::Token
+          # skip
+        else
+          raise "Unknown class: #{s}"
+        end
+      end
+    end
+
+    def set_lhs_and_rhs
+      @rules.each do |rule|
+        rule.lhs = token_to_symbol(rule._lhs) if rule._lhs
+
+        rule.rhs = rule._rhs.map do |t|
+          token_to_symbol(t)
+        end
+      end
+    end
+
+    # Rule inherits precedence from the last term in RHS.
+    #
+    # https://www.gnu.org/software/bison/manual/html_node/How-Precedence.html
+    def fill_default_precedence
+      @rules.each do |rule|
+        # Explicitly specified precedence has the highest priority
+        next if rule.precedence_sym
+
+        precedence_sym = nil
+        rule.rhs.each do |sym|
+          precedence_sym = sym if sym.term?
+        end
+
+        rule.precedence_sym = precedence_sym
+      end
+    end
+
+    def fill_symbols
+      fill_symbol_number
+      fill_nterm_type(@types)
+      fill_printer(@printers)
+      fill_destructor(@destructors)
+      fill_error_token(@error_tokens)
+      sort_by_number!
+    end
+
+    def fill_sym_to_rules
+      @rules.each do |rule|
+        key = rule.lhs.number
+        @sym_to_rules[key] ||= []
+        @sym_to_rules[key] << rule
+      end
+    end
+
+    def validate_rule_lhs_is_nterm!
+      errors = [] #: Array[String]
+
+      rules.each do |rule|
+        next if rule.lhs.nterm?
+
+        errors << "[BUG] LHS of #{rule.display_name} (line: #{rule.lineno}) is term. It should be nterm."
+      end
+
+      return if errors.empty?
+
+      raise errors.join("\n")
+    end
+
+    def set_locations
+      @locations = @locations || @rules.any? {|rule| rule.contains_at_reference? }
+    end
+  end
+end

data/ext/enclave/mruby/tools/lrama/lib/lrama/grammar_validator.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Lrama
+  class GrammarValidator
+    def initialize(grammar, states, logger)
+      @grammar = grammar
+      @states = states
+      @logger = logger
+    end
+
+    def valid?
+      conflicts_within_threshold?
+    end
+
+    private
+
+    def conflicts_within_threshold?
+      return true unless @grammar.expect
+
+      [sr_conflicts_within_threshold(@grammar.expect), rr_conflicts_within_threshold(0)].all?
+    end
+
+    def sr_conflicts_within_threshold(expected)
+      return true if expected == @states.sr_conflicts_count
+
+      @logger.error("shift/reduce conflicts: #{@states.sr_conflicts_count} found, #{expected} expected")
+      false
+    end
+
+    def rr_conflicts_within_threshold(expected)
+      return true if expected == @states.rr_conflicts_count
+
+      @logger.error("reduce/reduce conflicts: #{@states.rr_conflicts_count} found, #{expected} expected")
+      false
+    end
+  end
+end

data/ext/enclave/mruby/tools/lrama/lib/lrama/lexer/grammar_file.rb

@@ -0,0 +1,40 @@
+# rbs_inline: enabled
+# frozen_string_literal: true
+
+module Lrama
+  class Lexer
+    class GrammarFile
+      class Text < String
+        # @rbs () -> String
+        def inspect
+          length <= 50 ? super : "#{self[0..47]}...".inspect
+        end
+      end
+
+      attr_reader :path #: String
+      attr_reader :text #: String
+
+      # @rbs (String path, String text) -> void
+      def initialize(path, text)
+        @path = path
+        @text = Text.new(text).freeze
+      end
+
+      # @rbs () -> String
+      def inspect
+        "<#{self.class}: @path=#{path}, @text=#{text.inspect}>"
+      end
+
+      # @rbs (GrammarFile other) -> bool
+      def ==(other)
+        self.class == other.class &&
+        self.path == other.path
+      end
+
+      # @rbs () -> Array[String]
+      def lines
+        @lines ||= text.split("\n")
+      end
+    end
+  end
+end

data/ext/enclave/mruby/tools/lrama/lib/lrama/lexer/location.rb

@@ -0,0 +1,115 @@
+# rbs_inline: enabled
+# frozen_string_literal: true
+
+module Lrama
+  class Lexer
+    class Location
+      attr_reader :grammar_file #: GrammarFile
+      attr_reader :first_line #: Integer
+      attr_reader :first_column #: Integer
+      attr_reader :last_line #: Integer
+      attr_reader :last_column #: Integer
+
+      # @rbs (grammar_file: GrammarFile, first_line: Integer, first_column: Integer, last_line: Integer, last_column: Integer) -> void
+      def initialize(grammar_file:, first_line:, first_column:, last_line:, last_column:)
+        @grammar_file = grammar_file
+        @first_line = first_line
+        @first_column = first_column
+        @last_line = last_line
+        @last_column = last_column
+      end
+
+      # @rbs (Location other) -> bool
+      def ==(other)
+        self.class == other.class &&
+        self.grammar_file == other.grammar_file &&
+        self.first_line == other.first_line &&
+        self.first_column == other.first_column &&
+        self.last_line == other.last_line &&
+        self.last_column == other.last_column
+      end
+
+      # @rbs (Integer left, Integer right) -> Location
+      def partial_location(left, right)
+        offset = -first_column
+        new_first_line = -1
+        new_first_column = -1
+        new_last_line = -1
+        new_last_column = -1
+
+        _text.each.with_index do |line, index|
+          new_offset = offset + line.length + 1
+
+          if offset <= left && left <= new_offset
+            new_first_line = first_line + index
+            new_first_column = left - offset
+          end
+
+          if offset <= right && right <= new_offset
+            new_last_line = first_line + index
+            new_last_column = right - offset
+          end
+
+          offset = new_offset
+        end
+
+        Location.new(
+          grammar_file: grammar_file,
+          first_line: new_first_line, first_column: new_first_column,
+          last_line: new_last_line, last_column: new_last_column
+        )
+      end
+
+      # @rbs () -> String
+      def to_s
+        "#{path} (#{first_line},#{first_column})-(#{last_line},#{last_column})"
+      end
+
+      # @rbs (String error_message) -> String
+      def generate_error_message(error_message)
+        <<~ERROR.chomp
+          #{path}:#{first_line}:#{first_column}: #{error_message}
+          #{line_with_carets}
+        ERROR
+      end
+
+      # @rbs () -> String
+      def line_with_carets
+        <<~TEXT
+          #{text}
+          #{carets}
+        TEXT
+      end
+
+      private
+
+      # @rbs () -> String
+      def path
+        grammar_file.path
+      end
+
+      # @rbs () -> String
+      def blanks
+        (text[0...first_column] or raise "#{first_column} is invalid").gsub(/[^\t]/, ' ')
+      end
+
+      # @rbs () -> String
+      def carets
+        blanks + '^' * (last_column - first_column)
+      end
+
+      # @rbs () -> String
+      def text
+        @text ||= _text.join("\n")
+      end
+
+      # @rbs () -> Array[String]
+      def _text
+        @_text ||=begin
+          range = (first_line - 1)...last_line
+          grammar_file.lines[range] or raise "#{range} is invalid"
+        end
+      end
+    end
+  end
+end

data/ext/enclave/mruby/tools/lrama/lib/lrama/lexer/token/char.rb

@@ -0,0 +1,11 @@
+# rbs_inline: enabled
+# frozen_string_literal: true
+
+module Lrama
+  class Lexer
+    class Token
+      class Char < Token
+      end
+    end
+  end
+end

data/ext/enclave/mruby/tools/lrama/lib/lrama/lexer/token/ident.rb

@@ -0,0 +1,11 @@
+# rbs_inline: enabled
+# frozen_string_literal: true
+
+module Lrama
+  class Lexer
+    class Token
+      class Ident < Token
+      end
+    end
+  end
+end

data/ext/enclave/mruby/tools/lrama/lib/lrama/lexer/token/instantiate_rule.rb

@@ -0,0 +1,30 @@
+# rbs_inline: enabled
+# frozen_string_literal: true
+
+module Lrama
+  class Lexer
+    class Token
+      class InstantiateRule < Token
+        attr_reader :args #: Array[Lexer::Token]
+        attr_reader :lhs_tag #: Lexer::Token::Tag?
+
+        # @rbs (s_value: String, ?alias_name: String, ?location: Location, ?args: Array[Lexer::Token], ?lhs_tag: Lexer::Token::Tag?) -> void
+        def initialize(s_value:, alias_name: nil, location: nil, args: [], lhs_tag: nil)
+          super s_value: s_value, alias_name: alias_name, location: location
+          @args = args
+          @lhs_tag = lhs_tag
+        end
+
+        # @rbs () -> String
+        def rule_name
+          s_value
+        end
+
+        # @rbs () -> Integer
+        def args_count
+          args.count
+        end
+      end
+    end
+  end
+end

data/ext/enclave/mruby/tools/lrama/lib/lrama/lexer/token/tag.rb

@@ -0,0 +1,16 @@
+# rbs_inline: enabled
+# frozen_string_literal: true
+
+module Lrama
+  class Lexer
+    class Token
+      class Tag < Token
+        # @rbs () -> String
+        def member
+          # Omit "<>"
+          s_value[1..-2] or raise "Unexpected Tag format (#{s_value})"
+        end
+      end
+    end
+  end
+end