lrama 0.5.8 → 0.5.10

data/lib/lrama/lexer/token/user_code.rb

@@ -0,0 +1,64 @@
+require "strscan"
+
+module Lrama
+  class Lexer
+    class Token
+      class UserCode < Token
+        def references
+          @references ||= _references
+        end
+
+        private
+
+        def _references
+          scanner = StringScanner.new(s_value)
+          references = []
+
+          while !scanner.eos? do
+            case
+            when reference = scan_reference(scanner)
+              references << reference
+            when scanner.scan(/\/\*/)
+              scanner.scan_until(/\*\//)
+            else
+              scanner.getch
+            end
+          end
+
+          references
+        end
+
+        def scan_reference(scanner)
+          start = scanner.pos
+          case
+          # $ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$
+            tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil
+            return Lrama::Grammar::Reference.new(type: :dollar, name: "$", ex_tag: tag, first_column: start, last_column: scanner.pos - 1)
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1
+            tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil
+            return Lrama::Grammar::Reference.new(type: :dollar, index: Integer(scanner[2]), ex_tag: tag, first_column: start, last_column: scanner.pos - 1)
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil
+            return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[2], ex_tag: tag, first_column: start, last_column: scanner.pos - 1)
+          when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $expr.right, $expr-right, $<long>program (named reference with brackets)
+            tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil
+            return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[2], ex_tag: tag, first_column: start, last_column: scanner.pos - 1)
+
+          # @ references
+          # It need to wrap an identifier with brackets to use ".-" for identifiers
+          when scanner.scan(/@\$/) # @$
+            return Lrama::Grammar::Reference.new(type: :at, name: "$", first_column: start, last_column: scanner.pos - 1)
+          when scanner.scan(/@(\d+)/) # @1
+            return Lrama::Grammar::Reference.new(type: :at, index: Integer(scanner[1]), first_column: start, last_column: scanner.pos - 1)
+          when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets)
+            return Lrama::Grammar::Reference.new(type: :at, name: scanner[1], first_column: start, last_column: scanner.pos - 1)
+          when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @expr.right, @expr-right  (named reference with brackets)
+            return Lrama::Grammar::Reference.new(type: :at, name: scanner[1], first_column: start, last_column: scanner.pos - 1)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lrama/lexer/token.rb

@@ -1,84 +1,44 @@
-require 'lrama/lexer/token/type'
-
 module Lrama
   class Lexer
-    class Token
+    class Token < Struct.new(:s_value, :alias_name, :location, keyword_init: true)
 
-      attr_accessor :line, :column, :referred
-      # For User_code
-      attr_accessor :references
+      attr_accessor :referred
 
       def to_s
-        "#{super} line: #{line}, column: #{column}"
+        "#{super} location: #{location}"
       end
 
       def referred_by?(string)
-        [self.s_value, self.alias].include?(string)
+        [self.s_value, self.alias_name].compact.include?(string)
       end
 
       def ==(other)
-        self.class == other.class && self.type == other.type && self.s_value == other.s_value
+        self.class == other.class && self.s_value == other.s_value
       end
 
-      def numberize_references(lhs, rhs)
-        self.references.map! {|ref|
-          ref_name = ref[1]
-          if ref_name.is_a?(::String) && ref_name != '$'
-            value =
-              if lhs.referred_by?(ref_name)
-                '$'
-              else
-                index = rhs.find_index {|token| token.referred_by?(ref_name) }
-
-                if index
-                  index + 1
-                else
-                  raise "'#{ref_name}' is invalid name."
-                end
-              end
-            [ref[0], value, ref[2], ref[3], ref[4]]
-          else
-            ref
-          end
-        }
+      def first_line
+        location.first_line
       end
+      alias :line :first_line
 
-      @i = 0
-      @types = []
+      def first_column
+        location.first_column
+      end
+      alias :column :first_column
 
-      def self.define_type(name)
-        type = Type.new(id: @i, name: name.to_s)
-        const_set(name, type)
-        @types << type
-        @i += 1
+      def last_line
+        location.last_line
       end
 
-      # Token types
-      define_type(:P_expect)         # %expect
-      define_type(:P_define)         # %define
-      define_type(:P_printer)        # %printer
-      define_type(:P_error_token)    # %error-token
-      define_type(:P_lex_param)      # %lex-param
-      define_type(:P_parse_param)    # %parse-param
-      define_type(:P_initial_action) # %initial-action
-      define_type(:P_union)          # %union
-      define_type(:P_token)          # %token
-      define_type(:P_type)           # %type
-      define_type(:P_nonassoc)       # %nonassoc
-      define_type(:P_left)           # %left
-      define_type(:P_right)          # %right
-      define_type(:P_precedence)     # %precedence
-      define_type(:P_prec)           # %prec
-      define_type(:User_code)        # { ... }
-      define_type(:Tag)              # <int>
-      define_type(:Number)           # 0
-      define_type(:Ident_Colon)      # k_if:, k_if  : (spaces can be there)
-      define_type(:Ident)            # api.pure, tNUMBER
-      define_type(:Named_Ref)        # [foo]
-      define_type(:Semicolon)        # ;
-      define_type(:Bar)              # |
-      define_type(:String)           # "str"
-      define_type(:Char)             # '+'
+      def last_column
+        location.last_column
+      end
     end
   end
 end
+
+require 'lrama/lexer/token/char'
+require 'lrama/lexer/token/ident'
+require 'lrama/lexer/token/parameterizing'
+require 'lrama/lexer/token/tag'
+require 'lrama/lexer/token/user_code'

data/lib/lrama/lexer.rb

@@ -1,12 +1,14 @@
 require "strscan"
+require "lrama/lexer/location"
 require "lrama/lexer/token"
 
 module Lrama
   class Lexer
+    attr_reader :head_line, :head_column
     attr_accessor :status
     attr_accessor :end_symbol
 
-    SYMBOLS = %w(%{ %} %% { } \[ \] : \| ;)
+    SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';']
     PERCENT_TOKENS = %w(
       %union
       %token
@@ -24,12 +26,14 @@ module Lrama
       %precedence
       %prec
       %error-token
+      %empty
+      %code
     )
 
     def initialize(text)
       @scanner = StringScanner.new(text)
-      @head = @scanner.pos
-      @line = 1
+      @head_column = @head = @scanner.pos
+      @head_line = @line = 1
       @status = :initial
       @end_symbol = nil
     end
@@ -51,6 +55,13 @@ module Lrama
       @scanner.pos - @head
     end
 
+    def location
+      Location.new(
+        first_line: @head_line, first_column: @head_column,
+        last_line: @line, last_column: column
+      )
+    end
+
     def lex_token
       while !@scanner.eos? do
         case
@@ -60,11 +71,8 @@ module Lrama
           # noop
         when @scanner.scan(/\/\*/)
           lex_comment
-        when @scanner.scan(/\/\//)
-          @scanner.scan_until(/\n/)
-          newline
-        when @scanner.scan(/%empty/)
-          # noop
+        when @scanner.scan(/\/\/.*(?<newline>\n)?/)
+          newline if @scanner[:newline]
         else
           break
         end
@@ -80,18 +88,20 @@ module Lrama
         return [@scanner.matched, @scanner.matched]
       when @scanner.scan(/#{PERCENT_TOKENS.join('|')}/)
         return [@scanner.matched, @scanner.matched]
+      when @scanner.scan(/[\?\+\*]/)
+        return [@scanner.matched, @scanner.matched]
       when @scanner.scan(/<\w+>/)
-        return [:TAG, build_token(type: Token::Tag, s_value: @scanner.matched)]
+        return [:TAG, Lrama::Lexer::Token::Tag.new(s_value: @scanner.matched, location: location)]
       when @scanner.scan(/'.'/)
-        return [:CHARACTER, build_token(type: Token::Char, s_value: @scanner.matched)]
+        return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)]
       when @scanner.scan(/'\\\\'|'\\b'|'\\t'|'\\f'|'\\r'|'\\n'|'\\v'|'\\13'/)
-        return [:CHARACTER, build_token(type: Token::Char, s_value: @scanner.matched)]
-      when @scanner.scan(/"/)
-        return [:STRING, %Q("#{@scanner.scan_until(/"/)})]
+        return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)]
+      when @scanner.scan(/".*?"/)
+        return [:STRING, %Q(#{@scanner.matched})]
       when @scanner.scan(/\d+/)
         return [:INTEGER, Integer(@scanner.matched)]
       when @scanner.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)/)
-        token = build_token(type: Token::Ident, s_value: @scanner.matched)
+        token = Lrama::Lexer::Token::Ident.new(s_value: @scanner.matched, location: location)
         type =
           if @scanner.check(/\s*(\[\s*[a-zA-Z_.][-a-zA-Z0-9_.]*\s*\])?\s*:/)
             :IDENT_COLON
@@ -100,7 +110,7 @@ module Lrama
           end
         return [type, token]
       else
-        raise
+        raise ParseError, "Unexpected token: #{@scanner.peek(10).chomp}."
       end
     end
 
@@ -115,28 +125,30 @@ module Lrama
         when @scanner.scan(/}/)
           if nested == 0 && @end_symbol == '}'
             @scanner.unscan
-            return [:C_DECLARATION, build_token(type: Token::User_code, s_value: code, references: [])]
+            return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)]
           else
             code += @scanner.matched
             nested -= 1
           end
         when @scanner.check(/#{@end_symbol}/)
-          return [:C_DECLARATION, build_token(type: Token::User_code, s_value: code, references: [])]
+          return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)]
         when @scanner.scan(/\n/)
           code += @scanner.matched
           newline
-        when @scanner.scan(/"/)
-          matched = @scanner.scan_until(/"/)
-          code += %Q("#{matched})
-          @line += matched.count("\n")
-        when @scanner.scan(/'/)
-          matched = @scanner.scan_until(/'/)
-          code += %Q('#{matched})
+        when @scanner.scan(/".*?"/)
+          code += %Q(#{@scanner.matched})
+          @line += @scanner.matched.count("\n")
+        when @scanner.scan(/'.*?'/)
+          code += %Q(#{@scanner.matched})
         else
-          code += @scanner.getch
+          if @scanner.scan(/[^\"'\{\}\n#{@end_symbol}]+/)
+            code += @scanner.matched
+          else
+            code += @scanner.getch
+          end
         end
       end
-      raise
+      raise ParseError, "Unexpected code: #{code}."
     end
 
     private
@@ -155,17 +167,6 @@ module Lrama
       end
     end
 
-    def build_token(type:, s_value:, **options)
-      token = Token.new(type: type, s_value: s_value)
-      token.line = @head_line
-      token.column = @head_column
-      options.each do |attr, value|
-        token.public_send("#{attr}=", value)
-      end
-
-      token
-    end
-
     def newline
       @line += 1
       @head = @scanner.pos + 1

data/lib/lrama/option_parser.rb

@@ -58,6 +58,7 @@ module Lrama
         o.separator 'Tuning the Parser:'
         o.on('-S', '--skeleton=FILE', 'specify the skeleton to use') {|v| @options.skeleton = v }
         o.on('-t', 'reserved, do nothing') { }
+        o.on('--debug', 'display debugging outputs of internal parser') {|v| @options.debug = true }
         o.separator ''
         o.separator 'Output:'
         o.on('-H', '--header=[FILE]', 'also produce a header file named FILE') {|v| @options.header = true; @options.header_file = v }
@@ -108,7 +109,7 @@ module Lrama
     def validate_trace(trace)
       list = %w[
         none locations scan parse automaton bitsets
-        closure grammar resource sets muscles tools
+        closure grammar rules resource sets muscles tools
         m4-early m4 skeleton time ielr cex all
       ]
       h = {}

data/lib/lrama/options.rb

@@ -4,7 +4,8 @@ module Lrama
     attr_accessor :skeleton, :header, :header_file,
                   :report_file, :outfile,
                   :error_recovery, :grammar_file,
-                  :report_file, :trace_opts, :report_opts, :y
+                  :trace_opts, :report_opts, :y,
+                  :debug
 
     def initialize
       @skeleton = "bison/yacc.c"
@@ -14,7 +15,6 @@ module Lrama
       @outfile = "y.tab.c"
       @error_recovery = false
       @grammar_file = nil
-      @report_file = nil
       @trace_opts = nil
       @report_opts = nil
       @y = STDIN

data/lib/lrama/output.rb

@@ -186,9 +186,9 @@ module Lrama
       str = ""
 
       @context.states.rules.each do |rule|
-        next unless rule.code
+        next unless rule.token_code
 
-        code = rule.code
+        code = rule.token_code
         spaces = " " * (code.column - 1)
 
         str << <<-STR
@@ -349,6 +349,15 @@ module Lrama
       end
     end
 
+    # b4_percent_code_get
+    def percent_code(name)
+      @grammar.percent_codes.select do |percent_code|
+        percent_code.id.s_value == name
+      end.map do |percent_code|
+        percent_code.code.s_value
+      end.join
+    end
+
     private
 
     def eval_template(file, path)