adlint 2.6.14 → 3.0.0

data/lib/adlint/cpp/lexer.rb

@@ -1,7 +1,7 @@
 # Lexical analyzer which tokenizes C language source into pp-tokens.
 #
 # Author::    Yutaka Yanoh <mailto:yanoh@users.sourceforge.net>
-# Copyright:: Copyright (C) 2010-2012, OGIS-RI Co.,Ltd.
+# Copyright:: Copyright (C) 2010-2013, OGIS-RI Co.,Ltd.
 # License::   GPLv3+: GNU General Public License version 3 or later
 #
 # Owner::     Yutaka Yanoh <mailto:yanoh@users.sourceforge.net>
@@ -11,7 +11,7 @@
 #    /   |  / _  |/ /   / / | / /__  __/           Source Code Static Analyzer
 #   / /| | / / / / /   / /  |/ /  / /                   AdLint - Advanced Lint
 #  / __  |/ /_/ / /___/ / /|  /  / /
-# /_/  |_|_____/_____/_/_/ |_/  /_/   Copyright (C) 2010-2012, OGIS-RI Co.,Ltd.
+# /_/  |_|_____/_____/_/_/ |_/  /_/   Copyright (C) 2010-2013, OGIS-RI Co.,Ltd.
 #
 # This file is part of AdLint.
 #
@@ -37,17 +37,18 @@ module AdLint #:nodoc:
 module Cpp #:nodoc:
 
   class Lexer
-    def initialize(source)
-      source.on_cr_at_eol_found += lambda { |location|
-        on_cr_at_eol_found.invoke(location)
+    def initialize(src, traits)
+      src.on_cr_at_eol_found += lambda { |loc|
+        on_cr_at_eol_found.invoke(loc)
       }
-      source.on_eof_mark_at_eof_found += lambda { |location|
-        on_eof_mark_at_eof_found.invoke(location)
+      src.on_eof_mark_at_eof_found += lambda { |loc|
+        on_eof_mark_at_eof_found.invoke(loc)
       }
-      source.on_eof_newline_not_found += method(:notify_eof_newline_not_found)
+      src.on_eof_newline_not_found += method(:notify_eof_newline_not_found)
 
-      @content = SourceContent.lazy_new(source)
-      @state = Initial.new(self)
+      tab_width  = traits.of_project.coding_style.tab_width
+      @content   = SourceContent.lazy_new(src, tab_width)
+      @state     = Initial.new(self)
       @top_token = nil
     end
 
@@ -67,9 +68,9 @@ module Cpp #:nodoc:
 
     def next_token
       if @top_token
-        token = @top_token
+        tok = @top_token
         @top_token = nil
-        return token
+        return tok
       end
       @state.next_token
     end
@@ -104,17 +105,17 @@ module Cpp #:nodoc:
     def discard_heading_comments
       case
       when @content.check(/\/\*/)
-        location = @content.location
+        loc = @content.location
         comment = scan_block_comment(@content)
         unless comment.empty?
-          notify_block_comment_found(comment, location)
+          notify_block_comment_found(comment, loc)
           return true
         end
       when @content.check(/\/\//)
-        location = @content.location
+        loc = @content.location
         comment = scan_line_comment(@content)
         unless comment.empty?
-          notify_line_comment_found(comment, location)
+          notify_line_comment_found(comment, loc)
           return true
         end
       end
@@ -125,32 +126,32 @@ module Cpp #:nodoc:
       @state = next_state
     end
 
-    def notify_block_comment_found(comment, location)
-      on_block_comment_found.invoke(comment, location)
+    def notify_block_comment_found(comment, loc)
+      on_block_comment_found.invoke(comment, loc)
     end
 
-    def notify_line_comment_found(comment, location)
-      on_line_comment_found.invoke(comment, location)
+    def notify_line_comment_found(comment, loc)
+      on_line_comment_found.invoke(comment, loc)
     end
 
-    def notify_nested_block_comment_found(location)
-      on_nested_block_comment_found.invoke(location)
+    def notify_nested_block_comment_found(loc)
+      on_nested_block_comment_found.invoke(loc)
     end
 
-    def notify_unterminated_block_comment(location)
-      on_unterminated_block_comment.invoke(location)
+    def notify_unterminated_block_comment(loc)
+      on_unterminated_block_comment.invoke(loc)
     end
 
-    def notify_eof_newline_not_found(location)
-      on_eof_newline_not_found.invoke(location)
+    def notify_eof_newline_not_found(loc)
+      on_eof_newline_not_found.invoke(loc)
     end
 
-    def notify_unlexable_char_found(char, location)
-      on_unlexable_char_found.invoke(char, location)
+    def notify_unlexable_char_found(char, loc)
+      on_unlexable_char_found.invoke(char, loc)
     end
 
-    def notify_illformed_newline_escape_found(location)
-      on_illformed_newline_escape_found.invoke(location)
+    def notify_illformed_newline_escape_found(loc)
+      on_illformed_newline_escape_found.invoke(loc)
     end
 
     private
@@ -158,37 +159,37 @@ module Cpp #:nodoc:
       /^[ \t]*#[ \t]*(?:if|ifdef|ifndef|asm|else|elif|endif|endasm)\b/
     private_constant :GROUP_DIRECTIVE_RE
 
-    def scan_until_next_directive_or_comment(content)
-      content.scan(/.*?(?=#{GROUP_DIRECTIVE_RE}|\/\*|\/\/)/m)
+    def scan_until_next_directive_or_comment(cont)
+      cont.scan(/.*?(?=#{GROUP_DIRECTIVE_RE}|\/\*|\/\/)/m)
     end
 
-    def scan_block_comment(content)
-      result = ""
+    def scan_block_comment(cont)
+      comment = ""
       block_depth = 0
-      until content.empty?
-        location = content.location
+      until cont.empty?
+        loc = cont.location
         case
-        when content.scan(/\/\*/)
+        when cont.scan(/\/\*/)
           block_depth += 1
-          result += "/*"
-          notify_nested_block_comment_found(location) if block_depth > 1
-        when content.scan(/\*\//)
-          result += "*/"
+          comment += "/*"
+          notify_nested_block_comment_found(loc) if block_depth > 1
+        when cont.scan(/\*\//)
+          comment += "*/"
           break
         else
           return nil if block_depth == 0
-          if comment = content.scan(/.*?(?=\/\*|\*\/)/m)
-            result += comment
+          if scanned = cont.scan(/.*?(?=\/\*|\*\/)/m)
+            comment += scanned
           else
-            notify_unterminated_block_comment(location)
+            notify_unterminated_block_comment(loc)
           end
         end
       end
-      result
+      comment
     end
 
-    def scan_line_comment(content)
-      content.scan(/\/\/.*?(?=\n)/)
+    def scan_line_comment(cont)
+      cont.scan(/\/\/.*?(?=\n)/)
     end
   end
 
@@ -206,80 +207,80 @@ module Cpp #:nodoc:
       @lexer.discard_heading_comments
     end
 
-    def scan_escaped_newline(content)
-      location = content.location
+    def scan_escaped_newline(cont)
+      loc = cont.location
       case
-      when escaped_newline = content.scan(/\\\n/)
-        escaped_newline
-      when escaped_newline = content.scan(/\\[ \t]+\n/)
-        @lexer.notify_illformed_newline_escape_found(location)
-        escaped_newline
+      when escaped_nl = cont.scan(/\\\n/)
+        escaped_nl
+      when escaped_nl = cont.scan(/\\[ \t]+\n/)
+        @lexer.notify_illformed_newline_escape_found(loc)
+        escaped_nl
       else
         nil
       end
     end
 
-    def tokenize_pp_token(content)
-      location = content.location
+    def tokenize_pp_token(cont)
+      loc = cont.location
       case
-      when value = Language::C.scan_keyword(content),
-           value = Language::Cpp.scan_keyword(content)
-        Token.new(:PP_TOKEN, value, location, Language::C::KEYWORDS[value])
-      when value = Language::C.scan_char_constant(content),
-           value = Language::C.scan_floating_constant(content),
-           value = Language::C.scan_integer_constant(content)
-        Token.new(:PP_TOKEN, value, location, :CONSTANT)
-      when value = Language::C.scan_string_literal(content)
-        Token.new(:PP_TOKEN, value, location, :STRING_LITERAL)
-      when value = Language::C.scan_null_constant(content)
-        Token.new(:PP_TOKEN, value, location, :NULL)
-      when value = Language::C.scan_identifier(content)
-        Token.new(:PP_TOKEN, value, location, :IDENTIFIER)
-      when value = Language::Cpp.scan_punctuator(content)
-        Token.new(:PP_TOKEN, value, location, value)
+      when val = Language::C.scan_keyword(cont),
+           val = Language::Cpp.scan_keyword(cont)
+        Token.new(:PP_TOKEN, val, loc, Language::C::KEYWORDS[val])
+      when val = Language::C.scan_char_constant(cont),
+           val = Language::C.scan_floating_constant(cont),
+           val = Language::C.scan_integer_constant(cont)
+        Token.new(:PP_TOKEN, val, loc, :CONSTANT)
+      when val = Language::C.scan_string_literal(cont)
+        Token.new(:PP_TOKEN, val, loc, :STRING_LITERAL)
+      when val = Language::C.scan_null_constant(cont)
+        Token.new(:PP_TOKEN, val, loc, :NULL)
+      when val = Language::C.scan_identifier(cont)
+        Token.new(:PP_TOKEN, val, loc, :IDENTIFIER)
+      when val = Language::Cpp.scan_punctuator(cont)
+        Token.new(:PP_TOKEN, val, loc, val)
       else
         nil
       end
     end
 
-    def tokenize_new_line(content)
-      location = content.location
-      if value = content.scan(/\n/)
-        return Token.new(:NEW_LINE, value, location)
+    def tokenize_new_line(cont)
+      loc = cont.location
+      if val = cont.scan(/\n/)
+        return Token.new(:NEW_LINE, val, loc)
       end
       nil
     end
 
-    def tokenize_header_name(content)
-      location = content.location
-      if value = Language::Cpp.scan_system_header_name(content)
-        return Token.new(:SYS_HEADER_NAME, value, location)
-      elsif value = Language::Cpp.scan_user_header_name(content)
-        return Token.new(:USR_HEADER_NAME, value, location)
+    def tokenize_header_name(cont)
+      loc = cont.location
+      if val = Language::Cpp.scan_system_header_name(cont)
+        return Token.new(:SYS_HEADER_NAME, val, loc)
+      elsif val = Language::Cpp.scan_user_header_name(cont)
+        return Token.new(:USR_HEADER_NAME, val, loc)
       end
       nil
     end
 
-    def tokenize_identifier(content)
-      location = content.location
-      if value = Language::C.scan_identifier(content)
-        return Token.new(:IDENTIFIER, value, location)
+    def tokenize_identifier(cont)
+      loc = cont.location
+      if val = Language::C.scan_identifier(cont)
+        return Token.new(:IDENTIFIER, val, loc)
       end
       nil
     end
 
-    def tokenize_punctuator(content)
-      location = content.location
-      if punctuator = Language::Cpp.scan_punctuator(content)
-        return Token.new(punctuator, punctuator, location)
+    def tokenize_punctuator(cont)
+      loc = cont.location
+      if punctuator = Language::Cpp.scan_punctuator(cont)
+        return Token.new(punctuator, punctuator, loc)
       end
       nil
     end
 
-    def tokenize_extra_token(content)
+    def tokenize_extra_token(cont)
       # NOTE: #tokenize_pp_token can tokenize almost all types of tokens.
-      if token = tokenize_pp_token(content)
-        Token.new(:EXTRA_TOKEN, token.value, token.location)
+      if tok = tokenize_pp_token(cont)
+        Token.new(:EXTRA_TOKEN, tok.value, tok.location)
       else
         nil
       end
@@ -295,225 +296,225 @@ module Cpp #:nodoc:
       case
       when @lexer.content.check(/[ \t]*#/)
         case
-        when token = tokenize_if_directive(@lexer.content)
+        when tok = tokenize_if_directive(@lexer.content)
           @lexer.transit(InIfDirective.new(@lexer))
-        when token = tokenize_ifdef_directive(@lexer.content)
+        when tok = tokenize_ifdef_directive(@lexer.content)
           @lexer.transit(InIfdefDirective.new(@lexer))
-        when token = tokenize_ifndef_directive(@lexer.content)
+        when tok = tokenize_ifndef_directive(@lexer.content)
           @lexer.transit(InIfndefDirective.new(@lexer))
-        when token = tokenize_elif_directive(@lexer.content)
+        when tok = tokenize_elif_directive(@lexer.content)
           @lexer.transit(InElifDirective.new(@lexer))
-        when token = tokenize_else_directive(@lexer.content)
+        when tok = tokenize_else_directive(@lexer.content)
           @lexer.transit(InElseDirective.new(@lexer))
-        when token = tokenize_endif_directive(@lexer.content)
+        when tok = tokenize_endif_directive(@lexer.content)
           @lexer.transit(InEndifDirective.new(@lexer))
-        when token = tokenize_include_directive(@lexer.content)
+        when tok = tokenize_include_directive(@lexer.content)
           @lexer.transit(InIncludeDirective.new(@lexer))
-        when token = tokenize_include_next_directive(@lexer.content)
+        when tok = tokenize_include_next_directive(@lexer.content)
           @lexer.transit(InIncludeNextDirective.new(@lexer))
-        when token = tokenize_define_directive(@lexer.content)
+        when tok = tokenize_define_directive(@lexer.content)
           @lexer.transit(InDefineDirective.new(@lexer))
-        when token = tokenize_undef_directive(@lexer.content)
+        when tok = tokenize_undef_directive(@lexer.content)
           @lexer.transit(InUndefDirective.new(@lexer))
-        when token = tokenize_line_directive(@lexer.content)
+        when tok = tokenize_line_directive(@lexer.content)
           @lexer.transit(InLineDirective.new(@lexer))
-        when token = tokenize_error_directive(@lexer.content)
+        when tok = tokenize_error_directive(@lexer.content)
           @lexer.transit(InErrorDirective.new(@lexer))
-        when token = tokenize_pragma_directive(@lexer.content)
+        when tok = tokenize_pragma_directive(@lexer.content)
           @lexer.transit(InPragmaDirective.new(@lexer))
-        when token = tokenize_asm_directive(@lexer.content)
+        when tok = tokenize_asm_directive(@lexer.content)
           @lexer.transit(InAsmDirective.new(@lexer))
-        when token = tokenize_endasm_directive(@lexer.content)
+        when tok = tokenize_endasm_directive(@lexer.content)
           @lexer.transit(InEndasmDirective.new(@lexer))
         else
-          token = tokenize_null_directive(@lexer.content) ||
-                  tokenize_unknown_directive(@lexer.content)
+          tok = tokenize_null_directive(@lexer.content) ||
+                tokenize_unknown_directive(@lexer.content)
         end
       else
-        token = tokenize_text_line(@lexer.content)
+        tok = tokenize_text_line(@lexer.content)
       end
 
-      token
+      tok
     end
 
     private
-    def tokenize_if_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*if\b/)
-        return Token.new(:IF, value, location)
+    def tokenize_if_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*if\b/)
+        return Token.new(:IF, val, loc)
       end
       nil
     end
 
-    def tokenize_ifdef_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*ifdef\b/)
-        return Token.new(:IFDEF, value, location)
+    def tokenize_ifdef_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*ifdef\b/)
+        return Token.new(:IFDEF, val, loc)
       end
       nil
     end
 
-    def tokenize_ifndef_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*ifndef\b/)
-        return Token.new(:IFNDEF, value, location)
+    def tokenize_ifndef_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*ifndef\b/)
+        return Token.new(:IFNDEF, val, loc)
       end
       nil
     end
 
-    def tokenize_elif_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*elif\b/)
-        return Token.new(:ELIF, value, location)
+    def tokenize_elif_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*elif\b/)
+        return Token.new(:ELIF, val, loc)
       end
       nil
     end
 
-    def tokenize_else_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*else\b/)
-        return Token.new(:ELSE, value, location)
+    def tokenize_else_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*else\b/)
+        return Token.new(:ELSE, val, loc)
       end
       nil
     end
 
-    def tokenize_endif_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*endif\b/)
-        return Token.new(:ENDIF, value, location)
+    def tokenize_endif_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*endif\b/)
+        return Token.new(:ENDIF, val, loc)
       end
       nil
     end
 
-    def tokenize_include_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*include\b/)
-        return Token.new(:INCLUDE, value, location)
+    def tokenize_include_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*include\b/)
+        return Token.new(:INCLUDE, val, loc)
       end
       nil
     end
 
-    def tokenize_include_next_directive(content)
+    def tokenize_include_next_directive(cont)
       # NOTE: #include_next directive is a GCC extension.
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*include_next\b/)
-        return Token.new(:INCLUDE_NEXT, value, location)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*include_next\b/)
+        return Token.new(:INCLUDE_NEXT, val, loc)
       end
       nil
     end
 
-    def tokenize_define_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*define\b/)
-        return Token.new(:DEFINE, value, location)
+    def tokenize_define_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*define\b/)
+        return Token.new(:DEFINE, val, loc)
       end
       nil
     end
 
-    def tokenize_undef_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*undef\b/)
-        return Token.new(:UNDEF, value, location)
+    def tokenize_undef_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*undef\b/)
+        return Token.new(:UNDEF, val, loc)
       end
       nil
     end
 
-    def tokenize_line_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*line\b/)
-        return Token.new(:LINE, value, location)
+    def tokenize_line_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*line\b/)
+        return Token.new(:LINE, val, loc)
       end
       nil
     end
 
-    def tokenize_error_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*error\b/)
-        return Token.new(:ERROR, value, location)
+    def tokenize_error_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*error\b/)
+        return Token.new(:ERROR, val, loc)
       end
       nil
     end
 
-    def tokenize_pragma_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*pragma\b/)
-        return Token.new(:PRAGMA, value, location)
+    def tokenize_pragma_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*pragma\b/)
+        return Token.new(:PRAGMA, val, loc)
       end
       nil
     end
 
-    def tokenize_asm_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*asm\b/)
-        return Token.new(:ASM, value, location)
+    def tokenize_asm_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*asm\b/)
+        return Token.new(:ASM, val, loc)
       end
       nil
     end
 
-    def tokenize_endasm_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]*endasm\b/)
-        return Token.new(:ENDASM, value, location)
+    def tokenize_endasm_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]*endasm\b/)
+        return Token.new(:ENDASM, val, loc)
       end
       nil
     end
 
-    def tokenize_null_directive(content)
-      location = content.location
-      if value = content.scan(/[ \t]*#[ \t]\n/)
-        return Token.new(:NULL_DIRECTIVE, value, location)
+    def tokenize_null_directive(cont)
+      loc = cont.location
+      if val = cont.scan(/[ \t]*#[ \t]\n/)
+        return Token.new(:NULL_DIRECTIVE, val, loc)
       end
       nil
     end
 
-    def tokenize_unknown_directive(content)
-      location = content.location
-      value = content.scan(/[ \t]*#/)
-      until content.empty?
-        next if discard_heading_comments || scan_escaped_newline(content)
+    def tokenize_unknown_directive(cont)
+      loc = cont.location
+      val = cont.scan(/[ \t]*#/)
+      until cont.empty?
+        next if discard_heading_comments || scan_escaped_newline(cont)
 
         case
-        when str = content.scan(/.+?(?=\/\*|\/\/|\\[ \t]*\n|L?"|L?'|\n)/i)
-          value += str
-        when str = content.scan(/\n/)
-          value += str
+        when str = cont.scan(/.+?(?=\/\*|\/\/|\\[ \t]*\n|L?"|L?'|\n)/i)
+          val += str
+        when str = cont.scan(/\n/)
+          val += str
           break
         end
       end
-      Token.new(:UNKNOWN_DIRECTIVE, value, location)
+      Token.new(:UNKNOWN_DIRECTIVE, val, loc)
     end
 
-    def tokenize_text_line(content)
-      location = content.location
-      value = ""
+    def tokenize_text_line(cont)
+      loc = cont.location
+      val = ""
 
-      until content.empty?
-        if token = content.scan(/.*?(?=\/\*|\/\/|\\[ \t]*\n|L?"|L?'|\n)/i)
-          value += token
+      until cont.empty?
+        if tok = cont.scan(/.*?(?=\/\*|\/\/|\\[ \t]*\n|L?"|L?'|\n)/i)
+          val += tok
         end
 
-        if token = content.scan(/\n/)
-          value += token
+        if tok = cont.scan(/\n/)
+          val += tok
           break
         end
 
-        next if scan_escaped_newline(content)
+        next if scan_escaped_newline(cont)
 
         case
-        when content.check(/\/\*/)
+        when cont.check(/\/\*/)
           discard_heading_comments
-        when content.check(/\/\//)
+        when cont.check(/\/\//)
           discard_heading_comments
-        when content.check(/L?"/i)
-          string_literal = Language::C.scan_string_literal(content)
-          value += string_literal
-        when content.check(/L?'/i)
-          char_constant = Language::C.scan_char_constant(content)
-          value += char_constant
+        when cont.check(/L?"/i)
+          string_literal = Language::C.scan_string_literal(cont)
+          val += string_literal
+        when cont.check(/L?'/i)
+          char_constant = Language::C.scan_char_constant(cont)
+          val += char_constant
         end
       end
 
-      value.empty? ? nil : Token.new(:TEXT_LINE, value, location)
+      val.empty? ? nil : Token.new(:TEXT_LINE, val, loc)
     end
   end
 
@@ -524,27 +525,27 @@ module Cpp #:nodoc:
           next
         end
 
-        token = tokenize_pp_token(@lexer.content) ||
-                tokenize_new_line(@lexer.content) ||
-                tokenize_extra_token(@lexer.content)
+        tok = tokenize_pp_token(@lexer.content) ||
+              tokenize_new_line(@lexer.content) ||
+              tokenize_extra_token(@lexer.content)
 
-        if token
+        if tok
           break
         else
           @lexer.content.eat!
         end
       end
 
-      unless token
-        token = Token.new(:NEW_LINE, "\n", @lexer.content.location)
-        @lexer.notify_eof_newline_not_found(token.location)
+      unless tok
+        tok = Token.new(:NEW_LINE, "\n", @lexer.content.location)
+        @lexer.notify_eof_newline_not_found(tok.location)
       end
 
-      if token.type == :NEW_LINE
+      if tok.type == :NEW_LINE
         @lexer.transit(Initial.new(@lexer))
       end
 
-      token
+      tok
     end
   end
 
@@ -555,27 +556,27 @@ module Cpp #:nodoc:
           next
         end
 
-        token = tokenize_identifier(@lexer.content) ||
-                tokenize_new_line(@lexer.content)   ||
-                tokenize_extra_token(@lexer.content)
+        tok = tokenize_identifier(@lexer.content) ||
+              tokenize_new_line(@lexer.content)   ||
+              tokenize_extra_token(@lexer.content)
 
-        if token
+        if tok
           break
         else
           @lexer.content.eat!
         end
       end
 
-      unless token
-        token = Token.new(:NEW_LINE, "\n", @lexer.content.location)
-        @lexer.notify_eof_newline_not_found(token.location)
+      unless tok
+        tok = Token.new(:NEW_LINE, "\n", @lexer.content.location)
+        @lexer.notify_eof_newline_not_found(tok.location)
       end
 
-      if token.type == :NEW_LINE
+      if tok.type == :NEW_LINE
         @lexer.transit(Initial.new(@lexer))
       end
 
-      token
+      tok
     end
   end
 
@@ -590,26 +591,26 @@ module Cpp #:nodoc:
           next
         end
 
-        token = tokenize_new_line(@lexer.content) ||
-                tokenize_extra_token(@lexer.content)
+        tok = tokenize_new_line(@lexer.content) ||
+              tokenize_extra_token(@lexer.content)
 
-        if token
+        if tok
           break
         else
           @lexer.content.eat!
         end
       end
 
-      unless token
-        token = Token.new(:NEW_LINE, "\n", @lexer.content.location)
-        @lexer.notify_eof_newline_not_found(token.location)
+      unless tok
+        tok = Token.new(:NEW_LINE, "\n", @lexer.content.location)
+        @lexer.notify_eof_newline_not_found(tok.location)
       end
 
-      if token.type == :NEW_LINE
+      if tok.type == :NEW_LINE
         @lexer.transit(Initial.new(@lexer))
       end
 
-      token
+      tok
     end
   end
 
@@ -622,28 +623,28 @@ module Cpp #:nodoc:
           next
         end
 
-        token = tokenize_header_name(@lexer.content) ||
-                tokenize_pp_token(@lexer.content)    ||
-                tokenize_new_line(@lexer.content)    ||
-                tokenize_extra_token(@lexer.content)
+        tok = tokenize_header_name(@lexer.content) ||
+              tokenize_pp_token(@lexer.content)    ||
+              tokenize_new_line(@lexer.content)    ||
+              tokenize_extra_token(@lexer.content)
 
-        if token
+        if tok
           break
         else
           @lexer.content.eat!
         end
       end
 
-      unless token
-        token = Token.new(:NEW_LINE, "\n", @lexer.content.location)
-        @lexer.notify_eof_newline_not_found(token.location)
+      unless tok
+        tok = Token.new(:NEW_LINE, "\n", @lexer.content.location)
+        @lexer.notify_eof_newline_not_found(tok.location)
       end
 
-      if token.type == :NEW_LINE
+      if tok.type == :NEW_LINE
         @lexer.transit(Initial.new(@lexer))
       end
 
-      token
+      tok
     end
   end
 
@@ -661,38 +662,38 @@ module Cpp #:nodoc:
         tokenize_pp_tokens(@lexer.content)
       end
 
-      token = @tokens.shift
+      tok = @tokens.shift
       @lexer.transit(Initial.new(@lexer)) if @tokens.empty?
-      token
+      tok
     end
 
     private
-    def tokenize_macro_name(content)
-      until content.empty?
-        next if discard_heading_comments || scan_escaped_newline(content)
+    def tokenize_macro_name(cont)
+      until cont.empty?
+        next if discard_heading_comments || scan_escaped_newline(cont)
 
-        if token = tokenize_identifier(content)
-          @tokens.push(token)
+        if tok = tokenize_identifier(cont)
+          @tokens.push(tok)
           break
         else
-          content.eat!
+          cont.eat!
         end
       end
 
-      return unless content.check(/\(/)
+      return unless cont.check(/\(/)
 
       paren_depth = 0
-      until content.empty?
-        next if discard_heading_comments || scan_escaped_newline(content)
+      until cont.empty?
+        next if discard_heading_comments || scan_escaped_newline(cont)
 
-        if token = tokenize_identifier(content)
-          @tokens.push(token)
+        if tok = tokenize_identifier(cont)
+          @tokens.push(tok)
           next
         end
 
-        if token = tokenize_punctuator(content)
-          @tokens.push(token)
-          case token.type
+        if tok = tokenize_punctuator(cont)
+          @tokens.push(tok)
+          case tok.type
           when "("
             paren_depth += 1
           when ")"
@@ -702,38 +703,38 @@ module Cpp #:nodoc:
           next
         end
 
-        if token = tokenize_new_line(content)
-          @tokens.push(token)
+        if tok = tokenize_new_line(cont)
+          @tokens.push(tok)
           break
         end
 
-        content.eat!
+        cont.eat!
       end
     end
 
-    def tokenize_pp_tokens(content)
-      until content.empty?
-        next if discard_heading_comments || scan_escaped_newline(content)
+    def tokenize_pp_tokens(cont)
+      until cont.empty?
+        next if discard_heading_comments || scan_escaped_newline(cont)
 
-        token = tokenize_pp_token(content) || tokenize_new_line(content)
+        tok = tokenize_pp_token(cont) || tokenize_new_line(cont)
 
-        if token
-          @tokens.push(token)
-          if token.type == :NEW_LINE
+        if tok
+          @tokens.push(tok)
+          if tok.type == :NEW_LINE
             break
           end
         else
-          location = content.location
-          if eaten = content.eat! and eaten !~ /\A\s\z/
-            @lexer.notify_unlexable_char_found(eaten, location)
+          loc = cont.location
+          if eaten = cont.eat! and eaten !~ /\A\s\z/
+            @lexer.notify_unlexable_char_found(eaten, loc)
           end
         end
       end
 
       unless @tokens.last && @tokens.last.type == :NEW_LINE
-        token = Token.new(:NEW_LINE, "\n", content.location)
-        @lexer.notify_eof_newline_not_found(token.location)
-        @tokens.push(token)
+        tok = Token.new(:NEW_LINE, "\n", cont.location)
+        @lexer.notify_eof_newline_not_found(tok.location)
+        @tokens.push(tok)
       end
     end
   end
@@ -751,146 +752,151 @@ module Cpp #:nodoc:
   class InEndasmDirective < InElseDirective; end
 
   class StringToPPTokensLexer < StringLexer
+    def initialize(str, tab_width = 8)
+      super(str)
+      @tab_width = tab_width
+    end
+
     private
-    def create_context(str)
-      context = LexerContext.new(create_content(str))
+    def create_lexer_context(str)
+      lexer_ctxt = LexerContext.new(create_content(str))
 
-      class << context
+      class << lexer_ctxt
         attr_accessor :last_symbol
       end
 
-      context
+      lexer_ctxt
     end
 
     def create_content(str)
-      StringContent.new(str)
+      StringContent.new(str, @tab_width)
     end
 
-    def tokenize(context)
-      token_array = TokenArray.new
-      until context.content.empty?
-        next if tokenize_pp_token(context, token_array)
+    def tokenize(lexer_ctxt)
+      tok_ary = TokenArray.new
+      until lexer_ctxt.content.empty?
+        next if tokenize_pp_token(lexer_ctxt, tok_ary)
 
-        location = context.location
-        if new_line = context.content.scan(/\n/)
-          token_array.push(Token.new(:NEW_LINE, new_line, location))
+        loc = lexer_ctxt.location
+        if new_line = lexer_ctxt.content.scan(/\n/)
+          tok_ary.push(Token.new(:NEW_LINE, new_line, loc))
           break
         else
-          context.content.eat!
+          lexer_ctxt.content.eat!
         end
       end
-      token_array
+      tok_ary
     end
 
-    def tokenize_pp_token(context, token_array)
-      pp_token = tokenize_keyword(context)        ||
-                 tokenize_constant(context)       ||
-                 tokenize_string_literal(context) ||
-                 tokenize_null_constant(context)  ||
-                 tokenize_identifier(context)     ||
-                 tokenize_punctuator(context)     ||
-                 tokenize_backslash(context)
+    def tokenize_pp_token(lexer_ctxt, tok_ary)
+      pp_tok = tokenize_keyword(lexer_ctxt)        ||
+               tokenize_constant(lexer_ctxt)       ||
+               tokenize_string_literal(lexer_ctxt) ||
+               tokenize_null_constant(lexer_ctxt)  ||
+               tokenize_identifier(lexer_ctxt)     ||
+               tokenize_punctuator(lexer_ctxt)     ||
+               tokenize_backslash(lexer_ctxt)
 
-      if pp_token
-        token_array.push(pp_token)
+      if pp_tok
+        tok_ary.push(pp_tok)
         return true
       end
 
       false
     end
 
-    def tokenize_keyword(context)
-      location = context.location
+    def tokenize_keyword(lexer_ctxt)
+      loc = lexer_ctxt.location
 
-      keyword = Language::C.scan_keyword(context.content) ||
-                Language::Cpp.scan_keyword(context.content)
+      keyword = Language::C.scan_keyword(lexer_ctxt.content) ||
+                Language::Cpp.scan_keyword(lexer_ctxt.content)
 
       if keyword
-        context.last_symbol = :KEYWORD
-        Token.new(:PP_TOKEN, keyword, location, Language::C::KEYWORDS[keyword])
+        lexer_ctxt.last_symbol = :KEYWORD
+        Token.new(:PP_TOKEN, keyword, loc, Language::C::KEYWORDS[keyword])
       else
         nil
       end
     end
 
-    def tokenize_constant(context)
-      location = context.location
+    def tokenize_constant(lexer_ctxt)
+      loc = lexer_ctxt.location
 
       # NOTE: For extended bit-access operators.
-      return nil if context.last_symbol == :IDENTIFIER
+      return nil if lexer_ctxt.last_symbol == :IDENTIFIER
 
-      constant = Language::C.scan_char_constant(context.content)     ||
-                 Language::C.scan_floating_constant(context.content) ||
-                 Language::C.scan_integer_constant(context.content)
+      constant = Language::C.scan_char_constant(lexer_ctxt.content)     ||
+                 Language::C.scan_floating_constant(lexer_ctxt.content) ||
+                 Language::C.scan_integer_constant(lexer_ctxt.content)
 
       if constant
-        context.last_symbol = :CONSTANT
-        return Token.new(:PP_TOKEN, constant, location, :CONSTANT)
+        lexer_ctxt.last_symbol = :CONSTANT
+        return Token.new(:PP_TOKEN, constant, loc, :CONSTANT)
       end
 
       nil
     end
 
-    def tokenize_string_literal(context)
-      location = context.location
+    def tokenize_string_literal(lexer_ctxt)
+      loc = lexer_ctxt.location
 
-      string_literal = Language::C.scan_string_literal(context.content)
+      string_literal = Language::C.scan_string_literal(lexer_ctxt.content)
 
       if string_literal
-        context.last_symbol = :STRING_LITERAL
-        return Token.new(:PP_TOKEN, string_literal, location, :STRING_LITERAL)
+        lexer_ctxt.last_symbol = :STRING_LITERAL
+        return Token.new(:PP_TOKEN, string_literal, loc, :STRING_LITERAL)
       end
 
       nil
     end
 
-    def tokenize_null_constant(context)
-      location = context.location
+    def tokenize_null_constant(lexer_ctxt)
+      loc = lexer_ctxt.location
 
-      null_constant = Language::C.scan_null_constant(context.content)
+      null_constant = Language::C.scan_null_constant(lexer_ctxt.content)
 
       if null_constant
-        context.last_symbol = :NULL
-        return Token.new(:PP_TOKEN, null_constant, location, :NULL)
+        lexer_ctxt.last_symbol = :NULL
+        return Token.new(:PP_TOKEN, null_constant, loc, :NULL)
       end
 
       nil
     end
 
-    def tokenize_identifier(context)
-      location = context.location
+    def tokenize_identifier(lexer_ctxt)
+      loc = lexer_ctxt.location
 
-      identifier = Language::C.scan_identifier(context.content)
+      identifier = Language::C.scan_identifier(lexer_ctxt.content)
 
       if identifier
-        context.last_symbol = :IDENTIFIER
-        return Token.new(:PP_TOKEN, identifier, location, :IDENTIFIER)
+        lexer_ctxt.last_symbol = :IDENTIFIER
+        return Token.new(:PP_TOKEN, identifier, loc, :IDENTIFIER)
       end
 
       nil
     end
 
-    def tokenize_punctuator(context)
-      location = context.location
+    def tokenize_punctuator(lexer_ctxt)
+      loc = lexer_ctxt.location
 
-      punctuator = Language::Cpp.scan_punctuator(context.content)
+      punctuator = Language::Cpp.scan_punctuator(lexer_ctxt.content)
 
       if punctuator
-        context.last_symbol = :PUNCTUATOR
-        return Token.new(:PP_TOKEN, punctuator, location, punctuator)
+        lexer_ctxt.last_symbol = :PUNCTUATOR
+        return Token.new(:PP_TOKEN, punctuator, loc, punctuator)
       end
 
       nil
     end
 
-    def tokenize_backslash(context)
-      location = context.location
+    def tokenize_backslash(lexer_ctxt)
+      loc = lexer_ctxt.location
 
-      backslash = context.content.scan(/\\/)
+      backslash = lexer_ctxt.content.scan(/\\/)
 
       if backslash
-        context.last_symbol = :BACKSLASH
-        return Token.new(:PP_TOKEN, backslash, location, backslash)
+        lexer_ctxt.last_symbol = :BACKSLASH
+        return Token.new(:PP_TOKEN, backslash, loc, backslash)
       end
 
       nil
@@ -898,17 +904,14 @@ module Cpp #:nodoc:
   end
 
   class TextLineToPPTokensLexer < StringToPPTokensLexer
-    def initialize(text_line)
-      super(text_line.token.value)
+    def initialize(text_line, tab_width)
+      super(text_line.token.value, tab_width)
       @text_line = text_line
     end
 
     private
     def create_content(str)
-      StringContent.new(str,
-                        @text_line.location.fpath,
-                        @text_line.location.line_no,
-                        @text_line.location.column_no)
+      StringContent.new(str, @tab_width, *@text_line.location.to_a)
     end
   end