src_lexer 1.0.2 → 1.0.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MDExMjNmNTY4OTNmYTVlYzM3MDIyYzExY2I3YjE1NzMyMzVjY2UxMQ==
+    NjUyYWQ1MmYxNTM0ODdmM2JlNjA0MTJjZGVmZWI2NjMyZmFhNDlkMg==
   data.tar.gz: !binary |-
-    NzdjYjIyZmJmN2RiNjJhYzVlMjgzZWU1MjQ0MmFiZDJjMjMwMDQyNw==
+    NjZjYTZlZDgyOTFlNzZjZDE3Y2NjMDY5ZmQ5ZWY2NTJhN2EwODg1Nw==
 SHA512:
   metadata.gz: !binary |-
-    YmI2MjFlN2NiOWYwOTQyNzlkMGEyM2NjZjVmOGM1M2Y5N2Y1NDU5N2ExMzk4
-    OWFmZjIyYTg4N2ExY2IxNjEzMTE1OWFkMDZlNjQ4N2FmMDBiMWZhZTYzMTc1
-    ODI1OTNlNzY2YTM4ZDNiZTU2Njk1NzZhM2E4YWI1NzM5MGYyYTA=
+    ODI2N2RkMGU1MWFiN2Y5MzY0MWJmNTRiNTMzNjA0ODdkOTk4NjczMTc3NzEy
+    YzM2NzVlNGVlMjg5MDJjOWZjMGVjYzc3OTY5YTVhNDIwYTUwYjBiNDlmN2E5
+    ZjU2OWIxY2RhZDIwYmIxNzc5ZDYwMzVkYTI4MWY3OGY2YThlZjc=
   data.tar.gz: !binary |-
-    ODFjYThjOThlZTI2ODU5ODQyOGM5OTBkZGVkYTQ4ZTgzYTVkZjBhZDdiODIw
-    ZTAwNzk1ZTM4ODYzNzc5ZGM2N2JmOTg4ZjNkMWUzZjgzMmYxMDBmYzhhNzJk
-    Mjg1OTQ3MjA4N2JmZjIwOGYxZWViNzBlMDE4Y2ExMjBiMzg5ZTU=
+    YjFiOWQ3MmExZTRjM2IyODI0MzAwZjE2YWFiM2JhZjlmY2FjN2FiMDkwMTYx
+    MWU3MTk4YjMyMjRhMDdjMjUxZThkODVmYjZhZmE0MjNkODFlMWM2MTUyMDFi
+    Y2ZmNWNlNzg5OTIxM2FjMmQyOTQ4YjI1ZmU4OGVlYWQxMjU5YzY=

data/.gitignore

@@ -1,17 +1,17 @@
-*.gem
-*.rbc
-.bundle
-.config
-.yardoc
-Gemfile.lock
-InstalledFiles
-_yardoc
-coverage
-doc/
-lib/bundler/man
-pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -1,2 +1,2 @@
---format documentation
---color
+--format documentation
+--color

data/.travis.yml

@@ -1,3 +1,3 @@
-language: ruby
-rvm:
-  - 1.9.3
+language: ruby
+rvm:
+  - 1.9.3

data/Gemfile

@@ -1,4 +1,4 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in src_lexer.gemspec
-gemspec
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in src_lexer.gemspec
+gemspec

data/LICENSE.txt

@@ -1,22 +1,22 @@
-Copyright (c) 2014 kkikzk
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+Copyright (c) 2014 kkikzk
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -1,84 +1,84 @@
-# SrcLexer
-
-SrcLexer is a simple source file lexer.
-
-## Installation
-
-Add this line to your application's Gemfile:
-
-    gem 'src_lexer'
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install src_lexer
-
-## Usage
-
-    lexer = SrcLexer::Lexer.new(
-      ['struct', 'enum', 'true', 'false'], # kyewords
-      ['{', '}', '(', ')', ',', '==', '=', ';'], # symbols
-      ['"', '"'], # string literal markers
-      '//', # line comment marker
-      ['/*', '*/'] # multi line comment markers
-    )
-    
-    lexer.analyze(<<-'EOS')
-      // comment
-      enum ID {
-        First = 1,
-        Second = 1.5
-      }
-      /* comment
-         againe */
-      struct Data {
-        string name = "This is a name.";
-        ID id;
-      }
-      bool b = (true==false);
-    EOS
-    
-    lexer.pop_token # => ['enum', SrcLexer::Token.new('enum', 2, 3)]
-    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('ID', 2, 8)]
-    lexer.pop_token # => ['{', SrcLexer::Token.new('{', 2, 11)]
-    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('First', 3, 5)]
-    lexer.pop_token # => ['=', SrcLexer::Token.new('=', 3, 11)]
-    lexer.pop_token # => [:NUMBER, SrcLexer::Token.new('1', 3, 13)]
-    lexer.pop_token # => [',', SrcLexer::Token.new(',', 3, 14)]
-    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('Second', 4, 5)]
-    lexer.pop_token # => ['=', SrcLexer::Token.new('=', 4, 12)]
-    lexer.pop_token # => [:NUMBER, SrcLexer::Token.new('1.5', 4, 14)]
-    lexer.pop_token # => ['}', SrcLexer::Token.new('}', 5, 3)]
-    lexer.pop_token # => ['struct', SrcLexer::Token.new('struct', 8, 3)]
-    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('Data', 8, 10)]
-    lexer.pop_token # => ['{', SrcLexer::Token.new('{', 8, 15)]
-    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('string', 9, 5)]
-    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('name', 9, 12)]
-    lexer.pop_token # => ['=', SrcLexer::Token.new('=', 9, 17)]
-    lexer.pop_token # => [:STRING, SrcLexer::Token.new('"This is a name."', 9, 19)]
-    lexer.pop_token # => [';', SrcLexer::Token.new(';', 9, 36)]
-    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('ID', 10, 5)]
-    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('id', 10, 8)]
-    lexer.pop_token # => [';', SrcLexer::Token.new(';', 10, 10)]
-    lexer.pop_token # => ['}', SrcLexer::Token.new('}', 11, 3)]
-    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('bool', 12, 3)]
-    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('b', 12, 8)]
-    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('=', 12, 10)]
-    lexer.pop_token # => ['(', SrcLexer::Token.new('(', 12, 12)]
-    lexer.pop_token # => ['true', SrcLexer::Token.new('true', 12, 13)]
-    lexer.pop_token # => ['==', SrcLexer::Token.new('==', 12, 17)]
-    lexer.pop_token # => ['false', SrcLexer::Token.new('false', 12, 19)]
-    lexer.pop_token # => [')', SrcLexer::Token.new(')', 12, 24)]
-    lexer.pop_token # => [';', SrcLexer::Token.new(';', 12, 25)]
-    lexer.pop_token # => SrcLexer::Lexer::END_TOKEN
-
-## Contributing
-
-1. Fork it ( http://github.com/<my-github-username>/src_lexer/fork )
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+# SrcLexer
+
+SrcLexer is a simple source file lexer.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'src_lexer'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install src_lexer
+
+## Usage
+
+    lexer = SrcLexer::Lexer.new(
+      ['struct', 'enum', 'true', 'false'], # kyewords
+      ['{', '}', '(', ')', ',', '==', '=', ';'], # symbols
+      ['"', '"'], # string literal markers
+      '//', # line comment marker
+      ['/*', '*/'] # multi line comment markers
+    )
+    
+    lexer.analyze(<<-'EOS')
+      // comment
+      enum ID {
+        First = 1,
+        Second = 1.5
+      }
+      /* comment
+         againe */
+      struct Data {
+        string name = "This is a name.";
+        ID id;
+      }
+      bool b = (true==false);
+    EOS
+    
+    lexer.pop_token # => ['enum', SrcLexer::Token.new('enum', 2, 3)]
+    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('ID', 2, 8)]
+    lexer.pop_token # => ['{', SrcLexer::Token.new('{', 2, 11)]
+    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('First', 3, 5)]
+    lexer.pop_token # => ['=', SrcLexer::Token.new('=', 3, 11)]
+    lexer.pop_token # => [:NUMBER, SrcLexer::Token.new('1', 3, 13)]
+    lexer.pop_token # => [',', SrcLexer::Token.new(',', 3, 14)]
+    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('Second', 4, 5)]
+    lexer.pop_token # => ['=', SrcLexer::Token.new('=', 4, 12)]
+    lexer.pop_token # => [:NUMBER, SrcLexer::Token.new('1.5', 4, 14)]
+    lexer.pop_token # => ['}', SrcLexer::Token.new('}', 5, 3)]
+    lexer.pop_token # => ['struct', SrcLexer::Token.new('struct', 8, 3)]
+    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('Data', 8, 10)]
+    lexer.pop_token # => ['{', SrcLexer::Token.new('{', 8, 15)]
+    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('string', 9, 5)]
+    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('name', 9, 12)]
+    lexer.pop_token # => ['=', SrcLexer::Token.new('=', 9, 17)]
+    lexer.pop_token # => [:STRING, SrcLexer::Token.new('"This is a name."', 9, 19)]
+    lexer.pop_token # => [';', SrcLexer::Token.new(';', 9, 36)]
+    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('ID', 10, 5)]
+    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('id', 10, 8)]
+    lexer.pop_token # => [';', SrcLexer::Token.new(';', 10, 10)]
+    lexer.pop_token # => ['}', SrcLexer::Token.new('}', 11, 3)]
+    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('bool', 12, 3)]
+    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('b', 12, 8)]
+    lexer.pop_token # => [:IDENT, SrcLexer::Token.new('=', 12, 10)]
+    lexer.pop_token # => ['(', SrcLexer::Token.new('(', 12, 12)]
+    lexer.pop_token # => ['true', SrcLexer::Token.new('true', 12, 13)]
+    lexer.pop_token # => ['==', SrcLexer::Token.new('==', 12, 17)]
+    lexer.pop_token # => ['false', SrcLexer::Token.new('false', 12, 19)]
+    lexer.pop_token # => [')', SrcLexer::Token.new(')', 12, 24)]
+    lexer.pop_token # => [';', SrcLexer::Token.new(';', 12, 25)]
+    lexer.pop_token # => SrcLexer::Lexer::END_TOKEN
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/src_lexer/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -1,6 +1,6 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
-
-RSpec::Core::RakeTask.new(:spec)
-
-task :default => :spec
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/src_lexer

@@ -1,3 +1,3 @@
-#!/usr/bin/env ruby
-
-require 'src_lexer'
+#!/usr/bin/env ruby
+
+require 'src_lexer'

data/lib/src_lexer/version.rb

@@ -1,3 +1,3 @@
-module SrcLexer
-  VERSION = "1.0.2"
-end
+module SrcLexer
+  VERSION = "1.0.3"
+end

data/lib/src_lexer.rb

@@ -1,227 +1,227 @@
-# -*- encoding: utf-8 -*-
-require "src_lexer/version"
-
-module SrcLexer
-  class Token
-    attr_reader :str, :line_no, :char_no
-
-    def initialize(str, line_no, char_no)
-      @str = str
-      @line_no = line_no
-      @char_no = char_no
-    end
-
-    def ==(other_object)
-      @str == other_object.str && @line_no == other_object.line_no && @char_no == other_object.char_no
-    end
-  end
-
-  class Lexer
-    END_TOKEN = [false, nil]
-    NUMBER_REGEX = /^[\d]+[\.]?[\d]*\z/
-    STRING_REGEX = /^\"(.*)\"\z/m
-    attr_reader :keywords, :symbols, :string_literal_marker, :line_comment_marker, :comment_markers, :tokens, :str
-
-    def initialize(keywords, symbols, string_literal_marker, line_comment_marker, comment_markers)
-      @keywords = (keywords ? keywords.uniq.compact : [])
-      @symbols = (symbols ? symbols.uniq.compact : [])
-      @string_literal_marker = string_literal_marker
-      @line_comment_marker = line_comment_marker
-      @comment_markers = comment_markers
-    end
-
-    def analyze(str)
-      @str = str
-      tokenize
-    end
-
-    def pop_token
-      token = @tokens.shift
-      return END_TOKEN if token.nil?
-      case token[0]
-      when NUMBER_REGEX
-        [:NUMBER, Token.new(token[0], token[1], token[2])]
-      when STRING_REGEX
-        [:STRING, Token.new(token[0], token[1], token[2])]
-      else
-        [is_reserved?(token[0]) ? token[0] : :IDENT, Token.new(token[0], token[1], token[2])]
-      end
-    end
-
-    private
-
-    class PosInfo
-      attr_accessor :index, :line_no, :char_no
-      
-      def initialize
-        @index = 0
-        @line_no = 1
-        @char_no = 1
-      end
-    end
-
-    class StringIterator
-      def initialize(str)
-        @str = str
-        @current_pos = PosInfo.new
-        @marked_pos = PosInfo.new
-        mark_clear()
-      end
-
-      def mark_clear
-        @marked_pos.index = -1
-        @marked_pos.line_no = 0
-        @marked_pos.char_no = 0
-      end
-
-      def mark_set
-        @marked_pos = @current_pos.clone
-      end
-
-      def is(target_string)
-        return false if target_string.length.zero?
-        end_pos = (@current_pos.index + target_string.length - 1)
-        @str[@current_pos.index..end_pos] == target_string
-      end
-
-      def is_in(target_list)
-        target_list.find { |target| is(target) } != nil
-      end
-
-      def move_next
-        if /\n/.match @str[@current_pos.index]
-          @current_pos.line_no += 1
-          @current_pos.char_no = 1
-        else
-          @current_pos.char_no += 1
-        end
-        @current_pos.index += 1
-      end
-
-      def move_to_the_end_of_the_line
-        char_count_to_the_end_of_the_line = (@str[@current_pos.index..-1] =~ /$/) - 1
-        @current_pos.index += char_count_to_the_end_of_the_line
-        @current_pos.char_no += char_count_to_the_end_of_the_line
-      end
-
-      def move_to(target)
-        char_count_to_target = (@str[@current_pos.index..-1] =~ /#{Regexp.escape(target)}/m) + target.length - 1
-        chopped_string = @str[@current_pos.index..@current_pos.index + char_count_to_target]
-        @current_pos.index += char_count_to_target
-        match = /.*\n(.*)$/m.match(chopped_string)
-        p match[1].length if match
-        if match
-          @current_pos.char_no = match[1].length
-        else
-          @current_pos.char_no += char_count_to_target
-        end
-        @current_pos.line_no += chopped_string.each_char.select{|char| /\n/.match char}.length
-      end
-
-      def <(index)
-        @current_pos.index < index
-      end
-
-      def is_white_space
-        /\s/.match(@str[@current_pos.index])
-      end
-
-      def marked?
-        @marked_pos.index != -1
-      end
-
-      def shift
-        result = [@str[@marked_pos.index..(@current_pos.index - 1)], @marked_pos.line_no, @marked_pos.char_no]
-        mark_clear()
-        return result
-      end
-    end
-
-    def tokenize()
-      @tokens = []
-      iterator = StringIterator.new(@str)
-
-      while iterator < @str.length do
-        if iterator.is_white_space then
-          @tokens.push iterator.shift if iterator.marked?
-          iterator.move_next
-        elsif @line_comment_marker && iterator.is(@line_comment_marker) then
-          @tokens.push iterator.shift if iterator.marked?
-          iterator.move_to_the_end_of_the_line
-          iterator.move_next
-        elsif @comment_markers && iterator.is(@comment_markers[0]) then
-          @tokens.push iterator.shift if iterator.marked?
-          iterator.move_to(@comment_markers[1])
-          iterator.move_next
-        elsif @string_literal_marker && iterator.is(@string_literal_marker[0]) then
-          @tokens.push iterator.shift if iterator.marked?
-          iterator.mark_set
-          iterator.move_next
-          iterator.move_to(@string_literal_marker[1])
-          iterator.move_next
-          @tokens.push iterator.shift
-        elsif iterator.is_in(@symbols) then
-          @tokens.push iterator.shift if iterator.marked?
-          iterator.mark_set
-          @symbols.find { |symbol| iterator.is(symbol) }.length.times { iterator.move_next }
-          @tokens.push iterator.shift
-        elsif !iterator.marked? then
-          iterator.mark_set
-        else
-          iterator.move_next
-        end
-      end
-      @tokens.push iterator.shift if iterator.marked?
-      
-      return self
-    end
-
-    def is_reserved?(token)
-      @keywords.include?(token) || @symbols.include?(token)
-    end
-  end
-
-  class CSharpLexer < Lexer
-    def initialize
-      super(
-        [ # C# keywords
-          'abstract',   'as',       'base',       'bool',      'break',
-          'byte',       'case',     'catch',      'char',      'checked',
-          'class',      'const',    'continue',   'decimal',   'default',
-          'delegate',   'do',       'double',     'else',      'enum',
-          'event',      'explicit', 'extern',     'false',     'finally',
-          'fixed',      'float',    'for',        'foreach',   'goto',
-          'if',         'implicit', 'in',         'int',       'interface',
-          'internal',   'is',       'lock',       'long',      'namespace',
-          'new',        'null',     'object',     'operator',  'out',
-          'override',   'params',   'private',    'protected', 'public',
-          'readonly',   'ref',      'return',     'sbyte',     'sealed',
-          'short',      'sizeof',   'stackalloc', 'static',    'string',
-          'struct',     'switch',   'this',       'throw',     'true',
-          'try',        'typeof',   'uint',       'ulong',     'unchecked',
-          'unsafe',     'ushort',   'using',      'virtual',   'void',
-          'volatile',   'while',
-          # C# context keywords
-          'add',        'alias',    'ascending',  'async',     'await',
-          'descending', 'dynamic',  'from',       'get',       'global',
-          'group',      'into',     'join',       'let',       'orderby',
-          'partial',    'remove',   'select',     'set',       'value',
-          'var',        'where',    'yield'
-        ],
-        [
-          '<<=', '>>=', '<<',  '>>',  '<=',
-          '>=',  '==',  '!=',  '&&',  '||',
-          '??',  '+=',  '-=',  '*=',  '/=',
-          '%=',  '&=',  '|=',  '^=',  '=>',
-          '*',   '/',   '%',   '+',   '-',
-          '<',   '>',   '&',   '^',   '|',
-          '?',   ':',   '=',   '{',   '}',
-          '(',   ')',   '[',   ']',   ';',
-          ','
-        ],
-        ['"', '"'], # comment markers
-        '//', # line comment marker
-        ['/*', '*/']) # multi line comment markers
-    end
-  end
-end
+# -*- encoding: utf-8 -*-
+require "src_lexer/version"
+
+module SrcLexer
+  class Token
+    attr_reader :str, :line_no, :char_no
+
+    def initialize(str, line_no, char_no)
+      @str = str
+      @line_no = line_no
+      @char_no = char_no
+    end
+
+    def ==(other_object)
+      @str == other_object.str && @line_no == other_object.line_no && @char_no == other_object.char_no
+    end
+  end
+
+  class Lexer
+    END_TOKEN = [false, nil]
+    NUMBER_REGEX = /^[\d]+[\.]?[\d]*\z/
+    STRING_REGEX = /^\"(.*)\"\z/m
+    attr_reader :keywords, :symbols, :string_literal_marker, :line_comment_marker, :comment_markers, :tokens, :str
+
+    def initialize(keywords, symbols, string_literal_marker, line_comment_marker, comment_markers)
+      @keywords = (keywords ? keywords.uniq.compact : [])
+      @symbols = (symbols ? symbols.uniq.compact : [])
+      @string_literal_marker = string_literal_marker
+      @line_comment_marker = line_comment_marker
+      @comment_markers = comment_markers
+    end
+
+    def analyze(str)
+      @str = str
+      tokenize
+    end
+
+    def pop_token
+      token = @tokens.shift
+      return END_TOKEN if token.nil?
+      case token[0]
+      when NUMBER_REGEX
+        [:NUMBER, Token.new(token[0], token[1], token[2])]
+      when STRING_REGEX
+        [:STRING, Token.new(token[0], token[1], token[2])]
+      else
+        [is_reserved?(token[0]) ? token[0] : :IDENT, Token.new(token[0], token[1], token[2])]
+      end
+    end
+
+    private
+
+    class PosInfo
+      attr_accessor :index, :line_no, :char_no
+      
+      def initialize
+        @index = 0
+        @line_no = 1
+        @char_no = 1
+      end
+    end
+
+    class StringIterator
+      def initialize(str)
+        @str = str
+        @current_pos = PosInfo.new
+        @marked_pos = PosInfo.new
+        mark_clear()
+      end
+
+      def mark_clear
+        @marked_pos.index = -1
+        @marked_pos.line_no = 0
+        @marked_pos.char_no = 0
+      end
+
+      def mark_set
+        @marked_pos = @current_pos.clone
+      end
+
+      def is(target_string)
+        return false if target_string.length.zero?
+        end_pos = (@current_pos.index + target_string.length - 1)
+        @str[@current_pos.index..end_pos] == target_string
+      end
+
+      def is_in(target_list)
+        target_list.find { |target| is(target) } != nil
+      end
+
+      def move_next
+        if /\n/.match @str[@current_pos.index]
+          @current_pos.line_no += 1
+          @current_pos.char_no = 1
+        else
+          @current_pos.char_no += 1
+        end
+        @current_pos.index += 1
+      end
+
+      def move_to_the_end_of_the_line
+        char_count_to_the_end_of_the_line = (@str[@current_pos.index..-1] =~ /$/) - 1
+        @current_pos.index += char_count_to_the_end_of_the_line
+        @current_pos.char_no += char_count_to_the_end_of_the_line
+      end
+
+      def move_to(target)
+        char_count_to_target = (@str[@current_pos.index..-1] =~ /#{Regexp.escape(target)}/m) + target.length - 1
+        chopped_string = @str[@current_pos.index..@current_pos.index + char_count_to_target]
+        @current_pos.index += char_count_to_target
+        match = /.*\n(.*)$/m.match(chopped_string)
+        p match[1].length if match
+        if match
+          @current_pos.char_no = match[1].length
+        else
+          @current_pos.char_no += char_count_to_target
+        end
+        @current_pos.line_no += chopped_string.each_char.select{|char| /\n/.match char}.length
+      end
+
+      def <(index)
+        @current_pos.index < index
+      end
+
+      def is_white_space
+        /\s/.match(@str[@current_pos.index])
+      end
+
+      def marked?
+        @marked_pos.index != -1
+      end
+
+      def shift
+        result = [@str[@marked_pos.index..(@current_pos.index - 1)], @marked_pos.line_no, @marked_pos.char_no]
+        mark_clear()
+        return result
+      end
+    end
+
+    def tokenize()
+      @tokens = []
+      iterator = StringIterator.new(@str)
+
+      while iterator < @str.length do
+        if iterator.is_white_space then
+          @tokens.push iterator.shift if iterator.marked?
+          iterator.move_next
+        elsif @line_comment_marker && iterator.is(@line_comment_marker) then
+          @tokens.push iterator.shift if iterator.marked?
+          iterator.move_to_the_end_of_the_line
+          iterator.move_next
+        elsif @comment_markers && iterator.is(@comment_markers[0]) then
+          @tokens.push iterator.shift if iterator.marked?
+          iterator.move_to(@comment_markers[1])
+          iterator.move_next
+        elsif @string_literal_marker && iterator.is(@string_literal_marker[0]) then
+          @tokens.push iterator.shift if iterator.marked?
+          iterator.mark_set
+          iterator.move_next
+          iterator.move_to(@string_literal_marker[1])
+          iterator.move_next
+          @tokens.push iterator.shift
+        elsif iterator.is_in(@symbols) then
+          @tokens.push iterator.shift if iterator.marked?
+          iterator.mark_set
+          @symbols.find { |symbol| iterator.is(symbol) }.length.times { iterator.move_next }
+          @tokens.push iterator.shift
+        elsif !iterator.marked? then
+          iterator.mark_set
+        else
+          iterator.move_next
+        end
+      end
+      @tokens.push iterator.shift if iterator.marked?
+      
+      return self
+    end
+
+    def is_reserved?(token)
+      @keywords.include?(token) || @symbols.include?(token)
+    end
+  end
+
+  class CSharpLexer < Lexer
+    def initialize
+      super(
+        [ # C# keywords
+          'abstract',   'as',       'base',       'bool',      'break',
+          'byte',       'case',     'catch',      'char',      'checked',
+          'class',      'const',    'continue',   'decimal',   'default',
+          'delegate',   'do',       'double',     'else',      'enum',
+          'event',      'explicit', 'extern',     'false',     'finally',
+          'fixed',      'float',    'for',        'foreach',   'goto',
+          'if',         'implicit', 'in',         'int',       'interface',
+          'internal',   'is',       'lock',       'long',      'namespace',
+          'new',        'null',     'object',     'operator',  'out',
+          'override',   'params',   'private',    'protected', 'public',
+          'readonly',   'ref',      'return',     'sbyte',     'sealed',
+          'short',      'sizeof',   'stackalloc', 'static',    'string',
+          'struct',     'switch',   'this',       'throw',     'true',
+          'try',        'typeof',   'uint',       'ulong',     'unchecked',
+          'unsafe',     'ushort',   'using',      'virtual',   'void',
+          'volatile',   'while',
+          # C# context keywords
+          'add',        'alias',    'ascending',  'async',     'await',
+          'descending', 'dynamic',  'from',       'get',       'global',
+          'group',      'into',     'join',       'let',       'orderby',
+          'partial',    'remove',   'select',     'set',       'value',
+          'var',        'where',    'yield'
+        ],
+        [
+          '<<=', '>>=', '<<',  '>>',  '<=',
+          '>=',  '==',  '!=',  '&&',  '||',
+          '??',  '+=',  '-=',  '*=',  '/=',
+          '%=',  '&=',  '|=',  '^=',  '=>',
+          '*',   '/',   '%',   '+',   '-',
+          '<',   '>',   '&',   '^',   '|',
+          '?',   ':',   '=',   '{',   '}',
+          '(',   ')',   '[',   ']',   ';',
+          ','
+        ],
+        ['"', '"'], # comment markers
+        '//', # line comment marker
+        ['/*', '*/']) # multi line comment markers
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,3 +1,3 @@
-# -*- encoding: utf-8 -*-
-$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
-require 'src_lexer'
+# -*- encoding: utf-8 -*-
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'src_lexer'

data/spec/src_lexer_spec.rb

@@ -1,105 +1,115 @@
-# -*- encoding: utf-8 -*-
-require_relative './spec_helper'
-
-describe SrcLexer do
-  it 'should have a version number' do
-    SrcLexer::VERSION.should_not be_nil
-  end
-end
-
-describe SrcLexer::Lexer, 'with empty string' do  
-  it 'should return Lexer::END_TOKEN' do
-    sut = SrcLexer::Lexer.new(nil, nil, nil, nil, nil)
-    sut.analyze('')
-    sut.pop_token.should == SrcLexer::Lexer::END_TOKEN
-  end
-end
-
-describe SrcLexer::Lexer, 'with keyword definitions' do
-  it 'should recognize keywords' do
-    sut = SrcLexer::Lexer.new(['struct', 'enum'], nil, nil, nil, nil)
-    sut.analyze('struct structenum enum')
-    sut.pop_token.should == ['struct', SrcLexer::Token.new('struct', 1, 1)]
-    sut.pop_token.should == [:IDENT, SrcLexer::Token.new('structenum', 1, 8)]
-    sut.pop_token.should == ['enum', SrcLexer::Token.new('enum', 1, 19)]
-    sut.pop_token.should == SrcLexer::Lexer::END_TOKEN
-  end
-  it 'should reduce keyword duplication' do
-    sut = SrcLexer::Lexer.new(['struct', 'struct'], nil, nil, nil, nil)
-    sut.keywords.should == ['struct']
-  end
-  it 'should ignore nil keyword' do
-    sut = SrcLexer::Lexer.new(['struct', nil, 'enum'], nil, nil, nil, nil)
-    sut.keywords.should == ['struct', 'enum']
-  end
-end
-
-describe SrcLexer::Lexer, 'with symbol definitions' do
-  it 'should recognize symbols' do
-    sut = SrcLexer::Lexer.new(nil, ['..', ','], nil, nil, nil)
-    sut.analyze('.. A ,')
-    sut.pop_token.should == ['..', SrcLexer::Token.new('..', 1, 1)]
-    sut.pop_token.should == [:IDENT, SrcLexer::Token.new('A', 1, 4)]
-    sut.pop_token.should == [',', SrcLexer::Token.new(',', 1, 6)]
-    sut.pop_token.should == SrcLexer::Lexer::END_TOKEN
-  end
-  it 'should recognize symbols(,) if continues like "A,B"' do
-    sut = SrcLexer::Lexer.new(['A', 'B'], [','], nil, nil, nil)
-    sut.analyze('A,B')
-    sut.pop_token.should == ['A', SrcLexer::Token.new('A', 1, 1)]
-    sut.pop_token.should == [',', SrcLexer::Token.new(',', 1, 2)]
-    sut.pop_token.should == ['B', SrcLexer::Token.new('B', 1, 3)]
-    sut.pop_token.should == SrcLexer::Lexer::END_TOKEN
-  end
-  it 'should reduce symbol duplication' do
-    sut = SrcLexer::Lexer.new(nil, [',', ','], nil, nil, nil)
-    sut.symbols.should == [',']
-  end
-  it 'should ignore nil keyword' do
-    sut = SrcLexer::Lexer.new(nil, ['{', nil, '}'], nil, nil, nil)
-    sut.symbols.should == ['{', '}']
-  end
-end
-
-describe SrcLexer::Lexer, 'with line comment marker' do
-  it 'should recognize line comment' do
-    sut = SrcLexer::Lexer.new(nil, nil, nil, '//', nil)
-    sut.analyze(<<-'EOS')
-      A//comment
-      B
-    EOS
-    sut.pop_token.should == [:IDENT, SrcLexer::Token.new('A', 1, 7)]
-    sut.pop_token.should == [:IDENT, SrcLexer::Token.new('B', 2, 7)]
-    sut.pop_token.should == SrcLexer::Lexer::END_TOKEN
-  end
-  it 'should recognize multi line comment' do
-    sut = SrcLexer::Lexer.new(nil, nil, nil, '//', ['/*', '*/'])
-    sut.analyze(<<-'EOS')
-      A/*comment
-      B//still in comment*/C
-    EOS
-    sut.pop_token.should == [:IDENT, SrcLexer::Token.new('A', 1, 7)]
-    sut.pop_token.should == [:IDENT, SrcLexer::Token.new('C', 2, 28)]
-    sut.pop_token.should == SrcLexer::Lexer::END_TOKEN
-  end
-end
-
-describe SrcLexer::Lexer do
-  it 'should analyze number string' do
-    sut = SrcLexer::Lexer.new(nil, nil, nil, nil, nil)
-    sut.analyze('9 1.5')
-    sut.pop_token.should == [:NUMBER, SrcLexer::Token.new("9", 1, 1,)]
-    sut.pop_token.should == [:NUMBER, SrcLexer::Token.new("1.5", 1, 3)]
-    sut.pop_token.should == SrcLexer::Lexer::END_TOKEN
-  end
-  it 'should analyze string literal' do
-    sut = SrcLexer::Lexer.new(nil, nil, ['"', '"'], '//', ['/*', '*/'])
-    sut.analyze('A"//"B"/**/"C')
-    sut.pop_token.should == [:IDENT, SrcLexer::Token.new('A', 1, 1)]
-    sut.pop_token.should == [:STRING, SrcLexer::Token.new('"//"', 1, 2)]
-    sut.pop_token.should == [:IDENT, SrcLexer::Token.new('B', 1, 6)]
-    sut.pop_token.should == [:STRING, SrcLexer::Token.new('"/**/"', 1, 7)]
-    sut.pop_token.should == [:IDENT, SrcLexer::Token.new('C', 1, 13)]
-    sut.pop_token.should == SrcLexer::Lexer::END_TOKEN
-  end
-end
+# -*- encoding: utf-8 -*-
+require_relative './spec_helper'
+
+describe SrcLexer do
+  it 'should have a version number' do
+    expect(SrcLexer::VERSION).not_to eq(be_nil)
+  end
+end
+
+describe SrcLexer::Lexer, 'with empty string' do  
+  it 'should return Lexer::END_TOKEN' do
+    sut = SrcLexer::Lexer.new(nil, nil, nil, nil, nil)
+    sut.analyze('')
+    expect(sut.pop_token).to eq(SrcLexer::Lexer::END_TOKEN)
+  end
+end
+
+describe SrcLexer::Lexer, 'with keyword definitions' do
+  it 'should recognize keywords' do
+    sut = SrcLexer::Lexer.new(['struct', 'enum'], nil, nil, nil, nil)
+    sut.analyze('struct structenum enum')
+    expect(sut.pop_token).to eq(['struct', SrcLexer::Token.new('struct', 1, 1)])
+    expect(sut.pop_token).to eq([:IDENT, SrcLexer::Token.new('structenum', 1, 8)])
+    expect(sut.pop_token).to eq(['enum', SrcLexer::Token.new('enum', 1, 19)])
+    expect(sut.pop_token).to eq(SrcLexer::Lexer::END_TOKEN)
+  end
+  it 'should reduce keyword duplication' do
+    sut = SrcLexer::Lexer.new(['struct', 'struct'], nil, nil, nil, nil)
+    expect(sut.keywords).to eq(['struct'])
+  end
+  it 'should ignore nil keyword' do
+    sut = SrcLexer::Lexer.new(['struct', nil, 'enum'], nil, nil, nil, nil)
+    expect(sut.keywords).to eq(['struct', 'enum'])
+  end
+end
+
+describe SrcLexer::Lexer, 'with symbol definitions' do
+  it 'should recognize symbols' do
+    sut = SrcLexer::Lexer.new(nil, ['..', ','], nil, nil, nil)
+    sut.analyze('.. A ,')
+    expect(sut.pop_token).to eq(['..', SrcLexer::Token.new('..', 1, 1)])
+    expect(sut.pop_token).to eq([:IDENT, SrcLexer::Token.new('A', 1, 4)])
+    expect(sut.pop_token).to eq([',', SrcLexer::Token.new(',', 1, 6)])
+    expect(sut.pop_token).to eq(SrcLexer::Lexer::END_TOKEN)
+  end
+  it 'should recognize symbols(,) if continues like "A,B"' do
+    sut = SrcLexer::Lexer.new(['A', 'B'], [','], nil, nil, nil)
+    sut.analyze('A,B')
+    expect(sut.pop_token).to eq(['A', SrcLexer::Token.new('A', 1, 1)])
+    expect(sut.pop_token).to eq([',', SrcLexer::Token.new(',', 1, 2)])
+    expect(sut.pop_token).to eq(['B', SrcLexer::Token.new('B', 1, 3)])
+    expect(sut.pop_token).to eq(SrcLexer::Lexer::END_TOKEN)
+  end
+  it 'should recognize symbol(==) if symbol(=) defined' do
+    sut = SrcLexer::Lexer.new(nil, ['==', '='], nil, nil, nil)
+    sut.analyze('A = B == C')
+    expect(sut.pop_token).to eq([:IDENT, SrcLexer::Token.new('A', 1, 1)])
+    expect(sut.pop_token).to eq(['=', SrcLexer::Token.new('=', 1, 3)])
+    expect(sut.pop_token).to eq([:IDENT, SrcLexer::Token.new('B', 1, 5)])
+    expect(sut.pop_token).to eq(['==', SrcLexer::Token.new('==', 1, 7)])
+    expect(sut.pop_token).to eq([:IDENT, SrcLexer::Token.new('C', 1, 10)])
+    expect(sut.pop_token).to eq(SrcLexer::Lexer::END_TOKEN)
+  end
+  it 'should reduce symbol duplication' do
+    sut = SrcLexer::Lexer.new(nil, [',', ','], nil, nil, nil)
+    expect(sut.symbols).to eq([','])
+  end
+  it 'should ignore nil keyword' do
+    sut = SrcLexer::Lexer.new(nil, ['{', nil, '}'], nil, nil, nil)
+    expect(sut.symbols).to eq(['{', '}'])
+  end
+end
+
+describe SrcLexer::Lexer, 'with line comment marker' do
+  it 'should recognize line comment' do
+    sut = SrcLexer::Lexer.new(nil, nil, nil, '//', nil)
+    sut.analyze(<<-'EOS')
+      A//comment
+      B
+    EOS
+    expect(sut.pop_token).to eq([:IDENT, SrcLexer::Token.new('A', 1, 7)])
+    expect(sut.pop_token).to eq([:IDENT, SrcLexer::Token.new('B', 2, 7)])
+    expect(sut.pop_token).to eq(SrcLexer::Lexer::END_TOKEN)
+  end
+  it 'should recognize multi line comment' do
+    sut = SrcLexer::Lexer.new(nil, nil, nil, '//', ['/*', '*/'])
+    sut.analyze(<<-'EOS')
+      A/*comment
+      B//still in comment*/C
+    EOS
+    expect(sut.pop_token).to eq([:IDENT, SrcLexer::Token.new('A', 1, 7)])
+    expect(sut.pop_token).to eq([:IDENT, SrcLexer::Token.new('C', 2, 28)])
+    expect(sut.pop_token).to eq(SrcLexer::Lexer::END_TOKEN)
+  end
+end
+
+describe SrcLexer::Lexer do
+  it 'should analyze number string' do
+    sut = SrcLexer::Lexer.new(nil, nil, nil, nil, nil)
+    sut.analyze('9 1.5')
+    expect(sut.pop_token).to eq([:NUMBER, SrcLexer::Token.new("9", 1, 1,)])
+    expect(sut.pop_token).to eq([:NUMBER, SrcLexer::Token.new("1.5", 1, 3)])
+    expect(sut.pop_token).to eq(SrcLexer::Lexer::END_TOKEN)
+  end
+  it 'should analyze string literal' do
+    sut = SrcLexer::Lexer.new(nil, nil, ['"', '"'], '//', ['/*', '*/'])
+    sut.analyze('A"//"B"/**/"C')
+    expect(sut.pop_token).to eq([:IDENT, SrcLexer::Token.new('A', 1, 1)])
+    expect(sut.pop_token).to eq([:STRING, SrcLexer::Token.new('"//"', 1, 2)])
+    expect(sut.pop_token).to eq([:IDENT, SrcLexer::Token.new('B', 1, 6)])
+    expect(sut.pop_token).to eq([:STRING, SrcLexer::Token.new('"/**/"', 1, 7)])
+    expect(sut.pop_token).to eq([:IDENT, SrcLexer::Token.new('C', 1, 13)])
+    expect(sut.pop_token).to eq(SrcLexer::Lexer::END_TOKEN)
+  end
+end

data/src_lexer.gemspec

@@ -1,24 +1,24 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'src_lexer/version'
-
-Gem::Specification.new do |spec|
-  spec.name          = "src_lexer"
-  spec.version       = SrcLexer::VERSION
-  spec.authors       = ["kkikzk"]
-  spec.email         = ["kkikzk@gmail.com"]
-  spec.summary       = %q{A simple source file lexer}
-  spec.description   = ""
-  spec.homepage      = "https://github.com/kkikzk/src_lexer"
-  spec.license       = "MIT"
-
-  spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
-
-  spec.add_development_dependency "bundler", "~> 1.5"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
-end
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'src_lexer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "src_lexer"
+  spec.version       = SrcLexer::VERSION
+  spec.authors       = ["kkikzk"]
+  spec.email         = ["kkikzk@gmail.com"]
+  spec.summary       = %q{A simple source file lexer}
+  spec.description   = ""
+  spec.homepage      = "https://github.com/kkikzk/src_lexer"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: src_lexer
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - kkikzk
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-29 00:00:00.000000000 Z
+date: 2014-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler