sdl4r 0.9.9 → 0.9.11

data/lib/sdl4r/parser/reader.rb

@@ -1,184 +0,0 @@
-#!/usr/bin/env ruby -w
-# encoding: UTF-8
-
-#--
-# Simple Declarative Language (SDL) for Ruby
-# Copyright 2005 Ikayzo, inc.
-#
-# This program is free software. You can distribute or modify it under the
-# terms of the GNU Lesser General Public License version 2.1 as published by
-# the Free Software Foundation.
-#
-# This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
-# INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this program; if not, contact the Free Software Foundation, Inc.,
-# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-#++
-
-module SDL4R
-
-  class Parser
-
-    # Gives access to the characters read to the Parser.
-    # This class was designed to gather the handling of the UTF-8 issues in one place and shield the
-    # Parser class from these problems.
-    class Reader # :nodoc: all
-
-      # +io+  an open IO from which the characters are read.
-      def initialize(io)
-        raise ArgumentError, "io == nil" unless io
-
-        @io = io
-        @line = nil
-        @line_chars = nil
-        @line_no = -1
-        @pos = 0
-      end
-
-      # The current line no (zero-based)
-      attr_reader :line_no
-
-      # The position of the char currently pointed by this reader in the current line.
-      # This is not necessarily the position of the char you just got from a method, it might be
-      # the position of the next char, for instance.
-      attr_reader :pos
-      
-      attr_reader :line
-
-      def line_length
-        return @line_chars.nil? ? 0 : @line_chars.length
-      end
-
-      # Reads next line in stream skipping comment lines and blank lines.
-      #
-      # Returns the next line or nil at the end of the file.
-      def read_line
-        @line_chars = nil
-
-        while @line = read_raw_line()
-          # Skip empty and commented lines
-          break unless @line.empty? or @line =~ /^#/
-        end
-
-        return @line
-      end
-
-      # Returns the string that goes from the current position of this Reader to the end of the line
-      # or nil if the current position doesn't allow that.
-      def rest_of_line
-        return @line[@pos..-1]
-      end
-
-      # Indicates whether the end of file has been reached.
-      def eof?
-        return @line.nil?
-      end
-
-      # Indicates whether there are more characters in the current line after the current char
-      def more_chars_in_line?
-        return @pos < line_length - 1
-      end
-
-      # Returns whether the end of the current +line+ as been reached.
-      def eol?
-        return @pos >= line_length
-      end
-
-      # Skips the current line by going just after its end.
-      def skip_to_eol
-        @pos = line_length
-      end
-
-      # Skips the whitespaces that follow the current position.
-      def skip_whitespaces
-        while (@pos + 1) < line_length and
-            (@line[@pos + 1] == ?\s or @line[@pos + 1] == ?\t)
-          @pos += 1
-        end
-      end
-
-      # Returns the character at position +pos+ in the current line.
-      # Returns nil if there is no current line or if +pos+ is after the end of the line.
-      def get_line_char(pos)
-        if @line_chars and pos < line_length
-          return @line_chars[pos]
-        else
-          return nil
-        end
-      end
-
-      # Returns the character at the current position or nil after end-of-line or end-of -file.
-      def current_char
-        return get_line_char(@pos)
-      end
-
-      # Go to the next character in the line. This method doesn't skip to the next line once the
-      # reached eol.
-      def skip_char
-        @pos += 1 if @pos < line_length
-      end
-
-      # Returns the current char and go to the next.
-      # Returns nil if end-of-line or -file has been reached.
-      def read_char
-        c = current_char
-        skip_char()
-        c
-      end
-
-      # Returns to the previous char if possible.
-      def previous_char
-        @pos -= 1 if @pos >= 1
-      end
-
-      # Returns the next index of the expression (string, regexp, fixnum) in the current line,
-      # starting from after the current position if no position is specified.
-      def find_next_in_line(searched, start_pos = nil)
-        start_pos = @pos + 1 unless start_pos
-        return @line.index(searched, start_pos)
-      end
-
-      # Skips the specified position in the current line.
-      def skip_to(new_pos)
-        @pos = new_pos
-      end
-
-
-      # Returns a subpart of the current line starting from +from+ and stopping at +to+ (excluded).
-      def substring(from, to = -1)
-        return @line[from..to]
-      end
-
-      # Reads and returns a "raw" line including lines with comments and blank lines.
-      #
-      # Returns the next line or nil if at the end of the file.
-      #
-      # This method changes the value of @line, @lineNo and @pos.
-      def read_raw_line
-        @line = @io.gets()
-
-        # Remove a possible \r at the end of line
-        @line.gsub!(/\r+$/, "") if @line
-
-        @pos = 0;
-        @line_chars = nil
-        if @line
-          @line_no += 1
-          @line_chars = @line.scan(/./m)
-        end
-        return @line
-      end
-
-      # Closes this Reader and its underlying +IO+.
-      def close
-        @io.close
-      end
-
-    end
-    
-  end
-
-end

data/lib/sdl4r/parser/time_span_with_zone.rb

@@ -1,57 +0,0 @@
-#!/usr/bin/env ruby -w
-# encoding: UTF-8
-
-#--
-# Simple Declarative Language (SDL) for Ruby
-# Copyright 2005 Ikayzo, inc.
-#
-# This program is free software. You can distribute or modify it under the
-# terms of the GNU Lesser General Public License version 2.1 as published by
-# the Free Software Foundation.
-#
-# This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
-# INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this program; if not, contact the Free Software Foundation, Inc.,
-# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-#++
-
-module SDL4R
-
-  class Parser
-
-    # An intermediate object used to store a timeSpan or the time
-    # component of a date/time instance. The types are disambiguated at a later stage.
-    #
-    # +seconds+ can have a fraction
-    # +time_zone_offset+ is a fraction of a day (equal to nil if not specified)
-    class TimeSpanWithZone # :nodoc: all
-
-      private
-
-        SECONDS_IN_DAY = 24 * 60 * 60
-
-      public
-
-      def initialize(day, hour, minute, second, time_zone_offset)
-        @day = day
-        @hour = hour
-        @min = minute
-        @sec = second
-        @time_zone_offset = time_zone_offset
-      end
-
-      attr_reader :day, :hour, :min, :sec, :time_zone_offset
-
-      # Returns the UTC offset as a fraction of a day on the current machine
-      def TimeSpanWithZone.default_time_zone_offset
-        return Rational(Time.now.utc_offset, SECONDS_IN_DAY)
-      end
-
-    end
-
-  end
-
-end

data/lib/sdl4r/parser/token.rb

@@ -1,138 +0,0 @@
-#!/usr/bin/env ruby -w
-# encoding: UTF-8
-
-#--
-# Simple Declarative Language (SDL) for Ruby
-# Copyright 2005 Ikayzo, inc.
-#
-# This program is free software. You can distribute or modify it under the
-# terms of the GNU Lesser General Public License version 2.1 as published by
-# the Free Software Foundation.
-#
-# This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
-# INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this program; if not, contact the Free Software Foundation, Inc.,
-# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-#++
-
-module SDL4R
-  
-  require File.dirname(__FILE__) + '/time_span_with_zone'
-
-  class Parser
-
-    # An SDL token.
-    #
-    # @author Daniel Leuck, Philippe Vosges
-    #
-    class Token # :nodoc: all
-
-      def initialize(text, line = -1, position = -1)
-        @text = text
-        @line = line
-        @position = position
-        @size = text.length
-
-        begin
-          @type = nil
-          @object = nil
-
-          if text =~ /^["`]/
-            @type = :STRING
-            @object = Parser.parse_string(text)
-
-          elsif text =~ /^'/
-            @type = :CHARACTER
-            @object = text[1...-1]
-
-          elsif text == "null"
-            @type = :NULL
-            @object = nil
-
-          elsif text =~ /^true$|^on$/
-            @type = :BOOLEAN
-            @object = true
-
-          elsif text =~ /^false$|^off$/
-            @type = :BOOLEAN
-            @object = false
-
-          elsif text =~ /^\[/
-            @type=:BINARY
-            @object = Parser.parse_binary(text)
-
-          elsif text =~ /^-?\d+\/\d+\/\d+$/
-            @type = :DATE;
-            @object = Parser.parse_date_time(text)
-
-          elsif text =~ /^-?\d+d?:\d+/
-            @type = :TIME
-            @object = parse_time_span_with_zone(text)
-
-          elsif text =~ /^[\d\-\.]/
-            @type = :NUMBER
-            @object = Parser.parse_number(text)
-
-          else
-            case text[0]
-            when ?{
-              @type = :START_BLOCK
-            when ?}
-              @type = :END_BLOCK
-            when ?=
-              @type = :EQUALS
-            when ?:
-              @type = :COLON
-            when ?;
-              @type = :SEMICOLON
-            end
-          end
-
-        rescue ArgumentError
-          raise SdlParseError.new($!.message, @line, @position)
-        end
-
-        @type = :IDENTIFIER if @type.nil? # if all hope is lost, it's an identifier
-
-        @punctuation =
-          @type == :COLON || @type == :SEMICOLON || @type == :EQUALS ||
-          @type == :START_BLOCK || @type == :END_BLOCK
-        @literal = @type != :IDENTIFIER && !@punctuation
-      end
-
-      attr_reader :text, :type, :line, :position
-
-      def literal?
-        @literal
-      end
-
-      # Returns the Ruby object corresponding to this literal (or nil if it is
-      # not a literal).
-      def object_for_literal
-        return @object
-      end
-
-      def to_s
-        @type.to_s + " " + @text + " pos:" + @position.to_s
-      end
-
-      # This special parse method is used only by the Token class for
-      # tokens which are ambiguously either a TimeSpan or the time component
-      # of a date/time type
-      def parse_time_span_with_zone(literal)
-        raise ArgumentError("time span or date literal is nil") if literal.nil?
-
-        days, hours, minutes, seconds, time_zone_offset =
-          Parser.parse_time_span_and_time_zone(literal, true, true)
-
-        return Parser::TimeSpanWithZone.new(days, hours, minutes, seconds, time_zone_offset)
-      end
-      
-    end
-
-  end
-  
-end

data/lib/sdl4r/parser/tokenizer.rb

@@ -1,507 +0,0 @@
-#!/usr/bin/env ruby -w
-# encoding: UTF-8
-
-#--
-# Simple Declarative Language (SDL) for Ruby
-# Copyright 2005 Ikayzo, inc.
-#
-# This program is free software. You can distribute or modify it under the
-# terms of the GNU Lesser General Public License version 2.1 as published by
-# the Free Software Foundation.
-#
-# This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
-# INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this program; if not, contact the Free Software Foundation, Inc.,
-# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-#++
-
-
-module SDL4R
-
-  require File.dirname(__FILE__) + '/reader'
-  require File.dirname(__FILE__) + '/token'
-
-  class Parser
-
-    # Tokenizer of the SDL parser
-    class Tokenizer # :nodoc: all
-
-      TOKEN_TYPES = [
-        :IDENTIFIER,
-
-        # punctuation
-        :COLON, :SEMICOLON, :EQUALS, :START_BLOCK, :END_BLOCK,
-
-        # literals
-        :STRING, :CHARACTER, :BOOLEAN, :NUMBER, :DATE, :TIME, :BINARY, :NULL ]
-
-
-      # Creates an SDL tokenizer on the specified +IO+.
-      def initialize(io)
-        raise ArgumentError, "io == nil" if io.nil?
-
-        @reader = Parser::Reader.new(io)
-        @token_start = 0
-        @starting_escaped_quote_line = false
-        @tokens = nil
-        @token_text = nil
-      end
-
-      # Closes this Tokenizer and its underlying +Reader+.
-      def close
-        @reader.close
-      end
-
-      # Raises a SdlParseError.
-      def parse_error(description, line_no = nil, position = nil)
-        line_no = @reader.line_no unless line_no
-        # @reader.pos points to the *next* position (thence the "pos - 1")
-        position = @reader.pos - 1 unless position and position >= 0
-
-        # We add one because editors typically start with line 1 rather than 0...
-        raise SdlParseError.new(description, line_no + 1, position + 1, @reader.line)
-      end
-
-      # Close the reader and throw a SdlParseError using the format
-      # Was expecting X but got Y.
-      #
-      def expecting_but_got(expecting, got, line, position)
-        parse_error("Was expecting #{expecting} but got #{got}", line, position)
-      end
-
-      # Returns the next line as tokens or nil if the end of the stream has been reached.
-      # This method handles line continuations both within and outside String literals.
-      # The line of tokens is assigned to @tokens.
-      #
-      # Returns a logical line as a list of Tokens.
-      #
-      def read_line_tokens
-        begin
-          read_line_tokens_even_if_empty()
-        end until @tokens.nil? or !@tokens.empty?
-        return @tokens
-      end
-
-      def line_no
-        @reader.line_no
-      end
-
-      def pos
-        @reader.pos
-      end
-
-      def line
-        @reader.line
-      end
-
-      private
-
-      # Returns the next line as tokens or nil if the end of the stream has been reached.
-      # This method handles line continuations both within and outside String literals.
-      # The line of tokens is assigned to @tokens.
-      #
-      # Returns a logical line as a list of Tokens.
-      # Returns an empty array if the line was empty.
-      # Returns +nil+ if the end of the stream has been reached.
-      #
-      def read_line_tokens_even_if_empty
-        # Reset of the token-related fields
-        @tokens = nil
-        @token_text = nil
-        @token_start = nil
-
-        if @reader.eol? and not @reader.read_line()
-          return nil
-        end
-
-        @tokens = []
-        @token_start = @reader.pos
-
-        until @reader.eol?
-          if @token_text
-            @tokens << Token.new(@token_text, @reader.line_no, @token_start)
-            @token_text = nil
-            @token_start = @reader.pos
-          end
-
-          c = @reader.current_char
-          next_c = @reader.get_line_char(@reader.pos + 1)
-          
-          case c
-          when "\""
-            # handle "" style strings including line continuations
-            handle_double_quote_string()
-
-          when "'"
-            handle_character_literal()
-
-          when "{", "}", "=", ":", ";"
-            # handle punctuation
-            punctuation_token = Token.new(c, @reader.line_no, @reader.pos)
-            @token_text = nil
-
-            if punctuation_token.type == :SEMICOLON
-              @reader.skip_char()
-              break
-            else
-              @tokens << punctuation_token
-            end
-
-          when "#"
-            # skip : hash comment at end of line
-            @reader.skip_to_eol()
-
-          when "/"
-            # handle // and /**/ style comments
-            if next_c == "/"
-              # skip : // comment
-              @reader.skip_to_eol()
-            else
-              handle_slash_comment()
-            end
-
-          when "`"
-            # handle multiline `` style strings
-            handle_back_quote_string()
-
-          when "["
-            # handle binary literals
-            handle_binary_literal()
-
-          when "\s", "\t"
-            @reader.skip_whitespaces()
-
-          when "\\"
-            # line continuations (outside a string literal)
-            handle_line_continuation()
-            next # otherwise 1st char of the continuation line is skipped by skip_char()
-
-          when /^[0-9\-\.]$/
-            if c == "-" and next_c == "-"
-              # -- comments : ignore
-              @reader.skip_to_eol()
-            else
-              # handle numbers, dates, and time spans
-              handle_number_date_or_time_span()
-            end
-
-          when /^[a-zA-Z\$_]$/
-            # FIXME Here, the Java code specifies isJavaIdentifierStart() but
-            # this is not easily implemented (at least as of Ruby 1.8).
-            # So, we implement a subset of these characters.
-            handle_identifier()
-
-          when "\n", "\r" # end of line
-            @reader.skip_to_eol()
-
-          else
-            parse_error("Unexpected character '#{c}'")
-          end
-
-          @reader.skip_char()
-        end
-
-        if @token_text
-          @tokens << Token.new(@token_text, @reader.line_no, @token_start)
-        end
-
-        return @tokens
-      end
-
-      # Adds the current escaped character (represented by ((|c|))) to @token_text.
-      # This method assumes the previous char was a backslash.
-      #
-      def add_escaped_char_in_string(c)
-        case c
-        when "\\", "\""
-          @token_text << c
-        when "n"
-          @token_text << ?\n
-        when "r"
-          @token_text << ?\r
-        when "t"
-          @token_text << ?\t
-        else
-          parse_error("Illegal escape character in string literal: '#{c}'.")
-        end
-      end
-
-      def handle_double_quote_string
-        escaped = false
-        @starting_escaped_quote_line = false
-
-        @token_text = @reader.read_char()
-
-        until @reader.eol?
-          c = @reader.current_char
-
-          if "\s\t".include?(c) and @starting_escaped_quote_line
-            # skip the heading spaces (indentation) of a continued line
-          else
-            @starting_escaped_quote_line = false
-
-            if escaped
-              add_escaped_char_in_string(c)
-              escaped = false
-
-            elsif c == "\\"
-              # check for String broken across lines
-              if @reader.rest_of_line =~ /^\\\s*$/
-                handle_escaped_double_quoted_string()
-                next # as we are at the beginning of a new line
-              else
-                escaped = true
-              end
-
-            else
-              @token_text << c
-              if c == '"'
-                # end of double-quoted string detected
-                @tokens << Token.new(@token_text, @reader.line_no, @token_start)
-                @token_text = nil
-                return
-              end
-            end
-          end
-
-          @reader.skip_char()
-        end
-
-        # detection of ill-terminated literals
-        if @token_text =~ /^".*[^"]$/
-          parse_error(
-            "String literal \"#{@token_text}\" not terminated by end quote.",
-            @reader.line_no,
-            @reader.line_length)
-        else#if @token_text == '"'
-          parse_error("Orphan quote (unterminated string)", @reader.line_no, @reader.line_length)
-        end
-puts "end of chars"
-      end
-
-      def handle_escaped_double_quoted_string
-        # '\' can be followed by whitespaces
-        if @reader.rest_of_line =~ /^\\\s*$/
-          @reader.read_line()
-          parse_error("Escape at end of file.") if @reader.eof?
-
-          @starting_escaped_quote_line = true
-
-        else
-          parse_error(
-            "Malformed string literal - escape followed by whitespace " +
-              "followed by non-whitespace.")
-        end
-      end
-
-      def handle_character_literal
-        @reader.skip_char # skip the starting quote
-
-        parse_error("Got ' at end of line") if @reader.eol?
-
-        c2 = @reader.read_char()
-
-        if c2 == "\\"
-          parse_error("Got '\\ at end of line") if @reader.eol?
-
-          c3 = @reader.read_char()
-
-          parse_error("Got '\\#{c3} at end of line") if @reader.eol?
-
-          case c3
-          when "\\"
-            @tokens << Token.new("'\\'", @reader.line_no, @reader.pos)
-          when "'"
-            @tokens << Token.new("'''", @reader.line_no, @reader.pos)
-          when "n"
-            @tokens << Token.new("'\n'", @reader.line_no, @reader.pos)
-          when "r"
-            @tokens << Token.new("'\r'", @reader.line_no, @reader.pos)
-          when "t"
-            @tokens << Token.new("'\t'", @reader.line_no, @reader.pos)
-          else
-            parse_error("Illegal escape character #{@reader.current_char}")
-          end
-
-          if @reader.read_char != "'"
-            expecting_but_got("single quote (')", "\"#{@reader.current_char}\"")
-          end
-
-        else
-          @tokens << Token.new("'#{c2}'", @reader.line_no, @reader.pos)
-
-          parse_error("Got '#{c2} at end of line") if @reader.eol?
-          
-          if @reader.read_char != "'"
-            expecting_but_got(
-              "quote (')", "\"#{@reader.current_char}\"", @reader.line_no, @reader.pos)
-          end
-        end
-      end
-
-      def handle_slash_comment
-        if not @reader.more_chars_in_line?
-          parse_error("Got slash (/) at end of line.")
-        end
-
-        if @reader.get_line_char(@reader.pos + 1) == "*"
-          end_index = @reader.find_next_in_line("*/")
-          if end_index
-            # handle comment on same line
-            @reader.skip_to(end_index + 1)
-          else
-            # handle multiline comments
-            loop do
-              @reader.read_raw_line()
-              if @reader.eof?
-                parse_error("/* comment not terminated.", @reader.line_no, -2)
-              end
-
-              end_index = @reader.find_next_in_line("*/", 0)
-
-              if end_index
-                @reader.skip_to(end_index + 1)
-                break
-              end
-            end
-          end
-        elsif @reader.get_line_char(@reader.pos + 1) == "/"
-          parse_error("Got slash (/) in unexpected location.")
-        end
-      end
-
-      def handle_back_quote_string
-        end_index = @reader.find_next_in_line("`")
-
-        if end_index
-          # handle end quote on same line
-          @tokens << Token.new(@reader.substring(@reader.pos, end_index), @reader.line_no, @reader.pos)
-          @token_text = nil
-          @reader.skip_to(end_index)
-
-        else
-          @token_text = @reader.rest_of_line
-          @token_start = @reader.pos
-          # handle multiline quotes
-          loop do
-            @reader.read_raw_line()
-            if @reader.eof?
-              parse_error("` quote not terminated.", @reader.line_no, -2)
-            end
-
-            end_index = @reader.find_next_in_line("`", 0)
-            if end_index
-              @token_text << @reader.substring(0, end_index)
-              @reader.skip_to(end_index)
-              break
-            else
-              @token_text << @reader.line
-            end
-          end
-
-          @tokens << Token.new(@token_text, @reader.line_no, @token_start)
-          @token_text = nil
-        end
-      end
-
-      def handle_binary_literal
-        end_index = @reader.find_next_in_line("]")
-
-        if end_index
-          # handle end quote on same line
-          @tokens << Token.new(@reader.substring(@reader.pos, end_index), @reader.line_no, @reader.pos)
-          @token_text = nil
-          @reader.skip_to(end_index)
-        else
-          @token_text = @reader.substring(@reader.pos)
-          @token_start = @reader.pos
-          # handle multiline quotes
-          loop do
-            @reader.read_raw_line()
-            if @reader.eof?
-              parse_error("[base64] binary literal not terminated.", @reader.line_no, -2)
-            end
-
-            end_index = @reader.find_next_in_line("]", 0)
-            if end_index
-              @token_text << @reader.substring(0, end_index)
-              @reader.skip_to(end_index)
-              break
-            else
-              @token_text << @reader.line
-            end
-          end
-
-          @tokens << Token.new(@token_text, @reader.line_no, @token_start)
-          @token_text = nil
-        end
-      end
-
-      # handle a line continuation (not inside a string)
-      def handle_line_continuation
-        # backslash line continuation outside of a String literal
-        # can only occur at the end of a line
-        if not @reader.rest_of_line =~ /^\\\s*$/
-          parse_error("Line continuation (\\) before end of line")
-        else
-          @line = @reader.read_line()
-          unless @line
-            parse_error("Line continuation at end of file.", @reader.line_no, @reader.pos)
-          end
-        end
-      end
-
-      def handle_number_date_or_time_span
-        @token_start = @reader.pos
-        @token_text = ""
-
-        until @reader.eol?
-          c = @reader.current_char
-
-          if c =~ /[\w\.\-+:]/
-            @token_text << c
-          elsif c == "/" and not @reader.get_line_char(@reader.pos + 1) == "*"
-            @token_text << c
-          else
-            @reader.previous_char()
-            break
-          end
-
-          @reader.skip_char()
-        end
-
-        @tokens << Token.new(@token_text, @reader.line_no, @token_start)
-        @token_text = nil
-      end
-
-      def handle_identifier
-        @token_start = @reader.pos
-        @token_text = ""
-
-        until @reader.eol?
-          c = @reader.current_char
-
-          # FIXME here we are stricter than the Java version because there is no
-          # easy way to implement Character.isJavaIdentifierPart() in Ruby :)
-          if c =~ /[\w_$-\.]/
-            @token_text << c
-          else
-            @reader.previous_char()
-            break
-          end
-
-          @reader.skip_char()
-        end
-
-        @tokens << Token.new(@token_text, @reader.line_no, @token_start)
-        @token_text = nil
-      end
-
-    end
-
-  end
-  
-end