sdl4r 0.9.1 → 0.9.2
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +0 -0
- data/LICENSE +502 -0
- data/README +296 -1
- data/Rakefile +7 -4
- data/TODO.txt +33 -21
- data/doc/created.rid +1 -0
- data/doc/fr_class_index.html +36 -0
- data/doc/fr_file_index.html +39 -0
- data/doc/fr_method_index.html +141 -0
- data/doc/index.html +24 -0
- data/doc/rdoc-style.css +208 -0
- data/lib/sdl4r/parser.rb +1 -30
- data/lib/sdl4r/parser/reader.rb +175 -0
- data/lib/sdl4r/parser/time_span_with_zone.rb +52 -0
- data/lib/sdl4r/parser/token.rb +133 -0
- data/lib/sdl4r/parser/tokenizer.rb +506 -0
- data/lib/sdl4r/sdl.rb +126 -76
- data/lib/sdl4r/tag.rb +108 -54
- data/test/sdl4r/parser_test.rb +90 -19
- data/test/sdl4r/test.rb +93 -101
- data/test/sdl4r/test_basic_types.sdl +3 -3
- metadata +49 -37
- data/lib/scratchpad.rb +0 -49
- data/lib/sdl4r/reader.rb +0 -171
- data/lib/sdl4r/token.rb +0 -129
- data/lib/sdl4r/tokenizer.rb +0 -501
@@ -0,0 +1,175 @@
|
|
1
|
+
# Simple Declarative Language (SDL) for Ruby
|
2
|
+
# Copyright 2005 Ikayzo, inc.
|
3
|
+
#
|
4
|
+
# This program is free software. You can distribute or modify it under the
|
5
|
+
# terms of the GNU Lesser General Public License version 2.1 as published by
|
6
|
+
# the Free Software Foundation.
|
7
|
+
#
|
8
|
+
# This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
|
9
|
+
# INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
|
10
|
+
# See the GNU Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public License
|
13
|
+
# along with this program; if not, contact the Free Software Foundation, Inc.,
|
14
|
+
# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
15
|
+
|
16
|
+
module SDL4R
|
17
|
+
|
18
|
+
class Parser
|
19
|
+
|
20
|
+
# Gives access to the characters read to the Parser.
|
21
|
+
# This class was designed to gather the handling of the UTF-8 issues in one place and shield the
|
22
|
+
# Parser class from these problems.
|
23
|
+
class Reader
|
24
|
+
|
25
|
+
RUBY_1_8_OR_LESS = require 'jcode'
|
26
|
+
|
27
|
+
|
28
|
+
# +io+ an open IO from which the characters are read.
|
29
|
+
def initialize(io)
|
30
|
+
raise ArgumentError, "io == nil" if io.nil?
|
31
|
+
|
32
|
+
@io = io
|
33
|
+
@line = nil
|
34
|
+
@line_chars = nil
|
35
|
+
@line_no = 0
|
36
|
+
@pos = 0
|
37
|
+
end
|
38
|
+
|
39
|
+
attr_reader :line_no, :pos, :line;
|
40
|
+
|
41
|
+
def line_length
|
42
|
+
return @line_chars.nil? ? 0 : @line_chars.length
|
43
|
+
end
|
44
|
+
|
45
|
+
# Reads next line in stream skipping comment lines and blank lines.
|
46
|
+
#
|
47
|
+
# Returns the next line or nil at the end of the file.
|
48
|
+
def read_line
|
49
|
+
@line_chars = nil
|
50
|
+
|
51
|
+
while @line = read_raw_line()
|
52
|
+
# Skip empty and commented lines
|
53
|
+
break unless @line.empty? or @line =~ /^#/
|
54
|
+
end
|
55
|
+
|
56
|
+
return @line
|
57
|
+
end
|
58
|
+
|
59
|
+
# Returns the string that goes from the current position of this Reader to the end of the line
|
60
|
+
# or nil if the current position doesn't allow that.
|
61
|
+
def rest_of_line
|
62
|
+
return @line[@pos..-1]
|
63
|
+
end
|
64
|
+
|
65
|
+
# Indicates whether the end of file has been reached.
|
66
|
+
def end_of_file?
|
67
|
+
return @line.nil?
|
68
|
+
end
|
69
|
+
|
70
|
+
# Indicates whether there are more characters in the current line
|
71
|
+
def more_chars_in_line?
|
72
|
+
return @pos < line_length - 1
|
73
|
+
end
|
74
|
+
|
75
|
+
# Returns whether the end of the current +line+ as been reached.
|
76
|
+
def end_of_line?
|
77
|
+
return @pos >= line_length
|
78
|
+
end
|
79
|
+
|
80
|
+
# Skips the current line by going just after its end.
|
81
|
+
def skip_line
|
82
|
+
@pos = line_length
|
83
|
+
end
|
84
|
+
|
85
|
+
# Skips the whitespaces that follow the current position.
|
86
|
+
def skip_whitespaces
|
87
|
+
while (@pos + 1) < line_length and
|
88
|
+
(@line[@pos + 1] == ?\s or @line[@pos + 1] == ?\t)
|
89
|
+
@pos += 1
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
# Returns the character at position +pos+ in the current line.
|
94
|
+
# Returns nil if there is no current line or if +pos+ is after the end of the line.
|
95
|
+
def get_line_char(pos)
|
96
|
+
if @line_chars and pos < line_length
|
97
|
+
return @line_chars[pos]
|
98
|
+
else
|
99
|
+
return nil
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
# Returns the character at the current position or nil after end-of-line or end-of -file.
|
104
|
+
def current_char
|
105
|
+
return get_line_char(@pos)
|
106
|
+
end
|
107
|
+
|
108
|
+
# Go to the next character in the stream.
|
109
|
+
def skip_char
|
110
|
+
@pos += 1 if @pos < line_length
|
111
|
+
end
|
112
|
+
|
113
|
+
# Go to the next character and returns it (or nil if end-of-line or -file has been reached).
|
114
|
+
def read_char
|
115
|
+
skip_char()
|
116
|
+
return current_char
|
117
|
+
end
|
118
|
+
|
119
|
+
# Returns to the previous char if possible.
|
120
|
+
def previous_char
|
121
|
+
@pos -= 1 if @pos >= -1
|
122
|
+
end
|
123
|
+
|
124
|
+
# Returns the next index of the expression (string, regexp, fixnum) in the current line, starting
|
125
|
+
# from after the current position if no position is specified.
|
126
|
+
def find_next_in_line(searched, start_pos = nil)
|
127
|
+
start_pos = @pos + 1 unless start_pos
|
128
|
+
return @line.index(searched, start_pos)
|
129
|
+
end
|
130
|
+
|
131
|
+
# Skips the specified position in the current line.
|
132
|
+
def skip_to(new_pos)
|
133
|
+
@pos = new_pos
|
134
|
+
end
|
135
|
+
|
136
|
+
|
137
|
+
# Returns a subpart of the current line starting from +from+ and stopping at +to+ (excluded).
|
138
|
+
def substring(from, to = -1)
|
139
|
+
return @line[from..to]
|
140
|
+
end
|
141
|
+
|
142
|
+
# Reads and returns a "raw" line including lines with comments and blank lines.
|
143
|
+
#
|
144
|
+
# Returns the next line or nil if at the end of the file.
|
145
|
+
#
|
146
|
+
# This method changes the value of @line, @lineNo and @pos.
|
147
|
+
def read_raw_line
|
148
|
+
@line = @io.gets()
|
149
|
+
|
150
|
+
# We ensure that only \n is used as an end-of-line by replacing \r and \r\n.
|
151
|
+
if @line
|
152
|
+
if not @line.gsub!(/\r\n/m, "\n")
|
153
|
+
@line.gsub!(/\r/m, "\n")
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
@pos = 0;
|
158
|
+
@line_chars = nil
|
159
|
+
if @line
|
160
|
+
@line_no += 1
|
161
|
+
@line_chars = @line.scan(/./m)
|
162
|
+
end
|
163
|
+
return @line
|
164
|
+
end
|
165
|
+
|
166
|
+
# Closes this Reader and its underlying +IO+.
|
167
|
+
def close
|
168
|
+
@io.close
|
169
|
+
end
|
170
|
+
|
171
|
+
end
|
172
|
+
|
173
|
+
end
|
174
|
+
|
175
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# Simple Declarative Language (SDL) for Ruby
|
2
|
+
# Copyright 2005 Ikayzo, inc.
|
3
|
+
#
|
4
|
+
# This program is free software. You can distribute or modify it under the
|
5
|
+
# terms of the GNU Lesser General Public License version 2.1 as published by
|
6
|
+
# the Free Software Foundation.
|
7
|
+
#
|
8
|
+
# This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
|
9
|
+
# INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
|
10
|
+
# See the GNU Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public License
|
13
|
+
# along with this program; if not, contact the Free Software Foundation, Inc.,
|
14
|
+
# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
15
|
+
|
16
|
+
module SDL4R
|
17
|
+
|
18
|
+
class Parser
|
19
|
+
|
20
|
+
# An intermediate object used to store a timeSpan or the time
|
21
|
+
# component of a date/time instance. The types are disambiguated at a later stage.
|
22
|
+
#
|
23
|
+
# +seconds+ can have a fraction
|
24
|
+
# +time_zone_offset+ is a fraction of a day (equal to nil if not specified)
|
25
|
+
class TimeSpanWithZone
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
SECONDS_IN_DAY = 24 * 60 * 60
|
30
|
+
|
31
|
+
public
|
32
|
+
|
33
|
+
def initialize(day, hour, minute, second, time_zone_offset)
|
34
|
+
@day = day
|
35
|
+
@hour = hour
|
36
|
+
@min = minute
|
37
|
+
@sec = second
|
38
|
+
@time_zone_offset = time_zone_offset
|
39
|
+
end
|
40
|
+
|
41
|
+
attr_reader :day, :hour, :min, :sec, :time_zone_offset
|
42
|
+
|
43
|
+
# Returns the UTC offset as a fraction of a day on the current machine
|
44
|
+
def TimeSpanWithZone.default_time_zone_offset
|
45
|
+
return Rational(Time.now.utc_offset, SECONDS_IN_DAY)
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
@@ -0,0 +1,133 @@
|
|
1
|
+
# Simple Declarative Language (SDL) for Ruby
|
2
|
+
# Copyright 2005 Ikayzo, inc.
|
3
|
+
#
|
4
|
+
# This program is free software. You can distribute or modify it under the
|
5
|
+
# terms of the GNU Lesser General Public License version 2.1 as published by
|
6
|
+
# the Free Software Foundation.
|
7
|
+
#
|
8
|
+
# This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
|
9
|
+
# INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
|
10
|
+
# See the GNU Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public License
|
13
|
+
# along with this program; if not, contact the Free Software Foundation, Inc.,
|
14
|
+
# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
15
|
+
|
16
|
+
module SDL4R
|
17
|
+
|
18
|
+
require File.dirname(__FILE__) + '/time_span_with_zone'
|
19
|
+
|
20
|
+
class Parser
|
21
|
+
|
22
|
+
# An SDL token.
|
23
|
+
#
|
24
|
+
# @author Daniel Leuck, Philippe Vosges
|
25
|
+
#
|
26
|
+
class Token
|
27
|
+
|
28
|
+
def initialize(text, line = -1, position = -1)
|
29
|
+
@text = text
|
30
|
+
@line = line
|
31
|
+
@pos = position
|
32
|
+
@size = text.length
|
33
|
+
|
34
|
+
begin
|
35
|
+
@type = nil
|
36
|
+
@object = nil
|
37
|
+
|
38
|
+
if text =~ /^["`]/
|
39
|
+
@type = :STRING
|
40
|
+
@object = Parser.parse_string(text)
|
41
|
+
|
42
|
+
elsif text =~ /^'/
|
43
|
+
@type = :CHARACTER
|
44
|
+
@object = text[1...-1]
|
45
|
+
|
46
|
+
elsif text == "null"
|
47
|
+
@type = :NULL
|
48
|
+
@object = nil
|
49
|
+
|
50
|
+
elsif text =~ /^true$|^on$/
|
51
|
+
@type = :BOOLEAN
|
52
|
+
@object = true
|
53
|
+
|
54
|
+
elsif text =~ /^false$|^off$/
|
55
|
+
@type = :BOOLEAN
|
56
|
+
@object = false
|
57
|
+
|
58
|
+
elsif text =~ /^\[/
|
59
|
+
@type=:BINARY
|
60
|
+
@object = Parser.parse_binary(text)
|
61
|
+
|
62
|
+
elsif text =~ /^\d+\/\d+\/\d+$/
|
63
|
+
@type = :DATE;
|
64
|
+
@object = Parser.parse_date_time(text)
|
65
|
+
|
66
|
+
elsif text =~ /^-?\d+d?:\d+/
|
67
|
+
@type = :TIME
|
68
|
+
@object = parse_time_span_with_zone(text)
|
69
|
+
|
70
|
+
elsif text =~ /^[\d\-\.]/
|
71
|
+
@type = :NUMBER
|
72
|
+
@object = Parser.parse_number(text)
|
73
|
+
|
74
|
+
else
|
75
|
+
case text[0]
|
76
|
+
when ?{
|
77
|
+
@type = :START_BLOCK
|
78
|
+
when ?}
|
79
|
+
@type = :END_BLOCK
|
80
|
+
when ?=
|
81
|
+
@type = :EQUALS
|
82
|
+
when ?:
|
83
|
+
@type = :COLON
|
84
|
+
when ?;
|
85
|
+
@type = :SEMICOLON
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
rescue ArgumentError
|
90
|
+
raise SdlParseError.new($!.message, @line, @pos)
|
91
|
+
end
|
92
|
+
|
93
|
+
@type = :IDENTIFIER if @type.nil? # if all hope is lost, it's an identifier
|
94
|
+
|
95
|
+
@punctuation =
|
96
|
+
@type == :COLON || @type == :SEMICOLON || @type == :EQUALS ||
|
97
|
+
@type == :START_BLOCK || @type == :END_BLOCK
|
98
|
+
@literal = @type != :IDENTIFIER && !@punctuation
|
99
|
+
end
|
100
|
+
|
101
|
+
attr_reader :text, :type, :line, :position
|
102
|
+
|
103
|
+
def literal?
|
104
|
+
@literal
|
105
|
+
end
|
106
|
+
|
107
|
+
# Returns the Ruby object corresponding to this literal (or nil if it is
|
108
|
+
# not a literal).
|
109
|
+
def object_for_literal
|
110
|
+
return @object
|
111
|
+
end
|
112
|
+
|
113
|
+
def to_s
|
114
|
+
@type.to_s + " " + @text + " pos:" + @pos.to_s
|
115
|
+
end
|
116
|
+
|
117
|
+
# This special parse method is used only by the Token class for
|
118
|
+
# tokens which are ambiguously either a TimeSpan or the time component
|
119
|
+
# of a date/time type
|
120
|
+
def parse_time_span_with_zone(literal)
|
121
|
+
raise ArgumentError("time span or date literal is nil") if literal.nil?
|
122
|
+
|
123
|
+
days, hours, minutes, seconds, time_zone_offset =
|
124
|
+
Parser.parse_time_span_and_time_zone(literal, true, true)
|
125
|
+
|
126
|
+
return Parser::TimeSpanWithZone.new(days, hours, minutes, seconds, time_zone_offset)
|
127
|
+
end
|
128
|
+
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|
132
|
+
|
133
|
+
end
|
@@ -0,0 +1,506 @@
|
|
1
|
+
# Simple Declarative Language (SDL) for Ruby
|
2
|
+
# Copyright 2005 Ikayzo, inc.
|
3
|
+
#
|
4
|
+
# This program is free software. You can distribute or modify it under the
|
5
|
+
# terms of the GNU Lesser General Public License version 2.1 as published by
|
6
|
+
# the Free Software Foundation.
|
7
|
+
#
|
8
|
+
# This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
|
9
|
+
# INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
|
10
|
+
# See the GNU Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public License
|
13
|
+
# along with this program; if not, contact the Free Software Foundation, Inc.,
|
14
|
+
# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
15
|
+
|
16
|
+
|
17
|
+
module SDL4R
|
18
|
+
|
19
|
+
require File.dirname(__FILE__) + '/reader'
|
20
|
+
require File.dirname(__FILE__) + '/token'
|
21
|
+
|
22
|
+
class Parser
|
23
|
+
|
24
|
+
# Tokenizer of the SDL parser
|
25
|
+
class Tokenizer
|
26
|
+
|
27
|
+
TOKEN_TYPES = [
|
28
|
+
:IDENTIFIER,
|
29
|
+
|
30
|
+
# punctuation
|
31
|
+
:COLON, :SEMICOLON, :EQUALS, :START_BLOCK, :END_BLOCK,
|
32
|
+
|
33
|
+
# literals
|
34
|
+
:STRING, :CHARACTER, :BOOLEAN, :NUMBER, :DATE, :TIME, :BINARY, :NULL ]
|
35
|
+
|
36
|
+
|
37
|
+
# Creates an SDL tokenizer on the specified +IO+.
|
38
|
+
def initialize(io)
|
39
|
+
raise ArgumentError, "io == nil" if io.nil?
|
40
|
+
|
41
|
+
@reader = Parser::Reader.new(io)
|
42
|
+
@token_start = 0
|
43
|
+
@startEscapedQuoteLine = false
|
44
|
+
@tokens = nil
|
45
|
+
@tokenText = nil
|
46
|
+
end
|
47
|
+
|
48
|
+
# Closes this Tokenizer and its underlying +Reader+.
|
49
|
+
def close
|
50
|
+
@reader.close
|
51
|
+
end
|
52
|
+
|
53
|
+
# Close the reader and throw a SdlParseError.
|
54
|
+
def parse_error(description, line_no = nil, position = nil)
|
55
|
+
begin
|
56
|
+
@reader.close()
|
57
|
+
rescue IOError
|
58
|
+
# no recourse
|
59
|
+
end
|
60
|
+
|
61
|
+
line_no = @reader.line_no if line_no.nil?
|
62
|
+
position = @reader.pos if position.nil?
|
63
|
+
|
64
|
+
# We add one because editors typically start with line 1 and position 1
|
65
|
+
# rather than 0...
|
66
|
+
raise SdlParseError.new(description, line_no + 1, position + 1, @reader.line)
|
67
|
+
end
|
68
|
+
|
69
|
+
# Close the reader and throw a SdlParseError using the format
|
70
|
+
# Was expecting X but got Y.
|
71
|
+
#
|
72
|
+
def expecting_but_got(expecting, got, line, position)
|
73
|
+
parse_error("Was expecting #{expecting} but got #{got}", line, position)
|
74
|
+
end
|
75
|
+
|
76
|
+
# Returns the next line as tokens or nil if the end of the stream has been reached.
|
77
|
+
# This method handles line continuations both within and outside String literals.
|
78
|
+
# The line of tokens is assigned to @tokens.
|
79
|
+
#
|
80
|
+
# Returns a logical line as a list of Tokens.
|
81
|
+
#
|
82
|
+
def read_line_tokens
|
83
|
+
begin
|
84
|
+
read_line_tokens_even_if_empty()
|
85
|
+
end until @tokens.nil? or !@tokens.empty?
|
86
|
+
return @tokens
|
87
|
+
end
|
88
|
+
|
89
|
+
def line_no
|
90
|
+
@reader.line_no
|
91
|
+
end
|
92
|
+
|
93
|
+
def pos
|
94
|
+
@reader.pos
|
95
|
+
end
|
96
|
+
|
97
|
+
def line
|
98
|
+
@reader.line
|
99
|
+
end
|
100
|
+
|
101
|
+
private
|
102
|
+
|
103
|
+
# Returns the next line as tokens or nil if the end of the stream has been reached.
|
104
|
+
# This method handles line continuations both within and outside String literals.
|
105
|
+
# The line of tokens is assigned to @tokens.
|
106
|
+
#
|
107
|
+
# Returns a logical line as a list of Tokens.
|
108
|
+
# Returns an empty array if the line was empty.
|
109
|
+
#
|
110
|
+
def read_line_tokens_even_if_empty
|
111
|
+
@tokens = nil
|
112
|
+
@tokenText = nil
|
113
|
+
@token_start = nil
|
114
|
+
|
115
|
+
@reader.read_line() if @reader.end_of_line?
|
116
|
+
return @tokens unless @reader.line
|
117
|
+
|
118
|
+
@tokens = []
|
119
|
+
@token_start = @reader.pos
|
120
|
+
|
121
|
+
while not @reader.end_of_line?
|
122
|
+
if @tokenText
|
123
|
+
@tokens << Token.new(@tokenText, @reader.line_no, @token_start)
|
124
|
+
@tokenText = nil
|
125
|
+
end
|
126
|
+
|
127
|
+
c = @reader.current_char
|
128
|
+
next_c = @reader.get_line_char(@reader.pos + 1)
|
129
|
+
case c
|
130
|
+
when "\""
|
131
|
+
# handle "" style strings including line continuations
|
132
|
+
handle_double_quote_string()
|
133
|
+
|
134
|
+
when "'"
|
135
|
+
handle_character_literal()
|
136
|
+
|
137
|
+
when "{", "}", "=", ":", ";"
|
138
|
+
# handle punctuation
|
139
|
+
punctuation_token = Token.new(c, @reader.line_no, @reader.pos)
|
140
|
+
@tokenText = nil
|
141
|
+
|
142
|
+
if punctuation_token.type == :SEMICOLON
|
143
|
+
@reader.skip_char()
|
144
|
+
break
|
145
|
+
else
|
146
|
+
@tokens << punctuation_token
|
147
|
+
end
|
148
|
+
|
149
|
+
when "#"
|
150
|
+
# skip : hash comment at end of line
|
151
|
+
@reader.skip_line()
|
152
|
+
|
153
|
+
when "/"
|
154
|
+
# handle // and /**/ style comments
|
155
|
+
if next_c == "/"
|
156
|
+
# skip : // comment
|
157
|
+
@reader.skip_line()
|
158
|
+
else
|
159
|
+
handle_slash_comment()
|
160
|
+
end
|
161
|
+
|
162
|
+
when "`"
|
163
|
+
# handle multiline `` style strings
|
164
|
+
handle_back_quote_string()
|
165
|
+
|
166
|
+
when "["
|
167
|
+
# handle binary literals
|
168
|
+
handle_binary_literal()
|
169
|
+
|
170
|
+
when "\s", "\t"
|
171
|
+
@reader.skip_whitespaces()
|
172
|
+
|
173
|
+
when "\\"
|
174
|
+
# line continuations (outside a string literal)
|
175
|
+
handle_line_continuation();
|
176
|
+
|
177
|
+
when /^[0-9\-\.]$/
|
178
|
+
if c == "-" and next_c == "-"
|
179
|
+
# -- comments : ignore
|
180
|
+
@reader.skip_line()
|
181
|
+
else
|
182
|
+
# handle numbers, dates, and time spans
|
183
|
+
handle_number_date_or_time_span()
|
184
|
+
end
|
185
|
+
|
186
|
+
when /^[a-zA-Z\$_]$/
|
187
|
+
# FIXME Here, the Java code specifies isJavaIdentifierStart() but
|
188
|
+
# this is not easily implemented (at least as of Ruby 1.8).
|
189
|
+
# So, we implement a subset of these characters.
|
190
|
+
handle_identifier()
|
191
|
+
|
192
|
+
when "\n", "\r"
|
193
|
+
# end of line
|
194
|
+
@reader.skip_line()
|
195
|
+
|
196
|
+
else
|
197
|
+
parse_error("Unexpected character '#{c}'")
|
198
|
+
end
|
199
|
+
|
200
|
+
@reader.skip_char()
|
201
|
+
end
|
202
|
+
|
203
|
+
if @tokenText
|
204
|
+
@tokens << Token.new(@tokenText, @reader.line_no, @token_start)
|
205
|
+
end
|
206
|
+
|
207
|
+
return @tokens
|
208
|
+
end
|
209
|
+
|
210
|
+
# Adds the current escaped character (represented by ((|c|))) to @tokenText.
|
211
|
+
# This method assumes the previous char was a backslash.
|
212
|
+
#
|
213
|
+
def add_escaped_char_in_string(c)
|
214
|
+
case c
|
215
|
+
when "\\", "\""
|
216
|
+
@tokenText << c
|
217
|
+
when "n"
|
218
|
+
@tokenText << ?\n
|
219
|
+
when "r"
|
220
|
+
@tokenText << ?\r
|
221
|
+
when "t"
|
222
|
+
@tokenText << ?\t
|
223
|
+
else
|
224
|
+
parse_error("Illegal escape character in string literal: '#{c.chr}'.")
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
def handle_double_quote_string
|
229
|
+
escaped = false
|
230
|
+
@startEscapedQuoteLine = false
|
231
|
+
|
232
|
+
@tokenText = "\""
|
233
|
+
@reader.skip_char()
|
234
|
+
|
235
|
+
while not @reader.end_of_line?
|
236
|
+
c = @reader.current_char
|
237
|
+
|
238
|
+
if "\s\t".include?(c) and @startEscapedQuoteLine
|
239
|
+
# we continue
|
240
|
+
else
|
241
|
+
@startEscapedQuoteLine = false;
|
242
|
+
|
243
|
+
if escaped
|
244
|
+
add_escaped_char_in_string(c)
|
245
|
+
escaped = false
|
246
|
+
|
247
|
+
elsif c == "\\"
|
248
|
+
# check for String broken across lines
|
249
|
+
if @reader.rest_of_line =~ /^\\\s*$/
|
250
|
+
handle_escaped_double_quoted_string()
|
251
|
+
next # as we are at the beginning of a new line
|
252
|
+
else
|
253
|
+
escaped = true;
|
254
|
+
end
|
255
|
+
|
256
|
+
else
|
257
|
+
@tokenText << c
|
258
|
+
if c == "\""
|
259
|
+
# end of double-quoted string detected
|
260
|
+
@tokens << Token.new(@tokenText, @reader.line_no, @token_start)
|
261
|
+
@tokenText = nil
|
262
|
+
return
|
263
|
+
end
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
267
|
+
@reader.skip_char()
|
268
|
+
end
|
269
|
+
|
270
|
+
# detection of ill-terminated literals
|
271
|
+
if @tokenText =~ /^".*[^"]$/
|
272
|
+
parse_error(
|
273
|
+
"String literal \"#{@tokenText}\" not terminated by end quote.", @reader.line_no, @reader.line_length);
|
274
|
+
elsif @tokenText == "\""
|
275
|
+
parse_error("Orphan quote (unterminated string)", @reader.line_no, @reader.line_length);
|
276
|
+
end
|
277
|
+
end
|
278
|
+
|
279
|
+
def handle_escaped_double_quoted_string
|
280
|
+
# '\' can be followed by whitespaces
|
281
|
+
if @reader.rest_of_line =~ /^\\\s*$/
|
282
|
+
@reader.read_line()
|
283
|
+
parse_error("Escape at end of file.") if @reader.end_of_file?
|
284
|
+
|
285
|
+
@startEscapedQuoteLine = true
|
286
|
+
|
287
|
+
else
|
288
|
+
parse_error(
|
289
|
+
"Malformed string literal - escape followed by whitespace " +
|
290
|
+
"followed by non-whitespace.")
|
291
|
+
end
|
292
|
+
end
|
293
|
+
|
294
|
+
def handle_character_literal
|
295
|
+
if not @reader.more_chars_in_line?
|
296
|
+
parse_error("Got ' at end of line")
|
297
|
+
end
|
298
|
+
|
299
|
+
c2 = @reader.read_char()
|
300
|
+
|
301
|
+
if c2 == "\\"
|
302
|
+
if @reader.end_of_line?
|
303
|
+
parse_error("Got '\\ at end of line")
|
304
|
+
end
|
305
|
+
|
306
|
+
c3 = @reader.read_char()
|
307
|
+
|
308
|
+
if not @reader.more_chars_in_line?
|
309
|
+
parse_error("Got '\\#{c3} at end of line")
|
310
|
+
end
|
311
|
+
|
312
|
+
case c3
|
313
|
+
when "\\"
|
314
|
+
@tokens << Token.new("'\\'", @reader.line_no, @reader.pos)
|
315
|
+
when "'"
|
316
|
+
@tokens << Token.new("'''", @reader.line_no, @reader.pos)
|
317
|
+
when "n"
|
318
|
+
@tokens << Token.new("'\n'", @reader.line_no, @reader.pos)
|
319
|
+
when "r"
|
320
|
+
@tokens << Token.new("'\r'", @reader.line_no, @reader.pos)
|
321
|
+
when "t"
|
322
|
+
@tokens << Token.new("'\t'", @reader.line_no, @reader.pos)
|
323
|
+
else
|
324
|
+
parse_error("Illegal escape character #{@reader.current_char}")
|
325
|
+
end
|
326
|
+
|
327
|
+
@reader.skip_char()
|
328
|
+
if @reader.current_char != "'"
|
329
|
+
expecting_but_got("single quote (')", "\"#{@reader.current_char}\"")
|
330
|
+
end
|
331
|
+
else
|
332
|
+
@tokens << Token.new("'#{c2}'", @reader.line_no, @reader.pos)
|
333
|
+
if not @reader.more_chars_in_line?
|
334
|
+
parse_error("Got '#{c2} at end of line")
|
335
|
+
end
|
336
|
+
@reader.skip_char()
|
337
|
+
if @reader.current_char != "'"
|
338
|
+
expecting_but_got(
|
339
|
+
"quote (')", "\"#{@reader.current_char}\"", @reader.line_no, @reader.pos)
|
340
|
+
end
|
341
|
+
end
|
342
|
+
end
|
343
|
+
|
344
|
+
def handle_slash_comment
|
345
|
+
if not @reader.more_chars_in_line?
|
346
|
+
parse_error("Got slash (/) at end of line.")
|
347
|
+
end
|
348
|
+
|
349
|
+
if @reader.get_line_char(@reader.pos + 1) == "*"
|
350
|
+
end_index = @reader.find_next_in_line("*/")
|
351
|
+
if end_index
|
352
|
+
# handle comment on same line
|
353
|
+
@reader.skip_to(end_index + 1)
|
354
|
+
else
|
355
|
+
# handle multiline comments
|
356
|
+
loop do
|
357
|
+
@reader.read_raw_line()
|
358
|
+
if @reader.end_of_file?
|
359
|
+
parse_error("/* comment not terminated.", @reader.line_no, -2)
|
360
|
+
end
|
361
|
+
|
362
|
+
end_index = @reader.find_next_in_line("*/", 0)
|
363
|
+
|
364
|
+
if end_index
|
365
|
+
@reader.skip_to(end_index + 1)
|
366
|
+
break
|
367
|
+
end
|
368
|
+
end
|
369
|
+
end
|
370
|
+
elsif @reader.get_line_char(@reader.pos + 1) == "/"
|
371
|
+
parse_error("Got slash (/) in unexpected location.")
|
372
|
+
end
|
373
|
+
end
|
374
|
+
|
375
|
+
def handle_back_quote_string
|
376
|
+
end_index = @reader.find_next_in_line("`")
|
377
|
+
|
378
|
+
if end_index
|
379
|
+
# handle end quote on same line
|
380
|
+
@tokens << Token.new(@reader.substring(@reader.pos, end_index), @reader.line_no, @reader.pos)
|
381
|
+
@tokenText = nil
|
382
|
+
@reader.skip_to(end_index)
|
383
|
+
|
384
|
+
else
|
385
|
+
@tokenText = @reader.rest_of_line
|
386
|
+
@token_start = @reader.pos
|
387
|
+
# handle multiline quotes
|
388
|
+
loop do
|
389
|
+
@reader.read_raw_line()
|
390
|
+
if @reader.end_of_file?
|
391
|
+
parse_error("` quote not terminated.", @reader.line_no, -2)
|
392
|
+
end
|
393
|
+
|
394
|
+
end_index = @reader.find_next_in_line("`", 0)
|
395
|
+
if end_index
|
396
|
+
@tokenText << @reader.substring(0, end_index)
|
397
|
+
@reader.skip_to(end_index)
|
398
|
+
break
|
399
|
+
else
|
400
|
+
@tokenText << @reader.line
|
401
|
+
end
|
402
|
+
end
|
403
|
+
|
404
|
+
@tokens << Token.new(@tokenText, @reader.line_no, @token_start)
|
405
|
+
@tokenText = nil
|
406
|
+
end
|
407
|
+
end
|
408
|
+
|
409
|
+
def handle_binary_literal
|
410
|
+
end_index = @reader.find_next_in_line("]")
|
411
|
+
|
412
|
+
if end_index
|
413
|
+
# handle end quote on same line
|
414
|
+
@tokens << Token.new(@reader.substring(@reader.pos, end_index), @reader.line_no, @reader.pos)
|
415
|
+
@tokenText = nil
|
416
|
+
@reader.skip_to(end_index)
|
417
|
+
else
|
418
|
+
@tokenText = @reader.substring(@reader.pos)
|
419
|
+
@token_start = @reader.pos
|
420
|
+
# handle multiline quotes
|
421
|
+
loop do
|
422
|
+
@reader.read_raw_line()
|
423
|
+
if @reader.end_of_file?
|
424
|
+
parse_error("[base64] binary literal not terminated.", @reader.line_no, -2)
|
425
|
+
end
|
426
|
+
|
427
|
+
end_index = @reader.find_next_in_line("]", 0)
|
428
|
+
if end_index
|
429
|
+
@tokenText << @reader.substring(0, end_index)
|
430
|
+
@reader.skip_to(end_index)
|
431
|
+
break
|
432
|
+
else
|
433
|
+
@tokenText << @reader.line
|
434
|
+
end
|
435
|
+
end
|
436
|
+
|
437
|
+
@tokens << Token.new(@tokenText, @reader.line_no, @token_start)
|
438
|
+
@tokenText = nil
|
439
|
+
end
|
440
|
+
end
|
441
|
+
|
442
|
+
# handle a line continuation (not inside a string)
|
443
|
+
def handle_line_continuation
|
444
|
+
# backslash line continuation outside of a String literal
|
445
|
+
# can only occur at the end of a line
|
446
|
+
if not @reader.rest_of_line =~ /^\\\s*$/
|
447
|
+
parse_error("Line continuation (\\) before end of line")
|
448
|
+
else
|
449
|
+
@line = @reader.read_line()
|
450
|
+
if @line.nil?
|
451
|
+
parse_error("Line continuation at end of file.", @reader.line_no, @reader.pos)
|
452
|
+
end
|
453
|
+
end
|
454
|
+
end
|
455
|
+
|
456
|
+
def handle_number_date_or_time_span
|
457
|
+
@token_start = @reader.pos
|
458
|
+
@tokenText = ""
|
459
|
+
|
460
|
+
while not @reader.end_of_line?
|
461
|
+
c = @reader.current_char
|
462
|
+
|
463
|
+
if c =~ /[\w\.\-+:]/
|
464
|
+
@tokenText << c
|
465
|
+
elsif c == "/" and not @reader.get_line_char(@reader.pos + 1) == "*"
|
466
|
+
@tokenText << c
|
467
|
+
else
|
468
|
+
@reader.previous_char()
|
469
|
+
break
|
470
|
+
end
|
471
|
+
|
472
|
+
@reader.skip_char()
|
473
|
+
end
|
474
|
+
|
475
|
+
@tokens << Token.new(@tokenText, @reader.line_no, @token_start)
|
476
|
+
@tokenText = nil
|
477
|
+
end
|
478
|
+
|
479
|
+
def handle_identifier
|
480
|
+
@token_start = @reader.pos;
|
481
|
+
@tokenText = ""
|
482
|
+
|
483
|
+
while not @reader.end_of_line?
|
484
|
+
c = @reader.current_char
|
485
|
+
|
486
|
+
# FIXME here we are stricter than the Java version because there is no
|
487
|
+
# easy way to implement Character.isJavaIdentifierPart() in Ruby :)
|
488
|
+
if c =~ /[\w_$-\.]/
|
489
|
+
@tokenText << c
|
490
|
+
else
|
491
|
+
@reader.previous_char()
|
492
|
+
break
|
493
|
+
end
|
494
|
+
|
495
|
+
@reader.skip_char()
|
496
|
+
end
|
497
|
+
|
498
|
+
@tokens << Token.new(@tokenText, @reader.line_no, @token_start)
|
499
|
+
@tokenText = nil
|
500
|
+
end
|
501
|
+
|
502
|
+
end
|
503
|
+
|
504
|
+
end
|
505
|
+
|
506
|
+
end
|