sdl4r 0.9.1 → 0.9.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,175 @@
1
+ # Simple Declarative Language (SDL) for Ruby
2
+ # Copyright 2005 Ikayzo, inc.
3
+ #
4
+ # This program is free software. You can distribute or modify it under the
5
+ # terms of the GNU Lesser General Public License version 2.1 as published by
6
+ # the Free Software Foundation.
7
+ #
8
+ # This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
9
+ # INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
10
+ # See the GNU Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public License
13
+ # along with this program; if not, contact the Free Software Foundation, Inc.,
14
+ # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
15
+
16
+ module SDL4R
17
+
18
+ class Parser
19
+
20
+ # Gives access to the characters read to the Parser.
21
+ # This class was designed to gather the handling of the UTF-8 issues in one place and shield the
22
+ # Parser class from these problems.
23
+ class Reader
24
+
25
+ RUBY_1_8_OR_LESS = require 'jcode'
26
+
27
+
28
+ # +io+ an open IO from which the characters are read.
29
+ def initialize(io)
30
+ raise ArgumentError, "io == nil" if io.nil?
31
+
32
+ @io = io
33
+ @line = nil
34
+ @line_chars = nil
35
+ @line_no = 0
36
+ @pos = 0
37
+ end
38
+
39
+ attr_reader :line_no, :pos, :line;
40
+
41
+ def line_length
42
+ return @line_chars.nil? ? 0 : @line_chars.length
43
+ end
44
+
45
+ # Reads next line in stream skipping comment lines and blank lines.
46
+ #
47
+ # Returns the next line or nil at the end of the file.
48
+ def read_line
49
+ @line_chars = nil
50
+
51
+ while @line = read_raw_line()
52
+ # Skip empty and commented lines
53
+ break unless @line.empty? or @line =~ /^#/
54
+ end
55
+
56
+ return @line
57
+ end
58
+
59
+ # Returns the string that goes from the current position of this Reader to the end of the line
60
+ # or nil if the current position doesn't allow that.
61
+ def rest_of_line
62
+ return @line[@pos..-1]
63
+ end
64
+
65
+ # Indicates whether the end of file has been reached.
66
+ def end_of_file?
67
+ return @line.nil?
68
+ end
69
+
70
+ # Indicates whether there are more characters in the current line
71
+ def more_chars_in_line?
72
+ return @pos < line_length - 1
73
+ end
74
+
75
+ # Returns whether the end of the current +line+ as been reached.
76
+ def end_of_line?
77
+ return @pos >= line_length
78
+ end
79
+
80
+ # Skips the current line by going just after its end.
81
+ def skip_line
82
+ @pos = line_length
83
+ end
84
+
85
+ # Skips the whitespaces that follow the current position.
86
+ def skip_whitespaces
87
+ while (@pos + 1) < line_length and
88
+ (@line[@pos + 1] == ?\s or @line[@pos + 1] == ?\t)
89
+ @pos += 1
90
+ end
91
+ end
92
+
93
+ # Returns the character at position +pos+ in the current line.
94
+ # Returns nil if there is no current line or if +pos+ is after the end of the line.
95
+ def get_line_char(pos)
96
+ if @line_chars and pos < line_length
97
+ return @line_chars[pos]
98
+ else
99
+ return nil
100
+ end
101
+ end
102
+
103
+ # Returns the character at the current position or nil after end-of-line or end-of -file.
104
+ def current_char
105
+ return get_line_char(@pos)
106
+ end
107
+
108
+ # Go to the next character in the stream.
109
+ def skip_char
110
+ @pos += 1 if @pos < line_length
111
+ end
112
+
113
+ # Go to the next character and returns it (or nil if end-of-line or -file has been reached).
114
+ def read_char
115
+ skip_char()
116
+ return current_char
117
+ end
118
+
119
+ # Returns to the previous char if possible.
120
+ def previous_char
121
+ @pos -= 1 if @pos >= -1
122
+ end
123
+
124
+ # Returns the next index of the expression (string, regexp, fixnum) in the current line, starting
125
+ # from after the current position if no position is specified.
126
+ def find_next_in_line(searched, start_pos = nil)
127
+ start_pos = @pos + 1 unless start_pos
128
+ return @line.index(searched, start_pos)
129
+ end
130
+
131
+ # Skips the specified position in the current line.
132
+ def skip_to(new_pos)
133
+ @pos = new_pos
134
+ end
135
+
136
+
137
+ # Returns a subpart of the current line starting from +from+ and stopping at +to+ (excluded).
138
+ def substring(from, to = -1)
139
+ return @line[from..to]
140
+ end
141
+
142
+ # Reads and returns a "raw" line including lines with comments and blank lines.
143
+ #
144
+ # Returns the next line or nil if at the end of the file.
145
+ #
146
+ # This method changes the value of @line, @lineNo and @pos.
147
+ def read_raw_line
148
+ @line = @io.gets()
149
+
150
+ # We ensure that only \n is used as an end-of-line by replacing \r and \r\n.
151
+ if @line
152
+ if not @line.gsub!(/\r\n/m, "\n")
153
+ @line.gsub!(/\r/m, "\n")
154
+ end
155
+ end
156
+
157
+ @pos = 0;
158
+ @line_chars = nil
159
+ if @line
160
+ @line_no += 1
161
+ @line_chars = @line.scan(/./m)
162
+ end
163
+ return @line
164
+ end
165
+
166
+ # Closes this Reader and its underlying +IO+.
167
+ def close
168
+ @io.close
169
+ end
170
+
171
+ end
172
+
173
+ end
174
+
175
+ end
@@ -0,0 +1,52 @@
1
+ # Simple Declarative Language (SDL) for Ruby
2
+ # Copyright 2005 Ikayzo, inc.
3
+ #
4
+ # This program is free software. You can distribute or modify it under the
5
+ # terms of the GNU Lesser General Public License version 2.1 as published by
6
+ # the Free Software Foundation.
7
+ #
8
+ # This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
9
+ # INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
10
+ # See the GNU Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public License
13
+ # along with this program; if not, contact the Free Software Foundation, Inc.,
14
+ # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
15
+
16
+ module SDL4R
17
+
18
+ class Parser
19
+
20
+ # An intermediate object used to store a timeSpan or the time
21
+ # component of a date/time instance. The types are disambiguated at a later stage.
22
+ #
23
+ # +seconds+ can have a fraction
24
+ # +time_zone_offset+ is a fraction of a day (equal to nil if not specified)
25
+ class TimeSpanWithZone
26
+
27
+ private
28
+
29
+ SECONDS_IN_DAY = 24 * 60 * 60
30
+
31
+ public
32
+
33
+ def initialize(day, hour, minute, second, time_zone_offset)
34
+ @day = day
35
+ @hour = hour
36
+ @min = minute
37
+ @sec = second
38
+ @time_zone_offset = time_zone_offset
39
+ end
40
+
41
+ attr_reader :day, :hour, :min, :sec, :time_zone_offset
42
+
43
+ # Returns the UTC offset as a fraction of a day on the current machine
44
+ def TimeSpanWithZone.default_time_zone_offset
45
+ return Rational(Time.now.utc_offset, SECONDS_IN_DAY)
46
+ end
47
+
48
+ end
49
+
50
+ end
51
+
52
+ end
@@ -0,0 +1,133 @@
1
+ # Simple Declarative Language (SDL) for Ruby
2
+ # Copyright 2005 Ikayzo, inc.
3
+ #
4
+ # This program is free software. You can distribute or modify it under the
5
+ # terms of the GNU Lesser General Public License version 2.1 as published by
6
+ # the Free Software Foundation.
7
+ #
8
+ # This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
9
+ # INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
10
+ # See the GNU Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public License
13
+ # along with this program; if not, contact the Free Software Foundation, Inc.,
14
+ # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
15
+
16
+ module SDL4R
17
+
18
+ require File.dirname(__FILE__) + '/time_span_with_zone'
19
+
20
+ class Parser
21
+
22
+ # An SDL token.
23
+ #
24
+ # @author Daniel Leuck, Philippe Vosges
25
+ #
26
+ class Token
27
+
28
+ def initialize(text, line = -1, position = -1)
29
+ @text = text
30
+ @line = line
31
+ @pos = position
32
+ @size = text.length
33
+
34
+ begin
35
+ @type = nil
36
+ @object = nil
37
+
38
+ if text =~ /^["`]/
39
+ @type = :STRING
40
+ @object = Parser.parse_string(text)
41
+
42
+ elsif text =~ /^'/
43
+ @type = :CHARACTER
44
+ @object = text[1...-1]
45
+
46
+ elsif text == "null"
47
+ @type = :NULL
48
+ @object = nil
49
+
50
+ elsif text =~ /^true$|^on$/
51
+ @type = :BOOLEAN
52
+ @object = true
53
+
54
+ elsif text =~ /^false$|^off$/
55
+ @type = :BOOLEAN
56
+ @object = false
57
+
58
+ elsif text =~ /^\[/
59
+ @type=:BINARY
60
+ @object = Parser.parse_binary(text)
61
+
62
+ elsif text =~ /^\d+\/\d+\/\d+$/
63
+ @type = :DATE;
64
+ @object = Parser.parse_date_time(text)
65
+
66
+ elsif text =~ /^-?\d+d?:\d+/
67
+ @type = :TIME
68
+ @object = parse_time_span_with_zone(text)
69
+
70
+ elsif text =~ /^[\d\-\.]/
71
+ @type = :NUMBER
72
+ @object = Parser.parse_number(text)
73
+
74
+ else
75
+ case text[0]
76
+ when ?{
77
+ @type = :START_BLOCK
78
+ when ?}
79
+ @type = :END_BLOCK
80
+ when ?=
81
+ @type = :EQUALS
82
+ when ?:
83
+ @type = :COLON
84
+ when ?;
85
+ @type = :SEMICOLON
86
+ end
87
+ end
88
+
89
+ rescue ArgumentError
90
+ raise SdlParseError.new($!.message, @line, @pos)
91
+ end
92
+
93
+ @type = :IDENTIFIER if @type.nil? # if all hope is lost, it's an identifier
94
+
95
+ @punctuation =
96
+ @type == :COLON || @type == :SEMICOLON || @type == :EQUALS ||
97
+ @type == :START_BLOCK || @type == :END_BLOCK
98
+ @literal = @type != :IDENTIFIER && !@punctuation
99
+ end
100
+
101
+ attr_reader :text, :type, :line, :position
102
+
103
+ def literal?
104
+ @literal
105
+ end
106
+
107
+ # Returns the Ruby object corresponding to this literal (or nil if it is
108
+ # not a literal).
109
+ def object_for_literal
110
+ return @object
111
+ end
112
+
113
+ def to_s
114
+ @type.to_s + " " + @text + " pos:" + @pos.to_s
115
+ end
116
+
117
+ # This special parse method is used only by the Token class for
118
+ # tokens which are ambiguously either a TimeSpan or the time component
119
+ # of a date/time type
120
+ def parse_time_span_with_zone(literal)
121
+ raise ArgumentError("time span or date literal is nil") if literal.nil?
122
+
123
+ days, hours, minutes, seconds, time_zone_offset =
124
+ Parser.parse_time_span_and_time_zone(literal, true, true)
125
+
126
+ return Parser::TimeSpanWithZone.new(days, hours, minutes, seconds, time_zone_offset)
127
+ end
128
+
129
+ end
130
+
131
+ end
132
+
133
+ end
@@ -0,0 +1,506 @@
1
+ # Simple Declarative Language (SDL) for Ruby
2
+ # Copyright 2005 Ikayzo, inc.
3
+ #
4
+ # This program is free software. You can distribute or modify it under the
5
+ # terms of the GNU Lesser General Public License version 2.1 as published by
6
+ # the Free Software Foundation.
7
+ #
8
+ # This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
9
+ # INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
10
+ # See the GNU Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public License
13
+ # along with this program; if not, contact the Free Software Foundation, Inc.,
14
+ # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
15
+
16
+
17
+ module SDL4R
18
+
19
+ require File.dirname(__FILE__) + '/reader'
20
+ require File.dirname(__FILE__) + '/token'
21
+
22
+ class Parser
23
+
24
+ # Tokenizer of the SDL parser
25
+ class Tokenizer
26
+
27
+ TOKEN_TYPES = [
28
+ :IDENTIFIER,
29
+
30
+ # punctuation
31
+ :COLON, :SEMICOLON, :EQUALS, :START_BLOCK, :END_BLOCK,
32
+
33
+ # literals
34
+ :STRING, :CHARACTER, :BOOLEAN, :NUMBER, :DATE, :TIME, :BINARY, :NULL ]
35
+
36
+
37
+ # Creates an SDL tokenizer on the specified +IO+.
38
+ def initialize(io)
39
+ raise ArgumentError, "io == nil" if io.nil?
40
+
41
+ @reader = Parser::Reader.new(io)
42
+ @token_start = 0
43
+ @startEscapedQuoteLine = false
44
+ @tokens = nil
45
+ @tokenText = nil
46
+ end
47
+
48
+ # Closes this Tokenizer and its underlying +Reader+.
49
+ def close
50
+ @reader.close
51
+ end
52
+
53
+ # Close the reader and throw a SdlParseError.
54
+ def parse_error(description, line_no = nil, position = nil)
55
+ begin
56
+ @reader.close()
57
+ rescue IOError
58
+ # no recourse
59
+ end
60
+
61
+ line_no = @reader.line_no if line_no.nil?
62
+ position = @reader.pos if position.nil?
63
+
64
+ # We add one because editors typically start with line 1 and position 1
65
+ # rather than 0...
66
+ raise SdlParseError.new(description, line_no + 1, position + 1, @reader.line)
67
+ end
68
+
69
+ # Close the reader and throw a SdlParseError using the format
70
+ # Was expecting X but got Y.
71
+ #
72
+ def expecting_but_got(expecting, got, line, position)
73
+ parse_error("Was expecting #{expecting} but got #{got}", line, position)
74
+ end
75
+
76
+ # Returns the next line as tokens or nil if the end of the stream has been reached.
77
+ # This method handles line continuations both within and outside String literals.
78
+ # The line of tokens is assigned to @tokens.
79
+ #
80
+ # Returns a logical line as a list of Tokens.
81
+ #
82
+ def read_line_tokens
83
+ begin
84
+ read_line_tokens_even_if_empty()
85
+ end until @tokens.nil? or !@tokens.empty?
86
+ return @tokens
87
+ end
88
+
89
+ def line_no
90
+ @reader.line_no
91
+ end
92
+
93
+ def pos
94
+ @reader.pos
95
+ end
96
+
97
+ def line
98
+ @reader.line
99
+ end
100
+
101
+ private
102
+
103
+ # Returns the next line as tokens or nil if the end of the stream has been reached.
104
+ # This method handles line continuations both within and outside String literals.
105
+ # The line of tokens is assigned to @tokens.
106
+ #
107
+ # Returns a logical line as a list of Tokens.
108
+ # Returns an empty array if the line was empty.
109
+ #
110
+ def read_line_tokens_even_if_empty
111
+ @tokens = nil
112
+ @tokenText = nil
113
+ @token_start = nil
114
+
115
+ @reader.read_line() if @reader.end_of_line?
116
+ return @tokens unless @reader.line
117
+
118
+ @tokens = []
119
+ @token_start = @reader.pos
120
+
121
+ while not @reader.end_of_line?
122
+ if @tokenText
123
+ @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
124
+ @tokenText = nil
125
+ end
126
+
127
+ c = @reader.current_char
128
+ next_c = @reader.get_line_char(@reader.pos + 1)
129
+ case c
130
+ when "\""
131
+ # handle "" style strings including line continuations
132
+ handle_double_quote_string()
133
+
134
+ when "'"
135
+ handle_character_literal()
136
+
137
+ when "{", "}", "=", ":", ";"
138
+ # handle punctuation
139
+ punctuation_token = Token.new(c, @reader.line_no, @reader.pos)
140
+ @tokenText = nil
141
+
142
+ if punctuation_token.type == :SEMICOLON
143
+ @reader.skip_char()
144
+ break
145
+ else
146
+ @tokens << punctuation_token
147
+ end
148
+
149
+ when "#"
150
+ # skip : hash comment at end of line
151
+ @reader.skip_line()
152
+
153
+ when "/"
154
+ # handle // and /**/ style comments
155
+ if next_c == "/"
156
+ # skip : // comment
157
+ @reader.skip_line()
158
+ else
159
+ handle_slash_comment()
160
+ end
161
+
162
+ when "`"
163
+ # handle multiline `` style strings
164
+ handle_back_quote_string()
165
+
166
+ when "["
167
+ # handle binary literals
168
+ handle_binary_literal()
169
+
170
+ when "\s", "\t"
171
+ @reader.skip_whitespaces()
172
+
173
+ when "\\"
174
+ # line continuations (outside a string literal)
175
+ handle_line_continuation();
176
+
177
+ when /^[0-9\-\.]$/
178
+ if c == "-" and next_c == "-"
179
+ # -- comments : ignore
180
+ @reader.skip_line()
181
+ else
182
+ # handle numbers, dates, and time spans
183
+ handle_number_date_or_time_span()
184
+ end
185
+
186
+ when /^[a-zA-Z\$_]$/
187
+ # FIXME Here, the Java code specifies isJavaIdentifierStart() but
188
+ # this is not easily implemented (at least as of Ruby 1.8).
189
+ # So, we implement a subset of these characters.
190
+ handle_identifier()
191
+
192
+ when "\n", "\r"
193
+ # end of line
194
+ @reader.skip_line()
195
+
196
+ else
197
+ parse_error("Unexpected character '#{c}'")
198
+ end
199
+
200
+ @reader.skip_char()
201
+ end
202
+
203
+ if @tokenText
204
+ @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
205
+ end
206
+
207
+ return @tokens
208
+ end
209
+
210
+ # Adds the current escaped character (represented by ((|c|))) to @tokenText.
211
+ # This method assumes the previous char was a backslash.
212
+ #
213
+ def add_escaped_char_in_string(c)
214
+ case c
215
+ when "\\", "\""
216
+ @tokenText << c
217
+ when "n"
218
+ @tokenText << ?\n
219
+ when "r"
220
+ @tokenText << ?\r
221
+ when "t"
222
+ @tokenText << ?\t
223
+ else
224
+ parse_error("Illegal escape character in string literal: '#{c.chr}'.")
225
+ end
226
+ end
227
+
228
+ def handle_double_quote_string
229
+ escaped = false
230
+ @startEscapedQuoteLine = false
231
+
232
+ @tokenText = "\""
233
+ @reader.skip_char()
234
+
235
+ while not @reader.end_of_line?
236
+ c = @reader.current_char
237
+
238
+ if "\s\t".include?(c) and @startEscapedQuoteLine
239
+ # we continue
240
+ else
241
+ @startEscapedQuoteLine = false;
242
+
243
+ if escaped
244
+ add_escaped_char_in_string(c)
245
+ escaped = false
246
+
247
+ elsif c == "\\"
248
+ # check for String broken across lines
249
+ if @reader.rest_of_line =~ /^\\\s*$/
250
+ handle_escaped_double_quoted_string()
251
+ next # as we are at the beginning of a new line
252
+ else
253
+ escaped = true;
254
+ end
255
+
256
+ else
257
+ @tokenText << c
258
+ if c == "\""
259
+ # end of double-quoted string detected
260
+ @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
261
+ @tokenText = nil
262
+ return
263
+ end
264
+ end
265
+ end
266
+
267
+ @reader.skip_char()
268
+ end
269
+
270
+ # detection of ill-terminated literals
271
+ if @tokenText =~ /^".*[^"]$/
272
+ parse_error(
273
+ "String literal \"#{@tokenText}\" not terminated by end quote.", @reader.line_no, @reader.line_length);
274
+ elsif @tokenText == "\""
275
+ parse_error("Orphan quote (unterminated string)", @reader.line_no, @reader.line_length);
276
+ end
277
+ end
278
+
279
+ def handle_escaped_double_quoted_string
280
+ # '\' can be followed by whitespaces
281
+ if @reader.rest_of_line =~ /^\\\s*$/
282
+ @reader.read_line()
283
+ parse_error("Escape at end of file.") if @reader.end_of_file?
284
+
285
+ @startEscapedQuoteLine = true
286
+
287
+ else
288
+ parse_error(
289
+ "Malformed string literal - escape followed by whitespace " +
290
+ "followed by non-whitespace.")
291
+ end
292
+ end
293
+
294
+ def handle_character_literal
295
+ if not @reader.more_chars_in_line?
296
+ parse_error("Got ' at end of line")
297
+ end
298
+
299
+ c2 = @reader.read_char()
300
+
301
+ if c2 == "\\"
302
+ if @reader.end_of_line?
303
+ parse_error("Got '\\ at end of line")
304
+ end
305
+
306
+ c3 = @reader.read_char()
307
+
308
+ if not @reader.more_chars_in_line?
309
+ parse_error("Got '\\#{c3} at end of line")
310
+ end
311
+
312
+ case c3
313
+ when "\\"
314
+ @tokens << Token.new("'\\'", @reader.line_no, @reader.pos)
315
+ when "'"
316
+ @tokens << Token.new("'''", @reader.line_no, @reader.pos)
317
+ when "n"
318
+ @tokens << Token.new("'\n'", @reader.line_no, @reader.pos)
319
+ when "r"
320
+ @tokens << Token.new("'\r'", @reader.line_no, @reader.pos)
321
+ when "t"
322
+ @tokens << Token.new("'\t'", @reader.line_no, @reader.pos)
323
+ else
324
+ parse_error("Illegal escape character #{@reader.current_char}")
325
+ end
326
+
327
+ @reader.skip_char()
328
+ if @reader.current_char != "'"
329
+ expecting_but_got("single quote (')", "\"#{@reader.current_char}\"")
330
+ end
331
+ else
332
+ @tokens << Token.new("'#{c2}'", @reader.line_no, @reader.pos)
333
+ if not @reader.more_chars_in_line?
334
+ parse_error("Got '#{c2} at end of line")
335
+ end
336
+ @reader.skip_char()
337
+ if @reader.current_char != "'"
338
+ expecting_but_got(
339
+ "quote (')", "\"#{@reader.current_char}\"", @reader.line_no, @reader.pos)
340
+ end
341
+ end
342
+ end
343
+
344
+ def handle_slash_comment
345
+ if not @reader.more_chars_in_line?
346
+ parse_error("Got slash (/) at end of line.")
347
+ end
348
+
349
+ if @reader.get_line_char(@reader.pos + 1) == "*"
350
+ end_index = @reader.find_next_in_line("*/")
351
+ if end_index
352
+ # handle comment on same line
353
+ @reader.skip_to(end_index + 1)
354
+ else
355
+ # handle multiline comments
356
+ loop do
357
+ @reader.read_raw_line()
358
+ if @reader.end_of_file?
359
+ parse_error("/* comment not terminated.", @reader.line_no, -2)
360
+ end
361
+
362
+ end_index = @reader.find_next_in_line("*/", 0)
363
+
364
+ if end_index
365
+ @reader.skip_to(end_index + 1)
366
+ break
367
+ end
368
+ end
369
+ end
370
+ elsif @reader.get_line_char(@reader.pos + 1) == "/"
371
+ parse_error("Got slash (/) in unexpected location.")
372
+ end
373
+ end
374
+
375
+ def handle_back_quote_string
376
+ end_index = @reader.find_next_in_line("`")
377
+
378
+ if end_index
379
+ # handle end quote on same line
380
+ @tokens << Token.new(@reader.substring(@reader.pos, end_index), @reader.line_no, @reader.pos)
381
+ @tokenText = nil
382
+ @reader.skip_to(end_index)
383
+
384
+ else
385
+ @tokenText = @reader.rest_of_line
386
+ @token_start = @reader.pos
387
+ # handle multiline quotes
388
+ loop do
389
+ @reader.read_raw_line()
390
+ if @reader.end_of_file?
391
+ parse_error("` quote not terminated.", @reader.line_no, -2)
392
+ end
393
+
394
+ end_index = @reader.find_next_in_line("`", 0)
395
+ if end_index
396
+ @tokenText << @reader.substring(0, end_index)
397
+ @reader.skip_to(end_index)
398
+ break
399
+ else
400
+ @tokenText << @reader.line
401
+ end
402
+ end
403
+
404
+ @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
405
+ @tokenText = nil
406
+ end
407
+ end
408
+
409
+ def handle_binary_literal
410
+ end_index = @reader.find_next_in_line("]")
411
+
412
+ if end_index
413
+ # handle end quote on same line
414
+ @tokens << Token.new(@reader.substring(@reader.pos, end_index), @reader.line_no, @reader.pos)
415
+ @tokenText = nil
416
+ @reader.skip_to(end_index)
417
+ else
418
+ @tokenText = @reader.substring(@reader.pos)
419
+ @token_start = @reader.pos
420
+ # handle multiline quotes
421
+ loop do
422
+ @reader.read_raw_line()
423
+ if @reader.end_of_file?
424
+ parse_error("[base64] binary literal not terminated.", @reader.line_no, -2)
425
+ end
426
+
427
+ end_index = @reader.find_next_in_line("]", 0)
428
+ if end_index
429
+ @tokenText << @reader.substring(0, end_index)
430
+ @reader.skip_to(end_index)
431
+ break
432
+ else
433
+ @tokenText << @reader.line
434
+ end
435
+ end
436
+
437
+ @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
438
+ @tokenText = nil
439
+ end
440
+ end
441
+
442
+ # handle a line continuation (not inside a string)
443
+ def handle_line_continuation
444
+ # backslash line continuation outside of a String literal
445
+ # can only occur at the end of a line
446
+ if not @reader.rest_of_line =~ /^\\\s*$/
447
+ parse_error("Line continuation (\\) before end of line")
448
+ else
449
+ @line = @reader.read_line()
450
+ if @line.nil?
451
+ parse_error("Line continuation at end of file.", @reader.line_no, @reader.pos)
452
+ end
453
+ end
454
+ end
455
+
456
+ def handle_number_date_or_time_span
457
+ @token_start = @reader.pos
458
+ @tokenText = ""
459
+
460
+ while not @reader.end_of_line?
461
+ c = @reader.current_char
462
+
463
+ if c =~ /[\w\.\-+:]/
464
+ @tokenText << c
465
+ elsif c == "/" and not @reader.get_line_char(@reader.pos + 1) == "*"
466
+ @tokenText << c
467
+ else
468
+ @reader.previous_char()
469
+ break
470
+ end
471
+
472
+ @reader.skip_char()
473
+ end
474
+
475
+ @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
476
+ @tokenText = nil
477
+ end
478
+
479
+ def handle_identifier
480
+ @token_start = @reader.pos;
481
+ @tokenText = ""
482
+
483
+ while not @reader.end_of_line?
484
+ c = @reader.current_char
485
+
486
+ # FIXME here we are stricter than the Java version because there is no
487
+ # easy way to implement Character.isJavaIdentifierPart() in Ruby :)
488
+ if c =~ /[\w_$-\.]/
489
+ @tokenText << c
490
+ else
491
+ @reader.previous_char()
492
+ break
493
+ end
494
+
495
+ @reader.skip_char()
496
+ end
497
+
498
+ @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
499
+ @tokenText = nil
500
+ end
501
+
502
+ end
503
+
504
+ end
505
+
506
+ end