code-ruby 3.0.5 → 3.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +8 -0
- data/Gemfile.lock +1 -1
- data/VERSION +1 -1
- data/bin/code +27 -3
- data/lib/code/error.rb +10 -5
- data/lib/code/extensions/active_support.rb +9 -0
- data/lib/code/extensions/array.rb +7 -0
- data/lib/code/extensions/big_decimal.rb +9 -0
- data/lib/code/extensions/class.rb +7 -0
- data/lib/code/extensions/false_class.rb +7 -0
- data/lib/code/extensions/float.rb +9 -0
- data/lib/code/extensions/hash.rb +7 -0
- data/lib/code/extensions/integer.rb +9 -0
- data/lib/code/extensions/module.rb +7 -0
- data/lib/code/extensions/nil_class.rb +7 -0
- data/lib/code/extensions/nokogiri.rb +11 -0
- data/lib/code/extensions/object.rb +9 -0
- data/lib/code/extensions/string.rb +7 -0
- data/lib/code/extensions/symbol.rb +7 -0
- data/lib/code/extensions/true_class.rb +7 -0
- data/lib/code/extensions/word_number_comparaisons.rb +407 -0
- data/lib/code/format.rb +73 -56
- data/lib/code/node/code.rb +2 -1
- data/lib/code/node/statement.rb +4 -2
- data/lib/code/node/string.rb +1 -1
- data/lib/code/node/while.rb +8 -10
- data/lib/code/object/date.rb +37 -17
- data/lib/code/object/function.rb +23 -11
- data/lib/code/object/global.rb +5 -4
- data/lib/code/object/html.rb +169 -52
- data/lib/code/object/http.rb +8 -8
- data/lib/code/object/ics.rb +22 -18
- data/lib/code/object/identifier_list.rb +12 -10
- data/lib/code/object/list.rb +1 -5
- data/lib/code/object/super.rb +2 -1
- data/lib/code/object/time.rb +46 -44
- data/lib/code/parser.rb +342 -121
- data/lib/code-ruby.rb +16 -149
- data/spec/bin/code_spec.rb +32 -0
- data/spec/code/format_spec.rb +11 -10
- data/spec/code/node/call_spec.rb +1 -3
- data/spec/code/object/function_spec.rb +4 -8
- data/spec/code/object/http_spec.rb +20 -0
- data/spec/code/object/ics_spec.rb +5 -5
- data/spec/code/object/list_spec.rb +1 -1
- data/spec/code/parser_spec.rb +22 -0
- data/spec/code_spec.rb +347 -328
- metadata +18 -2
- data/applies +0 -0
data/lib/code/parser.rb
CHANGED
|
@@ -8,7 +8,8 @@ class Code
|
|
|
8
8
|
class Language
|
|
9
9
|
end
|
|
10
10
|
|
|
11
|
-
Token =
|
|
11
|
+
Token =
|
|
12
|
+
Data.define(:type, :value, :position, :newline_before, :space_before)
|
|
12
13
|
|
|
13
14
|
KEYWORDS = %w[
|
|
14
15
|
and
|
|
@@ -63,6 +64,13 @@ class Code
|
|
|
63
64
|
^=
|
|
64
65
|
=>
|
|
65
66
|
].sort_by(&:length).reverse.freeze
|
|
67
|
+
CONTINUATION_KEYWORDS = %w[or and rescue].freeze
|
|
68
|
+
POSTFIX_CONTINUATIONS = %w[. :: &.].freeze
|
|
69
|
+
HORIZONTAL_WHITESPACE = [" ", "\t"].freeze
|
|
70
|
+
NEWLINE_CHARACTERS = ["\n", "\r"].freeze
|
|
71
|
+
PUNCTUATION_CHARACTERS = %w[( ) [ ] { } , ? :].freeze
|
|
72
|
+
OPERATOR_CHARACTERS = %w[. & | = ! ~ + - * / % < > ^ × ÷].freeze
|
|
73
|
+
SUFFIX_PUNCTUATION = %w[! ?].freeze
|
|
66
74
|
|
|
67
75
|
ASSIGNMENT_RHS_MIN_BP = 20
|
|
68
76
|
|
|
@@ -144,8 +152,10 @@ class Code
|
|
|
144
152
|
until eof?
|
|
145
153
|
break if stop?(stop_keywords, stop_values)
|
|
146
154
|
|
|
155
|
+
previous_index = @index
|
|
147
156
|
statements << parse_expression
|
|
148
157
|
consume_newlines
|
|
158
|
+
ensure_parse_progress!(previous_index, "parsing code")
|
|
149
159
|
end
|
|
150
160
|
|
|
151
161
|
statements
|
|
@@ -156,13 +166,16 @@ class Code
|
|
|
156
166
|
left = nud(token)
|
|
157
167
|
|
|
158
168
|
loop do
|
|
169
|
+
previous_index = @index
|
|
159
170
|
token = current
|
|
160
171
|
break if token.type == :eof
|
|
172
|
+
|
|
161
173
|
if token.type == :newline
|
|
162
174
|
next_token = next_significant_token
|
|
163
175
|
break unless continuation_after_newline?(next_token)
|
|
164
176
|
|
|
165
177
|
skip_newlines
|
|
178
|
+
ensure_parse_progress!(previous_index, "skipping newlines")
|
|
166
179
|
next
|
|
167
180
|
end
|
|
168
181
|
|
|
@@ -172,6 +185,7 @@ class Code
|
|
|
172
185
|
break if call_like_postfix?(token) && !callable_expression?(left)
|
|
173
186
|
|
|
174
187
|
left = led_postfix(left)
|
|
188
|
+
ensure_parse_progress!(previous_index, "parsing postfix expression")
|
|
175
189
|
next
|
|
176
190
|
end
|
|
177
191
|
|
|
@@ -183,6 +197,7 @@ class Code
|
|
|
183
197
|
|
|
184
198
|
advance
|
|
185
199
|
left = led_infix(left, operator, right_bp)
|
|
200
|
+
ensure_parse_progress!(previous_index, "parsing infix expression")
|
|
186
201
|
end
|
|
187
202
|
|
|
188
203
|
left
|
|
@@ -237,7 +252,11 @@ class Code
|
|
|
237
252
|
when "!", "~", "+"
|
|
238
253
|
wrap_prefixed_expression(:negation, token.value, parse_expression(145))
|
|
239
254
|
when "-"
|
|
240
|
-
wrap_prefixed_expression(
|
|
255
|
+
wrap_prefixed_expression(
|
|
256
|
+
:unary_minus,
|
|
257
|
+
token.value,
|
|
258
|
+
parse_expression(159)
|
|
259
|
+
)
|
|
241
260
|
else
|
|
242
261
|
raise_parse_error("unexpected operator #{token.value.inspect}", token)
|
|
243
262
|
end
|
|
@@ -254,7 +273,10 @@ class Code
|
|
|
254
273
|
when ":"
|
|
255
274
|
parse_symbol_literal
|
|
256
275
|
else
|
|
257
|
-
raise_parse_error(
|
|
276
|
+
raise_parse_error(
|
|
277
|
+
"unexpected punctuation #{token.value.inspect}",
|
|
278
|
+
token
|
|
279
|
+
)
|
|
258
280
|
end
|
|
259
281
|
end
|
|
260
282
|
|
|
@@ -291,10 +313,10 @@ class Code
|
|
|
291
313
|
end
|
|
292
314
|
|
|
293
315
|
def led_infix(left, operator, right_bp)
|
|
316
|
+
skip_newlines
|
|
294
317
|
case operator
|
|
295
|
-
when "=", "+=", "-=", "*=", "/=", "%=", "<<=", ">>=", "&=", "|=",
|
|
296
|
-
"
|
|
297
|
-
skip_newlines
|
|
318
|
+
when "=", "+=", "-=", "*=", "/=", "%=", "<<=", ">>=", "&=", "|=", "^=",
|
|
319
|
+
"||=", "&&="
|
|
298
320
|
{
|
|
299
321
|
right_operation: {
|
|
300
322
|
left: left,
|
|
@@ -303,10 +325,14 @@ class Code
|
|
|
303
325
|
}
|
|
304
326
|
}
|
|
305
327
|
when "if", "unless", "while", "until", "rescue"
|
|
306
|
-
|
|
307
|
-
|
|
328
|
+
{
|
|
329
|
+
right_operation: {
|
|
330
|
+
left: left,
|
|
331
|
+
operator: operator,
|
|
332
|
+
right: parse_expression(right_bp)
|
|
333
|
+
}
|
|
334
|
+
}
|
|
308
335
|
when "?"
|
|
309
|
-
skip_newlines
|
|
310
336
|
middle = parse_expression
|
|
311
337
|
right =
|
|
312
338
|
if match?(:punctuation, ":")
|
|
@@ -316,7 +342,6 @@ class Code
|
|
|
316
342
|
end
|
|
317
343
|
{ ternary: { left: left, middle: middle, right: right } }
|
|
318
344
|
else
|
|
319
|
-
skip_newlines
|
|
320
345
|
right = parse_expression(right_bp)
|
|
321
346
|
|
|
322
347
|
if operator == "**"
|
|
@@ -341,7 +366,11 @@ class Code
|
|
|
341
366
|
skip_newlines
|
|
342
367
|
else_statement = parse_expression
|
|
343
368
|
else_body = parse_body(%w[elsif elsunless else end])
|
|
344
|
-
elses << {
|
|
369
|
+
elses << {
|
|
370
|
+
operator: else_operator,
|
|
371
|
+
statement: else_statement,
|
|
372
|
+
body: else_body
|
|
373
|
+
}
|
|
345
374
|
next
|
|
346
375
|
end
|
|
347
376
|
|
|
@@ -349,13 +378,10 @@ class Code
|
|
|
349
378
|
advance
|
|
350
379
|
skip_newlines
|
|
351
380
|
|
|
352
|
-
if match?(:keyword, "if") || match?(:keyword, "unless")
|
|
353
|
-
|
|
354
|
-
operator: "else",
|
|
355
|
-
body: [parse_if_expression(advance.value)]
|
|
356
|
-
}
|
|
381
|
+
elses << if match?(:keyword, "if") || match?(:keyword, "unless")
|
|
382
|
+
{ operator: "else", body: [parse_if_expression(advance.value)] }
|
|
357
383
|
else
|
|
358
|
-
|
|
384
|
+
{ operator: "else", body: parse_body(%w[end]) }
|
|
359
385
|
end
|
|
360
386
|
|
|
361
387
|
break
|
|
@@ -386,11 +412,7 @@ class Code
|
|
|
386
412
|
advance if match?(:keyword, "end")
|
|
387
413
|
|
|
388
414
|
{
|
|
389
|
-
while: {
|
|
390
|
-
operator: operator,
|
|
391
|
-
statement: statement,
|
|
392
|
-
body: body
|
|
393
|
-
}.compact
|
|
415
|
+
while: { operator: operator, statement: statement, body: body }.compact
|
|
394
416
|
}
|
|
395
417
|
end
|
|
396
418
|
|
|
@@ -467,19 +489,15 @@ class Code
|
|
|
467
489
|
if label_name_start?(current) && next_token_value == ":"
|
|
468
490
|
name = advance.value
|
|
469
491
|
advance
|
|
470
|
-
code = parse_optional_code([
|
|
492
|
+
code = parse_optional_code(%w[, }])
|
|
471
493
|
return { name_code: { name: name, code: code }.compact }
|
|
472
494
|
end
|
|
473
495
|
|
|
474
496
|
statement = parse_expression
|
|
475
497
|
|
|
476
|
-
if match?(:punctuation, ":")
|
|
477
|
-
advance
|
|
478
|
-
code = parse_optional_code([",", "}"])
|
|
479
|
-
{ statement_code: { statement: statement, code: code }.compact }
|
|
480
|
-
elsif match?(:operator, "=>")
|
|
498
|
+
if match?(:punctuation, ":") || match?(:operator, "=>")
|
|
481
499
|
advance
|
|
482
|
-
code = parse_optional_code([
|
|
500
|
+
code = parse_optional_code(%w[, }])
|
|
483
501
|
{ statement_code: { statement: statement, code: code }.compact }
|
|
484
502
|
else
|
|
485
503
|
{ code: wrap_code(statement) }
|
|
@@ -536,9 +554,9 @@ class Code
|
|
|
536
554
|
if label_name_start?(current) && next_token_value == ":"
|
|
537
555
|
name = advance.value
|
|
538
556
|
advance
|
|
539
|
-
{ name: name, value: parse_code(stop_values: [
|
|
557
|
+
{ name: name, value: parse_code(stop_values: %w[, )]) }
|
|
540
558
|
else
|
|
541
|
-
{ value: parse_code(stop_values: [
|
|
559
|
+
{ value: parse_code(stop_values: %w[, )]) }
|
|
542
560
|
end
|
|
543
561
|
end
|
|
544
562
|
|
|
@@ -607,15 +625,16 @@ class Code
|
|
|
607
625
|
if label_name_start?(current) && next_token_value == ":"
|
|
608
626
|
name = advance.value
|
|
609
627
|
advance
|
|
610
|
-
default = parse_optional_code([
|
|
628
|
+
default = parse_optional_code(%w[, ) |])
|
|
611
629
|
return { name: name, keyword: ":", default: default }.compact
|
|
612
630
|
end
|
|
613
631
|
|
|
614
|
-
name = advance.value if current.type == :identifier ||
|
|
632
|
+
name = advance.value if current.type == :identifier ||
|
|
633
|
+
keyword_name?(current)
|
|
615
634
|
default =
|
|
616
635
|
if match?(:operator, "=")
|
|
617
636
|
advance
|
|
618
|
-
parse_code(stop_values: [
|
|
637
|
+
parse_code(stop_values: %w[, ) |])
|
|
619
638
|
end
|
|
620
639
|
|
|
621
640
|
{
|
|
@@ -631,7 +650,7 @@ class Code
|
|
|
631
650
|
def parse_parameter_prefix
|
|
632
651
|
return unless current.type == :operator
|
|
633
652
|
|
|
634
|
-
|
|
653
|
+
advance.value if %w[* ** & .. ... .].include?(current.value)
|
|
635
654
|
end
|
|
636
655
|
|
|
637
656
|
def parse_optional_code(stop_values)
|
|
@@ -642,15 +661,11 @@ class Code
|
|
|
642
661
|
end
|
|
643
662
|
|
|
644
663
|
def attach_call_arguments(left, arguments)
|
|
645
|
-
update_terminal_call(left)
|
|
646
|
-
call[:arguments] = arguments
|
|
647
|
-
end
|
|
664
|
+
update_terminal_call(left) { |call| call[:arguments] = arguments }
|
|
648
665
|
end
|
|
649
666
|
|
|
650
667
|
def attach_call_block(left, block)
|
|
651
|
-
update_terminal_call(left)
|
|
652
|
-
call[:block] = block
|
|
653
|
-
end
|
|
668
|
+
update_terminal_call(left) { |call| call[:block] = block }
|
|
654
669
|
end
|
|
655
670
|
|
|
656
671
|
def update_terminal_call(left)
|
|
@@ -683,7 +698,11 @@ class Code
|
|
|
683
698
|
same_left_operation_group?(left[:left_operation], operator)
|
|
684
699
|
raw = left.deep_dup
|
|
685
700
|
target = raw[:left_operation][:others].last
|
|
686
|
-
target[:statement] = append_left_operation(
|
|
701
|
+
target[:statement] = append_left_operation(
|
|
702
|
+
target[:statement],
|
|
703
|
+
operator,
|
|
704
|
+
statement
|
|
705
|
+
)
|
|
687
706
|
raw
|
|
688
707
|
else
|
|
689
708
|
{
|
|
@@ -704,18 +723,30 @@ class Code
|
|
|
704
723
|
|
|
705
724
|
def operator_group(operator)
|
|
706
725
|
case operator
|
|
707
|
-
when ".", "::", "&."
|
|
708
|
-
|
|
709
|
-
when "
|
|
710
|
-
|
|
711
|
-
when "
|
|
712
|
-
|
|
713
|
-
when "
|
|
714
|
-
|
|
715
|
-
when "
|
|
716
|
-
|
|
717
|
-
when "
|
|
718
|
-
|
|
726
|
+
when ".", "::", "&."
|
|
727
|
+
:chain
|
|
728
|
+
when "or", "and"
|
|
729
|
+
:keyword_logic
|
|
730
|
+
when "||"
|
|
731
|
+
:or_operator
|
|
732
|
+
when "&&"
|
|
733
|
+
:and_operator
|
|
734
|
+
when "==", "===", "!=", "!==", "<=>", "=~", "~=", "!~"
|
|
735
|
+
:equality
|
|
736
|
+
when ">", "<", ">=", "<="
|
|
737
|
+
:comparison
|
|
738
|
+
when "|", "^"
|
|
739
|
+
:bitwise_or
|
|
740
|
+
when "&"
|
|
741
|
+
:bitwise_and
|
|
742
|
+
when "<<", ">>"
|
|
743
|
+
:shift
|
|
744
|
+
when "+", "-"
|
|
745
|
+
:addition
|
|
746
|
+
when "*", "/", "%", "×", "÷"
|
|
747
|
+
:multiplication
|
|
748
|
+
when "..", "..."
|
|
749
|
+
:range
|
|
719
750
|
else
|
|
720
751
|
operator
|
|
721
752
|
end
|
|
@@ -728,7 +759,12 @@ class Code
|
|
|
728
759
|
left_operation = right[:left_operation].deep_dup
|
|
729
760
|
{
|
|
730
761
|
left_operation: {
|
|
731
|
-
first: {
|
|
762
|
+
first: {
|
|
763
|
+
type => {
|
|
764
|
+
operator: operator,
|
|
765
|
+
right: left_operation[:first]
|
|
766
|
+
}
|
|
767
|
+
},
|
|
732
768
|
others: left_operation[:others]
|
|
733
769
|
}
|
|
734
770
|
}
|
|
@@ -752,28 +788,53 @@ class Code
|
|
|
752
788
|
end
|
|
753
789
|
|
|
754
790
|
def postfix_start?(token)
|
|
755
|
-
token.type == :punctuation && ["(", "[", "{"].include?(token.value) ||
|
|
756
|
-
token.type == :operator && [
|
|
757
|
-
token.type == :keyword && token.value == "do"
|
|
791
|
+
(token.type == :punctuation && ["(", "[", "{"].include?(token.value)) ||
|
|
792
|
+
(token.type == :operator && %w[. :: &.].include?(token.value)) ||
|
|
793
|
+
(token.type == :keyword && token.value == "do")
|
|
758
794
|
end
|
|
759
795
|
|
|
760
796
|
def call_like_postfix?(token)
|
|
761
|
-
token.type == :punctuation && [
|
|
762
|
-
token.type == :keyword && token.value == "do"
|
|
797
|
+
(token.type == :punctuation && %w[( {].include?(token.value)) ||
|
|
798
|
+
(token.type == :keyword && token.value == "do")
|
|
763
799
|
end
|
|
764
800
|
|
|
765
801
|
def infix_operator(token)
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
802
|
+
if token.type == :operator && INFIX_PRECEDENCE.key?(token.value)
|
|
803
|
+
return token.value
|
|
804
|
+
end
|
|
805
|
+
if token.type == :keyword && INFIX_PRECEDENCE.key?(token.value)
|
|
806
|
+
return token.value
|
|
807
|
+
end
|
|
808
|
+
|
|
809
|
+
token.value if token.type == :punctuation && token.value == "?"
|
|
769
810
|
end
|
|
770
811
|
|
|
771
812
|
def label_name_start?(token)
|
|
772
|
-
|
|
813
|
+
%i[identifier keyword].include?(token.type)
|
|
773
814
|
end
|
|
774
815
|
|
|
775
816
|
def keyword_name?(token)
|
|
776
|
-
token.type == :keyword &&
|
|
817
|
+
token.type == :keyword &&
|
|
818
|
+
!%w[
|
|
819
|
+
if
|
|
820
|
+
unless
|
|
821
|
+
while
|
|
822
|
+
until
|
|
823
|
+
loop
|
|
824
|
+
not
|
|
825
|
+
rescue
|
|
826
|
+
or
|
|
827
|
+
and
|
|
828
|
+
do
|
|
829
|
+
begin
|
|
830
|
+
else
|
|
831
|
+
elsif
|
|
832
|
+
elsunless
|
|
833
|
+
end
|
|
834
|
+
true
|
|
835
|
+
false
|
|
836
|
+
nothing
|
|
837
|
+
].include?(token.value)
|
|
777
838
|
end
|
|
778
839
|
|
|
779
840
|
def callable_expression?(expression)
|
|
@@ -790,7 +851,16 @@ class Code
|
|
|
790
851
|
end
|
|
791
852
|
|
|
792
853
|
def next_token_value
|
|
793
|
-
tokens.fetch(
|
|
854
|
+
tokens.fetch(
|
|
855
|
+
@index + 1,
|
|
856
|
+
Token.new(
|
|
857
|
+
type: :eof,
|
|
858
|
+
value: nil,
|
|
859
|
+
position: input.length,
|
|
860
|
+
newline_before: false,
|
|
861
|
+
space_before: false
|
|
862
|
+
)
|
|
863
|
+
).value
|
|
794
864
|
end
|
|
795
865
|
|
|
796
866
|
def advance
|
|
@@ -806,7 +876,9 @@ class Code
|
|
|
806
876
|
|
|
807
877
|
def expect(type, value = nil)
|
|
808
878
|
token = current
|
|
809
|
-
|
|
879
|
+
if token.type == type && (value.nil? || token.value == value)
|
|
880
|
+
return advance
|
|
881
|
+
end
|
|
810
882
|
|
|
811
883
|
expected = value || type
|
|
812
884
|
raise_parse_error("expected #{expected.inspect}", token)
|
|
@@ -823,26 +895,45 @@ class Code
|
|
|
823
895
|
def next_significant_token
|
|
824
896
|
index = @index
|
|
825
897
|
index += 1 while tokens[index]&.type == :newline
|
|
826
|
-
tokens.fetch(
|
|
898
|
+
tokens.fetch(
|
|
899
|
+
index,
|
|
900
|
+
Token.new(
|
|
901
|
+
type: :eof,
|
|
902
|
+
value: nil,
|
|
903
|
+
position: input.length,
|
|
904
|
+
newline_before: false,
|
|
905
|
+
space_before: false
|
|
906
|
+
)
|
|
907
|
+
)
|
|
827
908
|
end
|
|
828
909
|
|
|
829
910
|
def continuation_after_newline?(token)
|
|
830
911
|
return false if token.type == :eof
|
|
831
|
-
|
|
832
|
-
|
|
912
|
+
if token.type == :operator && INFIX_PRECEDENCE.key?(token.value)
|
|
913
|
+
return true
|
|
914
|
+
end
|
|
915
|
+
if token.type == :keyword && CONTINUATION_KEYWORDS.include?(token.value)
|
|
916
|
+
return true
|
|
917
|
+
end
|
|
833
918
|
return true if token.type == :punctuation && token.value == "?"
|
|
834
919
|
|
|
835
|
-
token.type == :operator &&
|
|
920
|
+
token.type == :operator && POSTFIX_CONTINUATIONS.include?(token.value)
|
|
836
921
|
end
|
|
837
922
|
|
|
838
923
|
def newline_postfix_continuation?(token)
|
|
839
|
-
token.type == :operator &&
|
|
924
|
+
token.type == :operator && POSTFIX_CONTINUATIONS.include?(token.value)
|
|
840
925
|
end
|
|
841
926
|
|
|
842
927
|
def consume_newlines
|
|
843
928
|
skip_newlines
|
|
844
929
|
end
|
|
845
930
|
|
|
931
|
+
def ensure_parse_progress!(previous_index, context)
|
|
932
|
+
return if @index > previous_index
|
|
933
|
+
|
|
934
|
+
raise_parse_error("parser made no progress while #{context}")
|
|
935
|
+
end
|
|
936
|
+
|
|
846
937
|
def raise_parse_error(message, token = current)
|
|
847
938
|
raise Error, "#{message} at #{token.position}"
|
|
848
939
|
end
|
|
@@ -856,20 +947,27 @@ class Code
|
|
|
856
947
|
while index < source.length
|
|
857
948
|
char = source[index]
|
|
858
949
|
|
|
859
|
-
if char
|
|
950
|
+
if HORIZONTAL_WHITESPACE.include?(char)
|
|
860
951
|
index += 1
|
|
861
952
|
space_before = true
|
|
862
953
|
next
|
|
863
954
|
end
|
|
864
955
|
|
|
865
|
-
if char
|
|
866
|
-
|
|
867
|
-
index
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
956
|
+
if NEWLINE_CHARACTERS.include?(char)
|
|
957
|
+
index +=
|
|
958
|
+
if char == "\r" && source[index + 1] == "\n"
|
|
959
|
+
2
|
|
960
|
+
else
|
|
961
|
+
1
|
|
962
|
+
end
|
|
963
|
+
|
|
964
|
+
tokens << Token.new(
|
|
965
|
+
type: :newline,
|
|
966
|
+
value: "\n",
|
|
967
|
+
position: index - 1,
|
|
968
|
+
newline_before: false,
|
|
969
|
+
space_before: false
|
|
970
|
+
)
|
|
873
971
|
newline_before = true
|
|
874
972
|
space_before = false
|
|
875
973
|
next
|
|
@@ -877,14 +975,16 @@ class Code
|
|
|
877
975
|
|
|
878
976
|
if char == "#"
|
|
879
977
|
index += 1
|
|
880
|
-
index += 1 while index < source.length &&
|
|
978
|
+
index += 1 while index < source.length &&
|
|
979
|
+
!NEWLINE_CHARACTERS.include?(source[index])
|
|
881
980
|
space_before = true unless newline_before
|
|
882
981
|
next
|
|
883
982
|
end
|
|
884
983
|
|
|
885
984
|
if source[index, 2] == "//"
|
|
886
985
|
index += 2
|
|
887
|
-
index += 1 while index < source.length &&
|
|
986
|
+
index += 1 while index < source.length &&
|
|
987
|
+
!NEWLINE_CHARACTERS.include?(source[index])
|
|
888
988
|
space_before = true unless newline_before
|
|
889
989
|
next
|
|
890
990
|
end
|
|
@@ -892,7 +992,7 @@ class Code
|
|
|
892
992
|
if source[index, 2] == "/*"
|
|
893
993
|
index += 2
|
|
894
994
|
while index < source.length && source[index, 2] != "*/"
|
|
895
|
-
newline_before ||=
|
|
995
|
+
newline_before ||= NEWLINE_CHARACTERS.include?(source[index])
|
|
896
996
|
index += 1
|
|
897
997
|
end
|
|
898
998
|
index += 2 if source[index, 2] == "*/"
|
|
@@ -901,7 +1001,19 @@ class Code
|
|
|
901
1001
|
end
|
|
902
1002
|
|
|
903
1003
|
if (string_data = scan_string(source, index))
|
|
904
|
-
|
|
1004
|
+
ensure_lex_progress!(
|
|
1005
|
+
index,
|
|
1006
|
+
string_data[:index],
|
|
1007
|
+
"scanning string",
|
|
1008
|
+
source
|
|
1009
|
+
)
|
|
1010
|
+
tokens << Token.new(
|
|
1011
|
+
type: :string,
|
|
1012
|
+
value: string_data[:parts],
|
|
1013
|
+
position: index,
|
|
1014
|
+
newline_before: newline_before,
|
|
1015
|
+
space_before: space_before
|
|
1016
|
+
)
|
|
905
1017
|
index = string_data[:index]
|
|
906
1018
|
newline_before = false
|
|
907
1019
|
space_before = false
|
|
@@ -909,7 +1021,19 @@ class Code
|
|
|
909
1021
|
end
|
|
910
1022
|
|
|
911
1023
|
if (symbol_data = scan_symbol(source, index))
|
|
912
|
-
|
|
1024
|
+
ensure_lex_progress!(
|
|
1025
|
+
index,
|
|
1026
|
+
symbol_data[:index],
|
|
1027
|
+
"scanning symbol",
|
|
1028
|
+
source
|
|
1029
|
+
)
|
|
1030
|
+
tokens << Token.new(
|
|
1031
|
+
type: :symbol,
|
|
1032
|
+
value: symbol_data[:value],
|
|
1033
|
+
position: index,
|
|
1034
|
+
newline_before: newline_before,
|
|
1035
|
+
space_before: space_before
|
|
1036
|
+
)
|
|
913
1037
|
index = symbol_data[:index]
|
|
914
1038
|
newline_before = false
|
|
915
1039
|
space_before = false
|
|
@@ -917,33 +1041,65 @@ class Code
|
|
|
917
1041
|
end
|
|
918
1042
|
|
|
919
1043
|
if (number_data = scan_number(source, index))
|
|
920
|
-
|
|
1044
|
+
ensure_lex_progress!(
|
|
1045
|
+
index,
|
|
1046
|
+
number_data[:index],
|
|
1047
|
+
"scanning number",
|
|
1048
|
+
source
|
|
1049
|
+
)
|
|
1050
|
+
tokens << Token.new(
|
|
1051
|
+
type: :number,
|
|
1052
|
+
value: number_data[:raw],
|
|
1053
|
+
position: index,
|
|
1054
|
+
newline_before: newline_before,
|
|
1055
|
+
space_before: space_before
|
|
1056
|
+
)
|
|
921
1057
|
index = number_data[:index]
|
|
922
1058
|
newline_before = false
|
|
923
1059
|
space_before = false
|
|
924
1060
|
next
|
|
925
1061
|
end
|
|
926
1062
|
|
|
927
|
-
operator =
|
|
1063
|
+
operator =
|
|
1064
|
+
MULTI_CHAR_OPERATORS.find do |candidate|
|
|
1065
|
+
source[index, candidate.length] == candidate
|
|
1066
|
+
end
|
|
928
1067
|
if operator
|
|
929
|
-
|
|
930
|
-
|
|
1068
|
+
tokens << Token.new(
|
|
1069
|
+
type: :operator,
|
|
1070
|
+
value: operator,
|
|
1071
|
+
position: index,
|
|
1072
|
+
newline_before: newline_before,
|
|
1073
|
+
space_before: space_before
|
|
1074
|
+
)
|
|
931
1075
|
index += operator.length
|
|
932
1076
|
newline_before = false
|
|
933
1077
|
space_before = false
|
|
934
1078
|
next
|
|
935
1079
|
end
|
|
936
1080
|
|
|
937
|
-
if
|
|
938
|
-
tokens << Token.new(
|
|
1081
|
+
if PUNCTUATION_CHARACTERS.include?(char)
|
|
1082
|
+
tokens << Token.new(
|
|
1083
|
+
type: :punctuation,
|
|
1084
|
+
value: char,
|
|
1085
|
+
position: index,
|
|
1086
|
+
newline_before: newline_before,
|
|
1087
|
+
space_before: space_before
|
|
1088
|
+
)
|
|
939
1089
|
index += 1
|
|
940
1090
|
newline_before = false
|
|
941
1091
|
space_before = false
|
|
942
1092
|
next
|
|
943
1093
|
end
|
|
944
1094
|
|
|
945
|
-
if
|
|
946
|
-
tokens << Token.new(
|
|
1095
|
+
if OPERATOR_CHARACTERS.include?(char)
|
|
1096
|
+
tokens << Token.new(
|
|
1097
|
+
type: :operator,
|
|
1098
|
+
value: char,
|
|
1099
|
+
position: index,
|
|
1100
|
+
newline_before: newline_before,
|
|
1101
|
+
space_before: space_before
|
|
1102
|
+
)
|
|
947
1103
|
index += 1
|
|
948
1104
|
newline_before = false
|
|
949
1105
|
space_before = false
|
|
@@ -952,8 +1108,20 @@ class Code
|
|
|
952
1108
|
|
|
953
1109
|
identifier = scan_identifier(source, index)
|
|
954
1110
|
if identifier
|
|
1111
|
+
ensure_lex_progress!(
|
|
1112
|
+
index,
|
|
1113
|
+
index + identifier.length,
|
|
1114
|
+
"scanning identifier",
|
|
1115
|
+
source
|
|
1116
|
+
)
|
|
955
1117
|
type = KEYWORDS.include?(identifier) ? :keyword : :identifier
|
|
956
|
-
tokens << Token.new(
|
|
1118
|
+
tokens << Token.new(
|
|
1119
|
+
type: type,
|
|
1120
|
+
value: identifier,
|
|
1121
|
+
position: index,
|
|
1122
|
+
newline_before: newline_before,
|
|
1123
|
+
space_before: space_before
|
|
1124
|
+
)
|
|
957
1125
|
index += identifier.length
|
|
958
1126
|
newline_before = false
|
|
959
1127
|
space_before = false
|
|
@@ -963,13 +1131,33 @@ class Code
|
|
|
963
1131
|
raise Error, "unexpected character #{char.inspect} at #{index}"
|
|
964
1132
|
end
|
|
965
1133
|
|
|
966
|
-
tokens << Token.new(
|
|
1134
|
+
tokens << Token.new(
|
|
1135
|
+
type: :eof,
|
|
1136
|
+
value: nil,
|
|
1137
|
+
position: source.length,
|
|
1138
|
+
newline_before: newline_before,
|
|
1139
|
+
space_before: space_before
|
|
1140
|
+
)
|
|
967
1141
|
tokens
|
|
968
1142
|
end
|
|
969
1143
|
|
|
1144
|
+
def ensure_lex_progress!(previous_index, next_index, context, source)
|
|
1145
|
+
return if next_index > previous_index
|
|
1146
|
+
|
|
1147
|
+
token =
|
|
1148
|
+
Token.new(
|
|
1149
|
+
type: :eof,
|
|
1150
|
+
value: nil,
|
|
1151
|
+
position: [previous_index, source.length].min,
|
|
1152
|
+
newline_before: false,
|
|
1153
|
+
space_before: false
|
|
1154
|
+
)
|
|
1155
|
+
raise_parse_error("lexer made no progress while #{context}", token)
|
|
1156
|
+
end
|
|
1157
|
+
|
|
970
1158
|
def scan_string(source, index)
|
|
971
1159
|
quote = source[index]
|
|
972
|
-
return unless
|
|
1160
|
+
return unless %w[' "].include?(quote)
|
|
973
1161
|
|
|
974
1162
|
parts = []
|
|
975
1163
|
text = +""
|
|
@@ -1023,18 +1211,12 @@ class Code
|
|
|
1023
1211
|
|
|
1024
1212
|
if char == "{"
|
|
1025
1213
|
depth += 1
|
|
1026
|
-
body << char
|
|
1027
|
-
i += 1
|
|
1028
1214
|
elsif char == "}"
|
|
1029
1215
|
depth -= 1
|
|
1030
|
-
return
|
|
1031
|
-
|
|
1032
|
-
body << char
|
|
1033
|
-
i += 1
|
|
1034
|
-
else
|
|
1035
|
-
body << char
|
|
1036
|
-
i += 1
|
|
1216
|
+
return body, i + 1 if depth.zero?
|
|
1037
1217
|
end
|
|
1218
|
+
body << char
|
|
1219
|
+
i += 1
|
|
1038
1220
|
end
|
|
1039
1221
|
|
|
1040
1222
|
[body, i]
|
|
@@ -1045,35 +1227,76 @@ class Code
|
|
|
1045
1227
|
return unless rest
|
|
1046
1228
|
|
|
1047
1229
|
if (match = /\A0[xX][0-9a-fA-F](?:_?[0-9a-fA-F])*/.match(rest))
|
|
1048
|
-
return
|
|
1230
|
+
return(
|
|
1231
|
+
{
|
|
1232
|
+
raw: {
|
|
1233
|
+
number: {
|
|
1234
|
+
base_16: match[0][2..].delete("_")
|
|
1235
|
+
}
|
|
1236
|
+
},
|
|
1237
|
+
index: index + match[0].length
|
|
1238
|
+
}
|
|
1239
|
+
)
|
|
1049
1240
|
end
|
|
1050
1241
|
|
|
1051
1242
|
if (match = /\A0[oO][0-7](?:_?[0-7])*/.match(rest))
|
|
1052
|
-
return
|
|
1243
|
+
return(
|
|
1244
|
+
{
|
|
1245
|
+
raw: {
|
|
1246
|
+
number: {
|
|
1247
|
+
base_8: match[0][2..].delete("_")
|
|
1248
|
+
}
|
|
1249
|
+
},
|
|
1250
|
+
index: index + match[0].length
|
|
1251
|
+
}
|
|
1252
|
+
)
|
|
1053
1253
|
end
|
|
1054
1254
|
|
|
1055
1255
|
if (match = /\A0[bB][01](?:_?[01])*/.match(rest))
|
|
1056
|
-
return
|
|
1256
|
+
return(
|
|
1257
|
+
{
|
|
1258
|
+
raw: {
|
|
1259
|
+
number: {
|
|
1260
|
+
base_2: match[0][2..].delete("_")
|
|
1261
|
+
}
|
|
1262
|
+
},
|
|
1263
|
+
index: index + match[0].length
|
|
1264
|
+
}
|
|
1265
|
+
)
|
|
1057
1266
|
end
|
|
1058
1267
|
|
|
1059
|
-
if (
|
|
1268
|
+
if (
|
|
1269
|
+
match =
|
|
1270
|
+
/\A[0-9](?:_?[0-9])*\.[0-9](?:_?[0-9])*(?:[eE][0-9](?:_?[0-9])*(?:\.[0-9](?:_?[0-9])*)?)?/.match(
|
|
1271
|
+
rest
|
|
1272
|
+
)
|
|
1273
|
+
)
|
|
1060
1274
|
decimal, exponent = match[0].split(/[eE]/, 2)
|
|
1061
1275
|
raw = { decimal: decimal.delete("_") }
|
|
1062
1276
|
raw[:exponent] = exponent_to_raw(exponent) if exponent
|
|
1063
|
-
return
|
|
1277
|
+
return(
|
|
1278
|
+
{ raw: { number: { decimal: raw } }, index: index + match[0].length }
|
|
1279
|
+
)
|
|
1064
1280
|
end
|
|
1065
1281
|
|
|
1066
|
-
if (
|
|
1282
|
+
if (
|
|
1283
|
+
match =
|
|
1284
|
+
/\A[0-9](?:_?[0-9])*(?:[eE][0-9](?:_?[0-9])*(?:\.[0-9](?:_?[0-9])*)?)?/.match(
|
|
1285
|
+
rest
|
|
1286
|
+
)
|
|
1287
|
+
)
|
|
1067
1288
|
whole, exponent = match[0].split(/[eE]/, 2)
|
|
1068
1289
|
raw = { whole: whole.delete("_") }
|
|
1069
1290
|
raw[:exponent] = exponent_to_raw(exponent) if exponent
|
|
1070
|
-
|
|
1291
|
+
{ raw: { number: { base_10: raw } }, index: index + match[0].length }
|
|
1071
1292
|
end
|
|
1072
1293
|
end
|
|
1073
1294
|
|
|
1074
1295
|
def exponent_to_raw(exponent)
|
|
1075
1296
|
exponent = exponent.delete("_")
|
|
1076
|
-
|
|
1297
|
+
if exponent.include?(".")
|
|
1298
|
+
return { number: { decimal: { decimal: exponent } } }
|
|
1299
|
+
end
|
|
1077
1300
|
|
|
1078
1301
|
{ number: { base_10: { whole: exponent } } }
|
|
1079
1302
|
end
|
|
@@ -1092,10 +1315,10 @@ class Code
|
|
|
1092
1315
|
|
|
1093
1316
|
while i < source.length
|
|
1094
1317
|
char = source[i]
|
|
1095
|
-
break if
|
|
1318
|
+
break if /\s/.match?(char)
|
|
1096
1319
|
break if "()[]{}.,:&|=~*/%<>^#".include?(char)
|
|
1097
1320
|
|
|
1098
|
-
if char
|
|
1321
|
+
if SUFFIX_PUNCTUATION.include?(char)
|
|
1099
1322
|
value << char
|
|
1100
1323
|
i += 1
|
|
1101
1324
|
break
|
|
@@ -1123,12 +1346,10 @@ class Code
|
|
|
1123
1346
|
|
|
1124
1347
|
while i < source.length
|
|
1125
1348
|
char = source[i]
|
|
1126
|
-
break if
|
|
1349
|
+
break if /\s/.match?(char)
|
|
1127
1350
|
|
|
1128
|
-
if char
|
|
1129
|
-
if value.empty? || source[i + 1] == "=" || source[i + 1] == "~"
|
|
1130
|
-
break
|
|
1131
|
-
end
|
|
1351
|
+
if SUFFIX_PUNCTUATION.include?(char)
|
|
1352
|
+
break if value.empty? || source[i + 1] == "=" || source[i + 1] == "~"
|
|
1132
1353
|
|
|
1133
1354
|
value << char
|
|
1134
1355
|
i += 1
|