soml-lang 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/soml/parser.rb CHANGED
@@ -24,15 +24,18 @@ module SOML
24
24
 
25
25
  DECIMAL_DIGIT = byte_table('0123456789')
26
26
 
27
+ # The first letters of the units, and of a day and a week.
28
+ DURATION_UNIT_START = byte_table('dhmnsuwDHMNSUW')
29
+
27
30
  # Exact as floats. The fast path divides by at most 10^14.
28
- POWERS_OF_TEN = Array.new(15) { |exponent| (10**exponent).to_f }.freeze
31
+ POWERS_OF_TEN = Array.new(15) { (10**it).to_f }.freeze
29
32
  LETTER = byte_table('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz')
30
33
 
31
- # The digits, the base, the name, its article, and the characters that an error does not name as an invalid digit. A lowercase hexadecimal digit is a digit in the wrong case, so it is not named. Shareable, so that a ractor can read it.
34
+ # The digits, the base, the name, its article, and a search for a character that an error names as an invalid digit, by the byte of the letter of the prefix. A lowercase hexadecimal digit is a digit in the wrong case, so it is not named. Shareable, so that a ractor can read it.
32
35
  RADIXES = Ractor.make_shareable({
33
- 'x' => [byte_table('0123456789ABCDEF'), 16, 'hexadecimal', 'A', /[0-9A-Fa-f_]/],
34
- 'o' => [byte_table('01234567'), 8, 'octal', 'An', /[0-7_]/],
35
- 'b' => [byte_table('01'), 2, 'binary', 'A', /[01_]/]
36
+ 0x78 => [byte_table('0123456789ABCDEF'), 16, 'hexadecimal', 'A', /[^0-9A-Fa-f_]/], # x
37
+ 0x6F => [byte_table('01234567'), 8, 'octal', 'An', /[^0-7_]/], # o
38
+ 0x62 => [byte_table('01'), 2, 'binary', 'A', /[^01_]/] # b
36
39
  })
37
40
 
38
41
  # The longest token the diagnostic regexes look at, so that a huge invalid token is described from its first part.
@@ -41,7 +44,7 @@ module SOML
41
44
  DURATION_UNIT_NAMES = DURATION_UNITS.keys.freeze
42
45
 
43
46
  INSTANT_PREFIX = /\A[0-9]{4}-[0-9]{2}-[0-9]{2}/
44
- INSTANT = /\A([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})(?:\.([0-9]+))?(Z|[+-]([0-9]{2}):([0-9]{2}))\z/
47
+ INSTANT = /\A[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?(?:Z|[+-][0-9]{2}:[0-9]{2})\z/
45
48
  LOCAL_DATE_TIME = /\A[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?\z/
46
49
  INSTANT_FORMAT = 'An instant is written as 2026-09-19T14:00:00Z, with an optional fraction of up to nine digits and an offset of Z or ±HH:MM'
47
50
  # Uppercase digits too, so that the value of such an escape is checked before its spelling.
@@ -49,7 +52,7 @@ module SOML
49
52
 
50
53
  # Every C0 control character except tab and line feed, and DEL. They are errors anywhere in a document. The set is for `String#count`, which looks up each byte in a table and is several times as fast as a regex search, so the regex only runs to find where an error is.
51
54
  CONTROL_CHARACTERS = "\x00-\x08\x0B-\x1F\x7F"
52
- CONTROL_CHARACTER = /[\x00-\x08\x0B-\x1F\x7F]/
55
+ CONTROL_CHARACTER = /[#{CONTROL_CHARACTERS}]/
53
56
 
54
57
  ESCAPED_STRING_SPECIAL = /[\n"\\]/
55
58
  KEY_QUOTING_HINT = '. A key that contains characters other than letters, digits, “_”, and “-” must be quoted'
@@ -151,7 +154,7 @@ module SOML
151
154
  # @param path [Array<String, Integer>]
152
155
  # @return [String]
153
156
  def self.describe_path(path)
154
- description = path.each_with_index.map do |segment, index|
157
+ description = path.map.with_index do |segment, index|
155
158
  if Integer === segment
156
159
  "[#{segment}]"
157
160
  else
@@ -159,29 +162,28 @@ module SOML
159
162
  end
160
163
  end.join
161
164
 
162
- abbreviate(description, 200).gsub(UNPRINTABLE) { |character| "\\u{#{character.ord.to_s(16)}}" }
165
+ abbreviate(description, 200).gsub(UNPRINTABLE) { "\\u{#{it.ord.to_s(16)}}" }
163
166
  end
164
167
 
165
168
  # Shorten text quoted in an error message, so a huge token does not make a huge message. The length is counted in UTF-16 code units, as the JS reference counts it, so that the two give the same message, and the cut never splits a character.
166
169
  def self.abbreviate(text, maximum_length = 40)
167
- # A character outside the BMP is two code units, and its UTF-8 lead byte is 0xF0 to 0xF4. `String#count` with that range of characters instead would build a table of a million entries.
168
- return text if text.length + text.b.count("\xF0-\xF4".b) <= maximum_length
170
+ return text if utf16_length(text) <= maximum_length
169
171
 
170
172
  length = 0
171
- kept = text.each_char.take_while { |character| (length += character.bytesize == 4 ? 2 : 1) <= maximum_length }
173
+ kept = text.each_char.take_while { (length += it.bytesize == 4 ? 2 : 1) <= maximum_length }
172
174
  "#{kept.join}…"
173
175
  end
174
176
 
177
+ # The length of `text` in UTF-16 code units, as JS counts it. A character outside the BMP is two code units, and its UTF-8 lead byte is 0xF0 to 0xF4. `String#count` with that range of characters instead would build a table of a million entries.
178
+ def self.utf16_length(text) = text.length + text.b.count("\xF0-\xF4".b)
179
+
175
180
  # The string as `JSON.stringify` writes it, as the JS reference shows a quoted key, so that a control character, such as a line feed, shows as its escape.
176
- def self.json_quote(text)
177
- escaped = text.gsub(/["\\\x00-\x1F]/) do |character|
178
- JSON_ESCAPES[character] || format('\\u%04x', character.ord)
179
- end
181
+ def self.json_quote(text) = "\"#{json_escape(text, /["\\\x00-\x1F]/)}\""
180
182
 
181
- "\"#{escaped}\""
182
- end
183
+ # `text` with the characters that `pattern` matches, which are control characters, `"`, or `\`, written as `JSON.stringify` writes them.
184
+ def self.json_escape(text, pattern) = text.gsub(pattern) { JSON_ESCAPES[it] || format('\\u%04x', it.ord) }
183
185
 
184
- # A value as Ruby names it in a conversion error, such as `nil` or `Integer` in "nil can't be coerced into Integer", in UTF-8. The class is read with `Kernel#class`, which a `BasicObject` does not have. {Duration} names a value the same way.
186
+ # A value as Ruby names it in a conversion error, such as `nil` or `Integer` in "nil can't be coerced into Integer", in UTF-8. The class is named with `Module#to_s`, as Ruby names it, because a class can define its own `to_s` that returns anything. {Duration} names a value the same way.
185
187
  #
186
188
  # @param value [Object]
187
189
  # @return [String]
@@ -190,10 +192,22 @@ module SOML
190
192
  when nil, true, false
191
193
  value.inspect
192
194
  else
193
- lossy_utf8(Kernel.instance_method(:class).bind_call(value).to_s)
195
+ Error.sanitize(lossy_utf8(Module.instance_method(:to_s).bind_call(class_of(value))))
194
196
  end
195
197
  end
196
198
 
199
+ # The class of a value, read with `Kernel#class`, which a `BasicObject` does not have.
200
+ #
201
+ # @param value [Object]
202
+ # @return [Class]
203
+ def self.class_of(value) = Kernel.instance_method(:class).bind_call(value)
204
+
205
+ # The name of a class, read with `Module#name`, because a class can define its own `name` that returns anything, or `nil` for an anonymous class.
206
+ #
207
+ # @param type [Module]
208
+ # @return [String, nil]
209
+ def self.module_name(type) = Module.instance_method(:name).bind_call(type)
210
+
197
211
  # Text in UTF-8, so that an error message can join it, such as a class name from a source file in Shift_JIS. Text that does not convert, such as a binary path from `ARGV` with `LANG=C`, or one in an encoding without a converter to UTF-8, has its bytes read as UTF-8, with what is not valid replaced.
198
212
  #
199
213
  # @param text [String]
@@ -214,7 +228,10 @@ module SOML
214
228
  # @return [String]
215
229
  def self.describe_argument(value)
216
230
  case value
217
- when String, Symbol, Integer, Float, Rational, nil, true, false
231
+ when String, Symbol
232
+ # `inspect` escapes a control character, but not a bidirectional control.
233
+ Error.sanitize(value.inspect)
234
+ when Integer, Float, Rational, nil, true, false
218
235
  value.inspect
219
236
  else
220
237
  conversion_name(value)
@@ -264,14 +281,8 @@ module SOML
264
281
  #
265
282
  # @return [SOML::Duration]
266
283
  def parse_duration_literal
267
- finish = 0
268
-
269
- while (byte = @source.getbyte(finish)) && NUMBER_BYTE[byte]
270
- finish += 1
271
- end
272
-
273
284
  # An empty source and an instant are not duration-like either.
274
- if finish < @length || !duration_like?(@source)
285
+ if number_end(0) < @length || !duration_like?(@source)
275
286
  fail_at("Expected a duration, such as 1h30m, but found #{@source.empty? ? 'nothing' : quote_found(abbreviate(@source))}", 0)
276
287
  end
277
288
 
@@ -322,10 +333,10 @@ module SOML
322
333
 
323
334
  # A sequence that the end of the input cuts short is reported as what it is.
324
335
  if truncated_utf8?(@source.byteslice(offset, @length - offset))
325
- raise ParseError.new('Incomplete UTF-8 sequence at the end of the input', valid, valid.bytesize, @path)
336
+ raise ParseError.new('Incomplete UTF-8 sequence at the end of the input', valid, offset, @path)
326
337
  end
327
338
 
328
- raise ParseError.new(format('Invalid UTF-8 byte 0x%02X', @source.getbyte(offset)), valid, valid.bytesize, @path)
339
+ raise ParseError.new(format('Invalid UTF-8 byte 0x%02X', @source.getbyte(offset)), valid, offset, @path)
329
340
  end
330
341
 
331
342
  # Whether some bytes could follow `rest` to make it one valid character.
@@ -348,13 +359,7 @@ module SOML
348
359
  end
349
360
  end
350
361
 
351
- def fail_at(reason, offset = @index)
352
- raise ParseError.new(reason, @source, offset, @path)
353
- end
354
-
355
- def fail_too_deep(offset = @index)
356
- fail_at("The document is nested more than #{MAX_DEPTH} levels deep", offset)
357
- end
362
+ def fail_at(reason, offset = @index) = raise(ParseError.new(reason, @source, offset, @path))
358
363
 
359
364
  # `bare-object = entry ( entry-sep entry )*`, where a separator is one or more line breaks.
360
365
  def parse_bare_object
@@ -512,7 +517,11 @@ module SOML
512
517
  # An entry whose value was left out, as in `a:` followed by `'b': 1` on the next line, reads the next key as the value. That failure is reported as what it is.
513
518
  def diagnose_missing_value(start, parent, key_start, comment_hint)
514
519
  # A YAML block sequence.
515
- fail_at('Expected a value, but found a “-” list. An array is written in brackets, as in [80, 443]', start) if @source.getbyte(start) == 0x2D && space?(@source.getbyte(start + 1))
520
+ first = @source.getbyte(start)
521
+ fail_at('Expected a value, but found a “-” list. An array is written in brackets, as in [80, 443]', start) if first == 0x2D && space?(@source.getbyte(start + 1))
522
+
523
+ # Only a key can be followed by its `:`. Reading anything else as a key would make an error, which counts the lines of the whole source, at every level of a nested value.
524
+ return unless first == 0x27 || first == 0x22 || (first && BARE_KEY_BYTE[first])
516
525
 
517
526
  @index = start
518
527
 
@@ -527,7 +536,6 @@ module SOML
527
536
 
528
537
  # Whitespace or the end follows the `:` of a bare key. A digit follows the `:` in an instant such as `2026-09-19T25:00:00Z`, whose own error is more precise. A quoted key cannot be part of a value, so anything may follow its `:`.
529
538
  following = @source.getbyte(@index + 1)
530
- first = @source.getbyte(start)
531
539
 
532
540
  return unless first == 0x27 || first == 0x22 || space_or_line_end?(following)
533
541
 
@@ -542,18 +550,12 @@ module SOML
542
550
  return '' if key_indentation.nil? || indentation.nil? || indentation <= key_indentation
543
551
 
544
552
  # The keys are written as in a document, so that the suggestion is valid, rather than as JSON strings like the rest of the message, whose escapes, such as `\b`, are not all valid.
545
- ". Indentation does not nest objects, so write #{abbreviate(format_key(parent), 200)}: {#{abbreviate(format_key(key), 200)}: …}"
546
- end
547
-
548
- # A key as a document writes it, bare when it can be, as {Generator} writes it.
549
- def format_key(key)
550
- SOML.bare_key?(key) ? key : Generator.new.generate_value(key, 0)
553
+ ". Indentation does not nest objects, so write #{abbreviate(Generator.format_key(parent), 200)}: {#{abbreviate(Generator.format_key(key), 200)}: …}"
551
554
  end
552
555
 
553
556
  def fail_missing_colon(key_start)
554
557
  source = @source
555
- following = @index
556
- following += 1 while space?(source.getbyte(following))
558
+ following = skip_spaces(@index)
557
559
  following_byte = source.getbyte(following)
558
560
 
559
561
  fail_at('Whitespace is not allowed between a key and its “:”') if following_byte == 0x3A && following > @index
@@ -562,11 +564,7 @@ module SOML
562
564
  if following_byte == 0x2F && source.getbyte(following + 1) == 0x2A
563
565
  comment_end = source.byteindex('*/', following + 2)
564
566
 
565
- if comment_end
566
- after_comment = comment_end + 2
567
- after_comment += 1 while space?(source.getbyte(after_comment))
568
- fail_at('A comment is not allowed between a key and its “:”', following) if source.getbyte(after_comment) == 0x3A
569
- end
567
+ fail_at('A comment is not allowed between a key and its “:”', following) if comment_end && source.getbyte(skip_spaces(comment_end + 2)) == 0x3A
570
568
  end
571
569
 
572
570
  colon = source.byteindex(':', following)
@@ -623,14 +621,12 @@ module SOML
623
621
  colon = @source.byteindex(':', @index)
624
622
  key = @source.byteslice(@index, colon - @index) if colon && colon < line_end(@index)
625
623
  # A longer key gets the hint without the example, as in JS, where the key and its `:` must fit in 1000 UTF-16 code units.
626
- key = nil if key && (key.length + key.b.count("\xF0-\xF4".b) >= MAX_DIAGNOSED_LENGTH || key.match?(NOT_QUOTABLE_KEY_CHARACTER))
624
+ key = nil if key && (Parser.utf16_length(key) >= MAX_DIAGNOSED_LENGTH || key.match?(NOT_QUOTABLE_KEY_CHARACTER))
627
625
  key.nil? ? KEY_QUOTING_HINT : "#{KEY_QUOTING_HINT}, as in '#{abbreviate(key)}'"
628
626
  end
629
627
 
630
628
  # `offset` is before the end of the source.
631
- def quotable_key_character?(offset)
632
- character_at(offset).match?(QUOTABLE_KEY_CHARACTER)
633
- end
629
+ def quotable_key_character?(offset) = character_at(offset).match?(QUOTABLE_KEY_CHARACTER)
634
630
 
635
631
  # A `.` after a key, as in `example.com: 1`. A key is never a path, so the `.` is meant either as part of the key or as nesting.
636
632
  def fail_dot_in_key(key_start)
@@ -652,7 +648,7 @@ module SOML
652
648
 
653
649
  nested = "#{words.last}: …"
654
650
 
655
- words[0...-1].reverse_each do |word|
651
+ words[...-1].reverse_each do |word|
656
652
  nested = "#{word}: {#{nested}}"
657
653
  end
658
654
 
@@ -660,9 +656,7 @@ module SOML
660
656
  end
661
657
 
662
658
  # Found text between curly quotes, as prose quotes it, with its control characters, such as a line feed, written as their JSON escapes, so that they show rather than become U+FFFD, and a backslash escaped, so that it does not read as one of them.
663
- def quote_found(text)
664
- "“#{text.gsub(/[\\\x00-\x1F]/) { JSON_ESCAPES[it] || format('\\u%04x', it.ord) }}”"
665
- end
659
+ def quote_found(text) = "“#{Parser.json_escape(text, /[\\\x00-\x1F]/)}”"
666
660
 
667
661
  def parse_object(depth)
668
662
  object = {}
@@ -678,7 +672,7 @@ module SOML
678
672
 
679
673
  # `{` or `[` at depth `depth`, then items, which the block reads, separated by a comma, a line break, or both, with an optional trailing comma, then the `closing` bracket. A comma must be on the line of the item before it.
680
674
  def parse_items(depth, closing, kind, item)
681
- fail_too_deep if depth > MAX_DEPTH
675
+ fail_at("The document is nested more than #{MAX_DEPTH} levels deep") if depth > MAX_DEPTH
682
676
 
683
677
  source = @source
684
678
  start = @index
@@ -774,10 +768,12 @@ module SOML
774
768
  unit_end += 1
775
769
  end
776
770
 
771
+ return if unit_end == unit_start
772
+
777
773
  unit = source.byteslice(unit_start, unit_end - unit_start)
778
774
 
779
775
  # A keyword after a number is a missing comma.
780
- return if unit.empty? || !value_end?(skip_spaces(unit_end)) || %w[true false null infinity].include?(unit)
776
+ return if !value_end?(skip_spaces(unit_end)) || %w[true false null infinity].include?(unit)
781
777
 
782
778
  number = source.byteslice(start, @index - start)
783
779
  # The duration is only suggested when it is valid, which a number with an exponent or a radix prefix, a fraction of a nanosecond, a negative zero, or a value outside the 64-bit range is not. The string is always offered, because a unit such as `m` may mean meters rather than minutes.
@@ -830,23 +826,8 @@ module SOML
830
826
 
831
827
  # The text from `start` that was most likely meant as one unquoted string, such as `John Smith`: up to the end of the line, a comma, a closing bracket, or a comment after a space or a tab, without the spaces and tabs at its end. It is read up to 1000 UTF-16 code units, as in JS, and never ends inside a character.
832
828
  def unquoted_text(start = @index)
833
- source = @source
834
- finish = start
835
- length = 0
836
-
837
- while (byte = source.getbyte(finish))
838
- # A continuation byte is part of the character before it.
839
- unless byte.between?(0x80, 0xBF)
840
- break if length >= MAX_DIAGNOSED_LENGTH || unquoted_text_end?(finish)
841
-
842
- # A character outside the BMP is two UTF-16 code units, and its UTF-8 lead byte is 0xF0 to 0xF4.
843
- length += byte >= 0xF0 ? 2 : 1
844
- end
845
-
846
- finish += 1
847
- end
848
-
849
- source.byteslice(start, skip_spaces_back(finish) - start)
829
+ finish = code_unit_end(start) { unquoted_text_end?(it) }
830
+ @source.byteslice(start, skip_spaces_back(finish) - start)
850
831
  end
851
832
 
852
833
  def unquoted_text_end?(offset)
@@ -877,14 +858,10 @@ module SOML
877
858
  end
878
859
 
879
860
  # The hint for a `//` comment, as in JavaScript.
880
- def slash_comment_hint
881
- @source.getbyte(@index) == 0x2F && @source.getbyte(@index + 1) == 0x2F ? '. A comment starts with “#”' : ''
882
- end
861
+ def slash_comment_hint = @source.getbyte(@index) == 0x2F && @source.getbyte(@index + 1) == 0x2F ? '. A comment starts with “#”' : ''
883
862
 
884
863
  # The example in a message that a string must be quoted, which quotes `text` as a `'...'` string. That cannot hold a `'`, so then there is no simple example.
885
- def quoting_example(text)
886
- text.include?("'") ? '' : ", as in '#{abbreviate(text)}'"
887
- end
864
+ def quoting_example(text) = text.include?("'") ? '' : ", as in '#{abbreviate(text)}'"
888
865
 
889
866
  def describe_here
890
867
  return 'the end of the document' if @index >= @length
@@ -893,9 +870,7 @@ module SOML
893
870
  character == "\n" ? 'a line break' : describe_character(character)
894
871
  end
895
872
 
896
- def character_at(offset)
897
- @source.byteslice(offset, 4)[0]
898
- end
873
+ def character_at(offset) = @source.byteslice(offset, 4)[0]
899
874
 
900
875
  def parse_string
901
876
  source = @source
@@ -952,17 +927,19 @@ module SOML
952
927
  return value
953
928
  end
954
929
 
955
- text, chunk_start = parse_escape(special)
956
- value << text
930
+ chunk_start = parse_escape(special, value)
957
931
  end
958
932
  end
959
933
 
960
- # Decodes the escape at `offset`, which is a backslash, and returns its text and the offset after it.
961
- def parse_escape(offset)
934
+ # Decodes the escape at `offset`, which is a backslash, onto `value`, and returns the offset after it.
935
+ def parse_escape(offset, value)
962
936
  source = @source
963
937
  byte = source.getbyte(offset + 1)
964
938
  simple = SIMPLE_ESCAPES[byte]
965
- return [simple, offset + 2] if simple
939
+ if simple
940
+ value << simple
941
+ return offset + 2
942
+ end
966
943
 
967
944
  if byte == 0x75 # u
968
945
  # The slice may cut a character in two, which a regex refuses to read.
@@ -979,7 +956,8 @@ module SOML
979
956
  fail_at('A Unicode escape uses lowercase hexadecimal digits', offset) if hex.match?(/[A-F]/)
980
957
  fail_at("A Unicode escape may not have leading zeros; write \\u{#{hex.sub(/\A0+(?=.)/, '')}}", offset) if hex.length > 1 && hex.start_with?('0')
981
958
 
982
- return [code_point.chr(Encoding::UTF_8), offset + 1 + match[0].bytesize]
959
+ value << code_point
960
+ return offset + 1 + match[0].bytesize
983
961
  end
984
962
 
985
963
  # A backslash at the end of the document is the same mistake as one at the end of a line.
@@ -1073,8 +1051,7 @@ module SOML
1073
1051
  line_start = offset
1074
1052
 
1075
1053
  while line_start < @length
1076
- delimiter_start = line_start
1077
- delimiter_start += 1 while space?(source.getbyte(delimiter_start))
1054
+ delimiter_start = skip_spaces(line_start)
1078
1055
  run_length = 0
1079
1056
  run_length += 1 while source.getbyte(delimiter_start + run_length) == quote
1080
1057
  return [line_start, delimiter_start] if run_length == length
@@ -1094,9 +1071,7 @@ module SOML
1094
1071
 
1095
1072
  while (index = text.byteindex('\\', chunk_start))
1096
1073
  value << text.byteslice(chunk_start, index - chunk_start)
1097
- escaped, finish = parse_escape(offset + index)
1098
- value << escaped
1099
- chunk_start = finish - offset
1074
+ chunk_start = parse_escape(offset + index, value) - offset
1100
1075
  end
1101
1076
 
1102
1077
  value << text.byteslice(chunk_start, text.bytesize - chunk_start)
@@ -1106,15 +1081,9 @@ module SOML
1106
1081
  plain = parse_plain_number
1107
1082
  return plain unless plain.nil?
1108
1083
 
1109
- source = @source
1110
1084
  start = @index
1111
- finish = start + 1
1112
-
1113
- while (byte = source.getbyte(finish)) && NUMBER_BYTE[byte]
1114
- finish += 1
1115
- end
1116
-
1117
- text = source.byteslice(start, finish - start)
1085
+ finish = number_end(start)
1086
+ text = @source.byteslice(start, finish - start)
1118
1087
  @index = finish
1119
1088
 
1120
1089
  return parse_instant(text, start) if text.match?(INSTANT_PREFIX)
@@ -1228,7 +1197,7 @@ module SOML
1228
1197
 
1229
1198
  # Whether `text` is a decimal int, a radix int, or a float, following the grammar. A run of digits may have single underscores between digits.
1230
1199
  def classify_number(text)
1231
- radix = text.getbyte(0) == 0x30 && RADIXES[text[1]]
1200
+ radix = text.getbyte(0) == 0x30 && RADIXES[text.getbyte(1)]
1232
1201
 
1233
1202
  if radix
1234
1203
  finish = skip_digits(text, 2, radix[0])
@@ -1288,14 +1257,12 @@ module SOML
1288
1257
  end
1289
1258
 
1290
1259
  # Returns the index after an integer part, which is `0` or digits without a leading zero, or `index` when there is none. Anything after a leading `0`, such as the `5` in `05`, is left for the caller to reject.
1291
- def skip_integer_part(text, index)
1292
- text.getbyte(index) == 0x30 ? index + 1 : skip_digits(text, index, DECIMAL_DIGIT)
1293
- end
1260
+ def skip_integer_part(text, index) = text.getbyte(index) == 0x30 ? index + 1 : skip_digits(text, index, DECIMAL_DIGIT)
1294
1261
 
1295
1262
  # More than 64 significant digits is out of range in any radix, which is decided before a huge digit string is converted.
1296
1263
  def parse_integer(text, start)
1297
1264
  digits = text.delete('_')
1298
- radix = RADIXES[digits[1]] if digits.bytesize > 1
1265
+ radix = RADIXES[digits.getbyte(1)]
1299
1266
  significant = digits.getbyte(0) == 0x2D ? 1 : 0
1300
1267
 
1301
1268
  if radix
@@ -1320,7 +1287,7 @@ module SOML
1320
1287
  # Negative zero is the same value as zero.
1321
1288
  return 0.0 if first.nil?
1322
1289
 
1323
- significant = trim_trailing_zeros(digits.byteslice(first, digits.bytesize - first))
1290
+ significant = digits.byteslice(first, digits.bytesize - first).rstrip('0')
1324
1291
  # An exponent with more than 18 digits is far outside the float range either way, which is decided before converting a huge digit string, which takes quadratic time.
1325
1292
  exponent_text = exponent_text.start_with?('-') ? '-1000000000000000000' : '1000000000000000000' if exponent_text && exponent_text.delete_prefix('-').bytesize > 18
1326
1293
  exponent = exponent_text.to_i - (fraction&.bytesize || 0) + (digits.bytesize - first - significant.bytesize)
@@ -1372,9 +1339,7 @@ module SOML
1372
1339
  end
1373
1340
 
1374
1341
  def parse_instant(text, start)
1375
- match = text.bytesize > MAX_DIAGNOSED_LENGTH ? nil : INSTANT.match(text)
1376
-
1377
- if match.nil?
1342
+ unless text.bytesize <= MAX_DIAGNOSED_LENGTH && text.match?(INSTANT)
1378
1343
  # A date, a space, and a time, as TOML and Python write a date and time.
1379
1344
  time_start = @index + 1
1380
1345
  finish = @source.getbyte(@index) == 0x20 && digit?(@source.getbyte(time_start)) ? number_end(time_start) : @index
@@ -1384,28 +1349,57 @@ module SOML
1384
1349
  fail_at(describe_bad_instant(text, time, is_whole_value, offset_after_space), start)
1385
1350
  end
1386
1351
 
1387
- year, month, day, hour, minute, second, fraction, offset, offset_hour, offset_minute = match.captures
1388
- check_instant(text, start, fraction.nil? || fraction.length <= 9, 'a fractional second has at most nine digits')
1389
- check_instant(text, start, year != '0000', 'the year must be 0001 to 9999')
1390
- check_instant(text, start, month.between?('01', '12'), 'the month must be 01 to 12')
1391
- last_day = days_in_month(year.to_i, month.to_i)
1392
- check_instant(text, start, day >= '01' && day.to_i <= last_day, "the day must be 01 to #{last_day} in that month")
1393
- check_instant(text, start, hour <= '23', 'the hour must be 00 to 23')
1394
- check_instant(text, start, minute <= '59', 'the minute must be 00 to 59')
1395
- check_instant(text, start, second <= '59', 'the second must be 00 to 59, and a leap second is not representable')
1396
-
1397
- seconds = (days_from_civil(year.to_i, month.to_i, day.to_i) * 86_400) + (hour.to_i * 3600) + (minute.to_i * 60) + second.to_i
1352
+ # The regex checked the layout, so each part is at a fixed offset, except the offset, which follows the optional fraction. Reading the digits as bytes makes no string for each part.
1353
+ year = digits_value(text, 0, 4)
1354
+ month = digits_value(text, 5, 2)
1355
+ day = digits_value(text, 8, 2)
1356
+ hour = digits_value(text, 11, 2)
1357
+ minute = digits_value(text, 14, 2)
1358
+ second = digits_value(text, 17, 2)
1359
+ # The offset, `Z` or `±HH:MM`, ends the instant, and a fraction is between the seconds and it.
1360
+ # `getbyte(-1)` made parsing a document of many instants a third slower, measured on Ruby 4.0 with YJIT.
1361
+ length = text.bytesize
1362
+ offset_start = text.getbyte(length - 1) == 0x5A ? length - 1 : length - 6
1363
+ fraction_length = offset_start == 19 ? 0 : offset_start - 20
1364
+
1365
+ fail_instant(text, start, 'a fractional second has at most nine digits') if fraction_length > 9
1366
+ fail_instant(text, start, 'the year must be 0001 to 9999') if year.zero?
1367
+ fail_instant(text, start, 'the month must be 01 to 12') unless month.between?(1, 12)
1368
+ last_day = days_in_month(year, month)
1369
+ fail_instant(text, start, "the day must be 01 to #{last_day} in that month") unless day.between?(1, last_day)
1370
+ fail_instant(text, start, 'the hour must be 00 to 23') if hour > 23
1371
+ fail_instant(text, start, 'the minute must be 00 to 59') if minute > 59
1372
+ fail_instant(text, start, 'the second must be 00 to 59, and a leap second is not representable') if second > 59
1373
+
1374
+ seconds = (days_from_civil(year, month, day) * 86_400) + (hour * 3600) + (minute * 60) + second
1375
+ sign = text.getbyte(offset_start)
1376
+
1377
+ # Not `Z`, so `±HH:MM`.
1378
+ if sign != 0x5A
1379
+ offset_hour = digits_value(text, offset_start + 1, 2)
1380
+ offset_minute = digits_value(text, offset_start + 4, 2)
1381
+ fail_instant(text, start, 'the offset hour must be 00 to 23') if offset_hour > 23
1382
+ fail_instant(text, start, 'the offset minute must be 00 to 59') if offset_minute > 59
1383
+ fail_instant(text, start, '-00:00 means “offset unknown” in RFC 3339, which is not representable; use Z or +00:00') if sign == 0x2D && offset_hour.zero? && offset_minute.zero?
1384
+ offset_seconds = (offset_hour * 3600) + (offset_minute * 60)
1385
+ seconds += sign == 0x2B ? -offset_seconds : offset_seconds
1386
+ end
1387
+
1388
+ fail_instant(text, start, 'in UTC it falls outside the years 0001 to 9999') unless seconds.between?(MIN_INSTANT_SECONDS, MAX_INSTANT_SECONDS)
1389
+ Time.at(seconds, fraction_length.zero? ? 0 : digits_value(text, 20, fraction_length) * (10**(9 - fraction_length)), :nsec).utc
1390
+ end
1391
+
1392
+ # The value of the `count` decimal digits at `offset` in `text`, which the caller checked are digits.
1393
+ def digits_value(text, offset, count)
1394
+ value = 0
1395
+ finish = offset + count
1398
1396
 
1399
- if offset != 'Z'
1400
- check_instant(text, start, offset_hour <= '23', 'the offset hour must be 00 to 23')
1401
- check_instant(text, start, offset_minute <= '59', 'the offset minute must be 00 to 59')
1402
- check_instant(text, start, offset != '-00:00', '-00:00 means “offset unknown” in RFC 3339, which is not representable; use Z or +00:00')
1403
- offset_seconds = (offset_hour.to_i * 3600) + (offset_minute.to_i * 60)
1404
- seconds += offset.start_with?('+') ? -offset_seconds : offset_seconds
1397
+ while offset < finish
1398
+ value = (value * 10) + text.getbyte(offset) - 0x30
1399
+ offset += 1
1405
1400
  end
1406
1401
 
1407
- check_instant(text, start, seconds.between?(MIN_INSTANT_SECONDS, MAX_INSTANT_SECONDS), 'in UTC it falls outside the years 0001 to 9999')
1408
- Time.at(seconds, fraction.nil? ? 0 : fraction.ljust(9, '0').to_i, :nsec).utc
1402
+ value
1409
1403
  end
1410
1404
 
1411
1405
  # A scan of the parts in order, so that each error names the part it is about.
@@ -1441,7 +1435,7 @@ module SOML
1441
1435
  fail_duration(text, start, 'an underscore must be between two digits') if text.getbyte(unit_start) == 0x5F
1442
1436
 
1443
1437
  # Trailing zeros change nothing, and leaving them out keeps the arithmetic small however many there are.
1444
- fraction = trim_trailing_zeros(text.byteslice(digits_end + 1, unit_start - digits_end - 1).delete('_'))
1438
+ fraction = text.byteslice(digits_end + 1, unit_start - digits_end - 1).delete('_').rstrip('0')
1445
1439
  end
1446
1440
 
1447
1441
  unit_end = unit_start
@@ -1457,10 +1451,9 @@ module SOML
1457
1451
  fail_duration(text, start, 'only the last part may have a fraction') if following == 0x2E && digit?(text.getbyte(unit_end))
1458
1452
 
1459
1453
  size = DURATION_UNITS[unit]
1460
- digits = text.byteslice(index, digits_end - index).delete('_')
1461
1454
 
1462
- # More than 19 digits is out of range in any unit, which is decided before a huge digit string is converted.
1463
- total += digits.bytesize > 19 ? INT64_MAX * 2 : digits.to_i * size
1455
+ # More than 19 digits, which with the single underscores that may be between them is more than 37 bytes, is out of range in any unit, which is decided before a huge digit string is converted. `to_i` reads the underscores.
1456
+ total += digits_end - index > 37 ? INT64_MAX * 2 : text.byteslice(index, digits_end - index).to_i * size
1464
1457
  index = unit_end
1465
1458
  previous_rank = rank
1466
1459
  end
@@ -1481,13 +1474,9 @@ module SOML
1481
1474
  Duration.new(nanoseconds: is_negative ? -total : total)
1482
1475
  end
1483
1476
 
1484
- def check_instant(text, start, is_valid, reason)
1485
- fail_at("Invalid instant #{abbreviate(text)}: #{reason}", start) unless is_valid
1486
- end
1477
+ def fail_instant(text, start, reason) = fail_at("Invalid instant #{abbreviate(text)}: #{reason}", start)
1487
1478
 
1488
- def fail_duration(text, start, reason)
1489
- fail_at("Invalid duration #{abbreviate(text)}: #{reason}", start)
1490
- end
1479
+ def fail_duration(text, start, reason) = fail_at("Invalid duration #{abbreviate(text)}: #{reason}", start)
1491
1480
 
1492
1481
  # Whether a token that is not a number or an instant was meant as a duration: its first letter could begin a unit, or a day or a week. A radix prefix, an exponent, and other letters, such as the `T` in `20260919T140000Z` or the `x` in `1.5x`, are left to the number errors.
1493
1482
  def duration_like?(text)
@@ -1505,22 +1494,12 @@ module SOML
1505
1494
  return false if byte.nil?
1506
1495
 
1507
1496
  # A year unit, as in `1y` or `2years`, is only one when the token or another part follows it, so that a word such as `100yen` stays a string to quote. The longest unit and the byte after it fit in 6 bytes.
1508
- 'dhmnsuwDHMNSUW'.include?(byte.chr) || text.byteslice(index, 6).match?(/\A(?:y|yrs?|years?)(?:\z|[0-9])/i)
1497
+ DURATION_UNIT_START[byte] || text.byteslice(index, 6).match?(/\A(?:y|yrs?|years?)(?:\z|[0-9])/i)
1509
1498
  end
1510
1499
 
1511
- def trim_trailing_zeros(digits)
1512
- finish = digits.bytesize
1513
- finish -= 1 while finish.positive? && digits.getbyte(finish - 1) == 0x30
1514
- digits.byteslice(0, finish)
1515
- end
1500
+ def line_end(offset) = @source.byteindex("\n", offset) || @length
1516
1501
 
1517
- def line_end(offset)
1518
- @source.byteindex("\n", offset) || @length
1519
- end
1520
-
1521
- def line_start(offset)
1522
- offset.zero? ? 0 : (@source.byterindex("\n", offset - 1) || -1) + 1
1523
- end
1502
+ def line_start(offset) = offset.zero? ? 0 : (@source.byterindex("\n", offset - 1) || -1) + 1
1524
1503
 
1525
1504
  # The number of spaces and tabs before `offset` on its line, or `nil` when something else comes before it.
1526
1505
  def indentation_at(offset)
@@ -1528,9 +1507,9 @@ module SOML
1528
1507
  skip_spaces(start) == offset ? offset - start : nil
1529
1508
  end
1530
1509
 
1531
- # The end of the number, instant, or duration that starts at `start`.
1510
+ # The end of the number, instant, or duration that starts at `start`: the end of the run of the bytes that one can hold.
1532
1511
  def number_end(start)
1533
- finish = start + 1
1512
+ finish = start
1534
1513
 
1535
1514
  while (byte = @source.getbyte(finish)) && NUMBER_BYTE[byte]
1536
1515
  finish += 1
@@ -1539,15 +1518,15 @@ module SOML
1539
1518
  finish
1540
1519
  end
1541
1520
 
1542
- # The offset after `maximum` UTF-16 code units from `start`, as JS slices the source. A character outside the BMP that JS cuts in two at the limit is kept whole, so that, like the lone surrogate JS keeps, it is still there to stop a match at the end.
1543
- def code_unit_end(start, maximum = MAX_DIAGNOSED_LENGTH)
1521
+ # The offset after `maximum` UTF-16 code units from `start`, as JS slices the source, or before the first character for which the block returns true. A character outside the BMP that JS cuts in two at the limit is kept whole, so that, like the lone surrogate JS keeps, it is still there to stop a match at the end.
1522
+ def code_unit_end(start)
1544
1523
  finish = start
1545
1524
  length = 0
1546
1525
 
1547
1526
  while (byte = @source.getbyte(finish))
1548
1527
  # A continuation byte is part of the character before it.
1549
1528
  unless byte.between?(0x80, 0xBF)
1550
- break if length >= maximum
1529
+ break if length >= MAX_DIAGNOSED_LENGTH || (block_given? && yield(finish))
1551
1530
 
1552
1531
  # A character outside the BMP is two UTF-16 code units, and its UTF-8 lead byte is 0xF0 to 0xF4.
1553
1532
  length += byte >= 0xF0 ? 2 : 1
@@ -1565,40 +1544,28 @@ module SOML
1565
1544
  byte.nil? || (VALUE_END_BYTE[byte] && (byte != 0x2F || (following = @source.getbyte(offset + 1)) == 0x2A || following == 0x2F))
1566
1545
  end
1567
1546
 
1568
- def byte_before(offset)
1569
- offset.positive? ? @source.getbyte(offset - 1) : nil
1570
- end
1547
+ def byte_before(offset) = offset.positive? ? @source.getbyte(offset - 1) : nil
1571
1548
 
1572
1549
  def skip_spaces(offset)
1573
1550
  offset += 1 while space?(@source.getbyte(offset))
1574
1551
  offset
1575
1552
  end
1576
1553
 
1577
- def space?(byte)
1578
- byte == 0x20 || byte == 0x09
1579
- end
1554
+ def space?(byte) = byte == 0x20 || byte == 0x09
1580
1555
 
1581
1556
  # Whether a byte is a space, a tab, a line feed, or the end.
1582
- def space_or_line_end?(byte)
1583
- byte.nil? || byte == 0x0A || space?(byte)
1584
- end
1557
+ def space_or_line_end?(byte) = byte.nil? || byte == 0x0A || space?(byte)
1585
1558
 
1586
1559
  def skip_spaces_back(offset)
1587
1560
  offset -= 1 while offset.positive? && space?(@source.getbyte(offset - 1))
1588
1561
  offset
1589
1562
  end
1590
1563
 
1591
- def digit?(byte)
1592
- !byte.nil? && byte >= 0x30 && byte <= 0x39
1593
- end
1564
+ def digit?(byte) = !byte.nil? && byte >= 0x30 && byte <= 0x39
1594
1565
 
1595
- def digit_or_underscore?(byte)
1596
- byte == 0x5F || digit?(byte)
1597
- end
1566
+ def digit_or_underscore?(byte) = byte == 0x5F || digit?(byte)
1598
1567
 
1599
- def words_with_spaces?(text)
1600
- text.each_byte.all? { |byte| space?(byte) || BARE_KEY_BYTE[byte] }
1601
- end
1568
+ def words_with_spaces?(text) = text.each_byte.all? { space?(it) || BARE_KEY_BYTE[it] }
1602
1569
 
1603
1570
  def days_in_month(year, month)
1604
1571
  if month == 2
@@ -1621,9 +1588,7 @@ module SOML
1621
1588
 
1622
1589
  def abbreviate(...) = Parser.abbreviate(...)
1623
1590
 
1624
- def format_code_point(code_point)
1625
- format('U+%04X', code_point)
1626
- end
1591
+ def format_code_point(code_point) = format('U+%04X', code_point)
1627
1592
 
1628
1593
  # Describe a character for an error message, such as `“$”` or `U+00A0 (NO-BREAK SPACE)`.
1629
1594
  def describe_character(character)
@@ -1644,13 +1609,14 @@ module SOML
1644
1609
  # A keyword hint is only for the word on its own. In `Yes please` or `Nan Goldin`, following it would leave text behind or change the value.
1645
1610
  lowercase = text == word ? word.downcase : ''
1646
1611
 
1647
- if %w[true false yes no on off].include?(lowercase)
1612
+ case lowercase
1613
+ when 'true', 'false', 'yes', 'no', 'on', 'off'
1648
1614
  "“#{word}” is not a value. Booleans are written true and false, in lowercase"
1649
- elsif %w[null nil none undefined].include?(lowercase)
1615
+ when 'null', 'nil', 'none', 'undefined'
1650
1616
  "“#{word}” is not a value. Null is written null, in lowercase"
1651
- elsif lowercase == 'nan'
1617
+ when 'nan'
1652
1618
  'NaN is not representable. Use null for a missing value'
1653
- elsif %w[inf infinity].include?(lowercase)
1619
+ when 'inf', 'infinity'
1654
1620
  "“#{word}” is not a value. Infinity is written infinity, in lowercase"
1655
1621
  else
1656
1622
  "Unexpected “#{abbreviate(word)}”. A string value must be quoted#{quoting_example(text)}"
@@ -1723,22 +1689,28 @@ module SOML
1723
1689
  return 'A number prefix is lowercase: 0x, 0o, or 0b' if text.match?(/\A-?0[BOX]/)
1724
1690
 
1725
1691
  # The whole number, not the part cut at the diagnosed length, because a cut can end before the digit or the underscore that the message is about. Each check below takes linear time.
1726
- if (match = /\A(-?)0([box])(.*)\z/.match(full_text))
1727
- sign, radix, digits = match.captures
1728
- _, _, name, article, valid_character = RADIXES[radix]
1692
+ sign_length = full_text.getbyte(0) == 0x2D ? 1 : 0
1693
+ radix = full_text.getbyte(sign_length) == 0x30 && RADIXES[full_text.getbyte(sign_length + 1)]
1729
1694
 
1730
- return "#{article} #{name} integer cannot have a sign, because it states a bit pattern rather than a quantity" if sign == '-'
1731
- return "Expected #{name} digits after “0#{radix}”" if digits.empty?
1695
+ if radix
1696
+ _, base, name, article, invalid_character = radix
1697
+ prefix_end = sign_length + 2
1698
+ digits = full_text.byteslice(prefix_end, full_text.bytesize - prefix_end)
1732
1699
 
1733
- if radix == 'x' && digits.match?(/[a-f]/) && !digits.match?(/[^0-9a-fA-F_]/)
1700
+ return "#{article} #{name} integer cannot have a sign, because it states a bit pattern rather than a quantity" if sign_length == 1
1701
+ return "Expected #{name} digits after “#{full_text.byteslice(0, 2)}”" if digits.empty?
1702
+
1703
+ # A search for one character, because a repeat over a long number needs memory for every character.
1704
+ character = digits[invalid_character]
1705
+
1706
+ if base == 16 && character.nil? && digits.match?(/[a-f]/)
1734
1707
  # The uppercase spelling is only suggested when it is valid, so a misplaced underscore is reported first, and a value outside the 64-bit range gets no example.
1735
- return 'An underscore in a number must be between two digits' unless digits.match?(/\A[0-9a-fA-F]+(?:_[0-9a-fA-F]+)*\z/)
1708
+ return 'An underscore in a number must be between two digits' if digits.start_with?('_') || digits.end_with?('_') || digits.include?('__')
1736
1709
 
1737
1710
  uppercase = "0x#{digits.upcase}"
1738
1711
  return valid_value?(uppercase) ? "Hexadecimal digits are uppercase: #{abbreviate(uppercase)}" : 'Hexadecimal digits are uppercase'
1739
1712
  end
1740
1713
 
1741
- character = digits.each_char.find { |each_character| !each_character.match?(valid_character) }
1742
1714
  return character.nil? ? 'An underscore in a number must be between two digits' : "Invalid #{name} digit “#{character}”"
1743
1715
  end
1744
1716
 
@@ -1835,11 +1807,11 @@ module SOML
1835
1807
  separator = 'The date and time separator in an instant is an uppercase “T”, not a space'
1836
1808
  instant = "#{date}T#{time}"
1837
1809
 
1838
- if instant.bytesize > MAX_DIAGNOSED_LENGTH
1839
- "#{separator}. #{INSTANT_FORMAT}"
1840
- elsif valid_value?(instant)
1810
+ is_short = instant.bytesize <= MAX_DIAGNOSED_LENGTH
1811
+
1812
+ if is_short && valid_value?(instant)
1841
1813
  "#{separator}, as in #{abbreviate(instant)}"
1842
- elsif is_whole_value && instant.match?(LOCAL_DATE_TIME) && valid_value?("#{instant}Z")
1814
+ elsif is_short && is_whole_value && instant.match?(LOCAL_DATE_TIME) && valid_value?("#{instant}Z")
1843
1815
  "#{separator}, and an instant needs the offset it was meant in, as in #{abbreviate(instant)}Z for UTC. A date and time without an offset is a local time, which is not an instant, so write it as a string, as in '#{abbreviate("#{date} #{time}")}'"
1844
1816
  else
1845
1817
  "#{separator}. #{INSTANT_FORMAT}"