tomlrb 1.3.0 → 2.0.2

data/lib/tomlrb/handler.rb

@@ -7,17 +7,24 @@ module Tomlrb
       @current = @output
       @stack = []
       @array_names = []
+      @current_table = []
+      @keys = Keys.new
       @symbolize_keys = options[:symbolize_keys]
     end
 
     def set_context(identifiers, is_array_of_tables: false)
+      if identifiers.empty?
+        raise ParseError, 'Array needs a name'
+      end
+
+      @current_table = identifiers.dup
+      @keys.add_table_key identifiers, is_array_of_tables
       @current = @output
 
       deal_with_array_of_tables(identifiers, is_array_of_tables) do |identifierz|
         identifierz.each do |k|
           k = k.to_sym if @symbolize_keys
           if @current[k].is_a?(Array)
-            @current[k] << {} if @current[k].empty?
             @current = @current[k].last
           else
             @current[k] ||= {}
@@ -28,7 +35,6 @@ module Tomlrb
     end
 
     def deal_with_array_of_tables(identifiers, is_array_of_tables)
-      identifiers.map!{|n| n.gsub("\"", '')}
       stringified_identifier = identifiers.join('.')
 
       if is_array_of_tables
@@ -43,20 +49,50 @@ module Tomlrb
       if is_array_of_tables
         last_identifier = last_identifier.to_sym if @symbolize_keys
         @current[last_identifier] ||= []
+        raise ParseError, "Cannot use key #{last_identifier} for both table and array at once" unless @current[last_identifier].respond_to?(:<<)
         @current[last_identifier] << {}
         @current = @current[last_identifier].last
       end
     end
 
     def assign(k)
-      k = k.to_sym if @symbolize_keys
-      @current[k] = @stack.pop
+      @keys.add_pair_key k, @current_table
+      current = @current
+      while key = k.shift
+        key = key.to_sym if @symbolize_keys
+        current = assign_key_path(current, key, k.empty?)
+      end
     end
 
     def push(o)
       @stack << o
     end
 
+    def push_inline(inline_arrays)
+      merged_inline = {}
+
+      inline_arrays.each do |inline_array|
+        current = merged_inline
+        value = inline_array.pop
+        inline_array.each_with_index do |inline_key, inline_index|
+          last_key = inline_index == inline_array.size - 1
+
+          if last_key
+            if current[inline_key].nil?
+              current[inline_key] = value
+            else
+              raise Key::KeyConflict, "Inline key #{inline_key} is already used"
+            end
+          else
+            current[inline_key] ||= {}
+            current = current[inline_key]
+          end
+        end
+      end
+
+      push(merged_inline)
+    end
+
     def start_(type)
       push([type])
     end
@@ -64,10 +100,166 @@ module Tomlrb
     def end_(type)
       array = []
       while (value = @stack.pop) != [type]
-        raise ParseError, 'Unclosed table' if value.nil?
+        raise ParseError, 'Unclosed table' if @stack.empty?
         array.unshift(value)
       end
       array
     end
+
+    def validate_value(value)
+      if value.nil?
+        raise ParseError, 'Value must be present'
+      end
+    end
+
+    private
+
+    def assign_key_path(current, key, key_emptied)
+      if key_emptied
+
+        raise ParseError, "Cannot overwrite value with key #{key}" unless current.kind_of?(Hash)
+        current[key] = @stack.pop
+        return current
+      end
+      current[key] ||= {}
+      current = current[key]
+      current
+    end
+  end
+
+  class Keys
+    def initialize
+      @keys = {}
+    end
+
+    def add_table_key(keys, is_array_of_tables = false)
+      self << [keys, [], is_array_of_tables]
+    end
+
+    def add_pair_key(keys, context)
+      self << [context, keys, false]
+    end
+
+    def <<(keys)
+      table_keys, pair_keys, is_array_of_tables = keys
+      current = @keys
+      current = append_table_keys(current, table_keys, pair_keys.empty?, is_array_of_tables)
+      append_pair_keys(current, pair_keys, table_keys.empty?, is_array_of_tables)
+    end
+
+    private
+
+    def append_table_keys(current, table_keys, pair_keys_empty, is_array_of_tables)
+      table_keys.each_with_index do |key, index|
+        declared = (index == table_keys.length - 1) && pair_keys_empty
+        if index == 0
+          current = find_or_create_first_table_key(current, key, declared, is_array_of_tables)
+        else
+          current = current << [key, :table, declared, is_array_of_tables]
+        end
+      end
+
+      current.clear_children if is_array_of_tables
+      current
+    end
+
+    def find_or_create_first_table_key(current, key, declared, is_array_of_tables)
+      existed = current[key]
+      if existed && existed.type == :pair
+        raise Key::KeyConflict, "Key #{key} is already used as #{existed.type} key"
+      end
+      if existed && existed.declared? && declared && ! is_array_of_tables
+        raise Key::KeyConflict, "Key #{key} is already used"
+      end
+      k = existed || Key.new(key, :table, declared)
+      current[key] = k
+      k
+    end
+
+    def append_pair_keys(current, pair_keys, table_keys_empty, is_array_of_tables)
+      pair_keys.each_with_index do |key, index|
+        declared = index == pair_keys.length - 1
+        if index == 0 && table_keys_empty
+          current = find_or_create_first_pair_key(current, key, declared, table_keys_empty)
+        else
+          key = current << [key, :pair, declared, is_array_of_tables]
+          current = key
+        end
+      end
+    end
+
+    def find_or_create_first_pair_key(current, key, declared, table_keys_empty)
+      existed = current[key]
+      if existed && (existed.type == :pair) && declared && table_keys_empty
+        raise Key::KeyConflict, "Key #{key} is already used"
+      end
+      k = Key.new(key, :pair, declared)
+      current[key] = k
+      k
+    end
+  end
+
+  class Key
+    class KeyConflict < ParseError; end
+
+    attr_reader :key, :type
+
+    def initialize(key, type, declared = false)
+      @key = key
+      @type = type
+      @declared = declared
+      @children = {}
+    end
+
+    def declared?
+      @declared
+    end
+
+    def <<(key_type_declared)
+      key, type, declared, is_array_of_tables = key_type_declared
+      existed = @children[key]
+      validate_already_declared_as_different_key(type, declared, existed)
+      validate_already_declared_as_non_array_table(type, is_array_of_tables, declared, existed)
+      validate_path_already_created_as_different_type(type, declared, existed)
+      validate_path_already_declared_as_different_type(type, declared, existed)
+      validate_already_declared_as_same_key(declared, existed)
+      @children[key] = existed || self.class.new(key, type, declared)
+    end
+
+    def clear_children
+      @children.clear
+    end
+
+    private
+
+    def validate_already_declared_as_different_key(type, declared, existed)
+      if existed && existed.declared? && existed.type != type
+        raise KeyConflict, "Key #{existed.key} is already used as #{existed.type} key"
+      end
+    end
+
+    def validate_already_declared_as_non_array_table(type, is_array_of_tables, declared, existed)
+      if declared && type == :table && existed && existed.declared? && ! is_array_of_tables
+        raise KeyConflict, "Key #{existed.key} is already used"
+      end
+    end
+
+    def validate_path_already_created_as_different_type(type, declared, existed)
+      if declared && (type == :table) && existed && (existed.type == :pair) && (! existed.declared?)
+        raise KeyConflict, "Key #{existed.key} is already used as #{existed.type} key"
+      end
+    end
+
+    def validate_path_already_declared_as_different_type(type, declared, existed)
+      if ! declared && (type == :pair) && existed && (existed.type == :pair) && existed.declared?
+        raise KeyConflict, "Key #{key} is already used as #{type} key"
+      end
+    end
+
+    def validate_already_declared_as_same_key(declared, existed)
+      if existed && ! existed.declared? && declared
+        raise KeyConflict, "Key #{existed.key} is already used as #{existed.type} key"
+      end
+    end
   end
 end

data/lib/tomlrb/local_date.rb

@@ -0,0 +1,33 @@
+require 'forwardable'
+
+module Tomlrb
+  class LocalDate
+    extend Forwardable
+
+    def_delegators :@time, :year, :month, :day
+
+    def initialize(year, month, day)
+      @time = Time.new(year, month, day, 0, 0, 0, '-00:00')
+    end
+
+    # @param offset see {LocalDateTime#to_time}
+    # @return [Time] 00:00:00 of the date
+    def to_time(offset='-00:00')
+      return @time if offset == '-00:00'
+      Time.new(year, month, day, 0, 0, 0, offset)
+    end
+
+    def to_s
+      @time.strftime('%F')
+    end
+
+    def ==(other)
+      other.kind_of?(self.class) &&
+        to_time == other.to_time
+    end
+
+    def inspect
+      "#<#{self.class}: #{to_s}>"
+    end
+  end
+end

data/lib/tomlrb/local_date_time.rb

@@ -0,0 +1,40 @@
+require 'forwardable'
+
+module Tomlrb
+  class LocalDateTime
+    extend Forwardable
+
+    def_delegators :@time, :year, :month, :day, :hour, :min, :sec, :usec, :nsec
+
+    def initialize(year, month, day, hour, min, sec) # rubocop:disable Metrics/ParameterLists
+      @time = Time.new(year, month, day, hour, min, sec, '-00:00')
+      @sec = sec
+    end
+
+    # @param offset [String, Symbol, Numeric, nil] time zone offset.
+    #   * when +String+, must be '+HH:MM' format, '-HH:MM' format, 'UTC', 'A'..'I' or 'K'..'Z'. Arguments excluding '+-HH:MM' are supporeted at Ruby >= 2.7.0
+    #   * when +Symbol+, must be +:dst+(for summar time for local) or +:std+(for standard time).
+    #   * when +Numeric+, it is time zone offset in second.
+    #   * when +nil+, local time zone offset is used.
+    # @return [Time]
+    def to_time(offset='-00:00')
+      return @time if offset == '-00:00'
+      Time.new(year, month, day, hour, min, @sec, offset)
+    end
+
+    def to_s
+      frac = (@sec - sec)
+      frac_str = frac == 0 ? '' : "#{frac.to_s[1..-1]}"
+      @time.strftime("%FT%T") << frac_str
+    end
+
+    def ==(other)
+      other.kind_of?(self.class) &&
+        to_time == other.to_time
+    end
+
+    def inspect
+      "#<#{self.class}: #{to_s}>"
+    end
+  end
+end

data/lib/tomlrb/local_time.rb

@@ -0,0 +1,38 @@
+require 'forwardable'
+
+module Tomlrb
+  class LocalTime
+    extend Forwardable
+
+    def_delegators :@time, :hour, :min, :sec, :usec, :nsec
+
+    def initialize(hour, min, sec)
+      @time = Time.new(0, 1, 1, hour, min, sec, '-00:00')
+      @sec = sec
+    end
+
+    # @param year [Integer]
+    # @param month [Integer]
+    # @param day [Integer]
+    # @param offset see {LocalDateTime#to_time}
+    # @return [Time] the time of the date specified by params
+    def to_time(year, month, day, offset='-00:00')
+      Time.new(year, month, day, hour, min, @sec, offset)
+    end
+
+    def to_s
+      frac = (@sec - sec)
+      frac_str = frac == 0 ? '' : "#{frac.to_s[1..-1]}"
+      @time.strftime("%T") << frac_str
+    end
+
+    def ==(other)
+      other.kind_of?(self.class) &&
+        @time == other.to_time(0, 1, 1)
+    end
+
+    def inspect
+      "#<#{self.class}: #{to_s}>"
+    end
+  end
+end

data/lib/tomlrb/parser.y

@@ -1,16 +1,19 @@
 class Tomlrb::GeneratedParser
-token IDENTIFIER STRING_MULTI STRING_BASIC STRING_LITERAL_MULTI STRING_LITERAL DATETIME INTEGER FLOAT FLOAT_INF FLOAT_NAN TRUE FALSE
+token IDENTIFIER STRING_MULTI STRING_BASIC STRING_LITERAL_MULTI STRING_LITERAL DATETIME LOCAL_TIME INTEGER NON_DEC_INTEGER FLOAT FLOAT_KEYWORD BOOLEAN NEWLINE EOS
 rule
   expressions
     | expressions expression
+    | expressions EOS
     ;
   expression
     : table
     | assignment
     | inline_table
+    | NEWLINE
     ;
   table
-    : table_start table_continued
+    : table_start table_continued NEWLINE
+    | table_start table_continued EOS
     ;
   table_start
     : '[' '[' { @handler.start_(:array_of_tables) }
@@ -27,55 +30,112 @@ rule
     | '.' table_continued
     ;
   table_identifier
-    : IDENTIFIER { @handler.push(val[0]) }
-    | STRING_BASIC { @handler.push(val[0]) }
-    | STRING_LITERAL { @handler.push(val[0]) }
-    | INTEGER { @handler.push(val[0]) }
-    | TRUE { @handler.push(val[0]) }
-    | FALSE { @handler.push(val[0]) }
+    : table_identifier '.' table_identifier_component { @handler.push(val[2]) }
+    | table_identifier '.' FLOAT { val[2].split('.').each { |k| @handler.push(k) } }
+    | FLOAT {
+      keys = val[0].split('.')
+      @handler.start_(:table)
+      keys.each { |key| @handler.push(key) }
+    }
+    | table_identifier_component { @handler.push(val[0]) }
+    ;
+  table_identifier_component
+    : IDENTIFIER
+    | STRING_BASIC { result = StringUtils.replace_escaped_chars(val[0]) }
+    | STRING_LITERAL
+    | INTEGER
+    | NON_DEC_INTEGER
+    | FLOAT_KEYWORD
+    | BOOLEAN
     ;
   inline_table
-    : inline_table_start inline_continued
+    : inline_table_start inline_table_end
+    | inline_table_start inline_continued inline_table_end
     ;
   inline_table_start
     : '{' { @handler.start_(:inline) }
     ;
+  inline_table_end
+    : '}' {
+      array = @handler.end_(:inline)
+      @handler.push_inline(array)
+    }
+    ;
   inline_continued
-    : '}' { array = @handler.end_(:inline); @handler.push(Hash[*array]) }
-    | inline_assignment_key inline_assignment_value inline_next
+    : inline_assignment
+    | inline_assignment inline_next
     ;
   inline_next
-    : '}' {
-      array = @handler.end_(:inline)
-      array.map!.with_index{ |n,i| i.even? ? n.to_sym : n } if @handler.symbolize_keys
-      @handler.push(Hash[*array])
+    : ',' inline_continued
+    ;
+  inline_assignment
+    : inline_assignment_key '=' value {
+      keys = @handler.end_(:inline_keys)
+      @handler.push(keys)
     }
-    | ',' inline_continued
     ;
   inline_assignment_key
-    : IDENTIFIER { @handler.push(val[0]) }
-    ;
-  inline_assignment_value
-    : '=' value
+    : inline_assignment_key '.' assignment_key_component { 
+      @handler.push(val[2]) 
+    }
+    | inline_assignment_key '.' FLOAT { val[2].split('.').each { |k| @handler.push(k) } }
+    | FLOAT {
+      keys = val[0].split('.')
+      @handler.start_(:inline_keys)
+      keys.each { |key| @handler.push(key) }
+    }
+    | assignment_key_component { 
+      @handler.start_(:inline_keys) 
+      @handler.push(val[0]) 
+    }
     ;
   assignment
-    : IDENTIFIER '=' value { @handler.assign(val[0]) }
-    | STRING_BASIC '=' value { @handler.assign(val[0]) }
-    | STRING_LITERAL '=' value { @handler.assign(val[0]) }
-    | INTEGER '=' value { @handler.assign(val[0]) }
-    | TRUE '=' value { @handler.assign(val[0]) }
-    | FALSE '=' value { @handler.assign(val[0]) }
+    : assignment_key '=' value EOS {
+      keys = @handler.end_(:keys)
+      value = keys.pop
+      @handler.validate_value(value)
+      @handler.push(value)
+      @handler.assign(keys)
+    }
+    | assignment_key '=' value NEWLINE {
+      keys = @handler.end_(:keys)
+      value = keys.pop
+      @handler.validate_value(value)
+      @handler.push(value)
+      @handler.assign(keys)
+    }
+    ;
+  assignment_key
+    : assignment_key '.' assignment_key_component { @handler.push(val[2]) }
+    | assignment_key '.' FLOAT { val[2].split('.').each { |k| @handler.push(k) } }
+    | FLOAT {
+      keys = val[0].split('.')
+      @handler.start_(:keys)
+      keys.each { |key| @handler.push(key) }
+    }
+    | assignment_key_component { @handler.start_(:keys); @handler.push(val[0]) }
+    ;
+  assignment_key_component
+    : IDENTIFIER
+    | STRING_BASIC { result = StringUtils.replace_escaped_chars(val[0]) }
+    | STRING_LITERAL
+    | INTEGER
+    | NON_DEC_INTEGER
+    | FLOAT_KEYWORD
+    | BOOLEAN
     ;
   array
     : start_array array_continued
     ;
   array_continued
-    : ']' { array = @handler.end_(:array); @handler.push(array) }
+    : ']' { array = @handler.end_(:array); @handler.push(array.compact) }
     | value array_next
+    | NEWLINE array_continued
     ;
   array_next
-    : ']' { array = @handler.end_(:array); @handler.push(array) }
+    : ']' { array = @handler.end_(:array); @handler.push(array.compact) }
     | ',' array_continued
+    | NEWLINE array_continued
     ;
   start_array
     : '[' { @handler.start_(:array) }
@@ -91,12 +151,42 @@ rule
     ;
   literal
     | FLOAT { result = val[0].to_f }
-    | FLOAT_INF { result = (val[0][0] == '-' ? -1 : 1) * Float::INFINITY }
-    | FLOAT_NAN { result = Float::NAN }
+    | FLOAT_KEYWORD {
+      v = val[0]
+      result = if v.end_with?('nan')
+                 Float::NAN
+               else
+                 (v[0] == '-' ? -1 : 1) * Float::INFINITY
+               end
+    }
     | INTEGER { result = val[0].to_i }
-    | TRUE   { result = true }
-    | FALSE  { result = false }
-    | DATETIME { result = Time.new(*val[0])}
+    | NON_DEC_INTEGER {
+      base = case val[0][1]
+             when "x" then 16
+             when "o" then 8
+             when "b" then 2
+             end
+      result = val[0].to_i(base)
+    }
+    | BOOLEAN { result = val[0] == 'true' ? true : false }
+    | DATETIME {
+      v = val[0]
+      result = if v[6].nil?
+                 if v[4].nil?
+                   LocalDate.new(v[0], v[1], v[2])
+                 else
+                   LocalDateTime.new(v[0], v[1], v[2], v[3] || 0, v[4] || 0, v[5].to_f)
+                 end
+               else
+                 # Patch for 24:00:00 which Ruby parses
+                 if v[3].to_i == 24 && v[4].to_i == 0 && v[5].to_i == 0
+                   v[3] = (v[3].to_i + 1).to_s
+                 end
+
+                 Time.new(v[0], v[1], v[2], v[3] || 0, v[4] || 0, v[5].to_f, v[6])
+               end
+    }
+    | LOCAL_TIME { result = LocalTime.new(*val[0]) }
     ;
   string
     : STRING_MULTI { result = StringUtils.replace_escaped_chars(StringUtils.multiline_replacements(val[0])) }

data/lib/tomlrb/scanner.rb

@@ -2,57 +2,73 @@ require 'strscan'
 
 module Tomlrb
   class Scanner
-    COMMENT = /#.*/
+    COMMENT = /#[^\u0000-\u0008\u000A-\u001F\u007F]*/
     IDENTIFIER = /[A-Za-z0-9_-]+/
-    SPACE = /[ \t\r\n]/
-    STRING_BASIC = /(["])(?:\\?.)*?\1/
-    STRING_MULTI = /"{3}([\s\S]*?"{3,4})/m
-    STRING_LITERAL = /(['])(?:\\?.)*?\1/
-    STRING_LITERAL_MULTI = /'{3}([\s\S]*?'{3})/m
+    SPACE = /[ \t]/
+    NEWLINE = /(?:[ \t]*(?:\r?\n)[ \t]*)+/
+    STRING_BASIC = /(["])(?:\\?[^\u0000-\u0008\u000A-\u001F\u007F])*?\1/
+    STRING_MULTI = /"{3}([^\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]*?(?<!\\)"{3,5})/m
+    STRING_LITERAL = /(['])(?:\\?[^\u0000-\u0008\u000A-\u001F\u007F])*?\1/
+    STRING_LITERAL_MULTI = /'{3}([^\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]*?'{3,5})/m
     DATETIME = /(-?\d{4})-(\d{2})-(\d{2})(?:(?:t|\s)(\d{2}):(\d{2}):(\d{2}(?:\.\d+)?))?(z|[-+]\d{2}:\d{2})?/i
-    FLOAT = /[+-]?(?:[0-9_]+\.[0-9_]*|\d+(?=[eE]))(?:[eE][+-]?[0-9_]+)?/
-    FLOAT_INF = /[+-]?inf/
-    FLOAT_NAN = /[+-]?nan/
+    LOCAL_TIME = /(\d{2}):(\d{2}):(\d{2}(?:\.\d+)?)/
+    FLOAT = /[+-]?(?:(?:\d|[1-9](?:_?\d)*)\.\d(?:_?\d)*|\d+(?=[eE]))(?:[eE][+-]?[0-9]+(_[0-9])*[0-9]*)?(?!\w)/
+    FLOAT_KEYWORD = /[+-]?(?:inf|nan)\b/
     INTEGER = /[+-]?([1-9](_?\d)*|0)(?![A-Za-z0-9_-]+)/
-    TRUE   = /true/
-    FALSE  = /false/
+    NON_DEC_INTEGER = /0(?:x[0-9A-Fa-f]+(?:_[0-9A-Fa-f])*[0-9A-Fa-f]*|o[0-7]+(?:_[0-7])*[0-7]*|b[01]+(?:_[01])*[01]*)/
+    BOOLEAN = /true|false/
+    SPACED_ARRAY_OF_TABLES_START = /^\[[ \t]+\[(#{IDENTIFIER}|#{STRING_BASIC}|#{STRING_LITERAL}|#{INTEGER}|#{NON_DEC_INTEGER}|#{FLOAT_KEYWORD}|#{BOOLEAN})\]\]$/
+    SPACED_ARRAY_OF_TABLES_END = /^\[\[(#{IDENTIFIER}|#{STRING_BASIC}|#{STRING_LITERAL}|#{INTEGER}|#{NON_DEC_INTEGER}|#{FLOAT_KEYWORD}|#{BOOLEAN})\][ \t]+\]$/
+    SPACED_ARRAY_OF_TABLES_BOTH = /^\[[ \t]+\[(#{IDENTIFIER}|#{STRING_BASIC}|#{STRING_LITERAL}|#{INTEGER}|#{NON_DEC_INTEGER}|#{FLOAT_KEYWORD}|#{BOOLEAN})\][ \t]+\]$/
 
     def initialize(io)
       @ss = StringScanner.new(io.read)
+      @eos = false
     end
 
     def next_token
-      return if @ss.eos?
-
       case
+      when @ss.scan(NEWLINE) then [:NEWLINE, nil]
+      when @ss.scan(SPACED_ARRAY_OF_TABLES_START) then raise ParseError.new("Array of tables has spaces in starting brackets")
+      when @ss.scan(SPACED_ARRAY_OF_TABLES_END) then raise ParseError.new("Array of tables has spaces in ending brackets")
+      when @ss.scan(SPACED_ARRAY_OF_TABLES_BOTH) then raise ParseError.new("Array of tables has spaces in starting and ending brackets")
       when @ss.scan(SPACE) then next_token
       when @ss.scan(COMMENT) then next_token
       when @ss.scan(DATETIME) then process_datetime
+      when @ss.scan(LOCAL_TIME) then process_local_time
       when text = @ss.scan(STRING_MULTI) then [:STRING_MULTI, text[3..-4]]
       when text = @ss.scan(STRING_BASIC) then [:STRING_BASIC, text[1..-2]]
       when text = @ss.scan(STRING_LITERAL_MULTI) then [:STRING_LITERAL_MULTI, text[3..-4]]
       when text = @ss.scan(STRING_LITERAL) then [:STRING_LITERAL, text[1..-2]]
       when text = @ss.scan(FLOAT) then [:FLOAT, text]
-      when text = @ss.scan(FLOAT_INF) then [:FLOAT_INF, text]
-      when text = @ss.scan(FLOAT_NAN) then [:FLOAT_NAN, text]
+      when text = @ss.scan(FLOAT_KEYWORD) then [:FLOAT_KEYWORD, text]
       when text = @ss.scan(INTEGER) then [:INTEGER, text]
-      when text = @ss.scan(TRUE)   then [:TRUE, text]
-      when text = @ss.scan(FALSE)  then [:FALSE, text]
+      when text = @ss.scan(NON_DEC_INTEGER) then [:NON_DEC_INTEGER, text]
+      when text = @ss.scan(BOOLEAN) then [:BOOLEAN, text]
       when text = @ss.scan(IDENTIFIER) then [:IDENTIFIER, text]
-      else
-        x = @ss.getch
-        [x, x]
+      when @ss.eos? then process_eos
+      else x = @ss.getch; [x, x]
       end
     end
 
     def process_datetime
-      if @ss[7].nil?
-        offset = '+00:00'
-      else
-        offset = @ss[7].gsub('Z', '+00:00')
+      if @ss[7]
+        offset = @ss[7].gsub(/[zZ]/, '+00:00')
       end
-      args = [@ss[1], @ss[2], @ss[3], @ss[4] || 0, @ss[5] || 0, @ss[6].to_f, offset]
+      args = [@ss[1], @ss[2], @ss[3], @ss[4], @ss[5], @ss[6], offset]
       [:DATETIME, args]
     end
+
+    def process_local_time
+      args = [@ss[1], @ss[2], @ss[3].to_f]
+      [:LOCAL_TIME, args]
+    end
+
+    def process_eos
+      return if @eos
+
+      @eos = true
+      [:EOS, nil]
+    end
   end
 end

data/lib/tomlrb/string_utils.rb

@@ -12,7 +12,13 @@ module Tomlrb
     }.freeze
 
     def self.multiline_replacements(str)
-      strip_spaces(str).gsub(/\\\n\s+/, '')
+      strip_spaces(str).gsub(/\\+\s*\n\s*/) {|matched|
+        if matched.match(/\\+/)[0].length.odd?
+          matched.gsub(/\\\s*\n\s*/, '')
+        else
+          matched
+        end
+      }
     end
 
     def self.replace_escaped_chars(str)