cton 0.1.1 → 0.3.0

data/lib/cton/decoder.rb

@@ -5,13 +5,15 @@ require "strscan"
 module Cton
   class Decoder
     TERMINATORS = [",", ";", ")", "]", "}"].freeze
+    KEY_VALUE_BOUNDARY_TOKENS = ["(", "[", "="].freeze
 
     def initialize(symbolize_names: false)
       @symbolize_names = symbolize_names
     end
 
     def decode(cton)
-      @scanner = StringScanner.new(cton.to_s)
+      @raw_string = cton.to_s
+      @scanner = StringScanner.new(@raw_string)
       skip_ws
 
       value = if key_ahead?
@@ -21,14 +23,28 @@ module Cton
               end
 
       skip_ws
-      raise ParseError, "Unexpected trailing data" unless @scanner.eos?
+      raise_error("Unexpected trailing data") unless @scanner.eos?
 
       value
     end
 
     private
 
-    attr_reader :symbolize_names, :scanner
+    attr_reader :symbolize_names, :scanner, :raw_string
+
+    def raise_error(message)
+      line, col = calculate_location(@scanner.pos)
+      raise ParseError, "#{message} at line #{line}, column #{col}"
+    end
+
+    def calculate_location(pos)
+      string = raw_string
+      consumed = string[0...pos]
+      line = consumed.count("\n") + 1
+      last_newline = consumed.rindex("\n")
+      col = last_newline ? pos - last_newline : pos + 1
+      [line, col]
+    end
 
     def parse_document
       result = {}
@@ -43,22 +59,20 @@ module Cton
 
     def parse_value_for_key
       skip_ws
-      if @scanner.scan(/\(/)
+      if @scanner.scan("(")
         parse_object
-      elsif @scanner.scan(/\[/)
+      elsif @scanner.scan("[")
         parse_array
-      elsif @scanner.scan(/=/)
+      elsif @scanner.scan("=")
         parse_scalar(allow_key_boundary: true)
       else
-        raise ParseError, "Unexpected token at position #{@scanner.pos}"
+        raise_error("Unexpected token")
       end
     end
 
     def parse_object
       skip_ws
-      if @scanner.scan(/\)/)
-        return {}
-      end
+      return {} if @scanner.scan(")")
 
       pairs = {}
       loop do
@@ -67,7 +81,8 @@ module Cton
         value = parse_value
         pairs[key] = value
         skip_ws
-        break if @scanner.scan(/\)/)
+        break if @scanner.scan(")")
+
         expect!(",")
         skip_ws
       end
@@ -92,7 +107,8 @@ module Cton
       fields = []
       loop do
         fields << parse_key_name
-        break if @scanner.scan(/\}/)
+        break if @scanner.scan("}")
+
         expect!(",")
       end
       fields
@@ -125,9 +141,9 @@ module Cton
 
     def parse_value(allow_key_boundary: false)
       skip_ws
-      if @scanner.scan(/\(/)
+      if @scanner.scan("(")
         parse_object
-      elsif @scanner.scan(/\[/)
+      elsif @scanner.scan("[")
         parse_array
       elsif @scanner.peek(1) == '"'
         parse_string
@@ -140,242 +156,88 @@ module Cton
       skip_ws
       return parse_string if @scanner.peek(1) == '"'
 
-      start_pos = @scanner.pos
-      
-      # If we allow key boundary, we need to be careful not to consume the next key
-      # This is the tricky part. The original implementation scanned ahead.
-      # With StringScanner, we can scan until a terminator or whitespace.
-      
+      @scanner.pos
+
       token = if allow_key_boundary
                 scan_until_boundary_or_terminator
               else
                 scan_until_terminator
               end
 
-      raise ParseError, "Empty value at #{start_pos}" if token.nil? || token.empty?
+      raise_error("Empty value") if token.nil? || token.empty?
 
       convert_scalar(token)
     end
 
     def scan_until_terminator
-      # Scan until we hit a terminator char, whitespace, or structure char
-      # Terminators: , ; ) ] }
-      # Structure: ( [ {
-      # Whitespace
-      
-      @scanner.scan(/[^,;\]\}\)\(\[\{\s]+/)
+      start_pos = @scanner.pos
+      end_pos = find_terminator_position(start_pos)
+      consume_slice(start_pos, end_pos)
     end
 
     def scan_until_boundary_or_terminator
-      # This is complex because "key=" looks like a scalar "key" followed by "="
-      # But "value" followed by "key=" means "value" ends before "key".
-      # The original logic used `next_key_index`.
-      
-      # Let's try to replicate the logic:
-      # Scan characters that are safe for keys/values.
-      # If we see something that looks like a key start, check if it is followed by [(=
-      
       start_pos = @scanner.pos
-      
-      # Fast path: scan until something interesting happens
-      chunk = @scanner.scan(/[0-9A-Za-z_.:-]+/)
-      return nil unless chunk
-      
-      # Now we might have consumed too much if the chunk contains a key.
-      # e.g. "valuekey=" -> chunk is "valuekey"
-      # We need to check if there is a split point inside `chunk` or if `chunk` itself is followed by [(=
-      
-      # Actually, the original logic was:
-      # Find the *first* position where a valid key starts AND is followed by [(=
-      
-      # Let's re-implement `next_key_index` logic but using the scanner's string
-      
-      rest_of_string = @scanner.string[@scanner.pos..-1]
-      # But we also need to consider the chunk we just scanned? 
-      # No, `scan_until_boundary_or_terminator` is called when we are at the start of a scalar.
-      
-      # Let's reset and do it properly.
-      @scanner.pos = start_pos
-      
-      full_scalar = scan_until_terminator
-      return nil unless full_scalar
-      
-      # Now check if `full_scalar` contains a key boundary
-      # A key boundary is a substring that matches SAFE_TOKEN and is followed by [(=
-      
-      # We need to look at `full_scalar` + whatever follows (whitespace?) + [(=
-      # But `scan_until_terminator` stops at whitespace.
-      
-      # If `full_scalar` is "valuekey", and next char is "=", then "key" is the key.
-      # But wait, "value" and "key" must be separated? 
-      # In CTON, "valuekey=..." is ambiguous if no separator.
-      # The README says: "Removing every newline makes certain inputs ambiguous... The default separator avoids that... You may pass separator: ''... decoding such strings is only safe if you can guarantee extra quoting or whitespace".
-      
-      # So if we are in `allow_key_boundary` mode (top level), we must look for embedded keys.
-      
-      # Let's look for the pattern inside the text we just consumed + lookahead.
-      # Actually, the original `next_key_index` scanned from the current position.
-      
-      # Let's implement a helper that searches for the boundary in the remaining string
-      # starting from `start_pos`.
-      
-      boundary_idx = find_key_boundary(start_pos)
-      
-      if boundary_idx
-        # We found a boundary at `boundary_idx`.
-        # The scalar ends at `boundary_idx`.
-        length = boundary_idx - start_pos
-        @scanner.pos = start_pos
-        token = @scanner.peek(length)
-        @scanner.pos += length
-        token
-      else
-        # No boundary found, so the whole thing we scanned is the token
-        # We already scanned it into `full_scalar` but we need to put the scanner in the right place.
-        # Wait, I reset the scanner.
-        @scanner.pos = start_pos + full_scalar.length
-        full_scalar
+      boundary_pos = find_key_boundary(start_pos)
+      end_pos = boundary_pos || find_terminator_position(start_pos)
+      consume_slice(start_pos, end_pos)
+    end
+
+    def consume_slice(start_pos, end_pos)
+      return nil if end_pos <= start_pos
+
+      token = raw_string.byteslice(start_pos, end_pos - start_pos)
+      @scanner.pos = end_pos
+      token
+    end
+
+    def find_terminator_position(start_pos)
+      str = raw_string
+      len = str.length
+      idx = start_pos
+
+      while idx < len
+        char = str[idx]
+        break if terminator?(char)
+
+        idx += 1
       end
+
+      idx
     end
 
     def find_key_boundary(from_index)
-      str = @scanner.string
+      str = raw_string
       len = str.length
       idx = from_index
-      
-      # We are looking for a sequence that matches SAFE_KEY followed by [(=
-      # But we are currently parsing a scalar.
-      
-      # Optimization: we only care about boundaries that appear *before* any terminator/whitespace.
-      # Because if we hit a terminator/whitespace, the scalar ends anyway.
-      
-      # So we only need to check inside the `scan_until_terminator` range?
-      # No, because "valuekey=" has no terminator/whitespace between value and key.
-      
+
       while idx < len
         char = str[idx]
-        
-        # If we hit a terminator or whitespace, we stop looking for boundaries 
-        # because the scalar naturally ends here.
-        if TERMINATORS.include?(char) || whitespace?(char) || "([{".include?(char)
-           return nil
-        end
-        
-        # Check if a key starts here
+
+        return nil if terminator?(char)
+
         if safe_key_char?(char)
-          # Check if this potential key is followed by [(=
-          # We need to scan this potential key
           key_end = idx
-          while key_end < len && safe_key_char?(str[key_end])
-            key_end += 1
-          end
-          
-          # Check what follows
-          next_char_idx = key_end
-          # Skip whitespace after key? No, keys are immediately followed by [(= usually?
-          # The original `next_key_index` did NOT skip whitespace after the key candidate.
-          # "next_char = @source[idx]" (where idx is after key)
-          
-          if next_char_idx < len
-             next_char = str[next_char_idx]
-             if ["(", "[", "="].include?(next_char)
-               # Found a boundary!
-               # But wait, is this the *start* of the scalar?
-               # If idx == from_index, then the scalar IS the key? No, that means we are at the start.
-               # If we are at the start, and it looks like a key, then it IS a key, so we should have parsed it as a key?
-               # No, `parse_scalar` is called when we expect a value.
-               # If we are parsing a document "key=valuekey2=value2", we are parsing "valuekey2".
-               # "key2" is the next key. So "value" is the scalar.
-               # So if idx > from_index, we found a split.
-               
-               return idx if idx > from_index
-             end
+          key_end += 1 while key_end < len && safe_key_char?(str[key_end])
+
+          if key_end < len && KEY_VALUE_BOUNDARY_TOKENS.include?(str[key_end]) && idx > from_index && boundary_start_allowed?(str[idx])
+            return idx
           end
-          
-          # If not a boundary, we continue scanning from inside the key?
-          # "valuekey=" -> at 'k', key is "key", followed by '=', so split at 'k'.
-          # "valukey=" -> at 'l', key is "lukey", followed by '=', so split at 'l'.
-          # This seems to imply we should check every position?
-          # The original code:
-          # if safe_key_char?(char)
-          #   start = idx
-          #   idx += 1 while ...
-          #   if start > from_index && ... return start
-          #   idx = start + 1  <-- This is important! It backtracks to check nested keys.
-          #   next
-          
-          # Yes, we need to check every position.
-          
-          # Optimization: The key must end at `key_end`.
-          # If `str[key_end]` is not [(=, then this `key_candidate` is not a key.
-          # But maybe a suffix of it is?
-          # e.g. "abc=" -> "abc" followed by "=". Split at start? No.
-          # "a" followed by "bc="? No.
-          
-          # Actually, if we find a valid key char, we scan to the end of the valid key chars.
-          # Let's say we have "abc=def".
-          # At 'a': key is "abc". Next is "=". "abc" is a key.
-          # If we are at start (from_index), then the whole thing is a key?
-          # But we are parsing a scalar.
-          # If `parse_scalar` sees "abc=", and `allow_key_boundary` is true.
-          # Does it mean "abc" is the scalar? Or "abc" is the next key?
-          # If "abc" is the next key, then the scalar before it is empty?
-          # "key=abc=def" -> key="key", value="abc", next_key="def"? No.
-          # "key=value next=val" -> value="value", next="next".
-          # "key=valuenext=val" -> value="value", next="next".
-          
-          # So if we find a key boundary at `idx`, it means the scalar ends at `idx`.
-          
-          # Let's stick to the original logic:
-          # Scan the maximal sequence of safe chars.
-          # If it is followed by [(=, then it IS a key.
-          # If it starts after `from_index`, then we found the boundary.
-          # If it starts AT `from_index`, then... what?
-          # If we are parsing a scalar, and we see "key=...", then the scalar is empty?
-          # That shouldn't happen if we called `parse_scalar`.
-          # Unless `parse_document` called `parse_value_for_key` -> `parse_scalar`.
-          # But `parse_document` calls `parse_key_name` first.
-          # So we are inside `parse_value`.
-          
-          # Example: "a=1b=2".
-          # parse "a", expect "=", parse value.
-          # value starts at "1".
-          # "1" is safe char. "1b" is safe.
-          # "b" is safe.
-          # At "1": max key is "1b". Next is "=". "1b" is a key? Yes.
-          # Is "1b" followed by "="? Yes.
-          # Does it start > from_index? "1" is at from_index. No.
-          # So "1b" is NOT a boundary.
-          # Continue to next char "b".
-          # At "b": max key is "b". Next is "=". "b" is a key.
-          # Does it start > from_index? Yes ("b" index > "1" index).
-          # So boundary is at "b".
-          # Scalar is "1".
-          
-          # So the logic is:
-          # For each char at `idx`:
-          #   If it can start a key:
-          #     Find end of key `end_key`.
-          #     If `str[end_key]` is [(= :
-          #       If `idx > from_index`: return `idx`.
-          #   idx += 1
-          
-          # But wait, "1b" was a key candidate.
-          # If we advanced `idx` to `end_key`, we would skip "b".
-          # So we must NOT advance `idx` to `end_key` blindly.
-          # We must check `idx`, then `idx+1`, etc.
-          
-          # But `safe_key_char?` is true for all chars in "1b".
-          # So we check "1...", then "b...".
-          
-          # Correct.
         end
-        
+
         idx += 1
       end
+
       nil
     end
 
+    def terminator?(char)
+      TERMINATORS.include?(char) || whitespace?(char) || ["(", "[", "{"].include?(char)
+    end
+
+    def boundary_start_allowed?(char)
+      !char.nil? && char.match?(/[A-Za-z_.:-]/)
+    end
+
     def convert_scalar(token)
       case token
       when "true" then true
@@ -396,22 +258,20 @@ module Cton
       expect!("\"")
       buffer = +""
       loop do
-        if @scanner.eos?
-          raise ParseError, "Unterminated string"
-        end
-        
+        raise_error("Unterminated string") if @scanner.eos?
+
         char = @scanner.getch
-        
-        if char == '\\'
+
+        if char == "\\"
           escaped = @scanner.getch
-          raise ParseError, "Invalid escape sequence" if escaped.nil?
+          raise_error("Invalid escape sequence") if escaped.nil?
           buffer << case escaped
-                    when 'n' then "\n"
-                    when 'r' then "\r"
-                    when 't' then "\t"
-                    when '"', '\\' then escaped
+                    when "n" then "\n"
+                    when "r" then "\r"
+                    when "t" then "\t"
+                    when '"', "\\" then escaped
                     else
-                      raise ParseError, "Unsupported escape sequence"
+                      raise_error("Unsupported escape sequence")
                     end
         elsif char == '"'
           break
@@ -425,16 +285,16 @@ module Cton
     def parse_key_name
       skip_ws
       token = @scanner.scan(/[0-9A-Za-z_.:-]+/)
-      raise ParseError, "Invalid key" if token.nil?
+      raise_error("Invalid key") if token.nil?
       symbolize_names ? token.to_sym : token
     end
 
     def parse_integer_literal
       token = @scanner.scan(/-?\d+/)
-      raise ParseError, "Expected digits" if token.nil?
+      raise_error("Expected digits") if token.nil?
       Integer(token, 10)
     rescue ArgumentError
-      raise ParseError, "Invalid length literal"
+      raise_error("Invalid length literal")
     end
 
     def symbolize_keys(row)
@@ -443,9 +303,9 @@ module Cton
 
     def expect!(char)
       skip_ws
-      unless @scanner.scan(Regexp.new(Regexp.escape(char)))
-        raise ParseError, "Expected #{char.inspect}, got #{@scanner.peek(1).inspect}"
-      end
+      return if @scanner.scan(Regexp.new(Regexp.escape(char)))
+
+      raise_error("Expected #{char.inspect}, got #{@scanner.peek(1).inspect}")
     end
 
     def skip_ws
@@ -453,18 +313,14 @@ module Cton
     end
 
     def whitespace?(char)
-      char == " " || char == "\t" || char == "\n" || char == "\r"
+      [" ", "\t", "\n", "\r"].include?(char)
     end
 
     def key_ahead?
-      # Check if the next token looks like a key followed by [(=
-      # We need to preserve position
       pos = @scanner.pos
       skip_ws
-      
-      # Scan a key
+
       if @scanner.scan(/[0-9A-Za-z_.:-]+/)
-        # Check what follows
         skip_ws
         next_char = @scanner.peek(1)
         result = ["(", "[", "="].include?(next_char)