toon-fu 4.1.4 → 4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a5a734c469d08f497c36883702683c8a1c7edc3245df9f886d5f6159ff26db1
-  data.tar.gz: 784178fff8819a4895e5252b8e356ffaddaceca1ff10d9d6067a73984d5ef8cc
+  metadata.gz: c7f87e2144c7adb60f949127ea36ed9e26b84fb1ba5e4cd6893d9fc46f3e859e
+  data.tar.gz: 84c74750d67f681645a7cd0e84b46675919f3e0549cc4e168306510b1f8b45bf
 SHA512:
-  metadata.gz: 4acf57be096b69fa1c675a97efb59fe63db7b66234e649305f5cb1dafa319465fa3ffb235d2aa252dd714861476720a910b7859e6916dc6ba45584b1d75a886c
-  data.tar.gz: 3ba5aa86ccfae68c53da20cfaa66a31d3ddee6b5e57af7c72f3ef884d36be45fe485b234973cd6cef2b31d1751d482d1f36b20a9d5eff746f02ff486ccc997e3
+  metadata.gz: a78c3f872ff747c74af86f91a5e945b8dcf1b2eb5d10ea9694ad2a0e3efccc938fe7652a5cfda36713163f0f39d965dbe75f1e77698bdea3aea124e81bb1b194
+  data.tar.gz: 1b6fa5a87123368f9a7e92ee1fb1f91fce1950d583967acf007542b6ea10168c8b67c282cb87e96009841a039b657a23c77c11d4e47535d192bde41f9e82aaad

data/README.md

@@ -138,7 +138,7 @@ ToonFu.decode("tags[3]: a,b")
 
 ### Options
 
-- `strict:` — `true` (default) checks what the spec requires: declared lengths, row widths, duplicate keys, indentation. Pass `false` to read a document that bends those rules — a declared length is ignored, a duplicate key takes its last value.
+- `strict:` — `true` (default) checks what the spec requires: declared lengths, row widths, duplicate keys, indentation. Pass `false` to read a document that bends those rules — a declared length is ignored, a duplicate key takes its last value, a stray over-indented line is skipped unless it is a bare token, which raises in either mode.
 - `indent_size:` — spaces per nesting level, default `2`.
 
 ```ruby
@@ -166,17 +166,17 @@ b08ea2d feat: ship a toon executable
 
 ## Compared with other Ruby TOON gems
 
-The only one that passes every spec fixture: toon-fu passes all 179 encode and all 359 decode fixtures. The table compares encoding — the 154 encode fixtures that use default options, which every gem can run; none of the others reads TOON back.
+The only one that passes every spec fixture: toon-fu passes all 160 encode and all 385 decode fixtures. The table compares encoding — the 138 encode fixtures that use default options, which every gem can run; none of the others reads TOON back.
 
 | Gem | Spec fixtures passed | Speed vs toon-fu |
 |---|---:|---:|
-| **toon-fu** | **154 / 154** | **1.00×** |
-| sorbet-toon 0.1.0 | 119 / 154 | 0.59× |
-| toon-ruby 0.1.1 | 117 / 154 | 0.62× |
-| toon_my_json 0.1.0 | 57 / 154 | 1.68× |
-| toon-format 0.1.2 | 45 / 154 | 1.18× |
+| **toon-fu** | **138 / 138** | **1.00×** |
+| sorbet-toon 0.1.0 | 104 / 138 | 0.63× |
+| toon-ruby 0.1.1 | 100 / 138 | 0.62× |
+| toon_my_json 0.1.0 | 48 / 138 | 1.62× |
+| toon-format 0.1.2 | 40 / 138 | 1.31× |
 
-The Ruby TOON encoders with more than 10,000 downloads, measured by [`benchmark/run.rb`](benchmark/run.rb) on Ruby 3.4.10 (2026-09-24). **Spec fixtures** are the spec's own encode fixtures that use default options. **Speed** is the geometric mean of encodes per second over five workloads — tables of 100 and 1000 rows, nested objects, a list of mixed objects, strings needing quotes — relative to toon-fu.
+The Ruby TOON encoders with more than 10,000 downloads, measured by [`benchmark/run.rb`](benchmark/run.rb) on Ruby 3.4.11 (2026-10-07). **Spec fixtures** are the spec's own encode fixtures that use default options. **Speed** is the geometric mean of encodes per second over five workloads — tables of 100 and 1000 rows, nested objects, a list of mixed objects, strings needing quotes — relative to toon-fu.
 
 What falls through the gaps:
 
@@ -203,7 +203,7 @@ The gem version tracks the TOON specification it implements:
 - `MAJOR.MINOR` is the spec version: `X.Y.Z` speaks TOON `X.Y`.
 - `PATCH` is the gem's own: fixes and improvements within the same dialect.
 
-This release speaks `toon-spec: 4.1`.
+This release speaks `toon-spec: 4.3`.
 
 Pin the spec line with `gem "toon-fu", "~> X.Y.0"`: it takes our fixes and keeps you on the dialect you speak. The gem badge above shows the current release; the spec-drift badge turns red when a newer spec is released and toon-fu has not caught up yet.
 

data/lib/toon_fu/decoder.rb

@@ -5,8 +5,11 @@ module ToonFu
   class Decoder
     # @param strict [Boolean] whether to enforce the spec's strict-mode
     #   checks; with +false+ duplicate keys resolve last-write-wins, leading
-    #   spaces that are not a multiple of indent_size are accepted, and a
-    #   tab in indentation is accepted and counts as one level
+    #   spaces that are not a multiple of indent_size are accepted, a tab in
+    #   indentation is accepted and counts as one level, a line of spaces and
+    #   tabs is blank, a scope's first line sets the scope's depth, an
+    #   over-indented line is skipped unless it is a bare token, and a
+    #   malformed or misplaced header is read as a key-value line
     # @param indent_size [Integer] spaces per nesting level
     # @raise [ArgumentError] when strict is not a boolean or indent_size is
     #   not a positive Integer
@@ -43,9 +46,10 @@ module ToonFu
     #   key without a colon; for a bare token line inside a scope; for
     #   content after a completed root form; for nesting beyond the depth
     #   limit; and, when strict, for a declared length or row width that
-    #   does not match, a malformed header, a duplicate key, a blank line
-    #   inside a header's scope, and indentation that is not a multiple of
-    #   indent_size
+    #   does not match, a malformed or misplaced header, a duplicate key, a
+    #   blank line inside a header's scope, a tab in indentation, indentation
+    #   that is not a multiple of indent_size, a depth jump, and an
+    #   over-indented line, an indented first line included
     def decode(text)
       Reader.new(@strict, @indent_size).read(utf8(text))
     end

data/lib/toon_fu/field_list.rb

@@ -37,16 +37,21 @@ module ToonFu
     end
 
     def split(entry)
+      entry = Tokens.trim(entry)
+      return [nil, nil] if entry.empty?
+
       brace = Tokens.brace_index(entry)
-      return [Tokens.key(entry), nil] if brace.nil?
+      return [Token.key(entry), nil] if brace.nil?
 
       name = entry[0...brace]
+      return [nil, nil] if name.empty? || name.match?(Tokens::TRAILING_WHITESPACE)
+
       group = entry[brace..]
       closing = Tokens.closing_brace(group)
       return [nil, nil] unless closing == group.length - 1
 
       nested = self.class.parse(group[1...closing], @delimiter)
-      nested.nil? ? [nil, nil] : [Tokens.key(name), nested]
+      nested.nil? ? [nil, nil] : [Token.key(name), nested]
     end
   end
 end

data/lib/toon_fu/header.rb

@@ -9,7 +9,7 @@ module ToonFu
 
     def self.parse(content)
       bracket = Tokens.bracket_index(content)
-      return nil if bracket.nil?
+      return nil if bracket.nil? || Tokens.colon_index(content).nil?
 
       new(content, bracket).parse
     end
@@ -20,9 +20,9 @@ module ToonFu
       @malformed = false
     end
 
-    # A line carrying a bracket segment is either a header, or an attempt at
-    # one that §6 makes a strict-mode error; only a line without one falls
-    # through to the key-value class untouched.
+    # A line whose first unquoted "[" precedes its first unquoted colon is an
+    # array-header line; one that fails the §6 grammar is malformed, a
+    # strict-mode error (§5.2).
     def malformed?
       @malformed
     end
@@ -30,7 +30,7 @@ module ToonFu
     def parse
       @key = @bracket.zero? ? nil : @content[0...@bracket]
       return nil unless @key.nil? || Tokens.whole_token?(@key)
-      return broken if @key&.match?(/\s\z/)
+      return broken if @key&.match?(Tokens::TRAILING_WHITESPACE)
       return broken unless (closing = @content.index("]", @bracket))
 
       return broken unless segment(@content[(@bracket + 1)...closing])

data/lib/toon_fu/lines.rb

@@ -29,8 +29,8 @@ module ToonFu
     BOM = "\u{feff}"
     COMMENT = /\A *#/
     TRAILING_SPACES = / +\z/
-    BLANK = /\A\s*\z/
     INDENTATION = /\A[ \t]*/
+    BLANK_WITH_TABS = /\A[ \t]*\z/
 
     def initialize(text, strict, indent_size)
       @strict = strict
@@ -69,7 +69,7 @@ module ToonFu
         line = raw.delete_suffix("\r").sub(TRAILING_SPACES, "")
         next if COMMENT.match?(line)
 
-        if BLANK.match?(line)
+        if blank?(line)
           blank = true
           next
         end
@@ -80,6 +80,10 @@ module ToonFu
       end
     end
 
+    def blank?(line)
+      @strict ? line.empty? : BLANK_WITH_TABS.match?(line)
+    end
+
     def build(line, number, blank)
       Line.new(line.sub(INDENTATION, ""), depth(line, number), number, line, blank)
     end

data/lib/toon_fu/reader.rb

@@ -8,6 +8,7 @@ module ToonFu
       @strict = strict
       @indent_size = indent_size
       @depth = 0
+      @in_span = false
     end
 
     def read(text)
@@ -21,18 +22,19 @@ module ToonFu
 
     def root
       line = @lines.peek
+      return object(-1, 0) unless line.depth.zero?
       return empty_array_root if line.content == "[]"
 
       header = usable(Header.parse(line.content), line)
       return keyless_root(header) if header && header.key.nil?
-      return scalar_root(line) if @lines.one? && Tokens.colon_index(line.content).nil?
+      return scalar_root(line) if @lines.one? && scalar?(line)
 
-      object(0)
+      object(-1, 0)
     end
 
     def keyless_root(header)
       @lines.next
-      value = header.fields? ? table(header, 1) : array(header, 1)
+      value = header.fields? ? table(header, 0) : array(header, 0)
       trailing
       value
     end
@@ -54,26 +56,30 @@ module ToonFu
       raise Error, "cannot decode content after the root form: #{@lines.peek.text}"
     end
 
-    def object(depth, result = {})
-      while (line = @lines.peek) && line.depth >= depth
-        if line.depth > depth
+    def object(parent, depth, result = {})
+      while (line = @lines.peek) && line.depth > parent
+        if line.depth == depth
+          take
+          key, value = field(line, depth)
+          store(result, key, value, line)
+        else
           orphan(line)
-          next
         end
-
-        @lines.next
-        key, value = field(line, depth)
-        store(result, key, value, line)
       end
       result
     end
 
     def orphan(line)
       raise Error, "cannot decode a line that belongs to no scope on line #{line.number}: #{line.text}" if @strict
+      raise Error, "cannot decode a bare token line inside a scope on line #{line.number}: #{line.text}" if scalar?(line)
 
       @lines.next
     end
 
+    def scalar?(line)
+      Tokens.colon_index(line.content).nil?
+    end
+
     def store(result, key, value, line)
       raise Error, "cannot decode a duplicate key on line #{line.number}: #{key.inspect}" if @strict && result.key?(key)
 
@@ -83,7 +89,7 @@ module ToonFu
     def field(line, depth)
       header = usable(Header.parse(line.content), line)
       return [Tokens.key(header.key), header_value(header, depth)] if header&.key
-      raise Error, "cannot decode a keyless array header in object field position on line #{line.number}: #{line.text}" if header
+      misplaced(line, "a keyless array header in object field position") if header
 
       colon = Tokens.colon_index(line.content)
       raise Error, "cannot decode a line without a colon after its key on line #{line.number}: #{line.text}" if colon.nil?
@@ -99,30 +105,23 @@ module ToonFu
       Token.decode(text)
     end
 
-    def nested(depth)
-      line = @lines.peek
-      return {} if line.nil? || line.depth <= depth
+    def nested(parent)
+      depth = content_depth(parent)
+      return {} if depth.nil?
 
-      guard(line)
-      descend { object(depth + 1) }
+      descend { object(parent, depth) }
     end
 
     def header_value(header, depth)
-      return table(header, depth + 1) if header.fields?
-      return [] if header.length.zero? && !items?(depth + 1)
-
-      array(header, depth + 1)
+      header.fields? ? table(header, depth) : array(header, depth)
     end
 
-    def items?(depth)
-      line = @lines.peek
-      !line.nil? && line.depth == depth
-    end
-
-    def array(header, depth)
+    def array(header, parent)
       return inline(header) unless header.inline.empty?
 
-      descend { list(header, depth) }
+      items = descend { gather(parent, stop: ->(line) { !line.item? }) { |line| item(line) } }
+      count(items.length, header, "item")
+      items
     end
 
     def inline(header)
@@ -131,46 +130,27 @@ module ToonFu
       cells
     end
 
-    def list(header, depth)
-      items = []
-      while (line = @lines.peek) && line.depth == depth && line.item?
-        span(line) unless items.empty?
-        @lines.next
-        items << item(line, depth)
-      end
-      scalar_line(@lines.peek, depth)
-      count(items.length, header, "item")
-      items
-    end
-
-    # A scalar line is valid only as a root primitive; inside a scope it is a
-    # structural error in strict and non-strict mode alike (§5.2, §14.2).
-    def scalar_line(line, depth)
-      return if line.nil? || line.depth != depth || Tokens.colon_index(line.content)
-
-      header = Header.parse(line.content)
-      return if header && !header.malformed?
-
-      raise Error, "cannot decode a bare token line inside a scope on line #{line.number}: #{line.text}"
-    end
-
-    def item(line, depth)
+    def item(line)
       rest = line.item_content
       return {} if rest.empty?
       return [] if rest == "[]"
 
       header = usable(Header.parse(rest), line)
-      return item_header(header, line, depth) if header
-      return item_object(rest, depth) if Tokens.colon_index(rest)
+      return item_header(header, line) if header
+      return item_object(rest, line.depth) if Tokens.colon_index(rest)
 
       Token.decode(Tokens.trim(rest))
     end
 
-    def item_header(header, line, depth)
-      return item_object(line.item_content, depth) if header.key
-      raise Error, "cannot decode a keyless fields-bearing header as a list item on line #{line.number}: #{line.text}" if header.fields?
+    def item_header(header, line)
+      return array(header, line.depth) unless header.key || header.fields?
+
+      misplaced(line, "a keyless fields-bearing header as a list item") unless header.key
+      item_object(line.item_content, line.depth)
+    end
 
-      array(header, depth + 1)
+    def misplaced(line, what)
+      raise Error, "cannot decode #{what} on line #{line.number}: #{line.text}" if @strict
     end
 
     def item_object(rest, depth)
@@ -178,20 +158,18 @@ module ToonFu
       key, value = descend { field(synthetic, depth + 1) }
       result = {}
       store(result, key, value, synthetic)
-      following = @lines.peek
-      span(following) if following && following.depth == depth + 1
-      descend { object(depth + 1, result) }
+      descend { object(depth, depth + 1, result) }
     end
 
-    def table(header, depth)
+    def table(header, parent)
       raise Error, "cannot decode a field name repeated in one field list: #{header.duplicate.inspect}" if @strict && header.duplicate
 
-      header.keyed? ? entries(header, depth) : rows(header, depth)
+      header.keyed? ? entries(header, parent) : rows(header, parent)
     end
 
-    def rows(header, depth)
+    def rows(header, parent)
       collected = descend do
-        gather(depth, stop: ->(line) { key_value_line?(line, header.delimiter) }) { |line| row(header, line) }
+        gather(parent, stop: ->(line) { key_value_line?(line, header.delimiter) }) { |line| row(header, line) }
       end
       count(collected.length, header, "row")
       collected
@@ -207,10 +185,10 @@ module ToonFu
       cell.nil? || colon < cell
     end
 
-    def entries(header, depth)
+    def entries(header, parent)
       result = {}
       collected = descend do
-        gather(depth) do |line|
+        gather(parent) do |line|
           colon = Tokens.colon_index(line.content)
           if colon.nil?
             raise Error, "cannot decode an entry row without a colon on line #{line.number}: #{line.text}" if @strict
@@ -226,22 +204,29 @@ module ToonFu
       result
     end
 
-    def gather(depth, stop: nil)
+    def gather(parent, stop: nil)
+      outer = @in_span
       collected = []
-      while (line = @lines.peek) && line.depth == depth
+      depth = content_depth(parent)
+      while depth && (line = @lines.peek) && line.depth > parent
+        if line.depth != depth
+          orphan(line)
+          next
+        end
         break if stop&.call(line)
 
-        span(line) unless collected.empty?
-        @lines.next
+        take
+        @in_span = true
         collected << yield(line)
       end
       collected
+    ensure
+      @in_span = outer
     end
 
-    def span(line)
-      return unless @strict && line.after_blank
-
-      raise Error, "cannot decode a blank line inside a header span, before line #{line.number}: #{line.text}"
+    def take
+      line = @lines.next
+      raise Error, "cannot decode a blank line inside a header span, before line #{line.number}: #{line.text}" if @strict && @in_span && line.after_blank
     end
 
     def row(header, line, text = line.content)
@@ -285,10 +270,12 @@ module ToonFu
       nil
     end
 
-    def guard(line)
-      return unless @strict && line.depth > @depth + 1
+    def content_depth(parent)
+      line = @lines.peek
+      return if line.nil? || line.depth <= parent
+      raise Error, "cannot decode a depth jump on line #{line.number}: #{line.text}" if @strict && line.depth > parent + 1
 
-      raise Error, "cannot decode a depth jump on line #{line.number}: #{line.text}"
+      line.depth
     end
 
     def descend

data/lib/toon_fu/string_literal.rb

@@ -19,6 +19,10 @@ module ToonFu
       quote?(string) ? quote(string) : string
     end
 
+    def root(string)
+      string.start_with?(Lines::BOM) ? quote(string) : encode(string)
+    end
+
     def key(string)
       BARE_KEY.match?(string) ? string : quote(string)
     end

data/lib/toon_fu/tokens.rb

@@ -2,7 +2,8 @@
 
 module ToonFu
   module Tokens
-    QUOTED = /"(?:[^"\\]|\\.)*"/
+    QUOTE_SPAN = /"(?:[^"\\]|\\.?)*"?/
+    TRAILING_WHITESPACE = /[[:space:]]\z/
     UNTIL_COLON = /[^":]+/
     UNTIL_BRACKET = /[^":\[]+/
     UNTIL_BRACE = /[^"{]+/
@@ -17,7 +18,7 @@ module ToonFu
       until scanner.eos?
         next if scanner.skip(pattern)
         return scanner.charpos if stops.include?(scanner.peek(1))
-        return nil unless scanner.skip(QUOTED)
+        scanner.skip(QUOTE_SPAN)
       end
       nil
     end
@@ -40,7 +41,7 @@ module ToonFu
       depth = 0
       scanner = StringScanner.new(text)
       until scanner.eos?
-        if scanner.skip(QUOTED)
+        if scanner.skip(QUOTE_SPAN)
           next
         elsif scanner.skip(/\{/)
           depth += 1
@@ -48,7 +49,7 @@ module ToonFu
           depth -= 1
           return scanner.charpos - 1 if depth.zero?
         else
-          scanner.skip(/[^"{}]+/) || scanner.skip(/./)
+          scanner.skip(/[^"{}]+/)
         end
       end
       nil
@@ -60,7 +61,7 @@ module ToonFu
       depth = 0
       scanner = StringScanner.new(text)
       until scanner.eos?
-        if (quoted = scanner.scan(QUOTED))
+        if (quoted = scanner.scan(QUOTE_SPAN))
           current << quoted
         elsif depth.zero? && scanner.skip(delimiter)
           entries << current
@@ -81,13 +82,13 @@ module ToonFu
       current = +""
       scanner = StringScanner.new(text)
       until scanner.eos?
-        if (quoted = scanner.scan(QUOTED))
+        if (quoted = scanner.scan(QUOTE_SPAN))
           current << quoted
         elsif scanner.skip(delimiter)
           cells << current
           current = +""
         else
-          current << (scanner.scan(UNTIL_CELL_END.fetch(delimiter)) || scanner.getch)
+          current << scanner.scan(UNTIL_CELL_END.fetch(delimiter))
         end
       end
       cells << current

data/lib/toon_fu/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ToonFu
-  VERSION = "4.1.4"
+  VERSION = "4.3.0"
 end

data/lib/toon_fu/writer.rb

@@ -23,6 +23,7 @@ module ToonFu
       case value
       when Hash then mapping("", value)
       when Array then array("", value)
+      when String then line(@strings.root(value))
       else line(scalar(value))
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: toon-fu
 version: !ruby/object:Gem::Version
-  version: 4.1.4
+  version: 4.3.0
 platform: ruby
 authors:
 - Yevhenii Hurin