toon-fu 4.1.2 → 4.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e095746e545c92bf6411c36b2042dde6d305deb9a1363d500d48877f09939b63
-  data.tar.gz: 937a2796b6a6f04afa2469994c94527473e93c66f575747adefea369691628d4
+  metadata.gz: 4b2af3b4dc45618bf09ddc862816f4bd173efc0a127d7a4554bdbc475d7be869
+  data.tar.gz: 502cc64707081cd0ba3a6b1b2577435fe0fc40716f4d7bca003ac61da65f407e
 SHA512:
-  metadata.gz: 979b0210dfed7025dc7be6ba5c32362d7293ea394cb9fbf495374a17d136a9e81d9f8af0419278b891d39bde60c731624050efd4bf5c6ee3dfcc7858d7347b5e
-  data.tar.gz: 481896ac9ff0f47a297e794e90ebd3aae04d3dfbe1267a0e5b8c264f5ef4385a1816b0d241d0bd2ace3047d7748d9b4eefa2487cc251b3f264c8054967552d2c
+  metadata.gz: '094c5bdf3d7f30170e955878a90c688241dafd4bfbc198aeaba4797e774cca2eb7af28bae402b6cd45d206c8b48913c0b09204a1ddbdd503c2e9337e9ee79239'
+  data.tar.gz: 88b2f6001a47e7c04eb0f86711ef1b4f8d0338e291bbc356bff66640c0135ce61c1d604c852a74196de0b2a5ee4c7b06932765e1121e5bf7e054956aaf591188

data/README.md

@@ -4,11 +4,11 @@
 [![Spec drift](https://github.com/hoblin/toon-fu/actions/workflows/spec-drift.yml/badge.svg)](https://github.com/hoblin/toon-fu/actions/workflows/spec-drift.yml)
 [![Gem](https://img.shields.io/gem/v/toon-fu)](https://rubygems.org/gems/toon-fu)
 
-[TOON](https://toonformat.dev/) (Token-Oriented Object Notation) encoder for Ruby, versioned by the spec it implements: the gem's `MAJOR.MINOR` is the TOON spec version it speaks.
+[TOON](https://toonformat.dev/) (Token-Oriented Object Notation) codec for Ruby, versioned by the spec it implements: the gem's `MAJOR.MINOR` is the TOON spec version it speaks.
 
 ## What is this?
 
-TOON is a compact, readable encoding of the JSON data model for LLM prompts: indentation instead of braces, quotes only where needed, and tables for arrays of uniform objects. toon-fu is written from the [specification](https://github.com/toon-format/spec) and runs the spec's reference fixtures as its conformance suite — every encode fixture of the spec version it implements passes.
+TOON is a compact, readable encoding of the JSON data model for LLM prompts: indentation instead of braces, quotes only where needed, and tables for arrays of uniform objects. toon-fu is written from the [specification](https://github.com/toon-format/spec) and runs the spec's reference fixtures as its conformance suite — every encode and decode fixture of the spec version it implements passes.
 
 ## Why?
 
@@ -111,9 +111,45 @@ For ActiveRecord models, pass `record.as_json` (or define `as_toon`).
 
 Everything else raises `ToonFu::Error` — including a `Struct` or `Data` without `as_toon`, circular references, and nesting too deep for the stack.
 
+## Reading TOON
+
+```ruby
+ToonFu.decode("users[2]{id,name}:\n  1,Ada\n  2,Bob")
+# => {"users" => [{"id" => 1, "name" => "Ada"}, {"id" => 2, "name" => "Bob"}]}
+```
+
+You get back plain Ruby: `Hash` with String keys, `Array`, `String`, `Integer`, `Float`, `true`, `false`, `nil`.
+
+Quoting decides the type, and an unquoted token is a number only if the spec says it is one:
+
+```ruby
+ToonFu.decode("a: 42")      # => {"a" => 42}
+ToonFu.decode('a: "42"')    # => {"a" => "42"}
+ToonFu.decode("a: 05")      # => {"a" => "05"}
+ToonFu.decode("a: +1")      # => {"a" => "+1"}
+```
+
+Malformed input raises `ToonFu::Error`:
+
+```ruby
+ToonFu.decode("tags[3]: a,b")
+# ToonFu::Error: cannot decode 2 values where the header declares 3
+```
+
+### Options
+
+- `strict:` — `true` (default) checks what the spec requires: declared lengths, row widths, duplicate keys, indentation. Pass `false` to read a document that bends those rules — a declared length is ignored, a duplicate key takes its last value.
+- `indent_size:` — spaces per nesting level, default `2`.
+
+```ruby
+ToonFu.decode(text, strict: false)
+decoder = ToonFu::Decoder.new(indent_size: 4) # reuse for many documents
+decoder.decode(text)
+```
+
 ## Compared with other Ruby TOON gems
 
-The only one that passes every spec fixture: toon-fu passes all 179 encode fixtures; the table compares the 154 that use default options, which every gem can run.
+The only one that passes every spec fixture: toon-fu passes all 179 encode and all 359 decode fixtures. The table compares encoding — the 154 encode fixtures that use default options, which every gem can run; none of the others reads TOON back.
 
 | Gem | Spec fixtures passed | Speed vs toon-fu |
 |---|---:|---:|

data/lib/toon_fu/decoder.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module ToonFu
+  # Decodes TOON with one set of options, for callers that decode many.
+  class Decoder
+    # @param strict [Boolean] whether to enforce the spec's strict-mode
+    #   checks; with +false+ duplicate keys resolve last-write-wins, leading
+    #   spaces that are not a multiple of indent_size are accepted, and a
+    #   tab in indentation is accepted and counts as one level
+    # @param indent_size [Integer] spaces per nesting level
+    # @raise [ArgumentError] when strict is not a boolean or indent_size is
+    #   not a positive Integer
+    def initialize(strict: true, indent_size: 2)
+      raise ArgumentError, "strict must be true or false, got #{strict.inspect}" unless [true, false].include?(strict)
+      raise ArgumentError, "indent_size must be a positive Integer, got #{indent_size.inspect}" unless indent_size.is_a?(Integer) && indent_size.positive?
+
+      @strict = strict
+      @indent_size = indent_size
+    end
+
+    # Decodes TOON into the JSON data model.
+    #
+    # Returns +Hash+ with String keys, +Array+, +String+, +Integer+,
+    # +Float+, +true+, +false+ or +nil+. An unquoted token is a number only
+    # when it matches the spec's own grammar, so +.5+, +1.+, +05+, +Infinity+
+    # and +1_000+ decode as strings.
+    #
+    # Out-of-range numbers: a token the grammar accepts but +Float+ cannot
+    # hold without losing its magnitude, such as +1e400+ or +1e-400+, raises
+    # rather than becoming infinity or zero. Plain digits become an exact
+    # +Integer+ of any size; a fraction or an exponent goes through +Float+
+    # and carries its 53-bit precision, even when the result is integral.
+    #
+    # Key order is the document's, except in the tabular and keyed tabular
+    # forms, where it is the header's field order at every level. Nesting
+    # deeper than 100 levels raises rather than exhausting the stack.
+    #
+    # @param text [String] UTF-8, or an encoding that transcodes to it
+    # @return [Hash, Array, String, Integer, Float, true, false, nil]
+    # @raise [Error] for text that is not a String or not valid UTF-8; for a
+    #   number outside the Float range; for an invalid escape, an
+    #   unterminated quoted string, or content after a closing quote; for a
+    #   key without a colon; for a bare token line inside a scope; for
+    #   content after a completed root form; for nesting beyond the depth
+    #   limit; and, when strict, for a declared length or row width that
+    #   does not match, a malformed header, a duplicate key, a blank line
+    #   inside a header's scope, and indentation that is not a multiple of
+    #   indent_size
+    def decode(text)
+      Reader.new(@strict, @indent_size).read(utf8(text))
+    end
+
+    private
+
+    def utf8(text)
+      raise Error, "cannot decode #{text.class}; pass a String" unless text.is_a?(String)
+
+      text = text.dup.force_encoding(Encoding::UTF_8) if text.encoding == Encoding::BINARY
+      text = text.encode(Encoding::UTF_8) unless text.encoding == Encoding::UTF_8
+      raise Error, "cannot decode text that is not valid UTF-8" unless text.valid_encoding?
+
+      text
+    rescue EncodingError => error
+      raise Error, "cannot decode text as UTF-8: #{error.message}"
+    end
+  end
+end

data/lib/toon_fu/field_list.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module ToonFu
+  class FieldList
+    def self.parse(text, delimiter)
+      new(text, delimiter).parse
+    end
+
+    def initialize(text, delimiter)
+      @text = text
+      @delimiter = delimiter
+    end
+
+    def parse
+      return nil if @text.empty?
+
+      entries = []
+      Tokens.split_fields(@text, @delimiter).each do |entry|
+        return nil if foreign?(entry)
+
+        name, nested = split(entry)
+        return nil if name.nil?
+
+        entries << [name, nested]
+      end
+      entries
+    end
+
+    private
+
+    # A field list is split by the bracket segment's delimiter alone, so
+    # another delimiter left unquoted in an entry is a header defect (§6).
+    # Quoting protects it, which is how the encoder emits {id,"a|b"}.
+    def foreign?(entry)
+      others = Encoder::DELIMITERS - [@delimiter]
+      Tokens.index_of(entry, Tokens::UNTIL_FOREIGN.fetch(@delimiter), others.join) ? true : false
+    end
+
+    def split(entry)
+      brace = Tokens.brace_index(entry)
+      return [Tokens.key(entry), nil] if brace.nil?
+
+      name = entry[0...brace]
+      group = entry[brace..]
+      closing = Tokens.closing_brace(group)
+      return [nil, nil] unless closing == group.length - 1
+
+      nested = self.class.parse(group[1...closing], @delimiter)
+      nested.nil? ? [nil, nil] : [Tokens.key(name), nested]
+    end
+  end
+end

data/lib/toon_fu/header.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module ToonFu
+  class Header
+    LENGTH = /\A(?:0|[1-9][0-9]*)\z/
+    DELIMITERS = {"" => ",", "\t" => "\t", "|" => "|"}.freeze
+
+    attr_reader :key, :length, :delimiter, :fields, :inline
+
+    def self.parse(content)
+      bracket = Tokens.bracket_index(content)
+      return nil if bracket.nil?
+
+      new(content, bracket).parse
+    end
+
+    def initialize(content, bracket)
+      @content = content
+      @bracket = bracket
+      @malformed = false
+    end
+
+    # A line carrying a bracket segment is either a header, or an attempt at
+    # one that §6 makes a strict-mode error; only a line without one falls
+    # through to the key-value class untouched.
+    def malformed?
+      @malformed
+    end
+
+    def parse
+      @key = @bracket.zero? ? nil : @content[0...@bracket]
+      return nil unless @key.nil? || Tokens.whole_token?(@key)
+      return broken if @key&.match?(/\s\z/)
+      return broken unless (closing = @content.index("]", @bracket))
+
+      return broken unless segment(@content[(@bracket + 1)...closing])
+      return broken unless tail(closing + 1)
+      # §6: a keyed header's field list is required, and a fields-bearing
+      # header carries no inline content.
+      return broken if @keyed && !fields?
+      return broken if fields? && !@inline.strip.empty?
+
+      self
+    end
+
+    def broken
+      @malformed = true
+      self
+    end
+
+    def keyed?
+      @keyed
+    end
+
+    def fields?
+      !@fields.nil?
+    end
+
+    def leaves
+      @leaves ||= walk(@fields)
+    end
+
+    # Names repeated across nesting levels are not duplicates (§9.3): only a
+    # repeat within one list counts.
+    def duplicate
+      return @duplicate if defined?(@duplicate)
+
+      @duplicate = repeated(@fields)
+    end
+
+    private
+
+    def segment(text)
+      @keyed = false
+      if text.end_with?(":")
+        @keyed = true
+        text = text[0...-1]
+      elsif (marker = text[-1]) && DELIMITERS.key?(marker) && !marker.empty?
+        @delimiter = DELIMITERS.fetch(marker)
+        text = text[0...-1]
+        if text.end_with?(":")
+          @keyed = true
+          text = text[0...-1]
+        end
+      end
+      @delimiter ||= ","
+      return false unless LENGTH.match?(text)
+
+      @length = Integer(text, 10)
+      true
+    end
+
+    def tail(index)
+      rest = @content[index..]
+      if rest.start_with?("{")
+        closing = Tokens.closing_brace(rest)
+        return false if closing.nil?
+
+        @fields = FieldList.parse(rest[1...closing], @delimiter)
+        return false if @fields.nil?
+
+        rest = rest[(closing + 1)..]
+      end
+      return false unless rest.start_with?(":")
+
+      @inline = rest[1..]
+      true
+    end
+
+    def walk(fields)
+      fields.flat_map { |name, nested| nested ? walk(nested) : [name] }
+    end
+
+    def repeated(fields)
+      names = fields.map(&:first)
+      duplicate = names.tally.find { |_, count| count > 1 }
+      return duplicate.first if duplicate
+
+      fields.filter_map { |_, nested| nested && repeated(nested) }.first
+    end
+  end
+end

data/lib/toon_fu/lines.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module ToonFu
+  class Line
+    ITEM = /\A-(?: |\z)/
+
+    attr_reader :content, :depth, :number, :text
+
+    attr_reader :after_blank
+
+    def initialize(content, depth, number, text = content, after_blank = false)
+      @content = content
+      @depth = depth
+      @number = number
+      @text = text
+      @after_blank = after_blank
+    end
+
+    def item?
+      ITEM.match?(@content)
+    end
+
+    def item_content
+      @content.sub(ITEM, "")
+    end
+  end
+
+  class Lines
+    BOM = "\u{feff}"
+    COMMENT = /\A *#/
+    TRAILING_SPACES = / +\z/
+    BLANK = /\A\s*\z/
+    INDENTATION = /\A[ \t]*/
+
+    def initialize(text, strict, indent_size)
+      @strict = strict
+      @indent_size = indent_size
+      @lines = significant(text.delete_prefix(BOM))
+      @index = 0
+    end
+
+    def empty?
+      @index >= @lines.length
+    end
+
+    def one?
+      @lines.length - @index == 1
+    end
+
+    def peek
+      @lines[@index]
+    end
+
+    def next
+      line = @lines[@index]
+      @index += 1
+      line
+    end
+
+    def number
+      peek ? peek.number : @lines.length
+    end
+
+    private
+
+    def significant(text)
+      blank = false
+      text.split("\n", -1).each_with_index.filter_map do |raw, offset|
+        line = raw.delete_suffix("\r").sub(TRAILING_SPACES, "")
+        next if COMMENT.match?(line)
+
+        if BLANK.match?(line)
+          blank = true
+          next
+        end
+
+        built = build(line, offset + 1, blank)
+        blank = false
+        built
+      end
+    end
+
+    def build(line, number, blank)
+      Line.new(line.sub(INDENTATION, ""), depth(line, number), number, line, blank)
+    end
+
+    # Non-strict mode accepts a tab as indentation (§12), where each tab
+    # counts as one level; strict mode rejects it.
+    def depth(line, number)
+      indent = line[INDENTATION]
+      raise Error, "cannot decode a line indented with a tab on line #{number}: #{line}" if @strict && indent.include?("\t")
+
+      spaces = indent.count(" ")
+      raise Error, "cannot decode an indentation of #{spaces} spaces on line #{number}, not a multiple of #{@indent_size}: #{line}" if @strict && !(spaces % @indent_size).zero?
+
+      spaces / @indent_size + indent.count("\t")
+    end
+  end
+end

data/lib/toon_fu/reader.rb

@@ -0,0 +1,303 @@
+# frozen_string_literal: true
+
+module ToonFu
+  class Reader
+    MAX_DEPTH = 100
+
+    def initialize(strict, indent_size)
+      @strict = strict
+      @indent_size = indent_size
+      @depth = 0
+    end
+
+    def read(text)
+      @lines = Lines.new(text, @strict, @indent_size)
+      return {} if @lines.empty?
+
+      root
+    end
+
+    private
+
+    def root
+      line = @lines.peek
+      return empty_array_root if line.content == "[]"
+
+      header = usable(Header.parse(line.content), line)
+      return keyless_root(header) if header && header.key.nil?
+      return scalar_root(line) if @lines.one? && Tokens.colon_index(line.content).nil?
+
+      object(0)
+    end
+
+    def keyless_root(header)
+      @lines.next
+      value = header.fields? ? table(header, 1) : array(header, 1)
+      trailing
+      value
+    end
+
+    def empty_array_root
+      @lines.next
+      trailing
+      []
+    end
+
+    def scalar_root(line)
+      @lines.next
+      Token.decode(Tokens.trim(line.content))
+    end
+
+    def trailing
+      return if @lines.empty?
+
+      raise Error, "cannot decode content after the root form: #{@lines.peek.text}"
+    end
+
+    def object(depth, result = {})
+      while (line = @lines.peek) && line.depth >= depth
+        if line.depth > depth
+          orphan(line)
+          next
+        end
+
+        @lines.next
+        key, value = field(line, depth)
+        store(result, key, value, line)
+      end
+      result
+    end
+
+    def orphan(line)
+      raise Error, "cannot decode a line that belongs to no scope on line #{line.number}: #{line.text}" if @strict
+
+      @lines.next
+    end
+
+    def store(result, key, value, line)
+      raise Error, "cannot decode a duplicate key on line #{line.number}: #{key.inspect}" if @strict && result.key?(key)
+
+      result[key] = value
+    end
+
+    def field(line, depth)
+      header = usable(Header.parse(line.content), line)
+      return [Tokens.key(header.key), header_value(header, depth)] if header&.key
+      raise Error, "cannot decode a keyless array header in object field position on line #{line.number}: #{line.text}" if header
+
+      colon = Tokens.colon_index(line.content)
+      raise Error, "cannot decode a line without a colon after its key on line #{line.number}: #{line.text}" if colon.nil?
+
+      key = Tokens.key(line.content[0...colon])
+      [key, field_value(Tokens.trim(line.content[(colon + 1)..]), depth)]
+    end
+
+    def field_value(text, depth)
+      return [] if text == "[]"
+      return nested(depth) if text.empty?
+
+      Token.decode(text)
+    end
+
+    def nested(depth)
+      line = @lines.peek
+      return {} if line.nil? || line.depth <= depth
+
+      guard(line)
+      descend { object(depth + 1) }
+    end
+
+    def header_value(header, depth)
+      return table(header, depth + 1) if header.fields?
+      return [] if header.length.zero? && !items?(depth + 1)
+
+      array(header, depth + 1)
+    end
+
+    def items?(depth)
+      line = @lines.peek
+      !line.nil? && line.depth == depth
+    end
+
+    def array(header, depth)
+      return inline(header) unless header.inline.empty?
+
+      descend { list(header, depth) }
+    end
+
+    def inline(header)
+      cells = Tokens.split(header.inline, header.delimiter).map { |cell| Token.decode(Tokens.trim(cell)) }
+      count(cells.length, header, "value")
+      cells
+    end
+
+    def list(header, depth)
+      items = []
+      while (line = @lines.peek) && line.depth == depth && line.item?
+        span(line) unless items.empty?
+        @lines.next
+        items << item(line, depth)
+      end
+      scalar_line(@lines.peek, depth)
+      count(items.length, header, "item")
+      items
+    end
+
+    # A scalar line is valid only as a root primitive; inside a scope it is a
+    # structural error in strict and non-strict mode alike (§5.2, §14.2).
+    def scalar_line(line, depth)
+      return if line.nil? || line.depth != depth || Tokens.colon_index(line.content)
+
+      header = Header.parse(line.content)
+      return if header && !header.malformed?
+
+      raise Error, "cannot decode a bare token line inside a scope on line #{line.number}: #{line.text}"
+    end
+
+    def item(line, depth)
+      rest = line.item_content
+      return {} if rest.empty?
+      return [] if rest == "[]"
+
+      header = usable(Header.parse(rest), line)
+      return item_header(header, line, depth) if header
+      return item_object(rest, depth) if Tokens.colon_index(rest)
+
+      Token.decode(Tokens.trim(rest))
+    end
+
+    def item_header(header, line, depth)
+      return item_object(line.item_content, depth) if header.key
+      raise Error, "cannot decode a keyless fields-bearing header as a list item on line #{line.number}: #{line.text}" if header.fields?
+
+      array(header, depth + 1)
+    end
+
+    def item_object(rest, depth)
+      synthetic = Line.new(rest, depth + 1, @lines.number)
+      key, value = descend { field(synthetic, depth + 1) }
+      result = {}
+      store(result, key, value, synthetic)
+      following = @lines.peek
+      span(following) if following && following.depth == depth + 1
+      descend { object(depth + 1, result) }
+    end
+
+    def table(header, depth)
+      raise Error, "cannot decode a field name repeated in one field list: #{header.duplicate.inspect}" if @strict && header.duplicate
+
+      header.keyed? ? entries(header, depth) : rows(header, depth)
+    end
+
+    def rows(header, depth)
+      collected = descend do
+        gather(depth, stop: ->(line) { key_value_line?(line, header.delimiter) }) { |line| row(header, line) }
+      end
+      count(collected.length, header, "row")
+      collected
+    end
+
+    # §9.3: at row depth a line whose first unquoted colon precedes its first
+    # unquoted delimiter is a key-value line, and the rows end before it.
+    def key_value_line?(line, delimiter)
+      colon = Tokens.colon_index(line.content)
+      return false if colon.nil?
+
+      cell = Tokens.index_of(line.content, Tokens::UNTIL_CELL_END.fetch(delimiter), delimiter)
+      cell.nil? || colon < cell
+    end
+
+    def entries(header, depth)
+      result = {}
+      collected = descend do
+        gather(depth) do |line|
+          colon = Tokens.colon_index(line.content)
+          if colon.nil?
+            raise Error, "cannot decode an entry row without a colon on line #{line.number}: #{line.text}" if @strict
+
+            next nil
+          end
+          [Tokens.key(line.content[0...colon]), row(header, line, line.content[(colon + 1)..]), line]
+        end
+      end
+      collected.compact!
+      count(collected.length, header, "entry row")
+      collected.each { |key, value, line| store(result, key, value, line) }
+      result
+    end
+
+    def gather(depth, stop: nil)
+      collected = []
+      while (line = @lines.peek) && line.depth == depth
+        break if stop&.call(line)
+
+        span(line) unless collected.empty?
+        @lines.next
+        collected << yield(line)
+      end
+      collected
+    end
+
+    def span(line)
+      return unless @strict && line.after_blank
+
+      raise Error, "cannot decode a blank line inside a header span, before line #{line.number}: #{line.text}"
+    end
+
+    def row(header, line, text = line.content)
+      cells = Tokens.split(text, header.delimiter)
+      cells = [] if cells.length == 1 && Tokens.trim(cells.first).empty?
+      width(cells.length, header, line)
+      shape(header.fields, cells.each)
+    end
+
+    def shape(fields, cells)
+      fields.each_with_object({}) do |(name, nested), result|
+        if nested
+          result[name] = shape(nested, cells)
+        else
+          cell = begin
+            cells.next
+          rescue StopIteration
+            next
+          end
+          result[name] = Token.decode(Tokens.trim(cell))
+        end
+      end
+    end
+
+    def count(actual, header, noun)
+      return unless @strict && actual != header.length
+
+      raise Error, "cannot decode #{actual} #{noun}#{"s" unless actual == 1} where the header declares #{header.length}"
+    end
+
+    def width(actual, header, line)
+      return unless @strict && actual != header.leaves.length
+
+      raise Error, "cannot decode #{actual} cells on line #{line.number} where the header declares #{header.leaves.length} field#{"s" unless header.leaves.length == 1}"
+    end
+
+    def usable(header, line)
+      return header unless header&.malformed?
+      raise Error, "cannot decode a malformed array header on line #{line.number}: #{line.text}" if @strict
+
+      nil
+    end
+
+    def guard(line)
+      return unless @strict && line.depth > @depth + 1
+
+      raise Error, "cannot decode a depth jump on line #{line.number}: #{line.text}"
+    end
+
+    def descend
+      @depth += 1
+      raise Error, "cannot decode nesting deeper than #{MAX_DEPTH} levels" if @depth > MAX_DEPTH
+
+      yield
+    ensure
+      @depth -= 1
+    end
+  end
+end

data/lib/toon_fu/token.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module ToonFu
+  class Token
+    NUMBER = /\A-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:e[+-]?[0-9]+)?\z/i
+    INTEGER = /\A-?(?:0|[1-9][0-9]*)\z/
+    ESCAPE = /\\(?:(["\\nrt])|u([0-9a-fA-F]{4})|(.|\z))/m
+    UNESCAPES = {'"' => '"', "\\" => "\\", "n" => "\n", "r" => "\r", "t" => "\t"}.freeze
+    SURROGATES = (0xd800..0xdfff)
+
+    def self.key(text)
+      text.start_with?('"') ? quoted(text) : text
+    end
+
+    def self.decode(text)
+      return quoted(text) if text.start_with?('"')
+      return nil if text == "null"
+      return true if text == "true"
+      return false if text == "false"
+      return number(text) if NUMBER.match?(text)
+
+      text
+    end
+
+    def self.quoted(text)
+      closing = closing_quote(text)
+      raise Error, "cannot decode an unterminated quoted string: #{text}" if closing.nil?
+      raise Error, "cannot decode content after a closing quote: #{text}" unless closing == text.length - 1
+
+      unescape(text[1...closing])
+    end
+
+    def self.closing_quote(text)
+      index = 1
+      while index < text.length
+        case text[index]
+        when "\\" then index += 2
+        when '"' then return index
+        else index += 1
+        end
+      end
+      nil
+    end
+
+    def self.unescape(body)
+      body.gsub(ESCAPE) do
+        if (literal = UNESCAPES[$1])
+          literal
+        elsif $2
+          codepoint($2.hex)
+        else
+          raise Error, "cannot decode the escape sequence \\#{$3}"
+        end
+      end
+    end
+
+    def self.codepoint(value)
+      raise Error, "cannot decode the lone surrogate escape \\u#{format("%04x", value)}" if SURROGATES.cover?(value)
+
+      value.chr(Encoding::UTF_8)
+    end
+
+    def self.number(text)
+      return Integer(text, 10) if INTEGER.match?(text)
+
+      float = Float(text)
+      raise Error, "cannot decode #{text} as a Float without losing its magnitude" if float.infinite? || (float.zero? && nonzero?(text))
+
+      float.zero? ? 0.0 : float
+    end
+
+    def self.nonzero?(text)
+      text.split(/e/i).first.match?(/[1-9]/)
+    end
+
+    private_class_method :quoted, :unescape, :codepoint, :number, :nonzero?
+  end
+end

data/lib/toon_fu/tokens.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module ToonFu
+  module Tokens
+    QUOTED = /"(?:[^"\\]|\\.)*"/
+    UNTIL_COLON = /[^":]+/
+    UNTIL_BRACKET = /[^":\[]+/
+    UNTIL_BRACE = /[^"{]+/
+    BRACE = /[{}]/
+    SURROUNDING_SPACES = /\A +| +\z/
+    UNTIL_CELL_END = Encoder::DELIMITERS.to_h { |d| [d, /[^"#{Regexp.escape(d)}]+/] }.freeze
+    UNTIL_FIELD_END = Encoder::DELIMITERS.to_h { |d| [d, /[^"{}#{Regexp.escape(d)}]+/] }.freeze
+    UNTIL_FOREIGN = Encoder::DELIMITERS.to_h { |d| [d, /[^"#{Regexp.escape((Encoder::DELIMITERS - [d]).join)}]+/] }.freeze
+
+    def self.index_of(text, pattern, stops)
+      scanner = StringScanner.new(text)
+      until scanner.eos?
+        next if scanner.skip(pattern)
+        return scanner.charpos if stops.include?(scanner.peek(1))
+        return nil unless scanner.skip(QUOTED)
+      end
+      nil
+    end
+
+    def self.colon_index(text)
+      index_of(text, UNTIL_COLON, ":")
+    end
+
+    def self.bracket_index(text)
+      index = index_of(text, UNTIL_BRACKET, "[:")
+      (index && text[index] == "[") ? index : nil
+    end
+
+    def self.brace_index(text)
+      index = index_of(text, UNTIL_BRACE, "{")
+      (index && text[index] == "{") ? index : nil
+    end
+
+    def self.closing_brace(text)
+      depth = 0
+      scanner = StringScanner.new(text)
+      until scanner.eos?
+        if scanner.skip(QUOTED)
+          next
+        elsif scanner.skip(/\{/)
+          depth += 1
+        elsif scanner.skip(/\}/)
+          depth -= 1
+          return scanner.charpos - 1 if depth.zero?
+        else
+          scanner.skip(/[^"{}]+/) || scanner.skip(/./)
+        end
+      end
+      nil
+    end
+
+    def self.split_fields(text, delimiter)
+      entries = []
+      current = +""
+      depth = 0
+      scanner = StringScanner.new(text)
+      until scanner.eos?
+        if (quoted = scanner.scan(QUOTED))
+          current << quoted
+        elsif depth.zero? && scanner.skip(delimiter)
+          entries << current
+          current = +""
+        elsif (brace = scanner.scan(BRACE))
+          depth += (brace == "{") ? 1 : -1
+          current << brace
+        else
+          current << (scanner.scan(UNTIL_FIELD_END.fetch(delimiter)) || scanner.getch)
+        end
+      end
+      entries << current
+      entries
+    end
+
+    def self.split(text, delimiter)
+      cells = []
+      current = +""
+      scanner = StringScanner.new(text)
+      until scanner.eos?
+        if (quoted = scanner.scan(QUOTED))
+          current << quoted
+        elsif scanner.skip(delimiter)
+          cells << current
+          current = +""
+        else
+          current << (scanner.scan(UNTIL_CELL_END.fetch(delimiter)) || scanner.getch)
+        end
+      end
+      cells << current
+      cells
+    end
+
+    def self.trim(text)
+      text.gsub(SURROUNDING_SPACES, "")
+    end
+
+    def self.whole_token?(text)
+      return true unless text.start_with?('"')
+
+      Token.closing_quote(text) == text.length - 1
+    end
+
+    def self.key(text)
+      Token.key(trim(text))
+    end
+  end
+end

data/lib/toon_fu/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ToonFu
-  VERSION = "4.1.2"
+  VERSION = "4.1.3"
 end

data/lib/toon_fu.rb

@@ -1,23 +1,32 @@
 # frozen_string_literal: true
 
 require "date"
+require "strscan"
 
 require_relative "toon_fu/version"
 require_relative "toon_fu/encoder"
+require_relative "toon_fu/decoder"
 require_relative "toon_fu/float_literal"
 require_relative "toon_fu/decimal_literal"
 require_relative "toon_fu/string_literal"
 require_relative "toon_fu/normalizer"
 require_relative "toon_fu/fields"
 require_relative "toon_fu/writer"
+require_relative "toon_fu/token"
+require_relative "toon_fu/tokens"
+require_relative "toon_fu/field_list"
+require_relative "toon_fu/header"
+require_relative "toon_fu/lines"
+require_relative "toon_fu/reader"
 require_relative "toon_fu/encodable"
 
 # TOON (Token-Oriented Object Notation) for Ruby.
 module ToonFu
-  # Raised when a value has no TOON representation; see {Encoder#encode}.
+  # Raised when a value has no TOON representation, or when TOON cannot be
+  # read; see {Encoder#encode} and {Decoder#decode}.
   class Error < StandardError; end
 
-  private_constant :FloatLiteral, :DecimalLiteral, :StringLiteral, :Normalizer, :Fields, :Writer
+  private_constant :FloatLiteral, :DecimalLiteral, :StringLiteral, :Normalizer, :Fields, :Writer, :Token, :Tokens, :FieldList, :Header, :Line, :Lines, :Reader
 
   # Encodes a value as TOON.
   #
@@ -43,6 +52,27 @@ module ToonFu
     (options.empty? ? DEFAULT_ENCODER : Encoder.new(**options)).encode(value)
   end
 
+  # Decodes TOON into the JSON data model.
+  #
+  #   ToonFu.decode("hello")                   # => "hello"
+  #   ToonFu.decode("\"42\"")                  # => "42"
+  #   ToonFu.decode("")                        # => {}
+  #   ToonFu.decode("[]")                      # => []
+  #   ToonFu.decode("name: Ada\nadmin: true")  # => {"name" => "Ada", "admin" => true}
+  #   ToonFu.decode("user:\n  id: 1")          # => {"user" => {"id" => 1}}
+  #   ToonFu.decode("tags[2]: a,b")            # => {"tags" => ["a", "b"]}
+  #   ToonFu.decode("[2]{id}:\n  1\n  2")      # => [{"id" => 1}, {"id" => 2}]
+  #   ToonFu.decode("[2:]{x}:\n  a: 1\n  b: 2") # => {"a" => {"x" => 1}, "b" => {"x" => 2}}
+  #
+  # @param text [String] see {Decoder#decode}
+  # @param options [Hash] see {Decoder#initialize}
+  # @return [Hash, Array, String, Integer, Float, true, false, nil]
+  # @raise [Error] see {Decoder#decode}
+  def self.decode(text, **options)
+    (options.empty? ? DEFAULT_DECODER : Decoder.new(**options)).decode(text)
+  end
+
   DEFAULT_ENCODER = Ractor.make_shareable(Encoder.new)
-  private_constant :DEFAULT_ENCODER
+  DEFAULT_DECODER = Ractor.make_shareable(Decoder.new)
+  private_constant :DEFAULT_ENCODER, :DEFAULT_DECODER
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: toon-fu
 version: !ruby/object:Gem::Version
-  version: 4.1.2
+  version: 4.1.3
 platform: ruby
 authors:
 - Yevhenii Hurin
@@ -25,12 +25,19 @@ files:
 - lib/toon-fu.rb
 - lib/toon_fu.rb
 - lib/toon_fu/decimal_literal.rb
+- lib/toon_fu/decoder.rb
 - lib/toon_fu/encodable.rb
 - lib/toon_fu/encoder.rb
+- lib/toon_fu/field_list.rb
 - lib/toon_fu/fields.rb
 - lib/toon_fu/float_literal.rb
+- lib/toon_fu/header.rb
+- lib/toon_fu/lines.rb
 - lib/toon_fu/normalizer.rb
+- lib/toon_fu/reader.rb
 - lib/toon_fu/string_literal.rb
+- lib/toon_fu/token.rb
+- lib/toon_fu/tokens.rb
 - lib/toon_fu/version.rb
 - lib/toon_fu/writer.rb
 homepage: https://github.com/hoblin/toon-fu