ruby-json-toon 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2eda6d64edfc1cc2a7f956141060c3705bf4a66e668ce10622f42c30ba2ff92b
-  data.tar.gz: f8dc21d8f08cc1751463d3b664c00c27f5a83720b715d0894be5e71c29ea0a12
+  metadata.gz: 239c3f74f2d761583b30c69d05882a1072d02c423b84446bbae9e7d241b09a2b
+  data.tar.gz: 4a9b92b5663e92fa1f99d3f61f9b80be0ad2b0fef1a5ebf16564573b3338e7b7
 SHA512:
-  metadata.gz: 435a729daee86264c5eab4917f6191fe0fc70646c7c49574a85a74c34b19ecebd58fd1e04fff136a36667346a2aa511f3a81c8f8732c4bcf6621ae2ccbc6ffd7
-  data.tar.gz: 00ae26b0633543fb121c7910efe3ab5d79b699d1143bd963e659fc2b3f94f8cbc9a1278ca81bbb65343476a05a44fb8d216aedd892ed108998451c9491d729a1
+  metadata.gz: c54ca35838a6697763813f709ee382162d5efdefae022ed00cba6849a63f3135e06af366b217552ea2c6a941941db82cf0069fe03a960f178ff7130fff8f470e
+  data.tar.gz: ed409049fb431d60f786edc49ace9dd827429f59b67f01e0b08c238e8cc2619cb1b704dbea26888e392a48e8894d2bb8e1e41ed7ef2c938460908cfd34e79591

data/CHANGELOG.md

@@ -21,11 +21,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Initial Encoder class implementation
 - Tests for JSON to TOON conversion
 
-##[0.2.0] - First tagged release
+## [0.2.0] - First Encoder release
 ### Added
- - Updated to run release workflow on tag pushes
+- Updated to run release workflow on tag pushes
 
-## [1.0.0] - TBD
+## [0.3.0] - Decoder implementation
+### Added
+- Added decoder implementation
+- Added ability to seperately require decoder using: require "toon_to_json"
 
 ### Added
 - Initial release

data/README.md

@@ -1,10 +1,25 @@
 # JSON to TOON
 
-A lightweight, zero-dependency Ruby library for converting JSON data to TOON (Token-Oriented Object Notation) format, achieving 30-60% token reduction for LLM applications.
+Lightweight Ruby library for converting JSON data to TOON (Token-Oriented Object Notation), achieving 30–60% token reduction for LLM applications.
+
+## Summary
+
+Convert JSON to TOON (Token-Oriented Object Notation)
+
+Authors: Jitendra Neema  
+Contact: jitendra.neema.8@gmail.com
+
+Homepage: https://github.com/jitendra-neema/ruby-json-toon  
+Documentation: https://rubydoc.info/gems/ruby-json-toon  
+Changelog: https://github.com/jitendra-neema/ruby-json-toon/blob/main/CHANGELOG.md  
+Bug tracker: https://github.com/jitendra-neema/ruby-json-toon/issues  
+Rubygems: https://rubygems.org/gems/ruby-json-toon
+
+Requires Ruby >= 2.7.0
 
 ## What is TOON?
 
-TOON (Token-Oriented Object Notation) is a compact, indentation-based data format optimized for LLM token efficiency. It uses 30-60% fewer tokens than JSON while remaining human-readable.
+TOON (Token-Oriented Object Notation) is a compact, indentation-based data format optimized for LLM token efficiency. It uses roughly 30–60% fewer tokens than JSON while remaining human-readable.
 
 ### Comparison
 
@@ -27,16 +42,22 @@ users[2]{id,name,role}:
 
 ## Installation
 
-Add to your Gemfile:
+Install the gem:
+
+```bash
+gem install ruby-json-toon
+```
+
+Or add to your Gemfile:
 
 ```ruby
-gem 'json_to_toon'
+gem 'ruby-json-toon'
 ```
 
-Or install directly:
+Require the library in your code (require path follows the library files):
 
-```bash
-gem install json_to_toon
+```ruby
+require 'json_to_toon'
 ```
 
 ## Quick Start
@@ -57,10 +78,6 @@ json_data = JSON.parse('{"users":[{"id":1,"name":"Alice"}]}')
 toon = JsonToToon.encode(json_data)
 ```
 
-## Documentation
-
-See full documentation at [rubydoc.info](https://rubydoc.info/gems/json_to_toon)
-
 ## Options
 
 ```ruby
@@ -73,8 +90,13 @@ JsonToToon.encode(data,
 
 ## Development
 
+Clone the repo, install dependencies, run tests, and build the gem:
+
 ```bash
-# Install dependencies
+git clone https://github.com/jitendra-neema/ruby-json-toon
+cd ruby-json-toon
+
+# Install development dependencies
 bundle install
 
 # Run tests
@@ -84,15 +106,18 @@ bundle exec rspec
 bundle exec rubocop
 
 # Build gem
-gem build json_to_toon.gemspec
+gem build ruby-json-toon.gemspec
 ```
 
+Development dependencies (from the gemspec): benchmark-ips, memory_profiler, rake, rspec, rubocop, rubocop-rake, rubocop-rspec, simplecov.
+
 ## License
 
-MIT License - see LICENSE file for details
+MIT License — see LICENSE file for details.
 
 ## Links
 
-- [TOON Specification](https://toonformat.dev)
-- [GitHub Repository](https://github.com/jitendra-neema/json_to_toon)
-- [Bug Tracker](https://github.com/jitendra-neema/json_to_toon/issues)
+- TOON Specification: https://toonformat.dev
+- Homepage / source: https://github.com/jitendra-neema/ruby-json-toon
+- Documentation: https://rubydoc.info/gems/ruby-json-toon
+- Bug tracker: https://github.com/jitendra-neema/ruby-json-toon/issues

data/lib/json_to_toon/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JsonToToon
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/lib/toon_to_json/decoder.rb

@@ -0,0 +1,527 @@
+# frozen_string_literal: true
+
+module ToonToJson
+  # Efficiently converts TOON format directly to JSON string
+  class Decoder
+    def decode(str)
+      return 'null' if str.nil? || str.empty?
+
+      lines = str.to_s.split("\n")
+
+      # Single-line primitive detection
+      if lines.length == 1
+        line = lines.first.strip
+
+        # Check if it's a primitive (not a TOON structure)
+        # TOON structures have:
+        # - Colons (key:value or key:)
+        # - List items (starts with "- " - note the space!)
+        # - Array headers (starts with [)
+
+        is_structure = line.include?(':') ||
+                       line.match?(/\A-\s/) ||  # "- " with space (list item)
+                       line.match?(/\A\[/)      # Array header
+
+        return primitive_to_json(line) unless is_structure
+      end
+
+      @lines = lines.map { |l| { raw: l, indent: leading_spaces(l), text: l.lstrip } }
+      @i = 0
+      @indent_unit = detect_indent_unit
+      @output = []
+
+      parse_block(0)
+      @output.join
+    end
+
+    private
+
+    def leading_spaces(line)
+      line[/^\s*/].size
+    end
+
+    def detect_indent_unit
+      prev = 0
+      @lines.each do |ln|
+        next if ln[:raw].strip.empty?
+        return ln[:indent] - prev if ln[:indent] > prev
+
+        prev = ln[:indent]
+      end
+      2
+    end
+
+    def parse_block(min_indent)
+      return parse_object_block(min_indent) if @i >= @lines.length
+
+      # Check if first line is array header
+      first_line = @lines[@i][:text]
+      if first_line.match?(/\A\[/)
+        array_header = parse_array_header(first_line)
+        if array_header && array_header[:key].nil?
+          # Root-level array
+          @i += 1
+          parse_array_body(array_header, @lines[@i - 1][:indent] + @indent_unit)
+          return
+        end
+      end
+
+      # Check if list format (starts with "- " with space)
+      start_i = @i
+      while start_i < @lines.length
+        ln = @lines[start_i]
+        break if ln[:raw].strip.empty?
+        break if ln[:indent] < min_indent
+
+        return parse_list_block(min_indent) if ln[:indent] >= min_indent && ln[:text].match?(/\A-\s/)
+
+        break if ln[:text].include?(':')
+
+        start_i += 1
+      end
+
+      parse_object_block(min_indent)
+    end
+
+    def parse_object_block(min_indent)
+      @output << '{'
+      first = true
+
+      while @i < @lines.length
+        ln = @lines[@i]
+        break if ln[:raw].strip.empty?
+        break if ln[:indent] < min_indent
+
+        # Array header
+        if (array_header = parse_array_header(ln[:text]))
+          @output << ',' unless first
+          first = false
+
+          key = array_header[:key]
+          @i += 1
+
+          if key
+            @output << json_string(key)
+            @output << ':'
+          end
+
+          parse_array_body(array_header, ln[:indent] + @indent_unit)
+          next
+        end
+
+        # Key-value pair
+        if (kv = parse_key_value_line(ln[:text]))
+          @output << ',' unless first
+          first = false
+
+          @output << json_string(kv[:key])
+          @output << ':'
+          @output << kv[:value]
+          @i += 1
+          next
+        end
+
+        # Key-only (nested object)
+        if (key = parse_key_only(ln[:text]))
+          @output << ',' unless first
+          first = false
+
+          @output << json_string(key)
+          @output << ':'
+          @i += 1
+          parse_block(ln[:indent] + @indent_unit)
+          next
+        end
+
+        @i += 1
+      end
+
+      @output << '}'
+    end
+
+    def parse_array_header(text)
+      m = text.match(/\A(?:(?<key>.+?)?)?\[(?<len>#?\d+)(?<marker>[\t|]?)\](?:\{(?<fields>[^}]*)\})?:(?:\s*(?<rest>.*))?\z/)
+      return nil unless m
+
+      key = m[:key]&.strip
+      key = parse_quoted_key(key) if key && !key.empty?
+
+      {
+        key: key,
+        length: m[:len].sub(/^#/, '').to_i,
+        fields: m[:fields],
+        inline: m[:rest],
+        marker: m[:marker]
+      }
+    end
+
+    def parse_array_body(header, child_indent)
+      @output << '['
+
+      # Inline values
+      if header[:inline] && !header[:inline].strip.empty?
+        delim = detect_delimiter(header[:marker], header[:fields])
+        values = split_with_quotes(header[:inline], delim)
+
+        values.each_with_index do |v, idx|
+          @output << ',' if idx > 0
+          @output << value_to_json(v.strip)
+        end
+
+        @output << ']'
+        return
+      end
+
+      # Tabular format
+      if header[:fields]
+        delim = detect_delimiter(header[:marker], header[:fields])
+        fields = split_with_quotes(header[:fields], delim)
+        first = true
+
+        while @i < @lines.length && @lines[@i][:indent] >= child_indent
+          row_text = @lines[@i][:text]
+          break if row_text.strip.empty?
+
+          @output << ',' unless first
+          first = false
+
+          values = split_with_quotes(row_text, delim)
+
+          @output << '{'
+          fields.each_with_index do |f, idx|
+            @output << ',' if idx > 0
+            @output << json_string(unquote_if_quoted(f.strip))
+            @output << ':'
+            @output << value_to_json(values[idx]&.strip || 'null')
+          end
+          @output << '}'
+
+          @i += 1
+        end
+
+        @output << ']'
+        return
+      end
+
+      if @i < @lines.length && @lines[@i][:indent] >= child_indent
+        peek_text = @lines[@i][:text]
+        peek_header = parse_array_header(peek_text)
+
+        if peek_header && peek_header[:key].nil?
+          parse_array_of_arrays(child_indent)
+          return
+        end
+      end
+
+      # List format - parse items directly
+      if @i < @lines.length && @lines[@i][:indent] >= child_indent &&
+         @lines[@i][:text].match?(/\A-\s/)
+        first = true
+        while @i < @lines.length
+          ln = @lines[@i]
+          break if ln[:raw].strip.empty?
+          break if ln[:indent] < child_indent
+          break unless ln[:text].match?(/\A-\s/)
+
+          @output << ',' unless first
+          first = false
+
+          after = ln[:text][2..]&.strip || ''
+
+          if after.empty?
+            @i += 1
+            if @i < @lines.length && @lines[@i][:indent] > ln[:indent]
+              parse_block(@lines[@i][:indent])
+            else
+              @output << '{}'
+            end
+            next
+          end
+
+          if (kv = parse_key_value_line(after))
+            @output << '{'
+            @output << json_string(kv[:key])
+            @output << ':'
+            @output << kv[:value]
+
+            @i += 1
+
+            if @i < @lines.length && @lines[@i][:indent] > ln[:indent]
+              child_ind = @lines[@i][:indent]
+
+              while @i < @lines.length && @lines[@i][:indent] >= child_ind
+                field_ln = @lines[@i]
+                break if field_ln[:text].match?(/\A-\s/)
+
+                if field_kv = parse_key_value_line(field_ln[:text])
+                  @output << ','
+                  @output << json_string(field_kv[:key])
+                  @output << ':'
+                  @output << field_kv[:value]
+                  @i += 1
+                elsif field_key = parse_key_only(field_ln[:text])
+                  @output << ','
+                  @output << json_string(field_key)
+                  @output << ':'
+                  @i += 1
+                  parse_block(field_ln[:indent] + @indent_unit)
+                else
+                  break
+                end
+              end
+            end
+
+            @output << '}'
+            next
+          end
+
+          if after.end_with?(':')
+            key = unquote_if_quoted(after[0...-1].strip)
+            @i += 1
+
+            @output << '{'
+            @output << json_string(key)
+            @output << ':'
+
+            if @i < @lines.length && @lines[@i][:indent] > ln[:indent]
+              parse_block(@lines[@i][:indent])
+            else
+              @output << '{}'
+            end
+
+            @output << '}'
+            next
+          end
+
+          @output << value_to_json(after)
+          @i += 1
+        end
+
+        @output << ']'
+        return
+      end
+
+      # Empty array
+      @output << ']'
+    end
+
+    def parse_array_of_arrays(child_indent)
+      first_item = true
+
+      while @i < @lines.length && @lines[@i][:indent] >= child_indent
+        child_text = @lines[@i][:text]
+        break if child_text.strip.empty?
+
+        child_header = parse_array_header(child_text)
+        break unless child_header && child_header[:key].nil? # Must be headerless array
+
+        @output << ',' unless first_item
+        first_item = false
+        @i += 1
+
+        parse_array_body(child_header, @lines[@i - 1][:indent] + @indent_unit)
+      end
+
+      @output << ']'
+    end
+
+    def split_with_quotes(text, delimiter)
+      return [text] if text.nil? || text.empty?
+
+      values = []
+      current = +''
+      in_quotes = false
+      i = 0
+
+      while i < text.length
+        c = text[i]
+
+        if c == '\\' && i + 1 < text.length
+          current << c << text[i + 1]
+          i += 2
+          next
+        end
+
+        if c == '"'
+          in_quotes = !in_quotes
+          current << c
+          i += 1
+          next
+        end
+
+        if c == delimiter && !in_quotes
+          values << current
+          current = +''
+          i += 1
+          next
+        end
+
+        current << c
+        i += 1
+      end
+
+      values << current unless current.empty?
+      values.map(&:strip)
+    end
+
+    def detect_delimiter(marker, fields)
+      return "\t" if marker == "\t"
+      return '|' if marker == '|'
+      return "\t" if fields&.include?("\t")
+      return '|' if fields&.include?('|')
+
+      ','
+    end
+
+    def parse_key_value_line(text)
+      key = nil
+      rest = nil
+
+      if text.start_with?('"')
+        idx = 1
+        while idx < text.length
+          if text[idx] == '\\' && idx + 1 < text.length
+            idx += 2
+            next
+          end
+          break if text[idx] == '"'
+
+          idx += 1
+        end
+        return nil if idx >= text.length
+
+        key = unescape_string(text[1...idx])
+        after = text[(idx + 1)..]&.lstrip
+        return nil unless after&.start_with?(':')
+
+        rest = after[1..]&.lstrip
+      elsif (cpos = text.index(':'))
+        key = text[0...cpos].strip
+        rest = text[(cpos + 1)..]&.lstrip
+      else
+        return nil
+      end
+
+      return nil if rest.nil? || rest.empty?
+
+      { key: unquote_if_quoted(key), value: value_to_json(rest) }
+    end
+
+    def parse_key_only(text)
+      return nil unless text.end_with?(':')
+
+      key_part = text[0...-1].strip
+      return nil if key_part.empty?
+
+      unquote_if_quoted(key_part)
+    end
+
+    def unquote_if_quoted(str)
+      return str unless str&.start_with?('"') && str.end_with?('"')
+
+      unescape_string(str[1...-1])
+    end
+
+    def parse_quoted_key(key)
+      key = key.strip
+      if key.start_with?('"') && key.end_with?('"')
+        unescape_string(key[1...-1])
+      else
+        key
+      end
+    end
+
+    def value_to_json(text)
+      return 'null' if text.nil? || text == 'null'
+
+      t = text.strip
+
+      return json_string(unescape_string(t[1...-1])) if t.start_with?('"') && t.end_with?('"')
+
+      return 'true' if t.casecmp('true').zero?
+      return 'false' if t.casecmp('false').zero?
+      return 'null' if t.casecmp('null').zero?
+
+      # Numbers
+      return t if t.match?(/\A-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?\z/)
+
+      json_string(t)
+    end
+
+    def primitive_to_json(token)
+      return 'null' if token.nil? || token.casecmp?('null')
+      return 'true' if token.casecmp?('true')
+      return 'false' if token.casecmp?('false')
+      return token if token.match?(/\A-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?\z/)
+
+      json_string(token)
+    end
+
+    def json_string(str)
+      return '""' if str.nil? || str.empty?
+
+      result = +'"'
+
+      str.each_char do |c|
+        result << case c
+                  when '"'  then '\\"'
+                  when '\\' then '\\\\'
+                  when "\n" then '\\n'
+                  when "\r" then '\\r'
+                  when "\t" then '\\t'
+                  when "\b" then '\\b'
+                  when "\f" then '\\f'
+                  else
+                    if c.ord < 32
+                      format('\\u%04x', c.ord)
+                    else
+                      c
+                    end
+                  end
+      end
+
+      result << '"'
+      result
+    end
+
+    def unescape_string(s)
+      result = +''
+      i = 0
+
+      while i < s.length
+        if s[i] == '\\' && i + 1 < s.length
+          case s[i + 1]
+          when 'n'  then result << "\n"
+          when 'r'  then result << "\r"
+          when 't'  then result << "\t"
+          when '"'  then result << '"'
+          when '\\' then result << '\\'
+          when 'b'  then result << "\b"
+          when 'f'  then result << "\f"
+          when 'u'
+            if i + 5 < s.length
+              hex = s[i + 2..i + 5]
+              begin
+                result << [hex.to_i(16)].pack('U')
+              rescue StandardError
+                result << s[i..i + 5]
+              end
+              i += 6
+              next
+            else
+              result << s[i] << s[i + 1]
+            end
+          else
+            result << s[i] << s[i + 1]
+          end
+          i += 2
+        else
+          result << s[i]
+          i += 1
+        end
+      end
+
+      result
+    end
+  end
+end

data/lib/toon_to_json/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ToonToJson
+  VERSION = '0.3.0'
+end

data/lib/toon_to_json.rb

@@ -0,0 +1,14 @@
+require_relative 'toon_to_json/decoder'
+
+module ToonToJson
+  class Error < StandardError; end
+
+  # Decode a TOON-formatted string into a Ruby object
+  #
+  # @param toon_str [String] The TOON-formatted string to decode
+  # @return [Object] The decoded Ruby object (Hash, Array, or primitive)
+
+  def self.decode(toon_str)
+    Decoder.new.decode(toon_str)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-json-toon
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Jitendra Neema
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-01-06 00:00:00.000000000 Z
+date: 2026-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: benchmark-ips
@@ -108,24 +108,10 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.22'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.22'
 description: Lightweight Ruby library for converting JSON data to TOON format, achieving
   30-60% token reduction for LLM applications
 email:
-- jitenra.neema.8@gmail.com
+- jitendra.neema.8@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -136,6 +122,9 @@ files:
 - lib/json_to_toon.rb
 - lib/json_to_toon/encoder.rb
 - lib/json_to_toon/version.rb
+- lib/toon_to_json.rb
+- lib/toon_to_json/decoder.rb
+- lib/toon_to_json/version.rb
 homepage: https://github.com/jitendra-neema/ruby-json-toon
 licenses:
 - MIT