ruby-json-toon 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2eda6d64edfc1cc2a7f956141060c3705bf4a66e668ce10622f42c30ba2ff92b
+  data.tar.gz: f8dc21d8f08cc1751463d3b664c00c27f5a83720b715d0894be5e71c29ea0a12
+SHA512:
+  metadata.gz: 435a729daee86264c5eab4917f6191fe0fc70646c7c49574a85a74c34b19ecebd58fd1e04fff136a36667346a2aa511f3a81c8f8732c4bcf6621ae2ccbc6ffd7
+  data.tar.gz: 00ae26b0633543fb121c7910efe3ab5d79b699d1143bd963e659fc2b3f94f8cbc9a1278ca81bbb65343476a05a44fb8d216aedd892ed108998451c9491d729a1

data/CHANGELOG.md

@@ -0,0 +1,31 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- Initial implementation of JSON to TOON converter
+- Support for all TOON format types (objects, arrays, primitives)
+- Configurable delimiters (comma, tab, pipe)
+- Configurable indentation
+- Optional length markers
+- Comprehensive test suite
+- Full TOON specification compliance
+
+## [0.1.0] - Initial publish
+### Added
+- Initial Encoder class implementation
+- Tests for JSON to TOON conversion
+
+##[0.2.0] - First tagged release
+### Added
+ - Updated to run release workflow on tag pushes
+
+## [1.0.0] - TBD
+
+### Added
+- Initial release

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 [Jitendra Neema]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,98 @@
+# JSON to TOON
+
+A lightweight, zero-dependency Ruby library for converting JSON data to TOON (Token-Oriented Object Notation) format, achieving 30-60% token reduction for LLM applications.
+
+## What is TOON?
+
+TOON (Token-Oriented Object Notation) is a compact, indentation-based data format optimized for LLM token efficiency. It uses 30-60% fewer tokens than JSON while remaining human-readable.
+
+### Comparison
+
+**JSON (87 tokens):**
+```json
+{
+  "users": [
+    {"id": 1, "name": "Alice", "role": "admin"},
+    {"id": 2, "name": "Bob", "role": "user"}
+  ]
+}
+```
+
+**TOON (31 tokens):**
+```
+users[2]{id,name,role}:
+  1,Alice,admin
+  2,Bob,user
+```
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem 'json_to_toon'
+```
+
+Or install directly:
+
+```bash
+gem install json_to_toon
+```
+
+## Quick Start
+
+```ruby
+require 'json_to_toon'
+
+# Convert Ruby hash to TOON
+data = { name: 'Ada', role: 'admin', active: true }
+toon = JsonToToon.encode(data)
+# Output:
+# name: Ada
+# role: admin
+# active: true
+
+# Convert JSON string to TOON
+json_data = JSON.parse('{"users":[{"id":1,"name":"Alice"}]}')
+toon = JsonToToon.encode(json_data)
+```
+
+## Documentation
+
+See full documentation at [rubydoc.info](https://rubydoc.info/gems/json_to_toon)
+
+## Options
+
+```ruby
+JsonToToon.encode(data,
+  indent: 2,         # Spaces per indentation level (default: 2)
+  delimiter: ',',    # Delimiter: ',' (default), "\t", or '|'
+  length_marker: '#' # Length marker or false (default: false)
+)
+```
+
+## Development
+
+```bash
+# Install dependencies
+bundle install
+
+# Run tests
+bundle exec rspec
+
+# Run linter
+bundle exec rubocop
+
+# Build gem
+gem build json_to_toon.gemspec
+```
+
+## License
+
+MIT License - see LICENSE file for details
+
+## Links
+
+- [TOON Specification](https://toonformat.dev)
+- [GitHub Repository](https://github.com/jitendra-neema/json_to_toon)
+- [Bug Tracker](https://github.com/jitendra-neema/json_to_toon/issues)

data/lib/json_to_toon/encoder.rb

@@ -0,0 +1,399 @@
+# frozen_string_literal: true
+
+require 'bigdecimal'
+
+module JsonToToon
+  class Encoder
+    DEFAULT_INDENT = 2
+    DELIMITER_COMMA = ','
+    DELIMITER_TAB = "\t"
+    DELIMITER_PIPE = '|'
+    VALID_DELIMITERS = [DELIMITER_COMMA, DELIMITER_TAB, DELIMITER_PIPE].freeze
+
+    attr_reader :indent_size, :delimiter, :length_marker
+
+    def initialize(options = {})
+      @indent_size = options[:indent] || DEFAULT_INDENT
+      @delimiter = options[:delimiter] || DELIMITER_COMMA
+      @length_marker = options[:length_marker] || false
+
+      validate_options!
+
+      @output = []
+      @visited = {}
+    end
+
+    def encode(value)
+      @output = []
+      @visited = {}
+      result = encode_value(value, 0)
+
+      # If encoding a primitive value at root, encode_value returns the
+      # formatted string but nothing is emitted to @output. In that case
+      # return the direct result.
+      return result.to_s if @output.empty? && result
+
+      @output.join("\n")
+    end
+
+    def delimiter_marker
+      case @delimiter
+      when DELIMITER_COMMA then ''
+      when DELIMITER_TAB then "\t"
+      when DELIMITER_PIPE then '|'
+      end
+    end
+
+    def length_prefix
+      @length_marker == '#' ? '#' : ''
+    end
+
+    private
+
+    def validate_options!
+      unless @indent_size.is_a?(Integer) && @indent_size.positive?
+        raise InvalidOptionError, "indent must be a positive integer, got: #{@indent_size.inspect}"
+      end
+
+      unless VALID_DELIMITERS.include?(@delimiter)
+        raise InvalidOptionError,
+              "delimiter must be one of #{VALID_DELIMITERS.map(&:inspect).join(', ')}, got: #{@delimiter.inspect}"
+      end
+
+      return if @length_marker == '#' || @length_marker == false
+
+      raise InvalidOptionError, "length_marker must be '#' or false, got: #{@length_marker.inspect}"
+    end
+
+    def encode_value(value, depth, key = nil)
+      check_circular_reference(value)
+
+      result = case value
+               when Hash then encode_hash(value, depth)
+               when Array then encode_array_with_key(value, depth, key)
+               else format_value(value)
+               end
+
+      unmark_visited(value)
+      result
+    end
+
+    def check_circular_reference(value)
+      return unless value.is_a?(Hash) || value.is_a?(Array)
+
+      object_id = value.object_id
+      raise CircularReferenceError, 'Circular reference detected' if @visited[object_id]
+
+      @visited[object_id] = true
+    end
+
+    def unmark_visited(value)
+      @visited.delete(value.object_id) if value.is_a?(Hash) || value.is_a?(Array)
+    end
+
+    def encode_hash(hash, depth)
+      return if hash.empty? && depth.zero?
+
+      hash.each do |key, value|
+        formatted_key = format_key(key)
+
+        if primitive?(value)
+          formatted_value = format_value(value)
+          emit_line(indent(depth) + "#{formatted_key}: #{formatted_value}")
+        elsif value.is_a?(Hash)
+          emit_line(indent(depth) + "#{formatted_key}:")
+          encode_value(value, depth + 1) unless value.empty?
+        elsif value.is_a?(Array)
+          encode_value(value, depth, formatted_key)
+        end
+      end
+    end
+
+    def encode_array_with_key(array, depth, key)
+      if array.empty?
+        header = key ? "#{key}[#{length_prefix}0]:" : "[#{length_prefix}0]:"
+        emit_line(indent(depth) + header)
+        return
+      end
+
+      if tabular_eligible?(array)
+        encode_tabular(array, depth, key)
+      elsif all_primitives?(array)
+        encode_inline(array, depth, key)
+      else
+        encode_list(array, depth, key)
+      end
+    end
+
+    # FIXED: Tabular eligibility with proper key checking
+    def tabular_eligible?(array)
+      return false if array.empty?
+      return false unless array.all? { |item| item.is_a?(Hash) && !item.empty? }
+
+      # Get first object's key set (sorted for comparison)
+      first_keys = array.first.keys.sort
+
+      # All objects must have exactly the same keys
+      return false unless array.all? { |item| item.keys.sort == first_keys }
+
+      # All values must be primitives
+      array.all? { |item| item.values.all? { |v| primitive?(v) } }
+    end
+
+    def all_primitives?(array)
+      array.all? { |item| primitive?(item) }
+    end
+
+    # FIXED: Tabular encoding with correct field order from first object
+    def encode_tabular(array, depth, key)
+      # Use first object's key order for header
+      field_order = array.first.keys
+      field_header = field_order.map { |k| format_key(k) }.join(delimiter)
+
+      length_str = "#{length_prefix}#{array.length}"
+      marker = delimiter_marker
+      header = if key
+                 "#{key}[#{length_str}#{marker}]{#{field_header}}:"
+               else
+                 "[#{length_str}#{marker}]{#{field_header}}:"
+               end
+
+      emit_line(indent(depth) + header)
+
+      array.each do |item|
+        # CRITICAL: Use field_order, not item's key order!
+        values = field_order.map { |k| format_value(item[k]) }
+        row = values.join(@delimiter)
+        emit_line(indent(depth + 1) + row)
+      end
+    end
+
+    def encode_inline(array, depth, key)
+      length_str = "#{length_prefix}#{array.length}"
+      marker = delimiter_marker
+      header = key ? "#{key}[#{length_str}#{marker}]:" : "[#{length_str}#{marker}]:"
+
+      values = array.map { |item| format_value(item) }
+      line = "#{indent(depth)}#{header} #{values.join(@delimiter)}"
+
+      emit_line(line)
+    end
+
+    # FIXED: List encoding with proper indentation
+    def encode_list(array, depth, key)
+      length_str = "#{length_prefix}#{array.length}"
+      header = key ? "#{key}[#{length_str}]:" : "[#{length_str}]:"
+      emit_line(indent(depth) + header)
+
+      array.each do |item|
+        encode_list_item(item, depth + 1)
+      end
+    end
+
+    # FIXED: Proper list item indentation
+    def encode_list_item(item, depth)
+      base_indent = indent(depth)
+      hyphen_line = "#{base_indent}- "
+      field_indent = "#{base_indent}  "
+
+      case item
+      when Hash
+        if item.empty?
+          emit_line(hyphen_line)
+        else
+          keys = item.keys
+          first_key = keys.first
+          formatted_key = format_key(first_key)
+
+          # Check if first value is an array (special case)
+          if item[first_key].is_a?(Array)
+            # TODO: Handle nested array as first field
+            # This is complex - needs special handling
+            encode_list_item_with_array_first(item, depth)
+          elsif item[first_key].is_a?(Hash)
+            # First field is nested object
+            emit_line("#{hyphen_line}#{formatted_key}:")
+            encode_value(item[first_key], depth + 1)
+
+            # Remaining fields
+            keys[1..].each do |k|
+              fk = format_key(k)
+              fv = format_value(item[k])
+              emit_line("#{field_indent}#{fk}: #{fv}")
+            end
+          else
+            # First field is primitive
+            fv = format_value(item[first_key])
+            emit_line("#{hyphen_line}#{formatted_key}: #{fv}")
+
+            # Remaining fields at same indent level
+            keys[1..].each do |k|
+              fk = format_key(k)
+              v = item[k]
+
+              if v.is_a?(Hash)
+                emit_line("#{field_indent}#{fk}:")
+                encode_value(v, depth + 2)
+              elsif v.is_a?(Array)
+                encode_value(v, depth + 1, fk)
+              else
+                fv = format_value(v)
+                emit_line("#{field_indent}#{fk}: #{fv}")
+              end
+            end
+          end
+        end
+      when Array
+        # Nested array in list - go through encode_value for circular checks
+        encode_value(item, depth, nil)
+      else
+        # Primitive
+        emit_line("#{hyphen_line}#{format_value(item)}")
+      end
+    end
+
+    # COMPLEX: Handle list item where first field is an array
+    def encode_list_item_with_array_first(item, depth)
+      # This is a complex edge case from the spec
+      # TODO: Implement proper handling per spec
+      keys = item.keys
+      first_key = keys.first
+
+      base_indent = indent(depth)
+      hyphen_line = "#{base_indent}- "
+      field_indent = "#{base_indent}  "
+
+      formatted_key = format_key(first_key)
+      emit_line("#{hyphen_line}#{formatted_key}:")
+      encode_value(item[first_key], depth + 1, nil)
+
+      keys[1..].each do |k|
+        fk = format_key(k)
+        fv = format_value(item[k])
+        emit_line("#{field_indent}#{fk}: #{fv}")
+      end
+    end
+
+    def primitive?(value)
+      value.nil? ||
+        value.is_a?(String) ||
+        value.is_a?(Numeric) ||
+        value == true ||
+        value == false ||
+        value.is_a?(Symbol)
+    end
+
+    def format_key(key)
+      key_str = key.to_s
+      needs_key_quotes?(key_str) ? quote_string(key_str) : key_str
+    end
+
+    def needs_key_quotes?(str)
+      return true if str.empty?
+      return true if str.start_with?('-')
+      return true if str.match?(/\A\d+\z/)
+      return true if str.match?(/[\s,:"\[\]{}\\]/)
+      return true if str.match?(/[\n\r\t]/)
+
+      false
+    end
+
+    def format_value(value)
+      case value
+      when nil then 'null'
+      when true then 'true'
+      when false then 'false'
+      when Integer then value.to_s
+      when Float then format_float(value)
+      when String then format_string(value)
+      when Symbol then format_string(value.to_s)
+      when Time, DateTime then quote_string(value.iso8601(3))
+      else 'null'
+      end
+    end
+
+    # FIXED: Better float formatting
+    def format_float(float)
+      return 'null' if float.nan? || float.infinite?
+      return '0' if float.zero?
+
+      # Handle scientific notation and very small/large numbers
+      str = if float.abs < 1e-10 || float.abs >= 1e15
+              BigDecimal(float, 10).to_s('F')
+            else
+              float.to_s
+            end
+
+      # Remove scientific notation if present
+      str = BigDecimal(str).to_s('F') if str.include?('e') || str.include?('E')
+
+      # Remove trailing zeros after decimal point
+      str = str.sub(/\.?0+\z/, '') if str.include?('.')
+
+      # Ensure no lone decimal point
+      str = str.sub(/\.\z/, '')
+
+      # Final -0 check
+      str = '0' if ['-0', '-0.0'].include?(str)
+
+      str
+    end
+
+    # FIXED: Correct string quoting - only leading/trailing spaces!
+    def format_string(str)
+      needs_string_quotes?(str) ? quote_string(str) : str
+    end
+
+    def needs_string_quotes?(str)
+      return true if str.empty?
+      return true if str.include?(@delimiter)
+      return true if str.include?(':')
+      return true if str.include?('"') || str.include?('\\')
+      return true if str.match?(/[\n\r\t]/)
+
+      # Quote strings that contain spaces (inner or leading/trailing)
+      return true if str.include?(' ')
+
+      return true if str == '-' || str.start_with?('- ')
+      return true if looks_like_boolean?(str)
+      return true if looks_like_null?(str)
+      return true if looks_like_number?(str)
+      return true if looks_like_structural?(str)
+
+      false
+    end
+
+    def looks_like_boolean?(str)
+      str.match?(/\A(true|false)\z/i)
+    end
+
+    def looks_like_null?(str)
+      str.match?(/\Anull\z/i)
+    end
+
+    def looks_like_number?(str)
+      str.match?(/\A-?\d+(\.\d+)?([eE][+-]?\d+)?\z/)
+    end
+
+    def looks_like_structural?(str)
+      str.match?(/\A\[.*\]\z/) || str.match?(/\A\{.*\}\z/)
+    end
+
+    def quote_string(str)
+      escaped = str.gsub('\\', '\\\\\\\\')
+                   .gsub('"', '\"')
+                   .gsub("\n", '\n')
+                   .gsub("\r", '\r')
+                   .gsub("\t", '\t')
+      "\"#{escaped}\""
+    end
+
+    def indent(depth)
+      ' ' * (depth * @indent_size)
+    end
+
+    def emit_line(content)
+      @output << content.rstrip
+    end
+  end
+end

data/lib/json_to_toon/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module JsonToToon
+  VERSION = '0.2.0'
+end

data/lib/json_to_toon.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative 'json_to_toon/version'
+require_relative 'json_to_toon/encoder'
+
+module JsonToToon
+  class Error < StandardError; end
+  class CircularReferenceError < Error; end
+  class InvalidOptionError < Error; end
+
+  # Convert a Ruby object to TOON format
+  #
+  # @param value [Object] The Ruby object to encode (Hash, Array, or primitive)
+  # @param options [Hash] Encoding options
+  # @option options [Integer] :indent Number of spaces per indentation level (default: 2)
+  # @option options [String] :delimiter Delimiter for arrays: ',' (default), "\t", or '|'
+  # @option options [String, false] :length_marker Length marker character or false (default: false)
+  # @return [String] TOON-formatted string with no trailing newline
+  # @raise [InvalidOptionError] If options are invalid
+  # @raise [CircularReferenceError] If circular references detected
+  #
+  # @example Basic usage
+  #   JsonToToon.encode({name: 'Ada', age: 30})
+  #   # => "name: Ada\nage: 30"
+  #
+  # @example With tab delimiter
+  #   JsonToToon.encode({tags: ['a', 'b']}, delimiter: "\t")
+  #   # => "tags[2\t]: a\tb"
+  def self.encode(value, options = {})
+    Encoder.new(options).encode(value)
+  end
+
+  # Alias for encode
+  def self.convert(value, options = {})
+    encode(value, options)
+  end
+end

metadata

@@ -0,0 +1,168 @@
+--- !ruby/object:Gem::Specification
+name: ruby-json-toon
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Jitendra Neema
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-01-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.12'
+- !ruby/object:Gem::Dependency
+  name: memory_profiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.50'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.50'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+description: Lightweight Ruby library for converting JSON data to TOON format, achieving
+  30-60% token reduction for LLM applications
+email:
+- jitenra.neema.8@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- lib/json_to_toon.rb
+- lib/json_to_toon/encoder.rb
+- lib/json_to_toon/version.rb
+homepage: https://github.com/jitendra-neema/ruby-json-toon
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/jitendra-neema/ruby-json-toon
+  source_code_uri: https://github.com/jitendra-neema/ruby-json-toon
+  changelog_uri: https://github.com/jitendra-neema/ruby-json-toon/blob/main/CHANGELOG.md
+  bug_tracker_uri: https://github.com/jitendra-neema/ruby-json-toon/issues
+  documentation_uri: https://rubydoc.info/gems/ruby-json-toon
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.19
+signing_key:
+specification_version: 4
+summary: Convert JSON to TOON (Token-Oriented Object Notation)
+test_files: []