toon-format 0.1.0

data/lib/toon_format/encoder.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+module ToonFormat
+  # Encodes Ruby objects to TOON format strings
+  class Encoder
+    DEFAULT_OPTIONS = {
+      delimiter: ",",
+      indent: 2,
+      length_marker: true
+    }.freeze
+
+    MAX_DEPTH = 100
+    MAX_ARRAY_SIZE = 100_000
+
+    # Encode Ruby object to TOON format string
+    #
+    # @param data [Object] Ruby object to encode
+    # @param options [Hash] Encoding options
+    # @return [String] TOON formatted string
+    def self.encode(data, options = {})
+      new(options).encode(data)
+    end
+
+    def initialize(options = {})
+      @options = DEFAULT_OPTIONS.merge(options)
+      @indent_level = 0
+      @visited = Set.new
+    end
+
+    # Main encode method
+    def encode(data, depth = 0)
+      check_depth(depth)
+      encode_value(data, depth)
+    end
+
+    private
+
+    def encode_value(data, depth)
+      case data
+      when NilClass then "null"
+      when TrueClass, FalseClass then data.to_s
+      when Numeric then encode_number(data)
+      when String then encode_string(data)
+      when Hash then encode_object(data, depth)
+      when Array then encode_array(data, depth)
+      else
+        raise EncodeError, "Unsupported type: #{data.class}"
+      end
+    end
+
+    def check_depth(depth)
+      raise EncodeError, "Maximum nesting depth #{MAX_DEPTH} exceeded" if depth > MAX_DEPTH
+    end
+
+    def encode_number(num)
+      return "0" if num.zero?
+      return "null" if num.respond_to?(:nan?) && num.nan?
+      return "null" if num.respond_to?(:infinite?) && num.infinite?
+
+      num.to_s
+    end
+
+    def encode_string(str)
+      return '""' if str.empty?
+      return quote(str) if needs_quoting?(str)
+
+      str
+    end
+
+    def encode_object(hash, depth)
+      return "{}" if hash.empty?
+
+      check_circular_reference(hash)
+
+      lines = hash.map do |key, value|
+        key_str = key.to_s
+
+        # Temporarily increase indent for nested values
+        @indent_level += 1
+        value_str = encode(value, depth + 1)
+        @indent_level -= 1
+
+        # Handle multi-line values (nested structures)
+        if value_str.include?("\n")
+          "#{current_indent}#{key_str}:\n#{value_str}"
+        else
+          "#{current_indent}#{key_str}: #{value_str}"
+        end
+      end
+
+      clear_circular_reference(hash)
+      lines.join("\n")
+    end
+
+    def check_circular_reference(obj)
+      obj_id = obj.object_id
+      raise EncodeError, "Circular reference detected" if @visited.include?(obj_id)
+
+      @visited.add(obj_id)
+    end
+
+    def clear_circular_reference(obj)
+      @visited.delete(obj.object_id)
+    end
+
+    def current_indent
+      " " * (@options[:indent] * @indent_level)
+    end
+
+    def indent_lines(text, additional_levels = 0)
+      indent_str = " " * (@options[:indent] * (@indent_level + additional_levels))
+      text.lines.map { |line| "#{indent_str}#{line}" }.join
+    end
+
+    def encode_array(array, depth)
+      return "[]" if array.empty?
+
+      check_array_size(array)
+      check_circular_reference(array)
+
+      result = if tabular?(array)
+                 encode_tabular_array(array, depth)
+               else
+                 encode_list_array(array, depth)
+               end
+
+      clear_circular_reference(array)
+      result
+    end
+
+    def check_array_size(array)
+      return unless array.size > MAX_ARRAY_SIZE
+
+      raise EncodeError, "Array size #{array.size} exceeds maximum #{MAX_ARRAY_SIZE}"
+    end
+
+    def primitive?(value)
+      value.is_a?(String) ||
+        value.is_a?(Numeric) ||
+        value.is_a?(TrueClass) ||
+        value.is_a?(FalseClass) ||
+        value.nil?
+    end
+
+    def tabular?(array)
+      return false unless array.all? { |el| el.is_a?(Hash) }
+      return false if array.empty?
+
+      keys = array.first.keys
+      array.all? do |element|
+        element.keys == keys &&
+          element.values.all? { |v| primitive?(v) }
+      end
+    end
+
+    def encode_tabular_array(array, _depth)
+      keys = array.first.keys
+      length_marker = @options[:length_marker] ? "[#{array.size},]" : ""
+      fields = keys.map(&:to_s).join(@options[:delimiter])
+      header = "#{current_indent}#{length_marker}{#{fields}}:"
+
+      rows = array.map do |row|
+        values = keys.map do |key|
+          value = row[key]
+          encode_primitive_value(value)
+        end
+        "#{current_indent}#{values.join(@options[:delimiter])}"
+      end
+
+      ([header] + rows).join("\n")
+    end
+
+    def encode_primitive_value(value)
+      case value
+      when NilClass then "null"
+      when TrueClass, FalseClass then value.to_s
+      when Numeric then encode_number(value)
+      when String then encode_string(value)
+      else
+        raise EncodeError, "Non-primitive value in tabular array: #{value.class}"
+      end
+    end
+
+    def encode_list_array(array, depth)
+      length_marker = @options[:length_marker] ? "[#{array.size}]" : "[]"
+      header = "#{current_indent}#{length_marker}:"
+
+      @indent_level += 1
+      elements = array.map do |item|
+        encoded = encode(item, depth + 1)
+        # If the encoded value doesn't already have indent (single line primitive), add it
+        if encoded.include?("\n")
+          encoded
+        else
+          "#{current_indent}#{encoded}"
+        end
+      end
+      @indent_level -= 1
+
+      ([header] + elements).join("\n")
+    end
+
+    QUOTE_PATTERN = /
+      \A\s |                      # Leading whitespace
+      \s\z |                      # Trailing whitespace
+      \A(null|true|false)\z |     # TOON keywords
+      \A-?\d+(\.\d+)?\z |         # Numeric pattern
+      [:\[\]{},]                  # Structural characters
+    /x
+
+    def needs_quoting?(str)
+      str.match?(QUOTE_PATTERN)
+    end
+
+    def quote(str)
+      # Escape any double quotes in the string
+      escaped = str.gsub('"', '\"')
+      "\"#{escaped}\""
+    end
+  end
+end

data/lib/toon_format/errors.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module ToonFormat
+  # Base error class for all ToonFormat errors
+  class Error < StandardError; end
+
+  # Raised when encoding fails
+  class EncodeError < Error; end
+
+  # Raised when decoding fails
+  class DecodeError < Error; end
+
+  # Raised when strict mode validation fails
+  class ValidationError < Error; end
+
+  # Raised when parsing encounters syntax errors
+  class ParseError < DecodeError
+    attr_reader :line_number, :column
+
+    def initialize(message, line: nil, column: nil)
+      @line_number = line
+      @column = column
+      super(format_message(message))
+    end
+
+    private
+
+    def format_message(message)
+      return message unless @line_number
+
+      location = " at line #{@line_number}"
+      location += ", column #{@column}" if @column
+      "#{message}#{location}"
+    end
+  end
+end

data/lib/toon_format/parser.rb

@@ -0,0 +1,269 @@
+# frozen_string_literal: true
+
+module ToonFormat
+  # Parses TOON format strings into Ruby objects
+  class Parser
+    def initialize(input, strict: true)
+      @input = input
+      @lines = input.split("\n")
+      @position = 0
+      @strict = strict
+    end
+
+    def parse
+      return nil if @lines.empty?
+
+      # Check if the entire input is an object (multiple key-value pairs at root level)
+      if looks_like_root_object?
+        parse_root_object
+      else
+        parse_value
+      end
+    end
+
+    def looks_like_root_object?
+      # If we have multiple lines and they all look like key-value pairs at the same indent level
+      return false if @lines.size < 2
+
+      @lines.all? do |line|
+        line.strip.empty? || line =~ KEY_VALUE_PATTERN || line =~ KEY_ONLY_PATTERN
+      end
+    end
+
+    def parse_root_object
+      result = {}
+
+      while current_line
+        line = current_line
+
+        if line.strip.empty?
+          advance
+          next
+        end
+
+        if line =~ KEY_VALUE_PATTERN
+          key = ::Regexp.last_match(1)
+          value_str = ::Regexp.last_match(2).strip
+          advance
+          # Check if value is an empty array or empty object
+          value = if value_str == "[]"
+                    []
+                  elsif value_str == "{}"
+                    {}
+                  else
+                    parse_primitive(value_str)
+                  end
+          result[key.to_sym] = value
+        elsif line =~ KEY_ONLY_PATTERN
+          key = ::Regexp.last_match(1)
+          advance
+          result[key.to_sym] = parse_nested_value
+        else
+          break
+        end
+      end
+
+      result
+    end
+
+    private
+
+    def current_line
+      @lines[@position]
+    end
+
+    def advance
+      @position += 1
+    end
+
+    def peek_line(offset = 1)
+      @lines[@position + offset]
+    end
+
+    TABULAR_ARRAY_PATTERN = /\A\s*\[(\d+),\]\{(.+)\}:/
+    LIST_ARRAY_PATTERN = /\A\s*\[(\d+)\]:/
+    EMPTY_ARRAY_PATTERN = /\A\s*\[\]/
+    KEY_VALUE_PATTERN = /\A\s*(\w+):\s*(.+)/
+    KEY_ONLY_PATTERN = /\A\s*(\w+):\s*$/
+
+    def parse_value
+      line = current_line
+      return nil if line.nil?
+
+      case line
+      when TABULAR_ARRAY_PATTERN
+        length = ::Regexp.last_match(1).to_i
+        fields = ::Regexp.last_match(2).split(",").map(&:strip)
+        advance
+        parse_tabular_array(length, fields)
+      when LIST_ARRAY_PATTERN
+        length = ::Regexp.last_match(1).to_i
+        advance
+        parse_list_array(length)
+      when EMPTY_ARRAY_PATTERN
+        advance
+        []
+      when KEY_ONLY_PATTERN
+        ::Regexp.last_match(1)
+        advance
+        parse_nested_value
+      when KEY_VALUE_PATTERN
+        key = ::Regexp.last_match(1)
+        value_str = ::Regexp.last_match(2).strip
+        advance
+        # Check if value is an empty array or empty object
+        value = if value_str == "[]"
+                  []
+                elsif value_str == "{}"
+                  {}
+                else
+                  parse_primitive(value_str)
+                end
+        { key.to_sym => value }
+      else
+        advance
+        parse_primitive(line.strip)
+      end
+    end
+
+    def parse_nested_value
+      # Parse a nested value (could be object or array)
+      line = current_line
+      return nil if line.nil?
+
+      case line
+      when TABULAR_ARRAY_PATTERN
+        length = ::Regexp.last_match(1).to_i
+        fields = ::Regexp.last_match(2).split(",").map(&:strip)
+        advance
+        parse_tabular_array(length, fields)
+      when LIST_ARRAY_PATTERN
+        length = ::Regexp.last_match(1).to_i
+        advance
+        parse_list_array(length)
+      when EMPTY_ARRAY_PATTERN
+        advance
+        []
+      else
+        parse_object_lines
+      end
+    end
+
+    def parse_object_lines
+      result = {}
+      base_indent = get_indent(current_line)
+
+      while current_line && get_indent(current_line) >= base_indent
+        if current_line =~ KEY_VALUE_PATTERN
+          key = ::Regexp.last_match(1)
+          value_str = ::Regexp.last_match(2)
+          advance
+          result[key.to_sym] = parse_primitive(value_str)
+        elsif current_line =~ KEY_ONLY_PATTERN
+          key = ::Regexp.last_match(1)
+          advance
+          result[key.to_sym] = parse_nested_value
+        else
+          break
+        end
+      end
+
+      result
+    end
+
+    def get_indent(line)
+      return 0 if line.nil?
+
+      line.match(/\A(\s*)/)[1].length
+    end
+
+    def parse_primitive(value)
+      stripped = value.strip
+
+      case stripped
+      when "null" then nil
+      when "true" then true
+      when "false" then false
+      when /\A-?\d+\z/ then stripped.to_i
+      when /\A-?\d+\.\d+\z/ then stripped.to_f
+      when /\A"(.*)"\z/ then unescape_string(::Regexp.last_match(1))
+      else stripped
+      end
+    end
+
+    def unescape_string(str)
+      # Unescape double quotes
+      str.gsub('\"', '"')
+    end
+
+    def parse_tabular_array(length, fields)
+      result = []
+
+      length.times do
+        line = current_line
+        raise ParseError.new("Unexpected end of input in tabular array", line: @position + 1) if line.nil?
+
+        values = split_row(line.strip)
+
+        if @strict && values.size != fields.size
+          raise ParseError.new(
+            "Field count mismatch: expected #{fields.size}, got #{values.size}",
+            line: @position + 1
+          )
+        end
+
+        row = {}
+        fields.each_with_index do |field, index|
+          row[field.to_sym] = parse_primitive(values[index] || "")
+        end
+
+        result << row
+        advance
+      end
+
+      result
+    end
+
+    def split_row(line)
+      # Split by comma, respecting quoted strings
+      values = []
+      current = ""
+      in_quotes = false
+
+      line.each_char do |char|
+        case char
+        when '"'
+          in_quotes = !in_quotes
+          current += char
+        when ","
+          if in_quotes
+            current += char
+          else
+            values << current
+            current = ""
+          end
+        else
+          current += char
+        end
+      end
+
+      values << current unless current.empty?
+      values.map(&:strip)
+    end
+
+    def parse_list_array(length)
+      result = []
+      get_indent(current_line)
+
+      length.times do
+        raise ParseError.new("Unexpected end of input in list array", line: @position + 1) if current_line.nil?
+
+        # Parse each element
+        element = parse_value
+        result << element
+      end
+
+      result
+    end
+  end
+end

data/lib/toon_format/rails/extensions.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module ToonFormat
+  module Rails
+    # Extensions for ActiveRecord models
+    module Extensions
+      # Convert model to TOON format
+      #
+      # @param options [Hash] Encoding options
+      # @return [String] TOON formatted string
+      def to_toon(**options)
+        ToonFormat.encode(as_json, **options)
+      end
+    end
+  end
+end

data/lib/toon_format/railtie.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+if defined?(Rails)
+  require "toon_format/rails/extensions"
+
+  module ToonFormat
+    class Railtie < Rails::Railtie
+      initializer "toon_format.active_record" do
+        ActiveSupport.on_load(:active_record) do
+          include ToonFormat::Rails::Extensions
+        end
+      end
+    end
+  end
+end

data/lib/toon_format/validator.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module ToonFormat
+  # Validates decoded data in strict mode
+  class Validator
+    # Validate decoded data against TOON string
+    #
+    # @param toon_string [String] Original TOON string
+    # @param decoded_data [Object] Decoded Ruby object
+    # @raise [ValidationError] If validation fails
+    def self.validate!(toon_string, decoded_data)
+      validate_array_lengths(toon_string, decoded_data)
+    end
+
+    def self.validate_array_lengths(toon_string, data)
+      # Extract all length markers from TOON string
+      length_markers = toon_string.scan(/\[(\d+)(?:,)?\]/).flatten.map(&:to_i)
+      arrays = find_non_empty_arrays(data)
+
+      # Only validate if we have length markers
+      return if length_markers.empty?
+
+      if length_markers.size != arrays.size
+        # This might be okay if there are nested arrays
+        # For now, just validate that we have at least as many arrays as markers
+        return if arrays.size >= length_markers.size
+
+        raise ValidationError,
+              "Length marker count (#{length_markers.size}) does not match array count (#{arrays.size})"
+      end
+
+      length_markers.each_with_index do |declared_length, index|
+        actual_length = arrays[index]&.size || 0
+
+        next if actual_length == declared_length
+
+        raise ValidationError,
+              "Array #{index}: declared length #{declared_length}, actual length #{actual_length}"
+      end
+    end
+
+    def self.find_non_empty_arrays(data, result = [])
+      case data
+      when Array
+        result << data unless data.empty?
+        data.each { |item| find_non_empty_arrays(item, result) }
+      when Hash
+        data.each_value { |value| find_non_empty_arrays(value, result) }
+      end
+
+      result
+    end
+
+    def self.find_arrays(data, result = [])
+      case data
+      when Array
+        result << data
+        data.each { |item| find_arrays(item, result) }
+      when Hash
+        data.each_value { |value| find_arrays(value, result) }
+      end
+
+      result
+    end
+
+    private_class_method :validate_array_lengths, :find_non_empty_arrays
+  end
+end

data/lib/toon_format/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ToonFormat
+  VERSION = "0.1.0"
+end

data/lib/toon_format.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require_relative "toon_format/version"
+require_relative "toon_format/errors"
+require_relative "toon_format/encoder"
+require_relative "toon_format/parser"
+require_relative "toon_format/decoder"
+require_relative "toon_format/validator"
+
+require_relative "toon_format/rails/extensions" if defined?(ActiveRecord::Base)
+require_relative "toon_format/railtie" if defined?(Rails)
+
+require "json"
+
+module ToonFormat
+  class << self
+    # Encode Ruby object to TOON format string
+    #
+    # @param data [Object] Ruby object to encode
+    # @param options [Hash] Encoding options
+    # @option options [String] :delimiter (',') Field delimiter
+    # @option options [Integer] :indent (2) Indentation spaces
+    # @option options [Boolean] :length_marker (true) Include array lengths
+    #
+    # @return [String] TOON formatted string
+    #
+    # @raise [ToonFormat::EncodeError] If data cannot be encoded
+    def encode(data, **options)
+      Encoder.encode(data, options)
+    end
+
+    # Decode TOON format string to Ruby object
+    #
+    # @param toon_string [String] TOON formatted string
+    # @param strict [Boolean] Enable strict validation (default: true)
+    #
+    # @return [Object] Decoded Ruby object
+    #
+    # @raise [ToonFormat::DecodeError] If decoding fails
+    # @raise [ToonFormat::ValidationError] If strict validation fails
+    def decode(toon_string, strict: true)
+      Decoder.decode(toon_string, strict: strict)
+    end
+
+    # Estimate token savings compared to JSON
+    #
+    # @param data [Object] Ruby object to analyze
+    #
+    # @return [Hash] Statistics including token counts and savings percentage
+    def estimate_savings(data)
+      json_str = JSON.generate(data)
+      toon_str = encode(data)
+
+      json_size = json_str.bytesize
+      toon_size = toon_str.bytesize
+
+      {
+        json_tokens: estimate_tokens(json_str),
+        toon_tokens: estimate_tokens(toon_str),
+        savings_percent: ((json_size - toon_size) / json_size.to_f * 100).round(1),
+        json_size: json_size,
+        toon_size: toon_size
+      }
+    end
+
+    private
+
+    def estimate_tokens(text)
+      # Simple approximation: ~4 characters per token
+      (text.bytesize / 4.0).ceil
+    end
+  end
+end

data/sig/toon/format.rbs

@@ -0,0 +1,6 @@
+module Toon
+  module Format
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end