toonify 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a4c3e9bd14eb81d809fcd9592c7e3366201d2bde28b79e259a90139d8b1b6b66
+  data.tar.gz: 39950dcc538a1d65c6d2f8f88e2af0bbc3c1ae27ac2253e0a2b9923f2e33a22d
+SHA512:
+  metadata.gz: c08cf024eabedc48f44601c32731eaafc5f7b06dbd9fd85aca5284676ebafc5d111f2c6678b4d31c5cb3f5e4024bfa479a2b56ec47220f3abfa0539c4a74e850
+  data.tar.gz: f547cd457555adc6aaea171904cde17d6c06b54a140e07f20c5c66b72f1905b4496ca760f55697372bb5ab660b51131ebcd080dd141d02f166c7d1cd9ccf5413

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/.rubocop.yml

@@ -0,0 +1,27 @@
+AllCops:
+  NewCops: enable
+  SuggestExtensions: false
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+    - 'toon.gemspec'
+
+Metrics/AbcSize:
+  Exclude:
+    - 'lib/toonify/decoder.rb'
+    - 'examples/**/*'
+
+Metrics/CyclomaticComplexity:
+  Exclude:
+    - 'lib/toonify/decoder.rb'
+
+Metrics/MethodLength:
+  Exclude:
+    - 'lib/toonify/decoder.rb'
+    - 'lib/toonify/encoder.rb'
+    - 'examples/**/*'
+
+Metrics/PerceivedComplexity:
+  Exclude:
+    - 'lib/toonify/decoder.rb'

data/Gemfile

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in toon.gemspec
+gemspec
+
+gem 'rake', '~> 13.0'
+
+# Development and test dependencies
+group :development, :test do
+  gem 'rspec', '~> 3.0'
+end
+
+gem 'rubocop', '~> 1.81', groups: %i[development test]

data/Gemfile.lock

@@ -0,0 +1,65 @@
+PATH
+  remote: .
+  specs:
+    toonify (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.3)
+    diff-lcs (1.6.2)
+    json (2.16.0)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    parallel (1.27.0)
+    parser (3.3.10.0)
+      ast (~> 2.4.1)
+      racc
+    prism (1.6.0)
+    racc (1.8.1)
+    rainbow (3.1.1)
+    rake (13.3.1)
+    regexp_parser (2.11.3)
+    rspec (3.13.2)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.6)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.7)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.6)
+    rubocop (1.81.7)
+      json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.48.0)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    ruby-progressbar (1.13.0)
+    unicode-display_width (3.2.0)
+      unicode-emoji (~> 4.1)
+    unicode-emoji (4.1.0)
+
+PLATFORMS
+  arm64-darwin-24
+
+DEPENDENCIES
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rubocop (~> 1.81)
+  toonify!
+
+BUNDLED WITH
+   2.7.2

data/README.md

@@ -0,0 +1,100 @@
+# TOON: Token-Oriented Object Notation
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+**TOON** (Token-Oriented Object Notation) is a lightweight, human-readable data serialization format designed to be **token-efficient for Large Language Models (LLMs)** while remaining easy for humans to read and write.
+
+It serves as a concise alternative to JSON, removing syntactic noise (like excessive quotes, braces, and brackets) to reduce token usage and improve clarity.
+
+## 🚀 Why TOON?
+
+### 1. Token Efficiency for AI/LLMs
+JSON is verbose. For LLMs (like GPT-4, Claude, Gemini), every character counts. TOON reduces the token footprint by eliminating structural overhead, which can lead to:
+*   **Lower Costs**: Fewer tokens processed means lower API bills.
+*   **Larger Context**: Fit more data into the model's context window.
+*   **Faster Generation**: Less syntax for the model to generate.
+
+### 2. Human Readability
+TOON looks like a clean configuration file or a summary report. It uses significant whitespace and minimal punctuation, making it ideal for:
+*   Logs and debug output.
+*   Configuration files.
+*   Data summaries for dashboards.
+
+## 📦 Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'toonify'
+```
+
+And then execute:
+
+```bash
+$ bundle install
+```
+
+Or install it yourself as:
+
+```bash
+$ gem install toonify
+```
+
+## 💻 Usage
+
+The `toonify` gem provides a simple API to convert between JSON and TOON.
+
+### Basic Conversion
+
+```ruby
+require 'toonify'
+
+# Input JSON
+json_data = '{"name": "Alice", "role": "Engineer", "active": true}'
+
+# Encode JSON -> TOON
+toon_output = Toon.encode(json_data)
+puts toon_output
+# Output:
+# name: Alice
+# role: Engineer
+# active: true
+
+# Decode TOON -> JSON
+json_output = Toon.decode(toon_output)
+puts json_output
+# Output: {"name":"Alice","role":"Engineer","active":true}
+```
+
+### Handling Complex Data
+
+TOON shines with nested structures and arrays. It automatically detects tabular data and formats it concisely.
+
+### Error Handling
+
+The converter is strict about input types to ensure reliability.
+
+```ruby
+begin
+  Toon.encode('invalid json')
+rescue ArgumentError => e
+  puts e.message # => "Invalid JSON input"
+end
+```
+
+## 🔍 Format Specification
+
+TOON uses a few simple rules:
+
+*   **Key-Value**: `key: value`
+*   **Nested Objects**: Indented blocks (YAML-style).
+*   **Primitive Arrays**: `key[count]: val1,val2,val3`
+*   **Object Arrays (Tabular)**: `key[count]{headers}:` followed by CSV-like rows.
+
+## 🤝 Contributing
+
+Bug reports and pull requests are welcome on GitHub at [https://github.com/ran010/toonify](https://github.com/ran010/toonify).
+
+## 📄 License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+task default: %i[]
+
+# RSpec task
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec)
+  task default: %i[spec]
+rescue LoadError
+  # rspec not available; leave default task as-is
+end

data/examples/gemini_token_comparison.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'json'
+require_relative '../lib/toonify'
+
+# ------------------------------------------------------------------------------
+# Configuration
+# ------------------------------------------------------------------------------
+API_KEY = 'api-key'
+MODEL_NAME = 'gemini-2.0-flash'
+API_URL = "https://generativelanguage.googleapis.com/v1beta/models/#{MODEL_NAME}:countTokens?key=#{API_KEY}"
+
+if API_KEY.nil? || API_KEY.empty?
+  puts 'Please set the GEMINI_API_KEY environment variable.'
+  puts "Example: export GEMINI_API_KEY='your_api_key'"
+  exit 1
+end
+
+# ------------------------------------------------------------------------------
+# Sample Data (Complex Nested Structure)
+# ------------------------------------------------------------------------------
+data = {
+  products: [
+    {
+      id: 101,
+      name: 'Super Gadget',
+      features: { wireless: true, battery: '24h', colors: %w[black white] },
+      reviews: [
+        { user: 'alice', rating: 5, comment: 'Amazing battery life!' },
+        { user: 'bob', rating: 4, comment: 'Good, but expensive.' }
+      ]
+    },
+    {
+      id: 102,
+      name: 'Budget Widget',
+      features: { wireless: false, battery: '12h', colors: ['gray'] },
+      reviews: [
+        { user: 'charlie', rating: 3, comment: 'It works, I guess.' }
+      ]
+    }
+  ],
+  metadata: {
+    generated_at: '2023-10-27T10:00:00Z',
+    source: 'inventory_system',
+    tags: %w[electronics gadgets sale]
+  }
+}
+
+# ------------------------------------------------------------------------------
+# Conversion
+# ------------------------------------------------------------------------------
+json_string = JSON.pretty_generate(data)
+toon_string = Toon.encode(json_string)
+
+puts "--- JSON Content (#{json_string.length} chars) ---"
+puts json_string
+puts "\n--- TOON Content (#{toon_string.length} chars) ---"
+puts toon_string
+puts "\n--------------------------------------------------"
+
+# ------------------------------------------------------------------------------
+# Token Counting Helper
+# ------------------------------------------------------------------------------
+def count_tokens(text_content)
+  # 1. Prepare URI and HTTP client
+  uri = URI(API_URL)
+  http = Net::HTTP.new(uri.host, uri.port)
+  http.use_ssl = true # Use SSL for HTTPS connection
+
+  # 2. Build the POST request
+  request = Net::HTTP::Post.new(uri)
+  request['Content-Type'] = 'application/json'
+
+  # 3. Prepare the JSON body structure
+  request_body = {
+    contents: [{
+      parts: [{
+        text: text_content
+      }]
+    }]
+  }
+
+  # Set the body as a JSON string
+  request.body = request_body.to_json
+
+  # 4. Send the request and handle the response
+  puts "Sending request to: #{API_URL}"
+  response = http.request(request)
+
+  if response.code == '200'
+    response_data = JSON.parse(response.body)
+    token_count = response_data['totalTokens']
+
+    puts "\n✅ Successfully counted tokens."
+    puts "Text: \"#{text_content}\""
+    puts "Total Tokens: #{token_count}"
+
+    token_count
+  else
+    puts "\n❌ Error calling Gemini API (HTTP #{response.code}):"
+    puts response.body
+    nil
+  end
+end
+
+# ------------------------------------------------------------------------------
+# Comparison
+# ------------------------------------------------------------------------------
+puts "Calculating tokens with #{MODEL_NAME}..."
+
+json_tokens = count_tokens(json_string)
+toon_tokens = count_tokens(toon_string)
+
+if json_tokens && toon_tokens
+  diff = json_tokens - toon_tokens
+  percent = (diff.to_f / json_tokens * 100).round(2)
+
+  puts "\nToken Usage Comparison:"
+  puts "JSON Tokens: #{json_tokens}"
+  puts "TOON Tokens: #{toon_tokens}"
+  puts "Savings:     #{diff} tokens (#{percent}%)"
+else
+  puts 'Failed to retrieve token counts.'
+end

data/lib/toonify/decoder.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Toonify
+  # Parses TOON (Token Oriented Object Notation) back into Ruby objects or JSON.
+  # Supports the subset produced by Toonify::Encoder:
+  # - key: value
+  # - key: (nested hash with indented lines)
+  # - key[n]: v1,v2,...  (array of primitives)
+  # - key[n]{col1,col2}:  (tabular array of hashes; rows are indented)
+  class Decoder
+    INDENT = '  '
+
+    def initialize(toon_string)
+      @lines = toon_string.to_s.lines.map(&:rstrip)
+      @index = 0
+    end
+
+    def parse
+      parse_top_level_hash
+    end
+
+    def decode
+      JSON.generate(parse)
+    end
+
+    def self.parse(toon_string)
+      new(toon_string).parse
+    end
+
+    def self.decode(toon_string)
+      new(toon_string).decode
+    end
+
+    private
+
+    def parse_top_level_hash
+      result = {}
+
+      while @index < @lines.length
+        line = @lines[@index]
+        @index += 1
+
+        next if line.nil? || line.strip.empty?
+
+        if (m = line.match(/^([^\[:{]+)\[(\d+)\]\{([^}]+)\}:$/))
+          key = m[1].strip
+          cols = m[3].split(',').map(&:strip)
+
+          rows = []
+          while peek_indented_line?
+            row_line = @lines[@index].lstrip
+            @index += 1
+            values = row_line.split(',').map { |t| parse_token(t) }
+            row = {}
+            cols.each_with_index do |col, i|
+              row[col] = values[i]
+            end
+            rows << row
+          end
+
+          result[key] = rows
+
+        elsif (m = line.match(/^([^\[]+)\[(\d+)\]:\s*(.*)$/))
+          key = m[1].strip
+          rest = m[3].strip
+          arr = rest.empty? ? [] : rest.split(',').map { |t| parse_token(t) }
+          result[key] = arr
+
+        elsif (m = line.match(/^([^:]+):\s*$/)) && !line.include?('{')
+          key = m[1].strip
+          nested = {}
+          while peek_indented_line?
+            nested_line = @lines[@index].lstrip
+            @index += 1
+            next unless (mm = nested_line.match(/^([^:]+):\s*(.*)$/))
+
+            nkey = mm[1].strip
+            nval = mm[2].strip
+            nested[nkey] = parse_token(nval)
+          end
+          result[key] = nested
+
+        elsif (m = line.match(/^([^:]+):\s*(.*)$/))
+          key = m[1].strip
+          val = m[2].strip
+          result[key] = parse_token(val)
+        end
+      end
+
+      result
+    end
+
+    def peek_indented_line?
+      return false if @index >= @lines.length
+
+      next_line = @lines[@index]
+      return false if next_line.nil?
+
+      next_line.start_with?(INDENT)
+    end
+
+    def parse_token(token)
+      t = token.to_s.strip
+      return nil if t == ''
+      return nil if t.downcase == 'null'
+      return true if t.downcase == 'true'
+      return false if t.downcase == 'false'
+
+      if t.match?(/\A-?\d+\z/)
+        return t.to_i
+      elsif t.match?(/\A-?\d+\.\d+\z/)
+        return t.to_f
+      end
+
+      return t[1..-2].gsub('\\"', '"') if t.start_with?('"') && t.end_with?('"') && t.length >= 2
+
+      t
+    end
+  end
+end

data/lib/toonify/encoder.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Toonify
+  # Encodes Ruby data structures into TOON format.
+  # Handles primitives, hashes, arrays, and nested structures.
+  class Encoder
+    INDENT = '  '
+
+    # Creates a new Encoder instance.
+    # @param data [Hash, Object] The data to encode.
+    def initialize(data)
+      @data = data
+    end
+
+    # Encodes the data into TOON format.
+    # @return [String] The TOON-formatted output.
+    def encode
+      result = format_hash(@data, 0)
+      result.join("\n")
+    end
+
+    private
+
+    # Formats a value based on its type.
+    # @param value [Object] The value to format.
+    # @param depth [Integer] Current indentation depth.
+    # @return [String] The formatted value.
+    def format_value(value, depth)
+      case value
+      when Hash
+        format_hash(value, depth)
+      when Array
+        format_array(value, depth)
+      else
+        value.to_s
+      end
+    end
+
+    # Formats a Hash into TOON format.
+    # @param hash [Hash] The hash to format.
+    # @param depth [Integer] Current indentation depth.
+    # @return [Array<String>] Array of formatted lines.
+    def format_hash(hash, depth)
+      lines = []
+
+      hash.each do |key, value|
+        case value
+        when Hash
+          lines << "#{key}:"
+          value.each do |k, v|
+            lines << "#{INDENT}#{k}: #{format_value(v, depth + 1)}"
+          end
+
+        when Array
+          lines << if uniform_hash_array?(value)
+                     format_tabular_array(key.to_s, value, depth)
+                   else
+                     "#{key}[#{value.size}]: #{value.join(',')}"
+                   end
+
+        else
+          lines << "#{key}: #{format_value(value, depth)}"
+        end
+      end
+
+      lines
+    end
+
+    # Formats an array.
+    # @param array [Array] The array to format.
+    # @param depth [Integer] Current indentation depth.
+    # @return [String] The formatted array.
+    def format_array(array, depth)
+      if uniform_hash_array?(array)
+        format_tabular_array('array', array, depth)
+      else
+        array.map { |item| format_value(item, depth) }.join(',')
+      end
+    end
+
+    # Checks if an array contains uniform hashes.
+    # @param array [Array] The array to check.
+    # @return [Boolean] True if uniform hashes, false otherwise.
+    def uniform_hash_array?(array)
+      return false if array.empty? || !array.all? { |item| item.is_a?(Hash) }
+
+      first_keys = array.first.keys.sort
+      array.all? { |item| item.keys.sort == first_keys }
+    end
+
+    # Formats an array of uniform hashes in tabular (CSV-like) format.
+    # @param key [String] The array key name.
+    # @param array [Array] The array of hashes.
+    # @param _depth [Integer] Current indentation depth (unused).
+    # @return [String] The formatted tabular array.
+    def format_tabular_array(key, array, _depth)
+      keys = array.first.keys.sort
+      header = keys.join(',')
+
+      formatted_lines = ["#{key}[#{array.size}]{#{header}}:", *array.map do |row|
+        values = keys.map { |k| row[k].to_s }
+        "#{INDENT}#{values.join(',')}"
+      end]
+      formatted_lines.join("\n")
+    end
+  end
+end

data/lib/toonify/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Toonify
+  VERSION = '0.1.0'
+end

data/lib/toonify.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative 'toonify/version'
+require_relative 'toonify/encoder'
+require_relative 'toonify/decoder'
+
+module Toonify
+  class Toon
+    # Public class method for easy conversion, useful for quick access.
+    # @param json_string [String] The raw JSON input.
+    # @return [String] The converted custom text output.
+    # @raise [ArgumentError] if the JSON input is invalid.
+    def self.encode(json_string)
+      new(json_string).encode
+    end
+
+    # Initializes the converter with the raw JSON string.
+    # @param json_string [String] The raw JSON input.
+    def initialize(json_string)
+      @json_string = json_string
+    end
+
+    # Parses the JSON and performs the transformation using the Encoder.
+    # @return [String] The converted custom text output.
+    def encode
+      data = JSON.parse(@json_string)
+      raise ArgumentError, 'Invalid JSON input' unless data.is_a?(Hash)
+
+      Encoder.new(data).encode
+    rescue JSON::ParserError, TypeError
+      raise ArgumentError, 'Invalid JSON input'
+    end
+
+    # Convenience: decode a TOON string back to JSON string
+    def self.decode(toon_string)
+      Decoder.decode(toon_string)
+    rescue StandardError
+      @json_string
+    end
+  end
+end

metadata

@@ -0,0 +1,54 @@
+--- !ruby/object:Gem::Specification
+name: toonify
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- ranjan
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: Toonify converts JSON data into a custom human-readable text format called
+  TOON.
+email:
+- ranjanbajra@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- examples/gemini_token_comparison.rb
+- lib/toonify.rb
+- lib/toonify/decoder.rb
+- lib/toonify/encoder.rb
+- lib/toonify/version.rb
+homepage: https://github.com/ran010/toonify
+licenses: []
+metadata:
+  homepage_uri: https://github.com/ran010/toonify
+  source_code_uri: https://github.com/ran010/toonify
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.7.2
+specification_version: 4
+summary: A simple JSON to custom text format converter.
+test_files: []