sorbet-toon 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fb0c6792c80943334b08014b87d6ee1b48a9341c69b58880a6b9a9f5f2eaf3ff
+  data.tar.gz: 86d67e78304e6afc6f0d5902171dbec0cc046e9cbba737e57335cb34a0da47c7
+SHA512:
+  metadata.gz: abf1f4ffe87f7e25d417563bcf1a28ef2f788552b5f9204a727af75c3cbaa81a3c43f5b306c6897740fcea2517fe165937a0a0cf45943e5aee0e0282aee6d441
+  data.tar.gz: 238e0569ff48361b6adbc273c5ca340d09099cd16be3947da828c57ccfb13fd4e53bf6b27e4910acd40d4fcc3e9b7426a259867cd691724c4e96f7c2ede18b55

data/LICENSE

@@ -0,0 +1,45 @@
+MIT License
+
+Copyright (c) 2025 Vicente Services SL
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+This project is a Ruby port of the original Python [DSPy library](https://github.com/stanfordnlp/dspy), which is licensed under the MIT License:
+
+MIT License
+
+Copyright (c) 2023 Stanford Future Data Systems
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/lib/sorbet/toon/README.md

@@ -0,0 +1,168 @@
+# Sorbet::Toon (alpha)
+
+Sorbet::Toon brings the official [TOON data format](https://github.com/toon-format/toon) to Ruby. It lets DSPy.rb—and any Sorbet-powered code—render compact prompt contracts and parse LLM responses using the same `DSPy::Signature` metadata that already powers JSON/BAML. Think of it as “typed Markdown tables for LLMs”: humans stay oriented, tokens stay low, and structs survive the round-trip.
+
+> **Status:** Codec + normalization + reconstruction + TOON mixins are merged. DSPy adapter + docs/tools are landing now; expect sharp edges until v0.1.0.
+
+---
+
+## Install & Load
+
+Until the gem ships on RubyGems, point Bundler at the repo:
+
+```ruby
+# Gemfile
+gem 'sorbet-toon', path: '.'   # replace with "~> 0.1" once published
+```
+
+```ruby
+require 'sorbet-runtime'
+require 'sorbet/toon'
+```
+
+Optional convenience:
+
+```ruby
+# Extensions are enabled automatically when sorbet/toon is required.
+# Structs already have #to_toon / .from_toon and enums gain helpers.
+```
+
+---
+
+## Encode + Decode Walkthrough
+
+```ruby
+class Source < T::Struct
+  prop :name, String
+  prop :url, String
+  prop :notes, T.nilable(String)
+end
+
+class ResearchSignature < DSPy::Signature
+  input  { const :query, String }
+  output do
+    const :summary, String
+    const :sources, T::Array[Source]
+  end
+end
+
+payload = ResearchSignature.output_struct_class.new(
+  summary: 'Recent AI papers',
+  sources: [
+    Source.new(name: 'Anthropic', url: 'https://anthropic.com', notes: nil),
+    Source.new(name: 'OpenAI', url: 'https://openai.com', notes: 'top pick')
+  ]
+)
+
+toon = Sorbet::Toon.encode(payload, signature: ResearchSignature, role: :output)
+# summary: Recent AI papers
+# sources[2]{name,url,notes}:
+#   Anthropic,https://anthropic.com,
+#   OpenAI,https://openai.com,"top pick"
+
+rehydrated = Sorbet::Toon.decode(toon, signature: ResearchSignature, role: :output)
+rehydrated.summary        # => "Recent AI papers"
+rehydrated.sources.last.notes  # => "top pick"
+```
+
+### Mixins (optional)
+
+```ruby
+toon_blob = payload.to_toon(include_type_metadata: true)
+# => struct instances gain #to_toon automatically
+
+decoded = Source.from_toon(%(- name: Ada\n  url: https://example.com))
+# => class methods .from_toon / enums too
+```
+
+### Config knobs
+
+```ruby
+Sorbet::Toon.configure do |config|
+  config.indent = 4
+  config.delimiter = Sorbet::Toon::Constants::PIPE
+  config.include_type_metadata = true
+end
+
+Sorbet::Toon.encode(value, delimiter: Sorbet::Toon::Constants::TAB) # per-call override
+```
+
+---
+
+## Signature Formatter & DSPy Adapter
+
+`Sorbet::Toon::SignatureFormatter` inspects `DSPy::Signature` definitions to produce human-friendly TOON guidance (field order, optional notes, tabular hints). `DSPy::Schema::SorbetToonAdapter` wires that into prompts:
+
+```ruby
+DSPy::Prompt.from_signature(
+  ResearchSignature,
+  schema_format: :toon,   # show TOON-friendly schema guidance
+  data_format:   :toon    # render inputs/outputs as TOON
+)
+```
+
+System prompts now read:
+
+```
+Your input schema fields (TOON order) are:
+- query (String) — Research question
+
+Your output schema fields (TOON order) are:
+- summary (String) — Key findings
+- sources (Array<Source>)
+    • Tabular columns: name, url, notes
+```
+
+User prompts embed real TOON values:
+
+```
+## Input Values
+```toon
+query: recent diffusion models
+```
+```
+
+LLMs reply with:
+
+```
+## Output values
+```toon
+summary: Diffusion models are accelerating image synthesis...
+sources[2]{name,url,notes}:
+  Anthropic,https://anthropic.com,
+  OpenAI,https://openai.com,"top pick"
+```
+
+Decoding uses the same adapter:
+
+```ruby
+DSPy::Schema::SorbetToonAdapter.parse_output(ResearchSignature, toon_blob)
+# => Hash with string keys (Predict converts to structs automatically)
+```
+
+`DSPy::LM.new(..., data_format: :toon)` enables automatic TOON parsing inside `DSPy::LM#chat`, so `Predict` keeps receiving hashes/structs without custom code.
+
+---
+
+## Troubleshooting
+
+| Symptom | Fix |
+| --- | --- |
+| `LoadError: cannot load such file -- sorbet/toon` | Ensure the gem is installed (via repo path or RubyGems) *before* loading DSPy. |
+| LLM responds with narrative text instead of TOON | Double-check the system prompt includes the TOON instructions. `data_format: :toon` forces `Prompt#render_user_prompt` to embed a ```toon``` block and explicitly asks for a TOON reply. |
+| `Sorbet::Toon::DecodeError` complaining about columns | Arrays of structs become tabular if every row shares the same keys. Make sure your struct props/names align and optional fields are either present in every row or omitted entirely. |
+| Need to include `_type` discriminator | `Sorbet::Toon.configure { |c| c.include_type_metadata = true }` or per-call option `Sorbet::Toon.encode(value, include_type_metadata: true)`. Useful for union/variant decoding. |
+| Optional fields showing as `null` | Normalizer already elides fully optional Sorbet props when nil. If you see `null`, the prop likely isn’t declared `T.nilable` or lacks a default. |
+
+---
+
+## Roadmap
+
+- [x] Codec + fixtures ported from the TypeScript reference.
+- [x] Sorbet-aware normalizer, decoder reconstruction, struct/enum mixins.
+- [x] Gem scaffold (`sorbet-toon.gemspec`, version file, README).
+- [ ] Signature formatter + DSPy adapter (data_format: :toon) **← in progress**
+- [ ] Full llms-full README (copy-pastable recipes + troubleshooting deep dive).
+- [ ] Integration specs mirroring the BAML suite.
+
+Feedback / bugs → issues or `hey@vicente.services`. Thank you!

data/lib/sorbet/toon/codec.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative 'errors'
+require_relative 'constants'
+require_relative 'encode/normalize'
+require_relative 'encode/encoders'
+require_relative 'decode/scanner'
+require_relative 'decode/decoders'
+
+module Sorbet
+  module Toon
+    module Codec
+      class << self
+        DEFAULT_INDENT = 2
+        DEFAULT_DELIMITER = Constants::DEFAULT_DELIMITER
+        DEFAULT_LENGTH_MARKER = false
+
+        def encode(input, indent: DEFAULT_INDENT, delimiter: DEFAULT_DELIMITER, length_marker: DEFAULT_LENGTH_MARKER)
+          normalized = Encode::Normalize.normalize(input)
+          Encode::Encoders.encode_value(
+            normalized,
+            indent: indent,
+            delimiter: delimiter,
+            length_marker: length_marker
+          )
+        end
+
+        def decode(input, **opts)
+          raise ArgumentError, 'Input must be a string' unless input.is_a?(String)
+
+          indent = extract_option(opts, :indent, DEFAULT_INDENT)
+          strict = extract_option(opts, :strict, true)
+
+          scan_result = Decode::Scanner.to_parsed_lines(input, indent, strict)
+          return {} if scan_result[:lines].empty?
+
+          cursor = Decode::LineCursor.new(scan_result[:lines], scan_result[:blank_lines])
+          Decode::Decoders.decode_value_from_lines(cursor, strict: strict)
+        end
+
+        private
+
+        def extract_option(options, key, default)
+          if options.key?(key)
+            options[key]
+          elsif options.key?(key.to_s)
+            options[key.to_s]
+          else
+            default
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sorbet/toon/config.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative 'constants'
+
+module Sorbet
+  module Toon
+    class Config
+      attr_accessor :include_type_metadata, :indent, :delimiter, :length_marker, :strict
+
+      def initialize
+        reset!
+      end
+
+      def reset!
+        @include_type_metadata = false
+        @indent = 2
+        @delimiter = Constants::DEFAULT_DELIMITER
+        @length_marker = false
+        @strict = true
+      end
+
+      def copy
+        copy = self.class.new
+        copy.include_type_metadata = include_type_metadata
+        copy.indent = indent
+        copy.delimiter = delimiter
+        copy.length_marker = length_marker
+        copy.strict = strict
+        copy
+      end
+
+      def resolve(overrides = {})
+        copy.apply(overrides)
+      end
+
+      def apply(overrides = {})
+        overrides.each do |key, value|
+          next if value.nil?
+
+          setter = "#{key}="
+          raise ArgumentError, "Unknown config option: #{key}" unless respond_to?(setter)
+
+          public_send(setter, value)
+        end
+
+        self
+      end
+    end
+  end
+end

data/lib/sorbet/toon/constants.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Sorbet
+  module Toon
+    module Constants
+      LIST_ITEM_MARKER = '-'
+      LIST_ITEM_PREFIX = '- '
+
+      COMMA = ','
+      COLON = ':'
+      SPACE = ' '
+      PIPE = '|'
+      HASH = '#'
+      TAB = "\t"
+
+      OPEN_BRACKET = '['
+      CLOSE_BRACKET = ']'
+      OPEN_BRACE = '{'
+      CLOSE_BRACE = '}'
+
+      NULL_LITERAL = 'null'
+      TRUE_LITERAL = 'true'
+      FALSE_LITERAL = 'false'
+
+      BACKSLASH = '\\'
+      DOUBLE_QUOTE = '"'
+      NEWLINE = "\n"
+      CARRIAGE_RETURN = "\r"
+
+      DELIMITERS = {
+        comma: COMMA,
+        tab: TAB,
+        pipe: PIPE
+      }.freeze
+
+      DEFAULT_DELIMITER = DELIMITERS[:comma]
+    end
+  end
+end

data/lib/sorbet/toon/decode/decoders.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+require_relative '../constants'
+require_relative '../errors'
+require_relative '../shared/string_utils'
+require_relative '../shared/literal_utils'
+require_relative 'scanner'
+require_relative 'parser'
+require_relative 'validation'
+
+module Sorbet
+  module Toon
+    module Decode
+      module Decoders
+        module_function
+
+        def decode_value_from_lines(cursor, **options)
+          first = cursor.peek
+          raise Sorbet::Toon::DecodeError, 'No content to decode' unless first
+
+          if Parser.array_header_after_hyphen?(first.content)
+            header_info = Parser.parse_array_header_line(first.content, Constants::DEFAULT_DELIMITER)
+            if header_info
+              cursor.advance
+              return decode_array_from_header(header_info[:header], header_info[:inline_values], cursor, 0, options)
+            end
+          end
+
+          if cursor.length == 1 && !key_value_line?(first)
+            return Parser.parse_primitive_token(first.content.strip)
+          end
+
+          decode_object(cursor, 0, options)
+        end
+
+        def key_value_line?(line)
+          content = line.content
+
+          if content.start_with?(Constants::DOUBLE_QUOTE)
+            closing = Shared::StringUtils.find_closing_quote(content, 0)
+            return false if closing == -1
+
+            return content[closing + 1..]&.include?(Constants::COLON)
+          end
+
+          content.include?(Constants::COLON)
+        end
+        private_class_method :key_value_line?
+
+        def decode_object(cursor, base_depth, options)
+          result = {}
+          computed_depth = nil
+
+          until cursor.at_end?
+            line = cursor.peek
+            break unless line
+            break if line.depth < base_depth
+
+            computed_depth ||= line.depth
+            break unless line.depth == computed_depth
+
+            key, value = decode_key_value_pair(line, cursor, computed_depth, options)
+            result[key] = value
+          end
+
+          result
+        end
+
+        def decode_key_value_pair(line, cursor, base_depth, options)
+          cursor.advance
+          key_value = decode_key_value(line.content, cursor, base_depth, options)
+          [key_value[:key], key_value[:value]]
+        end
+
+        def decode_key_value(content, cursor, base_depth, options)
+          array_header = Parser.parse_array_header_line(content, Constants::DEFAULT_DELIMITER)
+          if array_header && array_header[:header][:key]
+            value = decode_array_from_header(array_header[:header], array_header[:inline_values], cursor, base_depth, options)
+            return { key: array_header[:header][:key], value: value, follow_depth: base_depth + 1 }
+          end
+
+          key_info = Parser.parse_key_token(content, 0)
+          key = key_info[:key]
+          rest = content[key_info[:end]..]&.strip
+
+          if rest.nil? || rest.empty?
+            next_line = cursor.peek
+            if next_line && next_line.depth > base_depth
+              nested = decode_object(cursor, base_depth + 1, options)
+              return { key: key, value: nested, follow_depth: base_depth + 1 }
+            end
+            return { key: key, value: {}, follow_depth: base_depth + 1 }
+          end
+
+          value = Parser.parse_primitive_token(rest)
+          { key: key, value: value, follow_depth: base_depth + 1 }
+        end
+
+        def decode_array_from_header(header, inline_values, cursor, base_depth, options)
+          if inline_values
+            values = Parser.parse_delimited_values(inline_values, header[:delimiter])
+            primitives = Parser.map_row_values_to_primitives(values)
+            Validation.assert_expected_count(primitives.length, header[:length], 'inline array items', strict: options[:strict])
+            return primitives
+          end
+
+          if header[:fields] && !header[:fields].empty?
+            return decode_tabular_array(header, cursor, base_depth, options)
+          end
+
+          decode_list_array(header, cursor, base_depth, options)
+        end
+
+        def decode_list_array(header, cursor, base_depth, options)
+          items = []
+          item_depth = base_depth + 1
+          start_line = cursor.current&.line_number || 0
+          end_line = start_line
+
+          while !cursor.at_end? && items.length < header[:length]
+            line = cursor.peek
+            break unless line
+            break if line.depth < item_depth
+
+            if line.content == Constants::LIST_ITEM_MARKER || line.content.start_with?(Constants::LIST_ITEM_PREFIX)
+              item = decode_list_item(cursor, item_depth, options)
+              items << item
+              end_line = cursor.current&.line_number || end_line
+            else
+              break
+            end
+          end
+
+          Validation.assert_expected_count(items.length, header[:length], 'list array items', strict: options[:strict])
+          Validation.validate_no_blank_lines_in_range(start_line, end_line, cursor.blank_lines, strict: options[:strict], context: 'list array')
+          Validation.validate_no_extra_list_items(cursor, item_depth, header[:length])
+          items
+        end
+
+        def decode_tabular_array(header, cursor, base_depth, options)
+          rows = []
+          row_depth = base_depth + 1
+          start_line = cursor.current&.line_number || 0
+          end_line = start_line
+
+          while cursor.has_more_at_depth?(row_depth) && rows.length < header[:length]
+            line = cursor.peek
+            break unless line
+            break if line.depth != row_depth
+            break if line.content.start_with?(Constants::LIST_ITEM_PREFIX)
+
+            values = Parser.parse_delimited_values(line.content, header[:delimiter])
+            Validation.assert_expected_count(
+              values.length,
+              header[:fields].length,
+              'tabular row values',
+              strict: options[:strict]
+            )
+            primitives = Parser.map_row_values_to_primitives(values)
+            rows << primitives
+            cursor.advance
+            end_line = line.line_number
+          end
+
+          Validation.assert_expected_count(rows.length, header[:length], 'tabular rows', strict: options[:strict])
+          Validation.validate_no_blank_lines_in_range(start_line, end_line, cursor.blank_lines, strict: options[:strict], context: 'tabular array')
+          Validation.validate_no_extra_tabular_rows(cursor, row_depth, header)
+
+          rows.map do |primitives|
+            Hash[header[:fields].zip(primitives)]
+          end
+        end
+
+        def decode_list_item(cursor, base_depth, options)
+          line = cursor.next
+          raise Sorbet::Toon::DecodeError, 'Expected list item' unless line
+
+          return {} if line.content == Constants::LIST_ITEM_MARKER
+
+          unless line.content.start_with?(Constants::LIST_ITEM_PREFIX)
+            raise Sorbet::Toon::DecodeError, "Expected list item to start with \"#{Constants::LIST_ITEM_PREFIX}\""
+          end
+
+          after_hyphen = line.content[Constants::LIST_ITEM_PREFIX.length..] || ''
+          return {} if after_hyphen.strip.empty?
+
+          if Parser.array_header_after_hyphen?(after_hyphen)
+            header_info = Parser.parse_array_header_line(after_hyphen, Constants::DEFAULT_DELIMITER)
+            if header_info
+              return decode_array_from_header(header_info[:header], header_info[:inline_values], cursor, base_depth, options)
+            end
+          end
+
+          if Parser.object_first_field_after_hyphen?(after_hyphen)
+            return decode_object_from_list_item(line, cursor, base_depth, options)
+          end
+
+          Parser.parse_primitive_token(after_hyphen)
+        end
+
+        def decode_object_from_list_item(first_line, cursor, base_depth, options)
+          after_hyphen = first_line.content[Constants::LIST_ITEM_PREFIX.length..] || ''
+          key_value = decode_key_value(after_hyphen, cursor, base_depth, options)
+
+          obj = { key_value[:key] => key_value[:value] }
+          follow_depth = key_value[:follow_depth]
+
+          until cursor.at_end?
+            line = cursor.peek
+            break unless line
+            break if line.depth < follow_depth
+
+            if line.depth == follow_depth && !line.content.start_with?(Constants::LIST_ITEM_PREFIX)
+              k, v = decode_key_value_pair(line, cursor, follow_depth, options)
+              obj[k] = v
+            else
+              break
+            end
+          end
+
+          obj
+        end
+      end
+    end
+  end
+end