context_spook 1.6.0 → 1.7.0

data/lib/context_spook/toon/example.md

@@ -1,69 +1,39 @@
-# TOON (Token-Oriented Object Notation)
+# TOON format — quick reference
+#
+# key: value                  scalar (bare if safe)
+# key[2]:                     array with length
+#   - item                    list item
+# key[4]{col1,col2}:          tabular array (CSV rows)
+#   val1,val2                 one row per line
+#
+# Quoting: double-quote a string only if it contains
+# , { } [ ] # " \  or starts/ends with whitespace, or looks
+# like a number / true / false / null.
 
-It is a compact, human-readable serialization format that reduces JSON token
-count by 30-40% while maintaining lossless data fidelity. It uses CSV-style
-tabular arrays for uniform data structures and YAML-like indentation for nested
-objects, making it ideal for LLM input where token efficiency and parsing
-reliability matter.
-
-Key benefit: Same data, fewer tokens, better LLM understanding.
-
-**Original JSON:**
-```json
-{
-  "database": {
-    "name": "user_management",
-    "tables": [
-      {
-        "name": "users",
-        "columns": [
-          {
-            "name": "id",
-            "type": "integer",
-            "primary_key": true
-          },
-          {
-            "name": "email",
-            "type": "string",
-            "unique": true
-          }
-        ],
-        "row_count": 1000
-      },
-      {
-        "name": "orders",
-        "columns": [
-          {
-            "name": "id",
-            "type": "integer",
-            "primary_key": true
-          },
-          {
-            "name": "user_id",
-            "type": "integer"
-          }
-        ],
-        "row_count": 5000
-      }
-    ]
-  }
-}
-```
-
-**TOON equivalent:**
-```
+files:
+  "app/models/user.rb":
+    content[4]{linenumber,line}:
+      1,require 'json'
+      2,""
+      3,"# a comment line"
+      4,def parse(data)
+    size: 42
+    lines: 4
 database:
   name: user_management
-  tables[2]{name,row_count}:
-    users,1000
-    orders,5000
-  tables[2]{name,columns}:
-    users:
-      columns[2]{name,type,primary_key}:
-        id,integer,true
-        email,string,true
-    orders:
-      columns[2]{name,type}:
-        id,integer,true
-        user_id,integer,false
-```
+  tables[2]:
+    - name: users
+      columns[2]:
+        - name: id
+          type: integer
+          primary_key: true
+        - name: email
+          type: string
+      row_count: 1000
+    - name: orders
+      columns[2]:
+        - name: id
+          type: integer
+        - name: user_id
+          type: integer
+      row_count: 5000

data/lib/context_spook/toon.rb

@@ -1,13 +1,17 @@
-require 'toon'
+require 'json'
+require 'context_spook/toon/encoder'
 
 # The ContextSpook::TOON module provides TOON (Token-Oriented Object Notation)
-# serialization functionality for context objects.
+# v4 serialization functionality for context objects.
 module ContextSpook::TOON
-  # Converts the context object to TOON format.
+  # Converts the context object to TOON v4 format.
+  #
+  # Uses a fast JSON round-trip (C-optimized) to normalize the data,
+  # then walks the canonical Hash/Array tree with our native encoder.
   #
   # @return [String] the TOON-encoded representation of the context
   def to_toon
-    ::Toon.encode(as_json)
+    Encoder.encode(as_json)
   end
   memoize method: :to_toon
 
@@ -15,19 +19,13 @@ module ContextSpook::TOON
   #
   # @return [Integer] the size in bytes of the TOON representation
   def toon_size
-    to_toon.size
+    to_toon.bytesize
   end
 
-  # The toon_example method reads and returns the content of an example
-  # Markdown file that demonstrates the TOON (Token-Oriented Object Notation)
-  # format.
-  #
-  # This method is used to provide a sample representation of how context data
-  # can be formatted in TOON, which is an alternative serialization format
-  # supported by the ContextSpook library.
+  # Streams the TOON v4 output to the given IO.
   #
-  # @return [ String ] the content of the TOON example Markdown file
-  def toon_example
-    File.read(Pathname.new(__dir__) + 'toon/example.md')
+  # @param output [IO] the IO to write to (e.g. STDOUT, File, StringIO)
+  def to_toon_stream(output)
+    Encoder.encode(as_json, output: output)
   end
 end

data/lib/context_spook/utils.rb

@@ -6,15 +6,14 @@ module ContextSpook::Utils
   # The format_size method converts a byte size value into a human-readable
   # string with binary units.
   #
-  # This method takes a raw byte count and formats it using the Tins::Unit
-  # library to display the size with appropriate binary prefixes (KiB, MiB,
-  # etc.) and two decimal places.
+  # library to display the size with appropriate IEC prefixes (KiB, MiB,
+  # etc.) and one decimal place.
   #
   # @param context_size [ Integer ] the size in bytes to be formatted
   #
   # @return [ String ] the formatted size string with binary units
   def format_size(context_size)
-    Tins::Unit.format(context_size, format: '%.2f %U', unit: ?b, prefix: 1024)
+    Tins::Unit.format(context_size, format: '%.1f %U', unit: ?B, prefix: :iec_uc)
   end
 
   # The format_tokens method converts a token count into a human-readable
@@ -28,14 +27,20 @@ module ContextSpook::Utils
   end
 
   # The estimate_tokens method provides a crude estimation of token counts
-  # based on the byte size of the input content.
+  # from either a raw byte count or a string.
   #
-  # This follows a heuristic where one token is roughly equivalent to 3.5 bytes.
+  # This follows a heuristic where one token is roughly equivalent to 3.5
+  # bytes. A numeric argument is treated as a byte count directly; any other
+  # value is converted to a string and measured by its byte size.
   #
-  # @param text [ String ] the content to be estimated
+  # @param text_or_bytesize [ String, Integer ] the content to estimate
+  #   tokens for, either as a string or a precomputed byte count
   #
   # @return [ Integer ] the estimated number of tokens
-  def estimate_tokens(text)
-    (text.size.to_f / 3.5).ceil
+  def estimate_tokens(text_or_bytesize)
+    unless text_or_bytesize.is_a?(Numeric)
+      text_or_bytesize = text_or_bytesize.to_s.bytesize
+    end
+    (text_or_bytesize.to_f / 3.5).ceil
   end
 end

data/lib/context_spook/version.rb

@@ -1,6 +1,6 @@
 module ContextSpook
   # ContextSpook version
-  VERSION         = '1.6.0'
+  VERSION         = '1.7.0'
   VERSION_ARRAY   = VERSION.split('.').map(&:to_i) # :nodoc:
   VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
   VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:

data/spec/context_spook/generator_spec.rb

@@ -12,7 +12,7 @@ describe ContextSpook::Generator do
       expect_any_instance_of(described_class).to receive(:verbose_puts).
         with(/Built.*?31.0 T tokens.*?of JSON context in total/)
       my_context = ContextSpook.generate_context do
-        self.token_estimator = -> text { (text.size.to_f / Math::PI).ceil }
+        self.token_estimator = -> text { (text.bytesize.to_f / Math::PI).ceil }
         context do
           variable foo: 'bar'
           metadata version: '1.0'
@@ -92,12 +92,73 @@ describe ContextSpook::Generator do
       expect_any_instance_of(described_class).to receive(:output_context_size).
         and_call_original
       expect_any_instance_of(described_class).to receive(:verbose_puts).
-        with(/Built.*?Kb.*?KT.*?of JSON context in total/)
+        with(/Built.*?KiB.*?KT.*?of JSON context in total/)
       allow_any_instance_of(described_class::Context).to receive(:verbose_puts)
       my_context
     end
   end
 
+  describe 'manifest mode' do
+    let :manifest_context do
+      ContextSpook.generate_context(manifest: true, verbose: false) do
+        context do
+          file 'lib/context_spook.rb'
+        end
+      end
+    end
+
+    let :full_context do
+      ContextSpook.generate_context(manifest: false, verbose: false) do
+        context do
+          file 'lib/context_spook.rb'
+        end
+      end
+    end
+
+    it 'defaults to manifest: false' do
+      gen = described_class.send(:new)
+      expect(gen.manifest).to eq false
+    end
+
+    it 'stores manifest: true on Generator' do
+      gen = described_class.send(:new, manifest: true)
+      expect(gen.manifest).to eq true
+    end
+
+    it 'stamps manifest: true in metadata' do
+      expect(manifest_context.metadata[:manifest]).to eq true
+    end
+
+    it 'omits manifest key from metadata when false' do
+      expect(full_context.metadata).not_to have_key(:manifest)
+    end
+
+    it 'includes content and lines in non-manifest mode' do
+      file = full_context.files['lib/context_spook.rb']
+      expect(file[:content]).to be_a(Array)
+      expect(file[:content].first).to include(:linenumber, :line)
+      expect(file[:lines]).to be > 0
+    end
+
+    it 'omits content and lines in manifest mode' do
+      file = manifest_context.files['lib/context_spook.rb']
+      expect(file).not_to have_key(:content)
+      expect(file).not_to have_key(:lines)
+    end
+
+    it 'retains metadata fields in manifest mode' do
+      file = manifest_context.files['lib/context_spook.rb']
+      expect(file[:size]).to be > 0
+      expect(file[:tokens]).to be > 0
+      expect(file[:content_types]).to be_present
+    end
+
+    it 'produces smaller output in manifest mode' do
+      expect(manifest_context.to_json.bytesize).
+        to be < full_context.to_json.bytesize
+    end
+  end
+
   describe 'Context' do
     it 'can have variables' do
       expect(my_context.variables[:branch]).to be_present

data/spec/context_spook/toon_encoder_spec.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'json'
+
+# Conformance spec: drives ContextSpook::TOON::Encoder against the official
+# TOON v4 encode fixtures from the toon-format-spec repository.
+#
+# Each fixture file is a JSON document:
+#   { "tests": [ { "name", "input", "expected", ["options"], ["minSpecVersion"] } ] }
+#
+# We load every encode fixture, iterate the test cases, encode via our
+# encoder, and compare the output string.
+#
+# Fixtures with minSpecVersion above our target are skipped.
+
+describe ContextSpook::TOON::Encoder do
+  FIXTURES_DIR = File.expand_path('../fixtures/encode', __dir__)
+  MAX_SPEC_VERSION = '4.4'
+
+  describe '.new' do
+    it 'can output to IO' do
+      io = StringIO.new
+      encoder = described_class.new(output: io)
+      encoder.encode(%w[ hello world ], 0)
+      expect(io.string).to eq '[2]: hello,world'
+    end
+
+    it 'can output to StringArray' do
+      encoder = described_class.new
+      encoder.encode(%w[ hello world ], 0)
+      expect(encoder.to_s).to eq '[2]: hello,world'
+    end
+  end
+
+  def self.fixture_files
+    Dir[File.join(FIXTURES_DIR, '*.json')].sort
+  end
+
+  def self.version_ok?(fixture)
+    return true unless (ver = fixture['minSpecVersion'])
+    ver.version <= MAX_SPEC_VERSION.version
+  end
+
+  fixture_files.each do |path|
+    data = JSON.parse(File.read(path))
+    category = File.basename(path, '.json')
+
+    describe "fixture: #{category}" do
+      data['tests'].each do |test|
+        name = test['name']
+        input = JSON.parse(JSON.generate(test['input']))
+
+        # Skip tests for spec versions we haven't implemented yet
+        if self.version_ok?(test)
+          it name do
+            options = test['options'] || {}
+            delimiter = options['delimiter'] || ','
+            indent    = (options['indentSize'] || 2)
+
+            actual = described_class.encode(
+              input, indent: indent, delimiter:
+            )
+            expected = test['expected']
+
+            if actual == expected
+              # pass
+            else
+              # Pretty diff for debugging
+              expected_lines = expected.lines
+              actual_lines   = actual.lines
+              diff_lines = []
+              max_len = [expected_lines.size, actual_lines.size].max
+              (0...max_len).each do |i|
+                e = expected_lines[i] || '<missing>'
+                a = actual_lines[i] || '<missing>'
+                diff_lines << "  #{i + 1}: expected #{e.inspect}  got #{a.inspect}" \
+                  unless e == a
+              end
+              raise "TOON mismatch:\n#{diff_lines.join("\n")}"
+            end
+          end
+        else
+          skip "#{name} (requires spec v#{test['minSpecVersion']}, max: #{MAX_SPEC_VERSION})" do
+          end
+        end
+      end
+    end
+  end
+end

data/spec/context_spook/toon_spec.rb

@@ -20,22 +20,21 @@ describe ContextSpook::TOON do
 
     it 'produces valid TOON output' do
       toon_output = my_context.to_toon
-      expect(toon_output).to match(
-        %r{
-          \Afiles:\n
-          ^[ ]{2}"lib/context_spook\.rb":\n
-          ^[ ]{4}content:[ ]"[^"]+"\n
-          ^[ ]{4}size:[ ]120\n
-          ^[ ]{4}lines:[ ]5\n
-          ^[ ]{4}content_types\[2\]:[ ]application/x-ruby,text/x-ruby\n
-          ^commands:\n
-          ^metadata:\n
-          ^[ ]{2}version:[ ]"1\.0"\n
-          ^[ ]{2}format:[ ]JSON\n
-          ^variables:\n
-          ^[ ]{2}foo:[ ]bar\z
-        }x
+      expect(toon_output).to start_with(
+        "files:\n" \
+         "  \"lib/context_spook.rb\":\n" \
+        "    content[5]{linenumber,line}:\n" \
+        "      1,module ContextSpook\n"
       )
+      expect(toon_output).to include("    size: 120\n")
+      expect(toon_output).to include("    lines: 5\n")
+      expect(toon_output)
+        .to include("    content_types[2]: application/x-ruby,text/x-ruby\n")
+      expect(toon_output).to include("commands:\n")
+      expect(toon_output).to include("  version: \"1.0\"\n")
+      expect(toon_output).to include("  format: JSON\n")
+      expect(toon_output).to include("variables:\n")
+      expect(toon_output).to end_with("  foo: bar")
     end
   end
 
@@ -49,7 +48,7 @@ describe ContextSpook::TOON do
     it 'matches the size of the TOON output' do
       size = my_context.toon_size
       toon_output = my_context.to_toon
-      expect(size).to eq(toon_output.size)
+      expect(size).to eq(toon_output.bytesize)
     end
   end
 

data/spec/examples/README.md

@@ -0,0 +1,79 @@
+# TOON Examples
+
+Example TOON files, one per format feature – each filename names the feature it demonstrates. All files validate against the [reference implementation](https://github.com/toon-format/toon): valid examples decode in strict mode, invalid examples error, and every conversion pair is generated by the reference encoder and round-trip verified.
+
+## Valid Examples
+
+### Objects
+
+- [`valid/objects.toon`](valid/objects.toon) – Flat object with primitive values
+  - Key-value pairs across all data types: string, number, boolean, null
+  - Spec: §8
+
+- [`valid/nested-objects.toon`](valid/nested-objects.toon) – Multi-level nested objects
+  - Indentation-based nesting (2 spaces per level) instead of braces
+  - Spec: §8, §12
+
+### Arrays
+
+- [`valid/primitive-arrays.toon`](valid/primitive-arrays.toon) – Inline primitive arrays
+  - Compact inline form `[N]: item1,item2,item3` and the empty array `[]`
+  - Spec: §9.1
+
+- [`valid/mixed-arrays.toon`](valid/mixed-arrays.toon) – Mixed-type array (list form)
+  - One `-` item per element, used wherever neither inline nor tabular form applies
+  - Spec: §9.4
+
+### Comments
+
+- [`valid/comments.toon`](valid/comments.toon) – Full-line `#` comments
+  - Stripped by decoders in a lexical pre-pass, before row counting – decode-only; encoders never emit them
+  - Spec: §5.1
+
+### Delimiters
+
+- [`valid/pipe-delimiter.toon`](valid/pipe-delimiter.toon) – Pipe delimiter (`|`)
+  - Delimiter marker appears in both header `[N|]` and field list `{field|...}`
+  - Spec: §11
+
+- [`valid/tab-delimiter.toon`](valid/tab-delimiter.toon) – Tab delimiter (`\t`)
+  - Tab character appears in both header and between fields
+  - Spec: §11
+
+- [`valid/delimiter-scoping.toon`](valid/delimiter-scoping.toon) – Document vs active delimiter
+  - Tabular row cells split only on the active delimiter; object field values follow document delimiter quoting
+  - Spec: §11
+
+## Conversions
+
+Side-by-side JSON ↔ TOON pairs, one per tabular form:
+
+- [`conversions/tabular-arrays.json`](conversions/tabular-arrays.json) + [`conversions/tabular-arrays.toon`](conversions/tabular-arrays.toon)
+  - Uniform arrays of objects collapse into a table: fields declared once in the header, one row per element
+  - Spec: §9.3
+
+- [`conversions/nested-field-groups.json`](conversions/nested-field-groups.json) + [`conversions/nested-field-groups.toon`](conversions/nested-field-groups.toon)
+  - A uniform nested-object column folds into the header (`{id,customer{name,country},total}`) while rows stay flat
+  - Spec: §9.3
+
+- [`conversions/keyed-tabular-objects.json`](conversions/keyed-tabular-objects.json) + [`conversions/keyed-tabular-objects.toon`](conversions/keyed-tabular-objects.toon)
+  - Objects whose values are uniform objects become tables whose entry rows carry their own keys; the colon after the length (`[3:]`) marks the keyed header
+  - Spec: §9.5
+
+## Invalid Examples
+
+Files that intentionally violate TOON syntax rules and MUST error in strict mode:
+
+- [`invalid/length-mismatch.toon`](invalid/length-mismatch.toon) – Array length mismatch
+  - Declares `[3]` but provides only 2 rows
+  - Spec: §14.1
+
+- [`invalid/multiple-root-primitives.toon`](invalid/multiple-root-primitives.toon) – Multiple depth-0 primitives
+  - Two non-key-value lines at root depth
+  - Spec: §5, §14.2
+
+- [`invalid/delimiter-mismatch.toon`](invalid/delimiter-mismatch.toon) – Header delimiter mismatch
+  - Declares pipe delimiter in brackets (`[N|]`) but uses comma-separated fields (`{a,b}`)
+  - Spec: §6
+
+For reference test fixtures, see the [`tests/`](../tests/) directory.

data/spec/examples/conversions/keyed-tabular-objects.json

@@ -0,0 +1,19 @@
+{
+  "environments": {
+    "production": {
+      "region": "eu-central-1",
+      "replicas": 6,
+      "debug": false
+    },
+    "staging": {
+      "region": "eu-central-1",
+      "replicas": 2,
+      "debug": true
+    },
+    "development": {
+      "region": "local",
+      "replicas": 1,
+      "debug": true
+    }
+  }
+}

data/spec/examples/conversions/keyed-tabular-objects.toon

@@ -0,0 +1,4 @@
+environments[3:]{region,replicas,debug}:
+  production: eu-central-1,6,false
+  staging: eu-central-1,2,true
+  development: local,1,true

data/spec/examples/conversions/nested-field-groups.json

@@ -0,0 +1,28 @@
+{
+  "orders": [
+    {
+      "id": 1,
+      "customer": {
+        "name": "Ada",
+        "country": "DK"
+      },
+      "total": 99
+    },
+    {
+      "id": 2,
+      "customer": {
+        "name": "Bob",
+        "country": "UK"
+      },
+      "total": 149
+    },
+    {
+      "id": 3,
+      "customer": {
+        "name": "Cleo",
+        "country": "FR"
+      },
+      "total": 249
+    }
+  ]
+}

data/spec/examples/conversions/nested-field-groups.toon

@@ -0,0 +1,4 @@
+orders[3]{id,customer{name,country},total}:
+  1,Ada,DK,99
+  2,Bob,UK,149
+  3,Cleo,FR,249

data/spec/examples/conversions/tabular-arrays.json

@@ -0,0 +1,22 @@
+{
+  "users": [
+    {
+      "id": 1,
+      "name": "Ada",
+      "role": "admin",
+      "active": true
+    },
+    {
+      "id": 2,
+      "name": "Bob",
+      "role": "developer",
+      "active": true
+    },
+    {
+      "id": 3,
+      "name": "Cleo",
+      "role": "designer",
+      "active": false
+    }
+  ]
+}

data/spec/examples/conversions/tabular-arrays.toon

@@ -0,0 +1,4 @@
+users[3]{id,name,role,active}:
+  1,Ada,admin,true
+  2,Bob,developer,true
+  3,Cleo,designer,false

data/spec/examples/invalid/delimiter-mismatch.toon

@@ -0,0 +1,3 @@
+items[2|]{a,b}:
+  1|2
+  3|4

data/spec/examples/invalid/length-mismatch.toon

@@ -0,0 +1,3 @@
+items[3]{id,name}:
+  1,Ada
+  2,Bob

data/spec/examples/invalid/multiple-root-primitives.toon

@@ -0,0 +1,2 @@
+id 123
+name Ada

data/spec/examples/valid/comments.toon

@@ -0,0 +1,5 @@
+# Weekly export, generated 2025-01-15
+users[2]{id,name,role}:
+  # comment lines are stripped before row counting
+  1,Ada,admin
+  2,Bob,user

data/spec/examples/valid/delimiter-scoping.toon

@@ -0,0 +1,5 @@
+rows[1|]{id|value}:
+  1|a,b
+
+items[1|]:
+  - note: "a,b"

data/spec/examples/valid/mixed-arrays.toon

@@ -0,0 +1,5 @@
+items[3]:
+  - 1
+  - a: hello
+    b: world
+  - text value

data/spec/examples/valid/nested-objects.toon

@@ -0,0 +1,9 @@
+user:
+  id: 123
+  name: Ada Lovelace
+  contact:
+    email: ada@example.com
+    phone: +1-555-0100
+  settings:
+    theme: dark
+    notifications: true

data/spec/examples/valid/objects.toon

@@ -0,0 +1,6 @@
+id: 123
+name: Ada Lovelace
+active: true
+email: ada@example.com
+score: 98.5
+nickname: null

data/spec/examples/valid/pipe-delimiter.toon

@@ -0,0 +1,3 @@
+items[2|]{sku|name|qty|price}:
+  A1|Widget|2|9.99
+  B2|Gadget|1|14.5

data/spec/examples/valid/primitive-arrays.toon

@@ -0,0 +1,3 @@
+tags[3]: admin,ops,dev
+numbers[5]: 1,2,3,4,5
+empty: []

data/spec/examples/valid/tab-delimiter.toon

@@ -0,0 +1,3 @@
+items[2	]{sku	name	qty	price}:
+  A1	Widget	2	9.99
+  B2	Gadget	1	14.5