larsklevan-json 1.5.1

data/tests/test_json_encoding.rb

@@ -0,0 +1,65 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+require 'test/unit'
+require File.join(File.dirname(__FILE__), 'setup_variant')
+
+class TC_JSONEncoding < Test::Unit::TestCase
+  include JSON
+
+  def setup
+    @utf_8 = '["© ≠ €!"]'
+    @parsed = [ "© ≠ €!" ]
+    @generated = '["\u00a9 \u2260 \u20ac!"]'
+    if String.method_defined?(:encode)
+      @utf_16_data = [@parsed.first.encode('utf-16be', 'utf-8')]
+      @utf_8_ascii_8bit = @utf_8.dup.force_encoding(Encoding::ASCII_8BIT)
+      @utf_16be = @utf_8.encode('utf-16be', 'utf-8')
+      @utf_16be_ascii_8bit = @utf_16be.dup.force_encoding(Encoding::ASCII_8BIT)
+      @utf_16le = @utf_8.encode('utf-16le', 'utf-8')
+      @utf_16le_ascii_8bit = @utf_16le.dup.force_encoding(Encoding::ASCII_8BIT)
+      @utf_32be = @utf_8.encode('utf-32be', 'utf-8')
+      @utf_32be_ascii_8bit = @utf_32be.dup.force_encoding(Encoding::ASCII_8BIT)
+      @utf_32le = @utf_8.encode('utf-32le', 'utf-8')
+      @utf_32le_ascii_8bit = @utf_32le.dup.force_encoding(Encoding::ASCII_8BIT)
+    else
+      require 'iconv'
+      @utf_16_data = Iconv.iconv('utf-16be', 'utf-8', @parsed.first)
+      @utf_8_ascii_8bit = @utf_8.dup
+      @utf_16be, = Iconv.iconv('utf-16be', 'utf-8', @utf_8)
+      @utf_16be_ascii_8bit = @utf_16be.dup
+      @utf_16le, = Iconv.iconv('utf-16le', 'utf-8', @utf_8)
+      @utf_16le_ascii_8bit = @utf_16le.dup
+      @utf_32be, = Iconv.iconv('utf-32be', 'utf-8', @utf_8)
+      @utf_32be_ascii_8bit = @utf_32be.dup
+      @utf_32le, = Iconv.iconv('utf-32le', 'utf-8', @utf_8)
+      @utf_32le_ascii_8bit = @utf_32le.dup
+    end
+  end
+
+  def test_parse
+    assert_equal @parsed, JSON.parse(@utf_8)
+    assert_equal @parsed, JSON.parse(@utf_16be)
+    assert_equal @parsed, JSON.parse(@utf_16le)
+    assert_equal @parsed, JSON.parse(@utf_32be)
+    assert_equal @parsed, JSON.parse(@utf_32le)
+  end
+
+  def test_parse_ascii_8bit
+    assert_equal @parsed, JSON.parse(@utf_8_ascii_8bit)
+    assert_equal @parsed, JSON.parse(@utf_16be_ascii_8bit)
+    assert_equal @parsed, JSON.parse(@utf_16le_ascii_8bit)
+    assert_equal @parsed, JSON.parse(@utf_32be_ascii_8bit)
+    assert_equal @parsed, JSON.parse(@utf_32le_ascii_8bit)
+  end
+
+  def test_generate
+    assert_equal @generated, JSON.generate(@parsed, :ascii_only => true)
+    if defined?(::Encoding)
+      assert_equal @generated, JSON.generate(@utf_16_data, :ascii_only => true)
+    else
+      # XXX checking of correct utf8 data is not as strict (yet?) without :ascii_only
+      assert_raises(JSON::GeneratorError) { JSON.generate(@utf_16_data, :ascii_only => true) }
+    end
+  end
+end

data/tests/test_json_fixtures.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+require 'test/unit'
+require File.join(File.dirname(__FILE__), 'setup_variant')
+
+class TC_JSONFixtures < Test::Unit::TestCase
+  def setup
+    fixtures = File.join(File.dirname(__FILE__), 'fixtures/*.json')
+    passed, failed = Dir[fixtures].partition { |f| f['pass'] }
+    @passed = passed.inject([]) { |a, f| a << [ f, File.read(f) ] }.sort
+    @failed = failed.inject([]) { |a, f| a << [ f, File.read(f) ] }.sort
+  end
+
+  def test_passing
+    for name, source in @passed
+      begin
+        assert JSON.parse(source),
+          "Did not pass for fixture '#{name}': #{source.inspect}"
+      rescue => e
+        warn "\nCaught #{e.class}(#{e}) for fixture '#{name}': #{source.inspect}\n#{e.backtrace * "\n"}"
+        raise e
+      end
+    end
+  end
+
+  def test_failing
+    for name, source in @failed
+      assert_raises(JSON::ParserError, JSON::NestingError,
+        "Did not fail for fixture '#{name}': #{source.inspect}") do
+        JSON.parse(source)
+      end
+    end
+  end
+end

data/tests/test_json_generate.rb

@@ -0,0 +1,185 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+require 'test/unit'
+require File.join(File.dirname(__FILE__), 'setup_variant')
+
+class TC_JSONGenerate < Test::Unit::TestCase
+  include JSON
+
+  def setup
+    @hash = {
+      'a' => 2,
+      'b' => 3.141,
+      'c' => 'c',
+      'd' => [ 1, "b", 3.14 ],
+      'e' => { 'foo' => 'bar' },
+      'g' => "\"\0\037",
+      'h' => 1000.0,
+      'i' => 0.001
+    }
+    @json2 = '{"a":2,"b":3.141,"c":"c","d":[1,"b",3.14],"e":{"foo":"bar"},' +
+      '"g":"\\"\\u0000\\u001f","h":1000.0,"i":0.001}'
+    @json3 = <<'EOT'.chomp
+{
+  "a": 2,
+  "b": 3.141,
+  "c": "c",
+  "d": [
+    1,
+    "b",
+    3.14
+  ],
+  "e": {
+    "foo": "bar"
+  },
+  "g": "\"\u0000\u001f",
+  "h": 1000.0,
+  "i": 0.001
+}
+EOT
+  end
+
+  def test_generate
+    json = generate(@hash)
+    assert_equal(JSON.parse(@json2), JSON.parse(json))
+    parsed_json = parse(json)
+    assert_equal(@hash, parsed_json)
+    json = generate({1=>2})
+    assert_equal('{"1":2}', json)
+    parsed_json = parse(json)
+    assert_equal({"1"=>2}, parsed_json)
+    assert_raise(GeneratorError) { generate(666) }
+  end
+
+  def test_generate_pretty
+    json = pretty_generate(@hash)
+    assert_equal(JSON.parse(@json3), JSON.parse(json))
+    parsed_json = parse(json)
+    assert_equal(@hash, parsed_json)
+    json = pretty_generate({1=>2})
+    assert_equal(<<'EOT'.chomp, json)
+{
+  "1": 2
+}
+EOT
+    parsed_json = parse(json)
+    assert_equal({"1"=>2}, parsed_json)
+    assert_raise(GeneratorError) { pretty_generate(666) }
+  end
+
+  def test_fast_generate
+    json = fast_generate(@hash)
+    assert_equal(JSON.parse(@json2), JSON.parse(json))
+    parsed_json = parse(json)
+    assert_equal(@hash, parsed_json)
+    json = fast_generate({1=>2})
+    assert_equal('{"1":2}', json)
+    parsed_json = parse(json)
+    assert_equal({"1"=>2}, parsed_json)
+    assert_raise(GeneratorError) { fast_generate(666) }
+  end
+
+
+
+  def test_states
+    json = generate({1=>2}, nil)
+    assert_equal('{"1":2}', json)
+    s = JSON.state.new
+    assert s.check_circular?
+    assert s[:check_circular?]
+    h = { 1=>2 }
+    h[3] = h
+    assert_raises(JSON::NestingError) {  generate(h) }
+    assert_raises(JSON::NestingError) {  generate(h, s) }
+    s = JSON.state.new
+    a = [ 1, 2 ]
+    a << a
+    assert_raises(JSON::NestingError) {  generate(a, s) }
+    assert s.check_circular?
+    assert s[:check_circular?]
+  end
+
+  def test_pretty_state
+    state = PRETTY_STATE_PROTOTYPE.dup
+    assert_equal({
+      :allow_nan    => false,
+      :array_nl     => "\n",
+      :ascii_only   => false,
+      :depth        => 0,
+      :indent       => "  ",
+      :max_nesting  => 19,
+      :object_nl    => "\n",
+      :space        => " ",
+      :space_before => "",
+    }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s })
+  end
+
+  def test_safe_state
+    state = SAFE_STATE_PROTOTYPE.dup
+    assert_equal({
+      :allow_nan    => false,
+      :array_nl     => "",
+      :ascii_only   => false,
+      :depth        => 0,
+      :indent       => "",
+      :max_nesting  => 19,
+      :object_nl    => "",
+      :space        => "",
+      :space_before => "",
+    }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s })
+  end
+
+  def test_fast_state
+    state = FAST_STATE_PROTOTYPE.dup
+    assert_equal({
+      :allow_nan    => false,
+      :array_nl     => "",
+      :ascii_only   => false,
+      :depth        => 0,
+      :indent       => "",
+      :max_nesting  => 0,
+      :object_nl    => "",
+      :space        => "",
+      :space_before => "",
+    }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s })
+  end
+
+  def test_allow_nan
+    assert_raises(GeneratorError) { generate([JSON::NaN]) }
+    assert_equal '[NaN]', generate([JSON::NaN], :allow_nan => true)
+    assert_raises(GeneratorError) { fast_generate([JSON::NaN]) }
+    assert_raises(GeneratorError) { pretty_generate([JSON::NaN]) }
+    assert_equal "[\n  NaN\n]", pretty_generate([JSON::NaN], :allow_nan => true)
+    assert_raises(GeneratorError) { generate([JSON::Infinity]) }
+    assert_equal '[Infinity]', generate([JSON::Infinity], :allow_nan => true)
+    assert_raises(GeneratorError) { fast_generate([JSON::Infinity]) }
+    assert_raises(GeneratorError) { pretty_generate([JSON::Infinity]) }
+    assert_equal "[\n  Infinity\n]", pretty_generate([JSON::Infinity], :allow_nan => true)
+    assert_raises(GeneratorError) { generate([JSON::MinusInfinity]) }
+    assert_equal '[-Infinity]', generate([JSON::MinusInfinity], :allow_nan => true)
+    assert_raises(GeneratorError) { fast_generate([JSON::MinusInfinity]) }
+    assert_raises(GeneratorError) { pretty_generate([JSON::MinusInfinity]) }
+    assert_equal "[\n  -Infinity\n]", pretty_generate([JSON::MinusInfinity], :allow_nan => true)
+  end
+
+  def test_depth
+    ary = []; ary << ary
+    assert_equal 0, JSON::SAFE_STATE_PROTOTYPE.depth
+    assert_raises(JSON::NestingError) { JSON.generate(ary) }
+    assert_equal 0, JSON::SAFE_STATE_PROTOTYPE.depth
+    assert_equal 0, JSON::PRETTY_STATE_PROTOTYPE.depth
+    assert_raises(JSON::NestingError) { JSON.pretty_generate(ary) }
+    assert_equal 0, JSON::PRETTY_STATE_PROTOTYPE.depth
+    s = JSON.state.new
+    assert_equal 0, s.depth
+    assert_raises(JSON::NestingError) { ary.to_json(s) }
+    assert_equal 19, s.depth
+  end
+
+
+  def test_no_close_tag
+    assert_equal '["<\/script>"]', generate(["</script>"])
+    assert_equal '["<\/SCRIPT>"]', generate(["</SCRIPT>"])
+  end
+end

data/tests/test_json_string_matching.rb

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+require 'test/unit'
+require File.join(File.dirname(__FILE__), 'setup_variant')
+require 'stringio'
+require 'time'
+
+class TestJsonStringMatching < Test::Unit::TestCase
+  include JSON
+
+  class TestTime < ::Time
+    def self.json_create(string)
+      Time.parse(string)
+    end
+
+    def to_json(*)
+      %{"#{strftime('%FT%T%z')}"}
+    end
+
+    def ==(other)
+      to_i == other.to_i
+    end
+  end
+
+  def test_match_date
+    t = TestTime.new
+    t_json = [ t ].to_json
+    assert_equal [ t ],
+      JSON.parse(t_json,
+        :match_string => { /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{4}\Z/ => TestTime })
+    assert_equal [ t.strftime('%FT%T%z') ],
+      JSON.parse(t_json,
+        :match_string => { /\A\d{3}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{4}\Z/ => TestTime })
+    assert_equal [ t.strftime('%FT%T%z') ],
+      JSON.parse(t_json,
+        :match_string => { /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{4}\Z/ => TestTime },
+        :create_additions => false)
+  end
+end

data/tests/test_json_unicode.rb

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+require 'test/unit'
+require File.join(File.dirname(__FILE__), 'setup_variant')
+
+class TC_JSONUnicode < Test::Unit::TestCase
+  include JSON
+
+  def test_unicode
+    assert_equal '""', ''.to_json
+    assert_equal '"\\b"', "\b".to_json
+    assert_equal '"\u0001"', 0x1.chr.to_json
+    assert_equal '"\u001f"', 0x1f.chr.to_json
+    assert_equal '" "', ' '.to_json
+    assert_equal "\"#{0x7f.chr}\"", 0x7f.chr.to_json
+    utf8 = [ "© ≠ €! \01" ]
+    json = '["© ≠ €! \u0001"]'
+    assert_equal json, utf8.to_json(:ascii_only => false)
+    assert_equal utf8, parse(json)
+    json = '["\u00a9 \u2260 \u20ac! \u0001"]'
+    assert_equal json, utf8.to_json(:ascii_only => true)
+    assert_equal utf8, parse(json)
+    utf8 = ["\343\201\202\343\201\204\343\201\206\343\201\210\343\201\212"]
+    json = "[\"\343\201\202\343\201\204\343\201\206\343\201\210\343\201\212\"]"
+    assert_equal utf8, parse(json)
+    assert_equal json, utf8.to_json(:ascii_only => false)
+    utf8 = ["\343\201\202\343\201\204\343\201\206\343\201\210\343\201\212"]
+    assert_equal utf8, parse(json)
+    json = "[\"\\u3042\\u3044\\u3046\\u3048\\u304a\"]"
+    assert_equal json, utf8.to_json(:ascii_only => true)
+    assert_equal utf8, parse(json)
+    utf8 = ['საქართველო']
+    json = '["საქართველო"]'
+    assert_equal json, utf8.to_json(:ascii_only => false)
+    json = "[\"\\u10e1\\u10d0\\u10e5\\u10d0\\u10e0\\u10d7\\u10d5\\u10d4\\u10da\\u10dd\"]"
+    assert_equal json, utf8.to_json(:ascii_only => true)
+    assert_equal utf8, parse(json)
+    assert_equal '["Ã"]', JSON.generate(["Ã"], :ascii_only => false)
+    assert_equal '["\\u00c3"]', JSON.generate(["Ã"], :ascii_only => true)
+    assert_equal ["€"], JSON.parse('["\u20ac"]')
+    utf8 = ["\xf0\xa0\x80\x81"]
+    json = "[\"\xf0\xa0\x80\x81\"]"
+    assert_equal json, JSON.generate(utf8, :ascii_only => false)
+    assert_equal utf8, JSON.parse(json)
+    json = '["\ud840\udc01"]'
+    assert_equal json, JSON.generate(utf8, :ascii_only => true)
+    assert_equal utf8, JSON.parse(json)
+  end
+
+  def test_chars
+    (0..0x7f).each do |i|
+      json = '["\u%04x"]' % i
+      if RUBY_VERSION >= "1.9."
+        i = i.chr
+      end
+      assert_equal i, JSON.parse(json).first[0]
+      if i == ?\b
+        generated = JSON.generate(["" << i])
+        assert '["\b"]' == generated || '["\10"]' == generated
+      elsif [?\n, ?\r, ?\t, ?\f].include?(i)
+        assert_equal '[' << ('' << i).dump << ']', JSON.generate(["" << i])
+      elsif i.chr < 0x20.chr
+        assert_equal json, JSON.generate(["" << i])
+      end
+    end
+    assert_raise(JSON::GeneratorError) do
+      JSON.generate(["\x80"], :ascii_only => true)
+    end
+    assert_equal "\302\200", JSON.parse('["\u0080"]').first
+  end
+end

data/tools/fuzz.rb

@@ -0,0 +1,139 @@
+require 'json'
+
+require 'iconv'
+ISO_8859_1_TO_UTF8 = Iconv.new('utf-8', 'iso-8859-15')
+class ::String
+  def to_utf8
+    ISO_8859_1_TO_UTF8.iconv self
+  end
+end
+
+class Fuzzer
+  def initialize(n, freqs = {})
+    sum = freqs.inject(0.0) { |s, x| s + x.last }
+    freqs.each_key { |x| freqs[x] /= sum }
+    s = 0.0
+    freqs.each_key do |x|
+      freqs[x] = s .. (s + t = freqs[x])
+      s += t
+    end
+    @freqs = freqs
+    @n = n
+    @alpha = (0..0xff).to_a
+  end
+
+  def random_string
+    s = ''
+    30.times { s << @alpha[rand(@alpha.size)] }
+    s.to_utf8
+  end
+
+  def pick
+    r = rand
+    found = @freqs.find { |k, f| f.include? rand }
+    found && found.first
+  end
+
+  def make_pick
+    k = pick
+    case
+    when k == Hash, k == Array
+      k.new
+    when k == true, k == false, k == nil
+      k
+    when k == String
+      random_string
+    when k == Fixnum
+      rand(2 ** 30) - 2 ** 29
+    when k == Bignum
+      rand(2 ** 70) - 2 ** 69
+    end
+  end
+
+  def fuzz(current = nil)
+    if @n > 0
+      case current
+      when nil
+        @n -= 1
+        current = fuzz [ Hash, Array ][rand(2)].new
+      when Array
+        while @n > 0
+          @n -= 1
+          current << case p = make_pick
+          when Array, Hash
+            fuzz(p)
+          else
+            p
+          end
+        end
+      when Hash
+        while @n > 0
+          @n -= 1
+          current[random_string] = case p = make_pick
+          when Array, Hash
+            fuzz(p)
+          else
+            p
+          end
+        end
+      end
+    end
+    current
+  end
+end
+
+class MyState < JSON.state
+  WS = " \r\t\n"
+
+  def initialize
+    super(
+          :indent       => make_spaces,
+          :space        => make_spaces,
+          :space_before => make_spaces,
+          :object_nl    => make_spaces,
+          :array_nl     => make_spaces,
+          :max_nesting  => false
+         )
+  end
+
+  def make_spaces
+    s = ''
+    rand(1).times { s << WS[rand(WS.size)] }
+    s
+  end
+end
+
+n = (ARGV.shift || 500).to_i
+loop do
+  fuzzer = Fuzzer.new(n,
+                      Hash => 25,
+                      Array => 25,
+                      String => 10,
+                      Fixnum => 10,
+                      Bignum => 10,
+                      nil => 5,
+                      true => 5,
+                      false => 5
+                     )
+  o1 = fuzzer.fuzz
+  json = JSON.generate o1, MyState.new
+  if $DEBUG
+    puts "-" * 80
+    puts json, json.size
+  else 
+    puts json.size
+  end
+  begin
+    o2 = JSON.parse(json, :max_nesting => false)
+  rescue JSON::ParserError => e
+    puts "Caught #{e.class}: #{e.message}\n#{e.backtrace * "\n"}"
+    puts "o1 = #{o1.inspect}", "json = #{json}", "json_str = #{json.inspect}"
+    puts "locals = #{local_variables.inspect}"
+    exit
+  end
+  if o1 != o2
+    puts "mismatch", "o1 = #{o1.inspect}", "o2 = #{o2.inspect}",
+      "json = #{json}", "json_str = #{json.inspect}"
+    puts "locals = #{local_variables.inspect}"
+  end
+end