wankel 0.1.0

data/ext/wankel/yajl_helpers.h

@@ -0,0 +1,22 @@
+#ifndef YAJL_HELPERS
+#define YAJL_HELPERS
+
+#include <ruby.h>
+#include <yajl/yajl_common.h>
+#include <yajl/yajl_parse.h>
+#include <yajl/yajl_gen.h>
+	
+// Yajl Helpers ==============================================================
+void yajl_helper_check_status(yajl_handle handle, yajl_status status, int verbose, const unsigned char * jsonText, size_t jsonTextLength);
+void yajl_helper_check_gen_status(yajl_gen_status status);
+
+// Memory funcs
+void* yajl_helper_malloc(void *ctx, size_t);
+void* yajl_helper_realloc(void *ctx, void *ptr, size_t size);
+void  yajl_helper_free(void *ctx, void *ptr);
+// static yajl_alloc_funcs* yajl_helper_alloc_funcs();
+
+void yajl_configure(yajl_handle handle, VALUE options);
+void yajl_gen_configure(yajl_gen g, VALUE options);
+
+#endif

data/lib/wankel/ex_sax_parser.rb

@@ -0,0 +1,75 @@
+class Wankel::SaxParser
+
+  def on_map_start
+    puts 'start map'
+  end
+
+  def on_map_end
+    puts 'end map'
+  end
+
+  def on_map_key(key)
+    puts 'key'
+  end
+  
+  def on_null
+    puts "null"
+  end
+
+  def on_boolean(value)
+    puts value
+  end
+
+  def on_integer(i)
+    puts "integer"
+  end
+
+  def on_double(d)
+    puts "double"
+  end
+
+  def on_string(s)
+    puts "string"
+  end
+
+  def on_array_start
+    puts "start array"
+  end
+
+  def on_array_end
+    puts "end array"
+  end
+
+end
+
+
+class Wankel::SaxEncoder
+
+  def number
+  end
+  
+  def string
+  end
+  
+  def null
+  end
+  
+  def bool
+  end
+  
+  def map_open
+  end
+  
+  def map_close
+  end
+  
+  def array_open
+  end
+  
+  def array_close
+  end
+  
+  def complete
+  end
+
+end

data/lib/wankel.rb

@@ -0,0 +1,19 @@
+require 'wankel/wankel'
+
+class Wankel
+    DEFAULTS = {
+      :read_buffer_size => 8092,
+      :symbolize_keys => false,
+      :allow_comments => false,
+      :validate_strings => false,
+      :allow_trailing_garbage => false,
+      :multiple_values => false,
+      :allow_partial_values => false,
+      
+      :write_buffer_size => 8092,
+      :beautify => false,
+      :indent_string => '    ',
+      :validate_utf8 => false,
+      :escape_solidus => false
+    }
+end

data/test/encoding/encoding_test.rb

@@ -0,0 +1,230 @@
+require 'test_helper'
+require 'tmpdir'
+require 'zlib'
+
+class Dummy2
+  def to_json
+    "{\"hawtness\":true}"
+  end
+end
+
+class TheMindKiller
+  def to_json
+    nil
+  end
+end
+
+class TheMindKillerDuce
+  def to_s
+    nil
+  end
+end
+
+class Wankel::EncoderTest < ::Test::Unit::TestCase
+  FILES = Dir[File.dirname(__FILE__)+'/../../benchmark/subjects/*.json']
+  
+  FILES.each do |file|
+    if File.basename(file) != 'twitter_stream.json'
+      test "should encode #{File.basename(file)} to an StringIO" do
+         input = File.new(File.expand_path(file), 'r')
+         io = StringIO.new
+         encoder = Wankel::Encoder.new
+         hash = Wankel.parse(input)
+         encoder.encode(hash, io)
+         io.rewind
+         hash2 = Wankel.parse(io)
+         io.close
+         input.close
+         assert_equal(hash2, hash)
+       end
+     end
+   end
+  
+   FILES.each do |file|
+     test "should encode #{File.basename(file)} to a Zlib::GzipWriter" do
+      # we don't care about testing the stream subject as it has multiple JSON strings in it
+      if File.basename(file) != 'twitter_stream.json'
+         hash = File.open(File.expand_path(file), 'r') do |input|
+           Wankel.parse(input)
+         end
+         hash2 = Dir.mktmpdir do |tmp_dir|
+            output_filename = File.join(tmp_dir, 'output.json')
+            Zlib::GzipWriter.open(output_filename) do |writer|
+               Wankel.encode(hash, writer)
+            end
+            Zlib::GzipReader.open(output_filename) do |reader|
+               Wankel.parse(reader.read)
+            end
+         end
+         assert_equal(hash2, hash)
+      end
+     end
+   end
+  
+   FILES.each do |file|
+     test "should encode #{File.basename(file)} and return a String" do
+       # we don't care about testing the stream subject as it has multiple JSON strings in it
+       if File.basename(file) != 'twitter_stream.json'
+         input = File.new(File.expand_path(file), 'r')
+         encoder = Wankel::Encoder.new
+         hash = Wankel.parse(input)
+         output = encoder.encode(hash)
+         hash2 = Wankel.parse(output)
+         input.close
+         assert_equal(hash2, hash)
+       end
+     end
+   end
+  
+  test "should encode with :beautify turned on and a single space indent, to an IO" do
+    output = "{\n \"foo\": 1234\n}\n"
+    obj = {:foo => 1234}
+    io = StringIO.new
+    encoder = Wankel::Encoder.new(:beautify => true, :indent_string => ' ')
+    encoder.encode(obj, io)
+    io.rewind
+    assert_equal(output, io.read)
+  end
+  
+  test "should encode with :beautify turned on and a single space indent, and return a String" do
+    output = "{\n \"foo\": 1234\n}\n"
+    obj = {:foo => 1234}
+    encoder = Wankel::Encoder.new(:beautify => true, :indent_string => ' ')
+    result = encoder.encode(obj)
+    assert_equal(output, result)
+  end
+  
+  test "should encode with :beautify turned on and a tab character indent, to an IO" do
+    output = "{\n\t\"foo\": 1234\n}\n"
+    obj = {:foo => 1234}
+    io = StringIO.new
+    encoder = Wankel::Encoder.new(:beautify => true, :indent_string => "\t")
+    encoder.encode(obj, io)
+    io.rewind
+    assert_equal(output, io.read)
+  end
+  
+  test "should encode with :beautify turned on and a tab character indent, and return a String" do
+    output = "{\n\t\"foo\": 1234\n}\n"
+    obj = {:foo => 1234}
+    encoder = Wankel::Encoder.new(:beautify => true, :indent_string => "\t")
+    result = encoder.encode(obj)
+    assert_equal(output, result)
+  end
+  
+  test "should encode with it's class method with :beautify and a tab character indent options set, to an IO" do
+    output = "{\n\t\"foo\": 1234\n}\n"
+    obj = {:foo => 1234}
+    io = StringIO.new
+    Wankel.encode(obj, io, :beautify => true, :indent_string => "\t")
+    io.rewind
+    assert_equal(output, io.read)
+  end
+  
+  test "should encode with it's class method with :beautify and a tab character indent options set, and return a String" do
+    output = "{\n\t\"foo\": 1234\n}\n"
+    obj = {:foo => 1234}
+    result = Wankel.encode(obj, :beautify => true, :indent_string => "\t")
+    assert_equal(output, result)
+  end
+  
+  test "should encode multiple objects into a single stream, to an IO" do
+    io = StringIO.new
+    obj = {:foo => 1234}
+    encoder = Wankel::Encoder.new
+    5.times do
+      encoder.encode(obj, io)
+    end
+    io.rewind
+    output = "{\"foo\":1234}{\"foo\":1234}{\"foo\":1234}{\"foo\":1234}{\"foo\":1234}"
+    assert_equal(output, io.read)
+  end
+  
+  test "should encode multiple objects into a single stream, and return a String" do
+    obj = {:foo => 1234}
+    encoder = Wankel::Encoder.new
+    json_output = ''
+    5.times do
+      json_output << encoder.encode(obj)
+    end
+    output = "{\"foo\":1234}{\"foo\":1234}{\"foo\":1234}{\"foo\":1234}{\"foo\":1234}"
+    assert_equal(output, json_output)
+  end
+  
+  test "should encode all map keys as strings" do
+    assert_equal("{\"1\":1}", Wankel.encode({1=>1}))
+  end
+  
+  test "should check for and call #to_json if it exists on custom objects" do
+    d = Dummy2.new
+    assert_equal('{"foo":{"hawtness":true}}', Wankel.encode({:foo => d}))
+  end
+  
+  test "should encode a hash where the key and value can be symbols" do
+    assert_equal('{"foo":"bar"}', Wankel.encode({:foo => :bar}))
+  end
+  
+  test "should not encode NaN" do
+    assert_raises Wankel::EncodeError do
+      Wankel.encode(0.0/0.0)
+    end
+  end
+  
+  test "should not encode Infinity or -Infinity" do
+    assert_raises Wankel::EncodeError do
+      Wankel.encode(1.0/0.0)
+    end
+    assert_raises Wankel::EncodeError do
+      Wankel.encode(-1.0/0.0)
+    end
+  end
+  
+  test "should encode with unicode chars in the key" do
+    hash = {"浅草" => "<- those are unicode"}
+    assert_equal("{\"浅草\":\"<- those are unicode\"}", Wankel.encode(hash))
+  end
+
+  test "should return a string encoded in utf-8 if Encoding.default_internal is nil" do
+    Encoding.default_internal = nil
+    hash = {"浅草" => "<- those are unicode"}
+    assert_equal(Encoding.find('utf-8'), Wankel.encode(hash).encoding)
+  end
+
+  test "should return a string encoded in utf-8 even if Encoding.default_internal *is* set" do
+    Encoding.default_internal = Encoding.find('utf-8')
+    hash = {"浅草" => "<- those are unicode"}
+    assert_equal(Encoding.default_internal, Wankel.encode(hash).encoding)
+    
+    Encoding.default_internal = Encoding.find('us-ascii')
+    hash = {"浅草" => "<- those are unicode"}
+    assert_equal(Encoding.find('utf-8'), Wankel.encode(hash).encoding)
+  end
+  
+  test "should default to *not* escaping / characters" do
+    unsafe_encoder = Wankel::Encoder.new
+    assert_equal("\"</script>\"", unsafe_encoder.encode("</script>"))
+  end
+  
+  test "return value of #to_json must be a string" do
+    assert_raises TypeError do
+      Wankel.encode(TheMindKiller.new)
+    end
+  end
+  
+  test "return value of #to_s must be a string" do
+    assert_raises TypeError do
+      if TheMindKillerDuce.send(:method_defined?, :to_json)
+        TheMindKillerDuce.send(:undef_method, :to_json)
+      end
+      Wankel.encode(TheMindKillerDuce.new)
+    end
+  end
+  
+  test "output must be a an IO" do
+    assert_raises Wankel::EncodeError, "output must be a an IO" do
+      output = ""
+      Wankel.encode(TheMindKillerDuce.new, "")
+    end
+  end
+  
+end

data/test/encoding/sax_encoder_test.rb

@@ -0,0 +1,89 @@
+require 'stringio'
+
+class Wankel::SaxEncoderTest < ::Test::Unit::TestCase
+  
+  test 'default inherited from Wankel' do
+      encoder = Wankel::SaxEncoder.new(StringIO.new)
+      assert_equal(Wankel::DEFAULTS, encoder.instance_variable_get("@options"))
+  end
+  
+  test 'default options merged with options inherited from Wankel && Class' do
+      encoder = Wankel::SaxEncoder.new(StringIO.new, :hello => true, :metoo => false)
+      assert_equal(Wankel::DEFAULTS.merge({:hello => true, :metoo => false}), encoder.instance_variable_get("@options"))
+  end
+
+  test "number" do
+    output = StringIO.new
+    encoder = Wankel::SaxEncoder.new(output)
+    encoder.number(1234)
+    encoder.complete
+    assert_equal("1234", output.string)
+  end
+  
+  test "string" do
+    output = StringIO.new
+    encoder = Wankel::SaxEncoder.new(output)
+    encoder.string("hello world")
+    encoder.complete
+    assert_equal('"hello world"', output.string)
+  end
+    
+  test "null" do
+    output = StringIO.new
+    encoder = Wankel::SaxEncoder.new(output)
+    encoder.null
+    encoder.complete
+    assert_equal("null", output.string)
+  end
+  
+  test "bool(false)" do
+    output = StringIO.new
+    encoder = Wankel::SaxEncoder.new(output)
+    encoder.bool(false)
+    encoder.complete
+    assert_equal("false", output.string)
+  end
+  
+  test "bool(true)" do
+    output = StringIO.new
+    encoder = Wankel::SaxEncoder.new(output)
+    encoder.bool(true)
+    encoder.complete
+    assert_equal("true", output.string)
+  end
+      
+  test "map_open" do
+    output = StringIO.new
+    encoder = Wankel::SaxEncoder.new(output)
+    encoder.map_open
+    encoder.complete
+    assert_equal("{", output.string)
+  end
+  
+  test "map_close" do
+    output = StringIO.new
+    encoder = Wankel::SaxEncoder.new(output)
+    encoder.map_open
+    encoder.map_close
+    encoder.complete
+    assert_equal("{}", output.string)
+  end
+      
+  test "array_open" do
+    output = StringIO.new
+    encoder = Wankel::SaxEncoder.new(output)
+    encoder.array_open
+    encoder.complete
+    assert_equal("[", output.string)
+  end
+  
+  test "array_close" do
+    output = StringIO.new
+    encoder = Wankel::SaxEncoder.new(output)
+    encoder.array_open
+    encoder.array_close
+    encoder.complete
+    assert_equal("[]", output.string)
+  end
+
+end

data/test/parsing/active_support_test.rb

@@ -0,0 +1,66 @@
+# encoding: UTF-8
+require 'test_helper'
+
+class Wankel::ActiveSupportTest < ::Test::Unit::TestCase
+  
+  TESTS = {
+    %q({"returnTo":{"\/categories":"\/"}})        => {"returnTo" => {"/categories" => "/"}},
+    %q({"return\\"To\\":":{"\/categories":"\/"}}) => {"return\"To\":" => {"/categories" => "/"}},
+    %q({"returnTo":{"\/categories":1}})          => {"returnTo" => {"/categories" => 1}},
+    %({"returnTo":[1,"a"]})                    => {"returnTo" => [1, "a"]},
+    %({"returnTo":[1,"\\"a\\",", "b"]})        => {"returnTo" => [1, "\"a\",", "b"]},
+    %({"a": "'", "b": "5,000"})                  => {"a" => "'", "b" => "5,000"},
+    %({"a": "a's, b's and c's", "b": "5,000"})   => {"a" => "a's, b's and c's", "b" => "5,000"},
+    # multibyte
+    %({"matzue": "松江", "asakusa": "浅草"}) => {"matzue" => "松江", "asakusa" => "浅草"},
+    %({"a": "2007-01-01"})                       => {'a' => "2007-01-01"},
+    %({"a": "2007-01-01 01:12:34 Z"})            => {'a' => "2007-01-01 01:12:34 Z"},
+    # no time zone
+    %({"a": "2007-01-01 01:12:34"})              => {'a' => "2007-01-01 01:12:34"},
+    # needs to be *exact*
+    %({"a": " 2007-01-01 01:12:34 Z "})          => {'a' => " 2007-01-01 01:12:34 Z "},
+    %({"a": "2007-01-01 : it's your birthday"})  => {'a' => "2007-01-01 : it's your birthday"},
+    %([])    => [],
+    %({})    => {},
+    %({"a":1})     => {"a" => 1},
+    %({"a": ""})    => {"a" => ""},
+    %({"a":"\\""}) => {"a" => "\""},
+    %({"a": null})  => {"a" => nil},
+    %({"a": true})  => {"a" => true},
+    %({"a": false}) => {"a" => false},
+    %q({"a": "http:\/\/test.host\/posts\/1"}) => {"a" => "http://test.host/posts/1"},
+    %q({"a": "\u003cunicode\u0020escape\u003e"}) => {"a" => "<unicode escape>"},
+    %q({"a": "\\\\u0020skip double backslashes"}) => {"a" => "\\u0020skip double backslashes"},
+    %q({"a": "\u003cbr /\u003e"}) => {'a' => "<br />"},
+    %q({"b":["\u003ci\u003e","\u003cb\u003e","\u003cu\u003e"]}) => {'b' => ["<i>","<b>","<u>"]}
+  }
+
+  TESTS.each do |json, expected|
+    test "should be able to parse #{json} as an IO" do
+      assert_nothing_raised Wankel::ParseError do
+        Wankel.parse(StringIO.new(json))
+      end
+    end
+  end
+
+  TESTS.each do |json, expected|
+    test "should be able to parse #{json} as a string" do
+      assert_nothing_raised Wankel::ParseError do
+        Wankel.parse(json)
+      end
+    end
+  end
+
+  test "should fail parsing {: 1} as an IO" do
+    assert_raise Wankel::ParseError do
+      Wankel.parse(StringIO.new("{: 1}"))
+    end
+  end
+
+  test "should fail parsing {: 1} as a string" do
+    assert_raise Wankel::ParseError do
+      Wankel.parse("{: 1}")
+    end
+  end  
+  
+end

data/test/parsing/fixtures/fail.15.json

@@ -0,0 +1 @@
+["Illegal backslash escape: \x15"]

data/test/parsing/fixtures/fail.16.json

@@ -0,0 +1 @@
+["Illegal backslash escape: \'"]

data/test/parsing/fixtures/fail.17.json

@@ -0,0 +1 @@
+["Illegal backslash escape: \017"]

data/test/parsing/fixtures/fail.26.json

@@ -0,0 +1 @@
+["tab\   character\   in\  string\  "]

data/test/parsing/fixtures/fail11.json

@@ -0,0 +1 @@
+{"Illegal expression": 1 + 2}

data/test/parsing/fixtures/fail12.json

@@ -0,0 +1 @@
+{"Illegal invocation": alert()}

data/test/parsing/fixtures/fail13.json

@@ -0,0 +1 @@
+{"Numbers cannot have leading zeroes": 013}

data/test/parsing/fixtures/fail14.json

@@ -0,0 +1 @@
+{"Numbers cannot be hex": 0x14}

data/test/parsing/fixtures/fail19.json

@@ -0,0 +1 @@
+{"Missing colon" null}

data/test/parsing/fixtures/fail20.json

@@ -0,0 +1 @@
+{"Double colon":: null}

data/test/parsing/fixtures/fail21.json

@@ -0,0 +1 @@
+{"Comma instead of colon", null}

data/test/parsing/fixtures/fail22.json

@@ -0,0 +1 @@
+["Colon instead of comma": false]

data/test/parsing/fixtures/fail23.json

@@ -0,0 +1 @@
+["Bad value", truth]

data/test/parsing/fixtures/fail24.json

@@ -0,0 +1 @@
+['single quote']

data/test/parsing/fixtures/fail25.json

@@ -0,0 +1 @@
+["tab	character	in	string	"]

data/test/parsing/fixtures/fail27.json

@@ -0,0 +1,2 @@
+["line
+break"]

data/test/parsing/fixtures/fail28.json

@@ -0,0 +1,2 @@
+["line\
+break"]

data/test/parsing/fixtures/fail3.json

@@ -0,0 +1 @@
+{unquoted_key: "keys must be quoted"}

data/test/parsing/fixtures/fail4.json

@@ -0,0 +1 @@
+["extra comma",]

data/test/parsing/fixtures/fail5.json

@@ -0,0 +1 @@
+["double extra comma",,]

data/test/parsing/fixtures/fail6.json

@@ -0,0 +1 @@
+[   , "<-- missing value"]

data/test/parsing/fixtures/fail9.json

@@ -0,0 +1 @@
+{"Extra comma": true,}

data/test/parsing/fixtures/pass.array.json

@@ -0,0 +1,6 @@
+["foo",
+ "bar", "baz",
+ true,false,null,{"key":"value"},
+ [null,null,null,[]],
+ "\n\r\\"
+]

data/test/parsing/fixtures/pass.codepoints_from_unicode_org.json

@@ -0,0 +1 @@
+"\u004d\u0430\u4e8c\ud800\udf02"