yajl-ruby 1.3.0 → 1.4.1

data/ext/yajl/yajl_ext.h

@@ -53,10 +53,10 @@ static rb_encoding *utf8Encoding;
 #define RARRAY_LEN(s) (RARRAY(s)->len)
 #endif
 
-static VALUE cParseError, cEncodeError, mYajl, cParser, cEncoder;
+static VALUE cStandardError, cParseError, cEncodeError, mYajl, cParser, cProjector, cEncoder;
 static ID intern_io_read, intern_call, intern_keys, intern_to_s,
             intern_to_json, intern_has_key, intern_to_sym, intern_as_json;
-static ID sym_allow_comments, sym_check_utf8, sym_pretty, sym_indent, sym_terminator, sym_symbolize_keys, sym_symbolize_names, sym_html_safe;
+static ID sym_allow_comments, sym_check_utf8, sym_pretty, sym_indent, sym_terminator, sym_symbolize_keys, sym_symbolize_names, sym_html_safe, sym_entities;
 
 #define GetParser(obj, sval) Data_Get_Struct(obj, yajl_parser_wrapper, sval);
 #define GetEncoder(obj, sval) Data_Get_Struct(obj, yajl_encoder_wrapper, sval);

data/ext/yajl/yajl_lex.c

@@ -38,29 +38,25 @@
 #include <assert.h>
 #include <string.h>
 
-#ifdef YAJL_LEXER_DEBUG
-static const char *
-tokToStr(yajl_tok tok) 
-{
+const char *yajl_tok_name(yajl_tok tok) {
     switch (tok) {
         case yajl_tok_bool: return "bool";
         case yajl_tok_colon: return "colon";
         case yajl_tok_comma: return "comma";
         case yajl_tok_eof: return "eof";
         case yajl_tok_error: return "error";
-        case yajl_tok_left_brace: return "brace";
-        case yajl_tok_left_bracket: return "bracket";
+        case yajl_tok_left_brace: return "open_array";
+        case yajl_tok_left_bracket: return "open_object";
         case yajl_tok_null: return "null";
         case yajl_tok_integer: return "integer";
         case yajl_tok_double: return "double";
-        case yajl_tok_right_brace: return "brace";
-        case yajl_tok_right_bracket: return "bracket";
+        case yajl_tok_right_brace: return "close_array";
+        case yajl_tok_right_bracket: return "close_object";
         case yajl_tok_string: return "string";
         case yajl_tok_string_with_escapes: return "string_with_escapes";
     }
     return "unknown";
 }
-#endif
 
 /* Impact of the stream parsing feature on the lexer:
  *
@@ -740,6 +736,10 @@ yajl_tok yajl_lex_peek(yajl_lexer lexer, const unsigned char * jsonText,
     tok = yajl_lex_lex(lexer, jsonText, jsonTextLen, &offset,
                        &outBuf, &outLen);
 
+    if (tok == yajl_tok_eof) {
+        return tok;
+    }
+
     lexer->bufOff = bufOff;
     lexer->bufInUse = bufInUse;
     yajl_buf_truncate(lexer->buf, bufLen);

data/ext/yajl/yajl_lex.h

@@ -36,33 +36,34 @@
 #include "api/yajl_common.h"
 
 typedef enum {
-    yajl_tok_bool,         
-    yajl_tok_colon,
-    yajl_tok_comma,     
-    yajl_tok_eof,
-    yajl_tok_error,
-    yajl_tok_left_brace,     
-    yajl_tok_left_bracket,
-    yajl_tok_null,         
-    yajl_tok_right_brace,     
-    yajl_tok_right_bracket,
+    yajl_tok_bool,         // 0
+    yajl_tok_colon,        // 1
+    yajl_tok_comma,        // 2
+    yajl_tok_eof,          // 3
+    yajl_tok_error,        // 4
+    yajl_tok_left_brace,   // 5
+    yajl_tok_left_bracket, // 6
+    yajl_tok_null,         // 7
+    yajl_tok_right_brace,  // 8
+    yajl_tok_right_bracket, // 9
 
     /* we differentiate between integers and doubles to allow the
      * parser to interpret the number without re-scanning */
-    yajl_tok_integer, 
-    yajl_tok_double, 
+    yajl_tok_integer, // 10
+    yajl_tok_double,  // 11
 
     /* we differentiate between strings which require further processing,
      * and strings that do not */
-    yajl_tok_string,
-    yajl_tok_string_with_escapes,
+    yajl_tok_string, // 12
+    yajl_tok_string_with_escapes, // 13
 
     /* comment tokens are not currently returned to the parser, ever */
-    yajl_tok_comment
+    yajl_tok_comment // 14
 } yajl_tok;
 
 typedef struct yajl_lexer_t * yajl_lexer;
 
+const char *yajl_tok_name(yajl_tok tok);
 
 YAJL_API
 yajl_lexer yajl_lex_alloc(yajl_alloc_funcs * alloc,

data/lib/yajl.rb

@@ -23,6 +23,13 @@ module Yajl
     Encoder.encode(obj, args, &block)
   end
 
+  class Projector
+    def initialize(stream, read_bufsize=4096)
+      @stream = stream
+      @buffer_size = read_bufsize
+    end
+  end
+
   class Parser
     # A helper method for parse-and-forget use-cases
     #

data/lib/yajl/bzip2.rb

@@ -6,6 +6,6 @@ begin
   require 'bzip2' unless defined?(Bzip2)
   require 'yajl/bzip2/stream_reader.rb'
   require 'yajl/bzip2/stream_writer.rb'
-rescue LoadError => e
+rescue LoadError
   raise "Unable to load the bzip2 library. Is the bzip2-ruby gem installed?"
-end
+end

data/lib/yajl/http_stream.rb

@@ -4,6 +4,7 @@ require 'socket'
 require 'yajl'
 require 'yajl/version' unless defined? Yajl::VERSION
 require 'uri'
+require 'cgi'
 
 module Yajl
   # This module is for making HTTP requests to which the response bodies (and possibly requests in the near future)
@@ -101,7 +102,7 @@ module Yajl
           default_headers["Content-Type"] = opts["Content-Type"] || "application/x-www-form-urlencoded"
           body = opts.delete(:body)
           if body.is_a?(Hash)
-            body = body.keys.collect {|param| "#{URI.escape(param.to_s)}=#{URI.escape(body[param].to_s)}"}.join('&')
+            body = body.keys.collect {|param| "#{CGI.escape(param.to_s)}=#{CGI.escape(body[param].to_s)}"}.join('&')
           end
           default_headers["Content-Length"] = body.length
         end
@@ -161,7 +162,7 @@ module Yajl
           if block_given?
             chunkLeft = 0
             while !socket.eof? && (line = socket.gets)
-              break if line.match /^0.*?\r\n/
+              break if line.match(/^0.*?\r\n/)
               next if line == "\r\n"
               size = line.hex
               json = socket.read(size)

data/lib/yajl/version.rb

@@ -1,3 +1,3 @@
 module Yajl
-  VERSION = '1.3.0'
+  VERSION = '1.4.1'
 end

data/spec/encoding/encoding_spec.rb

@@ -275,11 +275,27 @@ describe "Yajl JSON encoder" do
     expect(safe_encoder.encode("</script>")).to eql("\"<\\/script>\"")
   end
 
+  it "should not encode characters with entities by default" do
+    expect(Yajl.dump("\u2028\u2029><&")).to eql("\"\u2028\u2029><&\"")
+  end
+
+  it "should encode characters with entities when enabled" do
+    expect(Yajl.dump("\u2028\u2029><&", entities: true)).to eql("\"\\u2028\\u2029\\u003E\\u003C\\u0026\"")
+  end
+
   it "should default to *not* escaping / characters" do
     unsafe_encoder = Yajl::Encoder.new
     expect(unsafe_encoder.encode("</script>")).not_to eql("\"<\\/script>\"")
   end
 
+  it "should encode slashes when enabled" do
+    unsafe_encoder = Yajl::Encoder.new(:entities => false)
+    safe_encoder   = Yajl::Encoder.new(:entities => true)
+
+    expect(unsafe_encoder.encode("</script>")).not_to eql("\"<\\/script>\"")
+    expect(safe_encoder.encode("</script>")).to eql("\"\\u003C\\/script\\u003E\"")
+  end
+
   it "return value of #to_json must be a string" do
     expect {
       Yajl::Encoder.encode(TheMindKiller.new)

data/spec/parsing/one_off_spec.rb

@@ -2,6 +2,13 @@
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper.rb')
 
 describe "One-off JSON examples" do
+  it "should not blow up with a bad surrogate trailer" do
+    # https://github.com/brianmario/yajl-ruby/issues/176
+    bad_json = "{\"e\":{\"\\uD800\\\\DC00\":\"a\"}}"
+
+    Yajl::Parser.new.parse(bad_json)
+  end
+
   it "should parse 23456789012E666 and return Infinity" do
     infinity = (1.0/0)
     silence_warnings do

data/spec/projection/project_file.rb

@@ -0,0 +1,41 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper.rb')
+
+require 'benchmark'
+require 'benchmark/memory'
+
+describe "file projection" do
+  it "projects file streams" do
+    schema = {
+      "forced" => nil,
+      "created" => nil,
+      "pusher" => {
+        "name" => nil,
+      },
+      "repository" => {
+        "name" => nil,
+        "full_name" => nil,
+      },
+      "ref" => nil,
+      "compare" => nil,
+      "commits" => {
+        "distinct" => nil,
+        "message" => nil,
+        "url" => nil,
+        "id" => nil,
+        "author" => {
+          "username" => nil,
+        }
+      }
+    }
+
+    file_path = ENV['JSON_FILE']
+    if file_path.nil? || file_path.empty?
+      return
+    end
+
+    Benchmark.memory { |x|
+      x.report("project (yajl)") { Yajl::Projector.new(File.open(file_path, 'r')).project(schema) }
+      x.compare!
+    }
+  end
+end

data/spec/projection/projection.rb

@@ -0,0 +1,498 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper.rb')
+
+require 'stringio'
+require 'json'
+
+describe "projection" do
+  it "should work" do
+    stream = StringIO.new('{"name": "keith", "age": 27}')
+    projector = Yajl::Projector.new(stream)
+    projection = projector.project({"name" => nil})
+    expect(projection['name']).to eql("keith")
+  end
+
+  it "should filter" do
+    stream = StringIO.new('{"name": "keith", "age": 27}')
+    projector = Yajl::Projector.new(stream)
+    projection = projector.project({"name" => nil})
+    expect(projection['age']).to eql(nil)
+  end
+
+  it "should raise an exception and not leak memory" do
+    stream = StringIO.new('foo')
+    projector = Yajl::Projector.new(stream)
+    expect {
+      projector.project({"name" => nil})
+    }.to raise_error(Yajl::ParseError)
+  end
+
+  it "should raise an exception and not segv" do
+    stream = StringIO.new('[,,,,]')
+    projector = Yajl::Projector.new(stream)
+    expect {
+      projector.project({"name" => nil})
+    }.to raise_error(Yajl::ParseError)
+  end
+
+  it "should raise an exception and not segv on colons" do
+    stream = StringIO.new('[::::]')
+    projector = Yajl::Projector.new(stream)
+    expect {
+      projector.project({"name" => nil})
+    }.to raise_error(Yajl::ParseError)
+  end
+
+  it "should behave the same way as the regular parser on bad tokens like comma" do
+    bad_json = '{"name": "keith", "age":, 27}'
+    stream = StringIO.new(bad_json)
+    projector = Yajl::Projector.new(stream)
+    expect {
+      projector.project({"name" => nil})
+    }.to raise_error(capture_exception_for(bad_json).class)
+  end
+
+  it "should behave the same way as the regular parser on bad tokens like colon" do
+    bad_json = '{"name": "keith", "age":: 27}'
+    stream = StringIO.new(bad_json)
+    projector = Yajl::Projector.new(stream)
+    expect {
+      projector.project({"name" => nil})
+    }.to raise_error(capture_exception_for(bad_json).class)
+  end
+
+  it "should behave the same way as the regular parser on not enough json" do
+    bad_json = '{"name": "keith", "age":'
+    stream = StringIO.new(bad_json)
+    projector = Yajl::Projector.new(stream)
+    expect {
+      projector.project({"name" => nil})
+    }.to raise_error(capture_exception_for(bad_json).class)
+  end
+
+  def capture_exception_for(bad_json)
+    Yajl::Parser.new.parse(bad_json)
+  rescue Exception => e
+    e
+  end
+
+  def project(schema, over: "", json: nil, stream: nil)
+    if stream.nil?
+      if json.nil?
+        json = over.to_json
+      end
+
+      stream = StringIO.new(json)
+    end
+
+    Yajl::Projector.new(stream).project(schema)
+  end
+
+  it "filters arrays" do
+    json = {
+      "users" => [
+        {
+          "name" => "keith",
+          "company" => "internet plumbing inc",
+          "department" => "janitorial",
+        },
+        {
+          "name" => "justin",
+          "company" => "big blue",
+          "department" => "programming?",
+        },
+        {
+          "name" => "alan",
+          "company" => "different colour of blue",
+          "department" => "drop bear containment",
+        }
+      ]
+    }.to_json
+
+    puts json
+
+    schema = {
+      # /users is an array of objects, each having many keys we only want name
+      "users" => {
+        "name" => nil,
+      }
+    }
+
+    expect(project(schema, json: json)).to eql({
+      "users" => [
+        { "name" => "keith" },
+        { "name" => "justin" },
+        { "name" => "alan" }
+      ]
+    })
+  end
+
+  it "filters top level arrays" do
+    json = [
+      {
+        "name" => "keith",
+        "personal detail" => "thing",
+      },
+      {
+        "name" => "cory",
+        "phone number" => "unknown",
+      }
+    ]
+
+    schema = {
+      "name" => nil,
+    }
+
+    expect(project(schema, over: json)).to eql([
+      { "name" => "keith" },
+      { "name" => "cory" },
+    ])
+  end
+
+  it "filters nested schemas" do
+    json = {
+      "foo" => 42,
+
+      "bar" => {
+        "name" => "keith",
+        "occupation" => "professional computering",
+        "age" => 26,
+        "hobbies" => [
+          "not computering",
+        ]
+      },
+
+      "qux" => {
+        "quux" => [
+          {
+            "name" => "Reactive X",
+            "members" => "many",
+          },
+          {
+            "name" => "lstoll",
+            "members" => "such",
+          },
+          {
+            "name" => "github",
+            "members" => "very",
+          },
+          {
+            "name" => "theleague",
+            "members" => "numerous",
+          }
+        ],
+
+        "corge" => {
+          "name" => "Brighton",
+          "address" =>"Buckingham Road",
+        },
+      },
+
+      "grault" => nil,
+
+      "waldo" => true,
+    }
+
+    schema = {
+      # include the /foo subtree (is a single number)
+      "foo" => nil,
+
+      # ignore the bar subtree (is an object)
+      # "bar" => ???
+
+      # include some of the /qux subtree (is an object)
+      "qux" => {
+        # include the whole /qux/quux subtree (is an array of objects)
+        "quux" => nil,
+
+        # include some of the /qux/corge subtree (is another object)
+        "corge" => {
+          # include name (is a string)
+          "name" => nil,
+          # include age (is missing from source doc)
+          "age" => nil,
+          # ignore address
+          # "address" => ???
+        },
+      },
+
+      # include the /grault subtree (is a null literal)
+      "grault" => nil,
+
+      # include the /waldo subtree (is a boolean literal)
+      "waldo" => nil,
+    }
+
+    expect(project(schema, over: json)).to eql({
+      "foo" => 42,
+
+      "qux" => {
+        "quux" => [
+          {
+            "name" => "Reactive X",
+            "members" => "many",
+          },
+          {
+            "name" => "lstoll",
+            "members" => "such",
+          },
+          {
+            "name" => "github",
+            "members" => "very",
+          },
+          {
+            "name" => "theleague",
+            "members" => "numerous",
+          }
+        ],
+
+        "corge" => {
+          "name" => "Brighton",
+        },
+      },
+
+      "grault" => nil,
+
+      "waldo" => true,
+    })
+  end
+
+  it "supports incompatible schemas" do
+    json = {
+      # surprise! the json doesn't include an object under the foo key
+      "foo" => 42,
+    }
+
+    schema = {
+      # include some of the /foo subtree
+      "foo" => {
+        # include the whole /foo/baz subtree
+        "baz" => nil,
+      }
+    }
+
+    # expect the 42 to be pulled out
+    expect(project(schema, over: json)).to eql({
+      "foo" => 42
+    })
+  end
+
+  it "supports nil schema" do
+    json = {
+      "foo" => "bar",
+    }
+
+    expect(project(nil, over: json)).to eql({
+      "foo" => "bar"
+    })
+  end
+
+  it "supports empty schema" do
+    json = {
+      "foo" => "bar",
+    }
+    expect(project({}, over: json)).to eql({})
+  end
+
+  it "supports object projection" do
+    json = {
+      "foo" => "bar",
+      "qux" => "quux",
+    }
+
+    schema = {
+      "foo" => nil,
+    }
+
+    expect(project(schema, over: json)).to eql({
+      "foo" => "bar"
+    })
+  end
+
+  it "projects the readme example" do
+    json = <<-EOJ
+    [
+      {
+        "user": {
+          "name": "keith",
+          "age": 26,
+          "jobs": [
+            {
+              "title": "director of overworking",
+              "company": "south coast software",
+              "department": "most"
+            },
+            {
+              "title": "some kind of computering",
+              "company": "github the website dot com",
+              "department": true
+            }
+          ]
+        },
+        "another key": {
+
+        },
+        "woah this document is huge": {
+
+        },
+        "many megabytes": {
+
+        },
+        "etc": {
+
+        }
+      }
+    ]
+EOJ
+
+    schema = {
+      "user" => {
+        "name" => nil,
+        "jobs" => {
+          "title" => nil,
+        },
+      },
+    }
+
+    expect(project(schema, json: json)).to eql([{
+      "user" => {
+        "name" => "keith",
+        "jobs" => [
+          { "title" => "director of overworking" },
+          { "title" => "some kind of computering" },
+        ]
+      }
+    }])
+  end
+
+  it "errors with invalid json" do
+    expect {
+      project({"b" => nil}, json: '{"a":, "b": 2}')
+    }.to raise_error(StandardError)
+  end
+
+  it "errors with ignored unbalanced object syntax" do
+    expect {
+      project({"b" => nil}, json: '{"a": {{, "b": 2}')
+    }.to raise_error(StandardError)
+  end
+
+  it "errors with accepted unbalanced object tokens" do
+    expect {
+      project({"a" => nil}, json: '{"a": {"b": 2}')
+    }.to raise_error(Yajl::ParseError)
+  end
+
+  it "errors when projecting if an object comma is missing" do
+    expect {
+      project({"a" => nil}, json: '{"a": 1 "b": 2}')
+    }.to raise_error(Yajl::ParseError)
+  end
+
+  it "errors when building if an object comma is missing" do
+    expect {
+      project(nil, json: '{"a": {"b": 2 "c": 3}}')
+    }.to raise_error(Yajl::ParseError)
+  end
+
+  it "errors when eof instead of simple value" do
+    expect {
+      project(nil, json: '[')
+    }.to raise_error(Yajl::ParseError)
+  end
+
+  it "errors when arrays don't have a comma between elements" do
+    expect {
+      project(nil, json: '[1 2]')
+    }.to raise_error(Yajl::ParseError)
+  end
+
+  it "supports parsing empty array" do
+    expect(project(nil, json: '[]')).to eql([])
+  end
+
+  it "supports parsing empty object" do
+    expect(project(nil, json: '{}')).to eql({})
+  end
+
+  it "reads a full buffer" do
+    json = "[" + "1,"*2046 + "1 ]"
+    expect(json.size).to eql(4096)
+    expect(project(nil, json: json)).to eql(Array.new(2047, 1))
+  end
+
+  it "reads into a second buffer" do
+    json = "[" + "1,"*2047 + "1 ]"
+    expect(json.size).to eql(4098)
+    expect(JSON.parse(json)).to eql(Array.new(2048, 1))
+    expect(project(nil, json: json)).to eql(Array.new(2048, 1))
+  end
+
+  it "supports parsing big strings" do
+    json = [
+      "a",
+      "b"*10_000,
+      "c",
+    ]
+    expect(project(nil, over: json)).to eql(json)
+  end
+
+  it "supports bigger read buffers" do
+    json = {
+      "a"*10_000 => "b"*10_000
+    }.to_json
+    stream = StringIO.new(json)
+    expect(Yajl::Projector.new(stream, 8192).project(nil)).to have_key("a"*10_000)
+  end
+
+  it "errors if starting with closing object" do
+    expect {
+      project(nil, json: '}')
+    }.to raise_error(Yajl::ParseError)
+  end
+
+  it "handles objects with utf16 escape sequences as keys" do
+    projection = project(nil, json: '{"\ud83d\ude00": "grinning face"}')
+    literal = {"😀" => "grinning face"}
+    expect(projection).to eql(literal)
+  end
+
+  it "handles objects with non-ascii utf8 bytes as keys" do
+    expect(project(nil, json: '{"😀": "grinning face"}')).to eql({"😀" => "grinning face"})
+  end
+
+  it "handles strings with utf16 escape sequences as object values" do
+    expect(project(nil, json: '{"grinning face": "\ud83d\ude00"}')).to eql({"grinning face" => "😀"})
+  end
+
+  it "handles strings with utf16 escape sequences as array values" do
+    projection = project(nil, json: '["\ud83d\ude00"]')
+    puts projection.first.inspect
+    puts projection.first.bytes
+
+    literal = ["😀"]
+    puts literal.first.inspect
+    puts literal.first.bytes
+
+    expect(projection).to eql(literal)
+  end
+
+  it "handles strings with non-ascii utf8 bytes as array values" do
+    projection = project(nil, json: '["😀"]')
+    puts projection.first.inspect
+    puts projection.first.bytes
+
+    literal = ["😀"]
+    puts literal.first.inspect
+    puts literal.first.bytes
+
+    expect(projection).to eql(literal)
+  end
+
+  it "ignores strings with utf16 escape sequences" do
+    expect(project({"grinning face with open mouth" => nil}, json: '{"grinning face": "\ud83d\ude00", "grinning face with open mouth": "\ud83d\ude03"}')).to eql({"grinning face with open mouth" => "😃"})
+  end
+
+  it "handles objects whose second key has escape sequences" do
+    expect(project(nil, json: '{"foo": "bar", "\ud83d\ude00": "grinning face"}')).to eql({"foo" => "bar", "😀" => "grinning face"})
+  end
+end