yajl-ruby 1.3.1 → 1.4.2

data/ext/yajl/yajl_ext.h

@@ -53,10 +53,10 @@ static rb_encoding *utf8Encoding;
 #define RARRAY_LEN(s) (RARRAY(s)->len)
 #endif
 
-static VALUE cParseError, cEncodeError, mYajl, cParser, cEncoder;
+static VALUE cStandardError, cParseError, cEncodeError, mYajl, cParser, cProjector, cEncoder;
 static ID intern_io_read, intern_call, intern_keys, intern_to_s,
             intern_to_json, intern_has_key, intern_to_sym, intern_as_json;
-static ID sym_allow_comments, sym_check_utf8, sym_pretty, sym_indent, sym_terminator, sym_symbolize_keys, sym_symbolize_names, sym_html_safe;
+static ID sym_allow_comments, sym_check_utf8, sym_pretty, sym_indent, sym_terminator, sym_symbolize_keys, sym_symbolize_names, sym_html_safe, sym_entities;
 
 #define GetParser(obj, sval) Data_Get_Struct(obj, yajl_parser_wrapper, sval);
 #define GetEncoder(obj, sval) Data_Get_Struct(obj, yajl_encoder_wrapper, sval);

data/ext/yajl/yajl_gen.c

@@ -178,7 +178,7 @@ yajl_gen_free(yajl_gen g)
     if (++(g->depth) >= YAJL_MAX_DEPTH) return yajl_max_depth_exceeded;
 
 #define DECREMENT_DEPTH \
-    if (--(g->depth) >= YAJL_MAX_DEPTH) return yajl_gen_error;
+    if (--(g->depth) >= YAJL_MAX_DEPTH) return yajl_depth_underflow;
 
 #define APPENDED_ATOM \
     switch (g->state[g->depth]) {                   \
@@ -230,6 +230,36 @@ yajl_gen_double(yajl_gen g, double number)
     return yajl_gen_status_ok;
 }
 
+yajl_gen_status
+yajl_gen_long(yajl_gen g, long val)
+{
+    char buf[32], *b = buf + sizeof buf;
+    unsigned int len = 0;
+    unsigned long uval;
+
+    ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
+
+    if (val < 0) {
+        g->print(g->ctx, "-", 1);
+        // Avoid overflow. This shouldn't happen because FIXNUMs are 1 bit less
+        // than LONGs, but good to be safe.
+        uval = 1 + (unsigned long)(-(val + 1));
+    } else {
+        uval = val;
+    }
+
+    do {
+        *--b = "0123456789"[uval % 10];
+        uval /= 10;
+        len++;
+    } while(uval);
+    g->print(g->ctx, b, len);
+
+    APPENDED_ATOM;
+    FINAL_NEWLINE;
+    return yajl_gen_status_ok;
+}
+
 yajl_gen_status
 yajl_gen_number(yajl_gen g, const char * s, unsigned int l)
 {
@@ -332,6 +362,10 @@ yajl_gen_get_buf(yajl_gen g, const unsigned char ** buf,
                  unsigned int * len)
 {
     if (g->print != (yajl_print_t)&yajl_buf_append) return yajl_gen_no_buf;
+    yajl_buf_state buf_err = yajl_buf_err((yajl_buf)g->ctx);
+    if (buf_err) {
+        return yajl_gen_alloc_error;
+    }
     *buf = yajl_buf_data((yajl_buf)g->ctx);
     *len = yajl_buf_len((yajl_buf)g->ctx);
     return yajl_gen_status_ok;

data/ext/yajl/yajl_lex.c

@@ -38,29 +38,26 @@
 #include <assert.h>
 #include <string.h>
 
-#ifdef YAJL_LEXER_DEBUG
-static const char *
-tokToStr(yajl_tok tok) 
-{
+const char *yajl_tok_name(yajl_tok tok) {
     switch (tok) {
         case yajl_tok_bool: return "bool";
         case yajl_tok_colon: return "colon";
         case yajl_tok_comma: return "comma";
+        case yajl_tok_comment: return "comment";
         case yajl_tok_eof: return "eof";
         case yajl_tok_error: return "error";
-        case yajl_tok_left_brace: return "brace";
-        case yajl_tok_left_bracket: return "bracket";
+        case yajl_tok_left_brace: return "open_array";
+        case yajl_tok_left_bracket: return "open_object";
         case yajl_tok_null: return "null";
         case yajl_tok_integer: return "integer";
         case yajl_tok_double: return "double";
-        case yajl_tok_right_brace: return "brace";
-        case yajl_tok_right_bracket: return "bracket";
+        case yajl_tok_right_brace: return "close_array";
+        case yajl_tok_right_bracket: return "close_object";
         case yajl_tok_string: return "string";
         case yajl_tok_string_with_escapes: return "string_with_escapes";
     }
     return "unknown";
 }
-#endif
 
 /* Impact of the stream parsing feature on the lexer:
  *
@@ -121,6 +118,8 @@ yajl_lex_alloc(yajl_alloc_funcs * alloc,
                unsigned int allowComments, unsigned int validateUTF8)
 {
     yajl_lexer lxr = (yajl_lexer) YA_MALLOC(alloc, sizeof(struct yajl_lexer_t));
+    if (!lxr)
+        return NULL;
     memset((void *) lxr, 0, sizeof(struct yajl_lexer_t));
     lxr->buf = yajl_buf_alloc(alloc);
     lxr->allowComments = allowComments;
@@ -636,7 +635,12 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText,
         lexer->bufInUse = 1;
         yajl_buf_append(lexer->buf, jsonText + startOffset, *offset - startOffset);
         lexer->bufOff = 0;
-        
+
+        if (yajl_buf_err(lexer->buf)) {
+            lexer->error = yajl_lex_alloc_failed;
+            return yajl_tok_error;
+        }
+
         if (tok != yajl_tok_eof) {
             *outBuf = yajl_buf_data(lexer->buf);
             *outLen = yajl_buf_len(lexer->buf);
@@ -703,6 +707,8 @@ yajl_lex_error_to_string(yajl_lex_error error)
         case yajl_lex_unallowed_comment:
             return "probable comment found in input text, comments are "
                    "not enabled.";
+        case yajl_lex_alloc_failed:
+            return "allocation failed";
     }
     return "unknown error code";
 }
@@ -740,6 +746,10 @@ yajl_tok yajl_lex_peek(yajl_lexer lexer, const unsigned char * jsonText,
     tok = yajl_lex_lex(lexer, jsonText, jsonTextLen, &offset,
                        &outBuf, &outLen);
 
+    if (tok == yajl_tok_eof) {
+        return tok;
+    }
+
     lexer->bufOff = bufOff;
     lexer->bufInUse = bufInUse;
     yajl_buf_truncate(lexer->buf, bufLen);

data/ext/yajl/yajl_lex.h

@@ -36,33 +36,34 @@
 #include "api/yajl_common.h"
 
 typedef enum {
-    yajl_tok_bool,         
-    yajl_tok_colon,
-    yajl_tok_comma,     
-    yajl_tok_eof,
-    yajl_tok_error,
-    yajl_tok_left_brace,     
-    yajl_tok_left_bracket,
-    yajl_tok_null,         
-    yajl_tok_right_brace,     
-    yajl_tok_right_bracket,
+    yajl_tok_bool,         // 0
+    yajl_tok_colon,        // 1
+    yajl_tok_comma,        // 2
+    yajl_tok_eof,          // 3
+    yajl_tok_error,        // 4
+    yajl_tok_left_brace,   // 5
+    yajl_tok_left_bracket, // 6
+    yajl_tok_null,         // 7
+    yajl_tok_right_brace,  // 8
+    yajl_tok_right_bracket, // 9
 
     /* we differentiate between integers and doubles to allow the
      * parser to interpret the number without re-scanning */
-    yajl_tok_integer, 
-    yajl_tok_double, 
+    yajl_tok_integer, // 10
+    yajl_tok_double,  // 11
 
     /* we differentiate between strings which require further processing,
      * and strings that do not */
-    yajl_tok_string,
-    yajl_tok_string_with_escapes,
+    yajl_tok_string, // 12
+    yajl_tok_string_with_escapes, // 13
 
     /* comment tokens are not currently returned to the parser, ever */
-    yajl_tok_comment
+    yajl_tok_comment // 14
 } yajl_tok;
 
 typedef struct yajl_lexer_t * yajl_lexer;
 
+const char *yajl_tok_name(yajl_tok tok);
 
 YAJL_API
 yajl_lexer yajl_lex_alloc(yajl_alloc_funcs * alloc,
@@ -119,7 +120,8 @@ typedef enum {
     yajl_lex_missing_integer_after_decimal,
     yajl_lex_missing_integer_after_exponent,
     yajl_lex_missing_integer_after_minus,
-    yajl_lex_unallowed_comment
+    yajl_lex_unallowed_comment,
+    yajl_lex_alloc_failed
 } yajl_lex_error;
 
 YAJL_API

data/ext/yajl/yajl_parser.c

@@ -134,6 +134,14 @@ yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText,
         return yajl_status_client_canceled;                       \
     }
 
+/* check for buffer error */
+#define _BUF_CHK(x)                                               \
+    if (yajl_buf_err(x)) {                                        \
+        yajl_bs_set(hand->stateStack, yajl_state_parse_error);    \
+        hand->parseError =                                        \
+            "allocation failed";                                  \
+        return yajl_status_alloc_failed;                          \
+    }
 
 yajl_status
 yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
@@ -161,7 +169,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
             /* for arrays and maps, we advance the state for this
              * depth, then push the state of the next depth.
              * If an error occurs during the parsing of the nesting
-             * enitity, the state at this level will not matter.
+             * entity, the state at this level will not matter.
              * a state that needs pushing will be anything other
              * than state_start */
             yajl_state stateToPush = yajl_state_start;
@@ -185,6 +193,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
                     if (hand->callbacks && hand->callbacks->yajl_string) {
                         yajl_buf_clear(hand->decodeBuf);
                         yajl_string_decode(hand->decodeBuf, buf, bufLen);
+                        _BUF_CHK(hand->decodeBuf);
                         _CC_CHK(hand->callbacks->yajl_string(
                                     hand->ctx, yajl_buf_data(hand->decodeBuf),
                                     yajl_buf_len(hand->decodeBuf)));
@@ -234,6 +243,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
                             long int i = 0;
                             yajl_buf_clear(hand->decodeBuf);
                             yajl_buf_append(hand->decodeBuf, buf, bufLen);
+                            _BUF_CHK(hand->decodeBuf);
                             buf = yajl_buf_data(hand->decodeBuf);
                             i = strtol((const char *) buf, NULL, 10);
                             if ((i == LONG_MIN || i == LONG_MAX) &&
@@ -261,6 +271,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
                             double d = 0.0;
                             yajl_buf_clear(hand->decodeBuf);
                             yajl_buf_append(hand->decodeBuf, buf, bufLen);
+                            _BUF_CHK(hand->decodeBuf);
                             buf = yajl_buf_data(hand->decodeBuf);
                             d = strtod((char *) buf, NULL);
                             if ((d == HUGE_VAL || d == -HUGE_VAL) &&
@@ -320,7 +331,9 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
                 }
             }
             if (stateToPush != yajl_state_start) {
-                yajl_bs_push(hand->stateStack, stateToPush);
+                if (yajl_bs_push(hand->stateStack, stateToPush)) {
+                    return yajl_status_alloc_failed;
+                }
             }
 
             goto around_again;
@@ -342,6 +355,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
                     if (hand->callbacks && hand->callbacks->yajl_map_key) {
                         yajl_buf_clear(hand->decodeBuf);
                         yajl_string_decode(hand->decodeBuf, buf, bufLen);
+                        _BUF_CHK(hand->decodeBuf);
                         buf = yajl_buf_data(hand->decodeBuf);
                         bufLen = yajl_buf_len(hand->decodeBuf);
                     }

data/lib/yajl/version.rb

@@ -1,3 +1,3 @@
 module Yajl
-  VERSION = '1.3.1'
+  VERSION = '1.4.2'
 end

data/lib/yajl.rb

@@ -23,6 +23,13 @@ module Yajl
     Encoder.encode(obj, args, &block)
   end
 
+  class Projector
+    def initialize(stream, read_bufsize=4096)
+      @stream = stream
+      @buffer_size = read_bufsize
+    end
+  end
+
   class Parser
     # A helper method for parse-and-forget use-cases
     #

data/spec/encoding/encoding_spec.rb

@@ -230,6 +230,16 @@ describe "Yajl JSON encoder" do
     expect(s.read).to eql("{\"foo\":\"bar\"}")
   end
 
+  it "should encode all integers correctly" do
+    0.upto(129).each do |b|
+      b = 1 << b
+      [b, b-1, b-2, b+1, b+2].each do |i|
+        expect(Yajl::Encoder.encode(i)).to eq(i.to_s)
+        expect(Yajl::Encoder.encode(-i)).to eq((-i).to_s)
+      end
+    end
+  end
+
   it "should not encode NaN" do
     expect {
       Yajl::Encoder.encode(0.0/0.0)
@@ -275,11 +285,27 @@ describe "Yajl JSON encoder" do
     expect(safe_encoder.encode("</script>")).to eql("\"<\\/script>\"")
   end
 
+  it "should not encode characters with entities by default" do
+    expect(Yajl.dump("\u2028\u2029><&")).to eql("\"\u2028\u2029><&\"")
+  end
+
+  it "should encode characters with entities when enabled" do
+    expect(Yajl.dump("\u2028\u2029><&", entities: true)).to eql("\"\\u2028\\u2029\\u003E\\u003C\\u0026\"")
+  end
+
   it "should default to *not* escaping / characters" do
     unsafe_encoder = Yajl::Encoder.new
     expect(unsafe_encoder.encode("</script>")).not_to eql("\"<\\/script>\"")
   end
 
+  it "should encode slashes when enabled" do
+    unsafe_encoder = Yajl::Encoder.new(:entities => false)
+    safe_encoder   = Yajl::Encoder.new(:entities => true)
+
+    expect(unsafe_encoder.encode("</script>")).not_to eql("\"<\\/script>\"")
+    expect(safe_encoder.encode("</script>")).to eql("\"\\u003C\\/script\\u003E\"")
+  end
+
   it "return value of #to_json must be a string" do
     expect {
       Yajl::Encoder.encode(TheMindKiller.new)

data/spec/projection/project_file.rb

@@ -0,0 +1,41 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper.rb')
+
+require 'benchmark'
+require 'benchmark/memory'
+
+describe "file projection" do
+  it "projects file streams" do
+    schema = {
+      "forced" => nil,
+      "created" => nil,
+      "pusher" => {
+        "name" => nil,
+      },
+      "repository" => {
+        "name" => nil,
+        "full_name" => nil,
+      },
+      "ref" => nil,
+      "compare" => nil,
+      "commits" => {
+        "distinct" => nil,
+        "message" => nil,
+        "url" => nil,
+        "id" => nil,
+        "author" => {
+          "username" => nil,
+        }
+      }
+    }
+
+    file_path = ENV['JSON_FILE']
+    if file_path.nil? || file_path.empty?
+      return
+    end
+
+    Benchmark.memory { |x|
+      x.report("project (yajl)") { Yajl::Projector.new(File.open(file_path, 'r')).project(schema) }
+      x.compare!
+    }
+  end
+end