ffi-yajl 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 948084f681256d2d301e01415ce82e335f1bee52
-  data.tar.gz: f749ab5a3e388d3c069825b00cd37c30796db6d9
+  metadata.gz: c6033cc1e00b955a9910f2bd2bcf36711fbbe842
+  data.tar.gz: cfc7023e68822af31e4e634024fa9477064516f6
 SHA512:
-  metadata.gz: fda05bb34154de35d5b9f2a9d3f3625267a2dd35ad3e048243a8cbf904397ae001462bbeb449e2d83d00564291a47152dff4414d36a217d7fdaf0bef5a288543
-  data.tar.gz: 9c47d061b3055164397a98666cf34786ff0341c0ccccb9fdb95655927150af3adbedc9d6ffcace61883cfb3f39e45102ee3fc990891cc5b4769de86f044a7f93
+  metadata.gz: 04cc3ba362d0d00cd1bca749b46e053cc24153725cc3a1475177f82f27ed9442483abff6ce11bba74ab691c7017c8e4aaf1339fb5f7ea593ec5b88b0b776b879
+  data.tar.gz: 73f5223e85d8ac1d15951eac652f02de799034b5698e0f4a2fe04e2e0e62401175a4deff8a687f85e7ca4e303ca041bdf19c9366be212ee0b0a12119424cff78

data/README.md

@@ -9,16 +9,109 @@ extension mechanisms, including both MRI native extensions and FFI in
 order to be compatible with as many Ruby implementations as possible
 while providing good performance where possible.
 
+## How to Install
+
+Install from the command-line:
+
+```
+gem install ffi-yajl
+```
+
+Or use a Gemfile:
+
+```
+gem 'ffi-yajl'
+```
+
+## Supported Ruby VMs:
+
+* Ruby MRI 1.9.3/2.0.0/2.1.x/2.2.x
+* rbx 2.2.x (possibly earlier)
+* Jruby 1.7.x (possibly earlier)
+
+Ruby 1.8.7 support was dropped in 2.2.0
+
+## Supported Distros:
+
+* Ubuntu 10.04 through 14.10
+* Debian 7.x
+* RHEL/CentOS/Oracle 5.x/6.x/7.x
+* Solaris 9/10/11 (gcc, sun compiler untested)
+* AIX 6.x/7.x (gcc or xlc)
+* Windows 2008r2/2012 (and Win2k/2k3 and consumer versions should work)
+
 ## Basic Usage
 
+Start by requiring it:
+
 ```ruby
 require 'ffi-yajl'
-json_out = FFI_Yajl::Encoder.encode( { "foo" => [ "bar", "baz" ] } )
-# => "{\"foo\":[\"bar\",\"baz\"]}"
-data_in = FFI_Yajl::Parser.parse( json_out )
-# => {"foo"=>["bar", "baz"]}
 ```
 
+You can encode and parse with class objects:
+
+```ruby
+options_hash = {}
+json = FFI_Yajl::Encoder.encode( {"foo"=>["bar","baz"]}, options_hash )
+hash = FFI_Yajl::Parser.parse( json, options_hash )
+```
+
+Or you can be more object oriented:
+
+```ruby
+options_hash = {}
+encoder = FFI_Yajl::Encoder.new( options_hash )
+json = encoder.encode( {"foo"=>["bar","baz"]} )
+parser = FFI_Yajl::Parser.new( options_hash )
+hash = parser.parse( json )
+```
+
+## Parser Options
+
+* `:check_utf8`
+* `:dont_validate_strings`
+* `:symbolize_keys` (default = false):  JSON keys are parsed into symbols instead of strings.
+* `:symbolize_names` (default = false):  Alias for `:symbolize_keys`.
+* `:allow_trailing_garbage`
+* `:allow_multiple_values`
+* `:allow_partial_values`
+* `:unique_key_checking` (default = false):  Will raise an exception if keys
+   are repeated in hashes in the input JSON.  Without this, repeated keys will
+   silently replace the previous key.
+
+## Encoder Options
+
+* `:pretty` (default = false):  Produces more human readable 'pretty' output.
+* `:validate_utf8` (default = true):  Will raise an exception when trying to
+   encode strings that are invalid UTF-8.  When set to false this still will
+   produce valid UTF-8 JSON but will replace invalid characters.
+
+## Forcing FFI or C Extension
+
+You can set the environment variable `FORCE_FFI_YAJL` to `ext` or `ffi` to
+control if the C extension or FFI bindings are used.
+
+## Yajl Library Packaging
+
+This library prefers to use the embedded yajl 2.x C library packaged in the
+libyajl2 gem.  In order to use the operating system yajl library (which must be
+yajl 2.x) the environment variable `USE_SYSTEM_LIBYAJL2` can be set before
+installing or bundling libyajl2.  This will force the libyajl2 gem to skip
+compiling its embedded library and the ffi-yajl gem will fallback to using the
+system yajl library.
+
+## No JSON Gem Compatiblity layer
+
+This library does not offer a feature to patch `#to_json` methods into all
+the ruby classes similarly to the JSON gem or yajl-ruby's `yajl/json_gem`
+compatibility API.  The differences in encoding between the JSON gem and the
+Yajl C library can produce different output on different systems and makes
+testing brittle.  Such a feature will not be included.  It was removed in
+this pull request and could be easily extracted to its own gem (we have
+no interest in maintaining that gem):
+
+https://github.com/chef/ffi-yajl/pull/47/files
+
 ## Why This Instead of X?
 
 yajl is the only JSON library we've found that has error messages that
@@ -36,15 +129,6 @@ could support non-MRI rubies) and we also needed some bug fixes in
 yajl2, but the maintainer wasn't able to devote enough time to the
 project to make these updates in a timeframe that worked for us.
 
-## Yajl Library Packaging
-
-This library prefers to use the embedded yajl 2.x C library packaged in the
-libyajl2 gem.  In order to use the operating system yajl library (which must be
-yajl 2.x) the environment variable `USE_SYSTEM_LIBYAJL2` can be set before
-installing or bundling libyajl2.  This will force the libyajl2 gem to skip
-compiling its embedded library and the ffi-yajl gem will fallback to using the
-system yajl library.
-
 ## Thanks
 
 This was initially going to be a clean rewrite of an ffi ruby wrapper around

data/ext/ffi_yajl/ext/encoder/encoder.c

@@ -7,9 +7,6 @@ static VALUE cYajl_Gen;
 
 /* FIXME: the json gem does a whole bunch of indirection around monkeypatching...  not sure if we need to as well... */
 
-#define CHECK_STATUS(call) \
-      if ((status = (call)) != yajl_gen_status_ok) { rb_funcall(mEncoder2, rb_intern("raise_error_for_status"), 1, INT2FIX(status)); }
-
 static VALUE mEncoder_do_yajl_encode(VALUE self, VALUE obj, VALUE yajl_gen_opts, VALUE json_opts) {
   ID sym_ffi_yajl = rb_intern("ffi_yajl");
   VALUE sym_yajl_gen_beautify = ID2SYM(rb_intern("yajl_gen_beautify"));
@@ -72,290 +69,285 @@ int rb_cHash_ffi_yajl_callback(VALUE key, VALUE val, VALUE extra) {
   return 0;
 }
 
-static VALUE rb_cHash_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+#define RB_FUNC0(call) rb_funcall(self, rb_intern(call), 0)
+
+/*
+ * wrappers around yajl_gen_* functions
+ */
+
+/* encode a c-string as a yajl string */
+VALUE gen_cstring(VALUE rb_yajl_gen, char *cptr, int len) {
   yajl_gen_status status;
-  VALUE extra;
   struct yajl_gen_t *yajl_gen;
   Data_Get_Struct(rb_yajl_gen, struct yajl_gen_t, yajl_gen);
 
-  if ( rb_hash_aref(state, rb_str_new2("processing_key")) == Qtrue ) {
-    ID sym_to_s = rb_intern("to_s");
-    VALUE str = rb_funcall(self, sym_to_s, 0);
-    char *cptr = RSTRING_PTR(str);
-    int len = RSTRING_LEN(str);
-
-    CHECK_STATUS(
-      yajl_gen_string(yajl_gen, (unsigned char *)cptr, len)
-    );
-  } else {
-    extra = rb_hash_new();  /* FIXME: reduce garbage */
+  if ((status = yajl_gen_string(yajl_gen, (unsigned char *)cptr, len)) != yajl_gen_status_ok) {
+    rb_funcall(mEncoder2, rb_intern("raise_error_for_status"), 2, INT2FIX(status), rb_str_new(cptr, len));
+  }
 
-    rb_hash_aset(extra, rb_str_new2("yajl_gen"), rb_yajl_gen);
+  return Qnil;
+}
 
-    rb_hash_aset(extra, rb_str_new2("state"), state);
+/* encode a ruby-sring as a yajl string */
+VALUE gen_string(VALUE rb_yajl_gen, VALUE str) {
+  char *cptr = RSTRING_PTR(str);
+  int len = RSTRING_LEN(str);
 
-    CHECK_STATUS(
-      yajl_gen_map_open(yajl_gen)
-    );
-    rb_hash_foreach(self, rb_cHash_ffi_yajl_callback, extra);
-    CHECK_STATUS(
-      yajl_gen_map_close(yajl_gen)
-    );
+  return gen_cstring(rb_yajl_gen, cptr, len);
+}
+
+/* calls #to_s on an object to encode it as a yajl string */
+static VALUE gen_string_to_s(VALUE rb_yajl_gen, VALUE self) {
+  return gen_string(rb_yajl_gen, RB_FUNC0("to_s"));
+}
+
+/* encode a ruby string as a yajl number (also used to embed already-rendered json from #to_json) */
+VALUE gen_number(VALUE rb_yajl_gen, VALUE str) {
+  yajl_gen_status status;
+  struct yajl_gen_t *yajl_gen;
+  Data_Get_Struct(rb_yajl_gen, struct yajl_gen_t, yajl_gen);
+  char *cptr = RSTRING_PTR(str);
+  int len = RSTRING_LEN(str);
+
+  if ((status = yajl_gen_number(yajl_gen, cptr, len)) != yajl_gen_status_ok) {
+    rb_funcall(mEncoder2, rb_intern("raise_error_for_status"), 2, INT2FIX(status), str);
   }
 
   return Qnil;
 }
 
-static VALUE rb_cArray_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+/* encode hash open */
+VALUE gen_map_open(VALUE rb_yajl_gen) {
   yajl_gen_status status;
-  ID sym_ffi_yajl = rb_intern("ffi_yajl");
-  long i;
-  VALUE val;
   struct yajl_gen_t *yajl_gen;
   Data_Get_Struct(rb_yajl_gen, struct yajl_gen_t, yajl_gen);
 
-  if ( rb_hash_aref(state, rb_str_new2("processing_key")) == Qtrue ) {
-    ID sym_to_s = rb_intern("to_s");
-    VALUE str = rb_funcall(self, sym_to_s, 0);
-    char *cptr = RSTRING_PTR(str);
-    int len = RSTRING_LEN(str);
-
-    CHECK_STATUS(
-      yajl_gen_string(yajl_gen, (unsigned char *)cptr, len)
-    );
-  } else {
-    CHECK_STATUS(
-      yajl_gen_array_open(yajl_gen)
-    );
-    for(i=0; i<RARRAY_LEN(self); i++) {
-      val = rb_ary_entry(self, i);
-      rb_funcall(val, sym_ffi_yajl, 2, rb_yajl_gen, state);
-    }
-    CHECK_STATUS(
-      yajl_gen_array_close(yajl_gen)
-    );
+  if ((status = yajl_gen_map_open(yajl_gen)) != yajl_gen_status_ok) {
+    rb_funcall(mEncoder2, rb_intern("raise_error_for_status"), 2, INT2FIX(status), rb_str_new2("{"));
   }
 
   return Qnil;
 }
 
-static VALUE rb_cNilClass_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+/* encode a hash close */
+VALUE gen_map_close(VALUE rb_yajl_gen) {
   yajl_gen_status status;
   struct yajl_gen_t *yajl_gen;
   Data_Get_Struct(rb_yajl_gen, struct yajl_gen_t, yajl_gen);
 
-  if ( rb_hash_aref(state, rb_str_new2("processing_key")) == Qtrue ) {
-    ID sym_to_s = rb_intern("to_s");
-    VALUE str = rb_funcall(self, sym_to_s, 0);
-    char *cptr = RSTRING_PTR(str);
-    int len = RSTRING_LEN(str);
-
-    CHECK_STATUS(
-      yajl_gen_string(yajl_gen, (unsigned char *)cptr, len)
-    );
-  } else {
-    CHECK_STATUS(
-      yajl_gen_null(yajl_gen)
-    );
+  if ((status = yajl_gen_map_close(yajl_gen)) != yajl_gen_status_ok) {
+    rb_funcall(mEncoder2, rb_intern("raise_error_for_status"), 2, INT2FIX(status), rb_str_new2("}"));
   }
 
   return Qnil;
 }
 
-static VALUE rb_cTrueClass_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+/* encode an array open */
+VALUE gen_array_open(VALUE rb_yajl_gen) {
   yajl_gen_status status;
   struct yajl_gen_t *yajl_gen;
   Data_Get_Struct(rb_yajl_gen, struct yajl_gen_t, yajl_gen);
 
-  if ( rb_hash_aref(state, rb_str_new2("processing_key")) == Qtrue ) {
-    ID sym_to_s = rb_intern("to_s");
-    VALUE str = rb_funcall(self, sym_to_s, 0);
-    char *cptr = RSTRING_PTR(str);
-    int len = RSTRING_LEN(str);
-
-    CHECK_STATUS(
-      yajl_gen_string(yajl_gen, (unsigned char *)cptr, len)
-    );
-  } else {
-    CHECK_STATUS(
-      yajl_gen_bool(yajl_gen, 1)
-    );
+  if ((status = yajl_gen_array_open(yajl_gen)) != yajl_gen_status_ok) {
+    rb_funcall(mEncoder2, rb_intern("raise_error_for_status"), 2, INT2FIX(status), rb_str_new2("["));
   }
 
   return Qnil;
 }
 
-static VALUE rb_cFalseClass_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+/* encode an array close */
+VALUE gen_array_close(VALUE rb_yajl_gen) {
   yajl_gen_status status;
   struct yajl_gen_t *yajl_gen;
   Data_Get_Struct(rb_yajl_gen, struct yajl_gen_t, yajl_gen);
 
-  if ( rb_hash_aref(state, rb_str_new2("processing_key")) == Qtrue ) {
-    ID sym_to_s = rb_intern("to_s");
-    VALUE str = rb_funcall(self, sym_to_s, 0);
-    char *cptr = RSTRING_PTR(str);
-    int len = RSTRING_LEN(str);
-
-    CHECK_STATUS(
-      yajl_gen_string(yajl_gen, (unsigned char *)cptr, len)
-    );
-  } else {
-    CHECK_STATUS(
-      yajl_gen_bool(yajl_gen, 0)
-    );
+  if ((status = yajl_gen_array_close(yajl_gen)) != yajl_gen_status_ok) {
+    rb_funcall(mEncoder2, rb_intern("raise_error_for_status"), 2, INT2FIX(status), rb_str_new2("]"));
   }
 
   return Qnil;
 }
 
-static VALUE rb_cFixnum_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+/* encode a json null */
+VALUE gen_null(VALUE rb_yajl_gen) {
   yajl_gen_status status;
-  ID sym_to_s = rb_intern("to_s");
-  VALUE str = rb_funcall(self, sym_to_s, 0);
-  char *cptr = RSTRING_PTR(str);
-  int len = RSTRING_LEN(str);
   struct yajl_gen_t *yajl_gen;
   Data_Get_Struct(rb_yajl_gen, struct yajl_gen_t, yajl_gen);
 
-  if (memcmp(cptr, "NaN", 3) == 0 || memcmp(cptr, "Infinity", 8) == 0 || memcmp(cptr, "-Infinity", 9) == 0) {
-    rb_raise(cEncodeError, "'%s' is an invalid number", cptr);
+  if ((status = yajl_gen_null(yajl_gen)) != yajl_gen_status_ok) {
+    rb_funcall(mEncoder2, rb_intern("raise_error_for_status"), 2, INT2FIX(status), rb_str_new2("null"));
   }
-  if ( rb_hash_aref(state, rb_str_new2("processing_key")) == Qtrue ) {
-    CHECK_STATUS(
-      yajl_gen_string(yajl_gen, (unsigned char *)cptr, len)
-    );
-  } else {
-    CHECK_STATUS(
-      yajl_gen_number(yajl_gen, cptr, len)
-    );
+
+  return Qnil;
+}
+
+/* encode a true value */
+VALUE gen_true(VALUE rb_yajl_gen) {
+  yajl_gen_status status;
+  struct yajl_gen_t *yajl_gen;
+  Data_Get_Struct(rb_yajl_gen, struct yajl_gen_t, yajl_gen);
+
+  if ((status = yajl_gen_bool(yajl_gen, 1)) != yajl_gen_status_ok) {
+    rb_funcall(mEncoder2, rb_intern("raise_error_for_status"), 2, INT2FIX(status), rb_str_new2("true"));
   }
 
   return Qnil;
 }
 
-static VALUE rb_cBignum_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+/* encode a false value */
+VALUE gen_false(VALUE rb_yajl_gen) {
   yajl_gen_status status;
-  ID sym_to_s = rb_intern("to_s");
-  VALUE str = rb_funcall(self, sym_to_s, 0);
-  char *cptr = RSTRING_PTR(str);
-  int len = RSTRING_LEN(str);
   struct yajl_gen_t *yajl_gen;
   Data_Get_Struct(rb_yajl_gen, struct yajl_gen_t, yajl_gen);
 
-  if (memcmp(cptr, "NaN", 3) == 0 || memcmp(cptr, "Infinity", 8) == 0 || memcmp(cptr, "-Infinity", 9) == 0) {
-    rb_raise(cEncodeError, "'%s' is an invalid number", cptr);
+  if ((status = yajl_gen_bool(yajl_gen, 0)) != yajl_gen_status_ok) {
+    rb_funcall(mEncoder2, rb_intern("raise_error_for_status"), 2, INT2FIX(status), rb_str_new2("false"));
   }
+
+  return Qnil;
+}
+
+/*
+ * <Object>#to_ffi_yajl() method calls
+ */
+
+static VALUE rb_cHash_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
   if ( rb_hash_aref(state, rb_str_new2("processing_key")) == Qtrue ) {
-    CHECK_STATUS(
-      yajl_gen_string(yajl_gen, (unsigned char *)cptr, len)
-    );
+    gen_string(rb_yajl_gen, rb_funcall(self, rb_intern("to_s"), 0));
   } else {
-    CHECK_STATUS(
-      yajl_gen_number(yajl_gen, cptr, len)
-    );
+
+    /* FIXME: i think this got refactored from something else and it is now pointless --
+       should just pass rb_yajl_gen directly instead of this 'extra' hash -- i don't
+       *think* this indirection is doing anything useful to mark memory against the GC */
+
+    VALUE extra = rb_hash_new();
+
+    rb_hash_aset(extra, rb_str_new2("yajl_gen"), rb_yajl_gen);
+
+    rb_hash_aset(extra, rb_str_new2("state"), state);
+
+    gen_map_open(rb_yajl_gen);
+
+    rb_hash_foreach(self, rb_cHash_ffi_yajl_callback, extra);
+
+    gen_map_close(rb_yajl_gen);
   }
 
   return Qnil;
 }
 
-static VALUE rb_cFloat_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
-  yajl_gen_status status;
-  ID sym_to_s = rb_intern("to_s");
-  VALUE str = rb_funcall(self, sym_to_s, 0);
-  char *cptr = RSTRING_PTR(str);
-  int len = RSTRING_LEN(str);
-  struct yajl_gen_t *yajl_gen;
-  Data_Get_Struct(rb_yajl_gen, struct yajl_gen_t, yajl_gen);
+static VALUE rb_cArray_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+  if ( rb_hash_aref(state, rb_str_new2("processing_key")) == Qtrue ) {
+    gen_string(rb_yajl_gen, rb_funcall(self, rb_intern("to_s"), 0));
+  } else {
+    VALUE val;
+    long i;
+    ID sym_ffi_yajl = rb_intern("ffi_yajl");
 
-  if (memcmp(cptr, "NaN", 3) == 0 || memcmp(cptr, "Infinity", 8) == 0 || memcmp(cptr, "-Infinity", 9) == 0) {
-    rb_raise(cEncodeError, "'%s' is an invalid number", cptr);
+    gen_array_open(rb_yajl_gen);
+
+    for(i=0; i<RARRAY_LEN(self); i++) {
+      val = rb_ary_entry(self, i);
+      rb_funcall(val, sym_ffi_yajl, 2, rb_yajl_gen, state);
+    }
+
+    gen_array_close(rb_yajl_gen);
   }
+
+  return Qnil;
+}
+
+static VALUE rb_cNilClass_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
   if ( rb_hash_aref(state, rb_str_new2("processing_key")) == Qtrue ) {
-    CHECK_STATUS(
-      yajl_gen_string(yajl_gen, (unsigned char *)cptr, len)
-    );
+    gen_cstring(rb_yajl_gen, "", sizeof("")-1);
   } else {
-    CHECK_STATUS(
-      yajl_gen_number(yajl_gen, cptr, len)
-    );
+    gen_null(rb_yajl_gen);
   }
 
   return Qnil;
 }
 
-static VALUE rb_cString_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
-  yajl_gen_status status;
-  struct yajl_gen_t *yajl_gen;
-  Data_Get_Struct(rb_yajl_gen, struct yajl_gen_t, yajl_gen);
+static VALUE rb_cTrueClass_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+  if ( rb_hash_aref(state, rb_str_new2("processing_key")) == Qtrue ) {
+    gen_cstring(rb_yajl_gen, "true", sizeof("true")-1);
+  } else {
+    gen_true(rb_yajl_gen);
+  }
 
-  CHECK_STATUS(
-    yajl_gen_string(yajl_gen, (unsigned char *)RSTRING_PTR(self), RSTRING_LEN(self))
-  );
+  return Qnil;
+}
+
+static VALUE rb_cFalseClass_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+  if ( rb_hash_aref(state, rb_str_new2("processing_key")) == Qtrue ) {
+    gen_cstring(rb_yajl_gen, "false", sizeof("false")-1);
+  } else {
+    gen_false(rb_yajl_gen);
+  }
 
   return Qnil;
 }
 
-static VALUE string_ffi_yajl(VALUE str, VALUE rb_yajl_gen, VALUE state) {
-  yajl_gen_status status;
+static VALUE rb_cFixnum_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+  VALUE str = rb_funcall(self, rb_intern("to_s"), 0);
   char *cptr = RSTRING_PTR(str);
-  int len = RSTRING_LEN(str);
-  struct yajl_gen_t *yajl_gen;
-  Data_Get_Struct(rb_yajl_gen, struct yajl_gen_t, yajl_gen);
 
-  CHECK_STATUS(
-    yajl_gen_string(yajl_gen, (unsigned char *)cptr, len)
-  );
+  if (memcmp(cptr, "NaN", sizeof("NaN")) == 0 || memcmp(cptr, "Infinity", sizeof("Infinity")) == 0 || memcmp(cptr, "-Infinity", sizeof("-Infinity")) == 0) {
+    rb_raise(cEncodeError, "'%s' is an invalid number", cptr);
+  }
+
+  if ( rb_hash_aref(state, rb_str_new2("processing_key")) == Qtrue ) {
+    gen_string(rb_yajl_gen, str);
+  } else {
+    gen_number(rb_yajl_gen, str);
+  }
 
   return Qnil;
 }
 
-/* calls #to_s on an object to encode it */
-static VALUE object_to_s_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
-  ID sym_to_s = rb_intern("to_s");
-  VALUE str = rb_funcall(self, sym_to_s, 0);
-  return string_ffi_yajl(str, rb_yajl_gen, state);
+static VALUE rb_cBignum_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+  return rb_cFixnum_ffi_yajl(self, rb_yajl_gen, state);
+}
+
+static VALUE rb_cFloat_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+  return rb_cFixnum_ffi_yajl(self, rb_yajl_gen, state);
+}
+
+static VALUE rb_cString_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
+  return gen_string(rb_yajl_gen, self);
 }
 
 static VALUE rb_cSymbol_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
-  return object_to_s_ffi_yajl(self, rb_yajl_gen, state);
+  return gen_string_to_s(rb_yajl_gen, self);
 }
 
 static VALUE rb_cDate_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
-  return object_to_s_ffi_yajl(self, rb_yajl_gen, state);
+  return gen_string_to_s(rb_yajl_gen, self);
 }
 
 static VALUE rb_cTime_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
   ID sym_strftime = rb_intern("strftime");
   VALUE str = rb_funcall(self, sym_strftime, 1, rb_str_new2("%Y-%m-%d %H:%M:%S %z"));
-  return string_ffi_yajl(str, rb_yajl_gen, state);
+
+  return gen_string(rb_yajl_gen, str);
 }
 
 static VALUE rb_cStringIO_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
-  ID sym_read = rb_intern("read");
-  VALUE str = rb_funcall(self, sym_read, 0);
-  return string_ffi_yajl(str, rb_yajl_gen, state);
+  return gen_string(rb_yajl_gen, RB_FUNC0("read"));
 }
 
 static VALUE rb_cDateTime_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
-  return object_to_s_ffi_yajl(self, rb_yajl_gen, state);
+  return gen_string_to_s(rb_yajl_gen, self);
 }
 
 static VALUE rb_cObject_ffi_yajl(VALUE self, VALUE rb_yajl_gen, VALUE state) {
-  yajl_gen_status status;
   ID sym_to_json = rb_intern("to_json");
-  VALUE str;
-
   if ( rb_hash_aref(state, rb_str_new2("processing_key")) != Qtrue && rb_respond_to(self, sym_to_json) ) {
-    VALUE json_opts =  rb_hash_aref(state, rb_str_new2("json_opts"));
-    struct yajl_gen_t *yajl_gen;
-    Data_Get_Struct(rb_yajl_gen, struct yajl_gen_t, yajl_gen);
-
-    str = rb_funcall(self, sym_to_json, 1, json_opts);
-    CHECK_STATUS(
-      yajl_gen_number(yajl_gen, (char *)RSTRING_PTR(str), RSTRING_LEN(str))
-    );
+    VALUE json_opts = rb_hash_aref(state, rb_str_new2("json_opts"));
+    VALUE str = rb_funcall(self, sym_to_json, 1, json_opts);
+
+    gen_number(rb_yajl_gen, str);
   } else {
-    object_to_s_ffi_yajl(self, rb_yajl_gen, state);
+    gen_string_to_s(rb_yajl_gen, self);
   }
 
   return Qnil;

data/lib/ffi_yajl/encoder.rb

@@ -40,7 +40,8 @@ module FFI_Yajl
 
       # call either the ext or ffi hook
       str = do_yajl_encode(obj, yajl_gen_opts, opts)
-      str.force_encoding('UTF-8') if defined? Encoding
+      # we can skip cleaning the whole string for utf-8 issues if we have yajl validate as we go
+      str.encode!("utf-8", "binary", :undef => :replace) unless yajl_gen_opts[:yajl_gen_validate_utf8]
       str
     end
 
@@ -53,7 +54,9 @@ module FFI_Yajl
       @opts ||= {}
     end
 
-    def self.raise_error_for_status(status)
+    def self.raise_error_for_status(status, token=nil)
+      # scrub token to valid utf-8 since we may be issuing an exception on an invalid utf-8 token
+      token = token.to_s.encode("utf-8", "binary", :undef => :replace)
       case status
       when 1 # yajl_gen_keys_must_be_strings
         raise FFI_Yajl::EncodeError, "YAJL internal error: attempted use of non-string object as key"
@@ -68,7 +71,7 @@ module FFI_Yajl
       when 6 # yajl_gen_no_buf
         raise FFI_Yajl::EncodeError, "YAJL internal error: yajl_gen_get_buf was called, but a print callback was specified, so no internal buffer is available"
       when 7 # yajl_gen_invalid_string
-        raise FFI_Yajl::EncodeError, "Invalid UTF-8 string: cannot encode to UTF-8"
+        raise FFI_Yajl::EncodeError, "Invalid UTF-8 string '#{token}': cannot encode to UTF-8"
       else
         raise FFI_Yajl::EncodeError, "Unknown YAJL Error (#{status}), please report this as a bug"
       end

data/lib/ffi_yajl/ffi/encoder.rb

@@ -70,11 +70,11 @@ class Hash
     if state[:processing_key]
       str = self.to_s
       if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     else
       if ( status = FFI_Yajl.yajl_gen_map_open(yajl_gen) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, '{')
       end
       self.each do |key, value|
         # Perf Fix: mutate state hash rather than creating new copy
@@ -84,7 +84,7 @@ class Hash
         value.ffi_yajl(yajl_gen, state)
       end
       if ( status = FFI_Yajl.yajl_gen_map_close(yajl_gen) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, '}')
       end
     end
   end
@@ -95,17 +95,17 @@ class Array
     if state[:processing_key]
       str = self.to_s
       if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     else
       if ( status = FFI_Yajl.yajl_gen_array_open(yajl_gen) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, '[')
       end
       self.each do |value|
         value.ffi_yajl(yajl_gen, state)
       end
       if ( status = FFI_Yajl.yajl_gen_array_close(yajl_gen) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, ']')
       end
     end
   end
@@ -113,14 +113,14 @@ end
 
 class NilClass
   def ffi_yajl(yajl_gen, state)
+    str = self.to_s
     if state[:processing_key]
-      str = self.to_s
       if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     else
       if ( status = FFI_Yajl.yajl_gen_null(yajl_gen) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     end
   end
@@ -128,14 +128,14 @@ end
 
 class TrueClass
   def ffi_yajl(yajl_gen, state)
+    str = self.to_s
     if state[:processing_key]
-      str = self.to_s
       if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     else
       if ( status = FFI_Yajl.yajl_gen_bool(yajl_gen, 1) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     end
   end
@@ -143,14 +143,14 @@ end
 
 class FalseClass
   def ffi_yajl(yajl_gen, state)
+    str = self.to_s
     if state[:processing_key]
-      str = self.to_s
       if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     else
       if ( status = FFI_Yajl.yajl_gen_bool(yajl_gen, 0) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     end
   end
@@ -164,11 +164,11 @@ class Fixnum
     end
     if state[:processing_key]
       if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     else
       if ( status = FFI_Yajl.yajl_gen_integer(yajl_gen, self) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     end
   end
@@ -182,11 +182,11 @@ class Bignum
     end
     if state[:processing_key]
       if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     else
       if ( status = FFI_Yajl.yajl_gen_number(yajl_gen, str, str.bytesize) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     end
   end
@@ -200,11 +200,11 @@ class Float
     end
     if state[:processing_key]
       if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     else
       if ( status = FFI_Yajl.yajl_gen_number(yajl_gen, str, str.bytesize) ) != 0
-        FFI_Yajl::Encoder.raise_error_for_status(status)
+        FFI_Yajl::Encoder.raise_error_for_status(status, str)
       end
     end
   end
@@ -214,7 +214,7 @@ class Symbol
   def ffi_yajl(yajl_gen, state)
     str = self.to_s
     if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
-      FFI_Yajl::Encoder.raise_error_for_status(status)
+      FFI_Yajl::Encoder.raise_error_for_status(status, str)
     end
   end
 end
@@ -222,7 +222,7 @@ end
 class String
   def ffi_yajl(yajl_gen, state)
     if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, self, self.bytesize) ) != 0
-      FFI_Yajl::Encoder.raise_error_for_status(status)
+      FFI_Yajl::Encoder.raise_error_for_status(status, self)
     end
   end
 end
@@ -231,7 +231,7 @@ class StringIO
   def ffi_yajl(yajl_gen, state)
     str = self.read
     if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
-      FFI_Yajl::Encoder.raise_error_for_status(status)
+      FFI_Yajl::Encoder.raise_error_for_status(status, str)
     end
   end
 end
@@ -240,7 +240,7 @@ class Date
   def ffi_yajl(yajl_gen, state)
     str = self.to_s
     if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
-      FFI_Yajl::Encoder.raise_error_for_status(status)
+      FFI_Yajl::Encoder.raise_error_for_status(status, str)
     end
   end
 end
@@ -249,16 +249,16 @@ class Time
   def ffi_yajl(yajl_gen, state)
     str = self.strftime "%Y-%m-%d %H:%M:%S %z"
     if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
-      FFI_Yajl::Encoder.raise_error_for_status(status)
+      FFI_Yajl::Encoder.raise_error_for_status(status, str)
     end
   end
 end
 
-class DateTime
+class DateTime < Date
   def ffi_yajl(yajl_gen, state)
     str = self.to_s
     if ( status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize) ) != 0
-      FFI_Yajl::Encoder.raise_error_for_status(status)
+      FFI_Yajl::Encoder.raise_error_for_status(status, str)
     end
   end
 end
@@ -267,15 +267,15 @@ end
 class Object
   def ffi_yajl(yajl_gen, state)
     if !state[:processing_key] && self.respond_to?(:to_json)
-      json = self.to_json(state[:json_opts])
+      str = self.to_json(state[:json_opts])
       # #yajl_gen_number outputs a string without quotes around it
-      status = FFI_Yajl.yajl_gen_number(yajl_gen, json, json.bytesize)
+      status = FFI_Yajl.yajl_gen_number(yajl_gen, str, str.bytesize)
     else
       str = self.to_s
       status = FFI_Yajl.yajl_gen_string(yajl_gen, str, str.bytesize)
     end
     if ( status ) != 0
-      FFI_Yajl::Encoder.raise_error_for_status(status)
+      FFI_Yajl::Encoder.raise_error_for_status(status, str)
     end
   end
 end

data/lib/ffi_yajl/version.rb

@@ -21,5 +21,5 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 module FFI_Yajl
-  VERSION = "2.1.0"
+  VERSION = "2.2.0"
 end

data/spec/ffi_yajl/encoder_spec.rb

@@ -187,7 +187,7 @@ describe "FFI_Yajl::Encoder" do
     }
 
     it "raises an error on invalid json" do
-      expect{ encoder.encode(ruby) }.to raise_error(FFI_Yajl::EncodeError, "Invalid UTF-8 string: cannot encode to UTF-8")
+      expect{ encoder.encode(ruby) }.to raise_error(FFI_Yajl::EncodeError, /Invalid UTF-8 string 'Elan Ruusam.e': cannot encode to UTF-8/)
     end
 
     context "when validate_utf8 is off" do
@@ -196,6 +196,14 @@ describe "FFI_Yajl::Encoder" do
       it "does not raise an error" do
         expect{ encoder.encode(ruby) }.not_to raise_error
       end
+
+      it "returns utf8" do
+        expect( encoder.encode(ruby).encoding ).to eq(Encoding::UTF_8)
+      end
+
+      it "returns valid utf8" do
+        expect( encoder.encode(ruby).valid_encoding? ).to be true
+      end
     end
   end
 end

data/spec/spec_helper.rb

@@ -35,7 +35,6 @@ require 'ffi_yajl'
 
 RSpec.configure do |c|
   c.filter_run_excluding :unix_only => true unless RUBY_PLATFORM !~ /mswin|mingw|windows/
-  c.filter_run_excluding :ruby_gte_19 => true unless RUBY_VERSION.to_f >= 1.9
   c.filter_run_excluding :ruby_gte_193 => true unless RUBY_VERSION.to_f >= 2.0 || RUBY_VERSION =~ /^1\.9\.3/
 
   c.order = 'random'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ffi-yajl
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Lamont Granquist
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-20 00:00:00.000000000 Z
+date: 2015-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -177,7 +177,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="