yaji 0.0.1 → 0.0.2

data/HISTORY.markdown

@@ -0,0 +1,7 @@
+=== 0.0.2 / 2011-08-19
+
+* Integration with curb gem
+
+=== 0.0.1 / 2011-08-19
+
+* Initial public release

data/README.markdown

@@ -3,6 +3,18 @@ Yet another JSON iterator
 
 YAJI is a ruby wrapper to YAJL providing iterator interface to streaming JSON parser.
 
+INSTALL
+-------
+
+This gem depend on [yajl][1]. So you need development headers installed
+on your system to build this gem. For Debian GNU/Linux family it will be something like:
+
+    sudo apt-get install libyajl-dev
+
+Now you ready to install YAJI gem:
+
+    gem install yaji
+
 USAGE
 -----
 
@@ -13,6 +25,17 @@ YAJI::Parser initializer accepts `IO` instance or `String`.
     YAJI::Parser.new('{"foo":"bar"}')
     YAJI::Parser.new(File.open('data.json'))
 
+There is integration with [curb][2], so you can pass `Curl::Easy` instance to
+as input for parser.
+
+    require 'curl'
+    curl = Curl::Easy.new('http://avsej.net/test.json')
+    parser = YAJI::Parser.new(curl)
+    parser.each.to_a.first  #=> {"foo"=>"bar", "baz"=>{"nums"=>[42, 3.1415]}}
+
+There no strict requirement though, it could be any instance responding
+to `#on_body` and `#perform`.
+
 Parser instance provides two iterators to get JSON data: event-oriented
 and object-oriented. `YAJI::Parser#parse` yields tuple `[path, event,
 value] describing some parser event. For example, this code
@@ -66,6 +89,7 @@ code above will print two lines:
 You can use this iterator when the data is huge and you'd like to allow
 GC to collect yielded object before parser finish its job.
 
+
 LICENSE
 -------
 
@@ -82,3 +106,6 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+
+[1]: http://lloyd.github.com/yajl/
+[2]: https://rubygems.org/gems/curb/

data/ext/yaji/parser_ext.c

@@ -22,8 +22,21 @@
 
 #define RB_P(OBJ) \
 	rb_funcall(rb_stderr, rb_intern("print"), 1, rb_funcall(OBJ, rb_intern("object_id"), 0)); \
+	rb_funcall(rb_stderr, rb_intern("print"), 1, rb_str_new_cstr(" ")); \
+	rb_funcall(rb_stderr, rb_intern("print"), 1, rb_funcall(OBJ, rb_intern("class"), 0)); \
+	rb_funcall(rb_stderr, rb_intern("print"), 1, rb_str_new_cstr(" ")); \
 	rb_funcall(rb_stderr, rb_intern("puts"), 1, rb_funcall(OBJ, rb_intern("inspect"), 0));
 
+#define RERAISE_PARSER_ERROR(parser) \
+{ \
+	unsigned char* emsg = yajl_get_error(parser->handle, 1, \
+			(const unsigned char*)RSTRING_PTR(p->chunk), \
+			RSTRING_LEN(p->chunk)); \
+	VALUE errobj = rb_exc_new2(c_parse_error, (const char*) emsg); \
+	yajl_free_error(parser->handle, emsg); \
+	rb_exc_raise(errobj); \
+}
+
 static int yaji_null(void *ctx)
 {
 	yaji_parser* p = (yaji_parser*) DATA_PTR(ctx);
@@ -143,6 +156,26 @@ static int yaji_end_array(void *ctx)
 	return STATUS_CONTINUE;
 }
 
+static VALUE rb_yaji_parser_parse_chunk(VALUE chunk, VALUE self)
+{
+	yajl_status rc;
+	yaji_parser* p = (yaji_parser*) DATA_PTR(self);
+	const char* buf = RSTRING_PTR(chunk);
+	unsigned int len = RSTRING_LEN(chunk);
+	int i;
+
+	p->events = rb_ary_new();
+	p->chunk = chunk;
+	rc = yajl_parse(p->handle, (const unsigned char*)buf, len);
+	if (rc == yajl_status_error) {
+		RERAISE_PARSER_ERROR(p);
+	}
+	for (i=0; i<RARRAY_LEN(p->events); i++) {
+		rb_funcall(p->parser_cb, id_call, 1, RARRAY_PTR(p->events)[i]);
+	}
+	return rb_funcall(chunk, id_bytesize, 0, NULL);
+}
+
 static VALUE rb_yaji_parser_new(int argc, VALUE *argv, VALUE klass)
 {
 	yaji_parser* p;
@@ -155,12 +188,16 @@ static VALUE rb_yaji_parser_new(int argc, VALUE *argv, VALUE klass)
 	p->symbolize_keys = 0;
 	p->rbufsize = Qnil;
 	p->input = Qnil;
+	p->parser_cb = Qnil;
 
 	rb_scan_args(argc, argv, "11", &p->input, &opts);
 	if (TYPE(p->input) == T_STRING) {
 		p->input = rb_class_new_instance(1, &p->input, c_stringio);
+	} else if (rb_respond_to(p->input, id_perform) && rb_respond_to(p->input, id_on_body)) {
+		rb_block_call(p->input, id_on_body, 0, NULL, rb_yaji_parser_parse_chunk, obj);
 	} else if (!rb_respond_to(p->input, id_read)) {
-		rb_raise(c_parse_error, "input must be a String or IO");
+		rb_raise(c_parse_error, "input must be a String or IO or "
+				"something responding to #perform and #on_body e.g. Curl::Easy");
 	}
 	if (!NIL_P(opts)) {
 		Check_Type(opts, T_HASH);
@@ -190,50 +227,35 @@ static VALUE rb_yaji_parser_init(int argc, VALUE *argv, VALUE self)
 	return self;
 }
 
-#define RERAISE_PARSER_ERROR(parser, chunk, len) \
-{ \
-	unsigned char* emsg = yajl_get_error(parser, 1, chunk, len); \
-	VALUE errobj = rb_exc_new2(c_parse_error, (const char*) emsg); \
-	yajl_free_error(parser, emsg); \
-	rb_exc_raise(errobj); \
-}
-
 static VALUE rb_yaji_parser_parse(int argc, VALUE* argv, VALUE self)
 {
 	yajl_status rc;
-	int i;
 	yaji_parser* p = (yaji_parser*) DATA_PTR(self);
-	VALUE rbuf, proc;
-	const char* chunk = NULL;
-	unsigned int len = 0;
+	int i;
 
-	rb_scan_args(argc, argv, "00&", &proc);
+	rb_scan_args(argc, argv, "00&", &p->parser_cb);
 	RETURN_ENUMERATOR(self, argc, argv);
 
-	rbuf = rb_str_new(NULL, 0);
 	p->path = rb_ary_new();
 	p->path_str = rb_str_new("", 0);
-	while (rb_funcall(p->input, id_read, 2, p->rbufsize, rbuf) != Qnil) {
-		chunk = RSTRING_PTR(rbuf);
-		len = RSTRING_LEN(rbuf);
-
-		p->events = rb_ary_new();
-		rc = yajl_parse(p->handle, (const unsigned char*)chunk, len);
-		if (rc == yajl_status_error) {
-			RERAISE_PARSER_ERROR(p->handle, (const unsigned char*)chunk, len);
-		}
-		for (i=0; i<RARRAY_LEN(p->events); i++) {
-			rb_funcall(proc, id_call, 1, RARRAY_PTR(p->events)[i]);
+	p->chunk = Qnil;
+
+	if (rb_respond_to(p->input, id_perform)) {
+		rb_funcall(p->input, id_perform, 0);
+	} else {
+		p->chunk = rb_str_new(NULL, 0);
+		while (rb_funcall(p->input, id_read, 2, p->rbufsize, p->chunk) != Qnil) {
+			rb_yaji_parser_parse_chunk(p->chunk, self);
 		}
 	}
 
 	p->events = rb_ary_new();
 	rc = yajl_parse_complete(p->handle);
 	if (rc == yajl_status_insufficient_data || rc == yajl_status_error) {
-		RERAISE_PARSER_ERROR(p->handle, (const unsigned char*)chunk, len);
+		RERAISE_PARSER_ERROR(p);
 	}
 	for (i=0; i<RARRAY_LEN(p->events); i++) {
-		rb_funcall(proc, id_call, 1, RARRAY_PTR(p->events)[i]);
+		rb_funcall(p->parser_cb, id_call, 1, RARRAY_PTR(p->events)[i]);
 	}
 
 	return Qnil;
@@ -326,6 +348,8 @@ static void rb_yaji_parser_mark(void *parser)
 		rb_gc_mark(p->events);
 		rb_gc_mark(p->path);
 		rb_gc_mark(p->path_str);
+		rb_gc_mark(p->parser_cb);
+		rb_gc_mark(p->chunk);
 	}
 }
 
@@ -345,7 +369,9 @@ void Init_parser_ext() {
 	id_call = rb_intern("call");
 	id_read = rb_intern("read");
 	id_parse = rb_intern("parse");
-	id_rewind = rb_intern("rewind");
+	id_perform = rb_intern("perform");
+	id_on_body = rb_intern("on_body");
+	id_bytesize = rb_intern("bytesize");
 
 	sym_allow_comments = ID2SYM(rb_intern("allow_comments"));
 	sym_check_utf8 = ID2SYM(rb_intern("check_utf8"));

data/ext/yaji/parser_ext.h

@@ -41,7 +41,7 @@ static rb_encoding *utf8_encoding;
 
 static VALUE m_yaji, c_yaji_parser, c_parse_error, c_stringio;
 
-static ID id_call, id_read, id_parse, id_rewind;
+static ID id_call, id_read, id_parse, id_perform, id_on_body, id_bytesize;
 static ID sym_allow_comments, sym_check_utf8, sym_symbolize_keys,
 	  sym_read_buffer_size, sym_null, sym_boolean, sym_number, sym_string,
 	  sym_hash_key, sym_start_hash, sym_end_hash, sym_start_array,
@@ -79,6 +79,8 @@ typedef struct {
 	VALUE events;
 	VALUE path;
 	VALUE path_str;
+	VALUE parser_cb;
+	VALUE chunk;
 	yajl_handle handle;
 	yajl_parser_config config;
 } yaji_parser;

data/lib/yaji/version.rb

@@ -18,5 +18,5 @@
 #
 
 module YAJI
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/test/test_parser.rb

@@ -1,5 +1,6 @@
 require 'minitest/autorun'
 require 'yaji'
+require 'curb'
 
 class TestParser < MiniTest::Unit::TestCase
 
@@ -136,6 +137,14 @@ class TestParser < MiniTest::Unit::TestCase
     assert expected, objects
   end
 
+  def test_it_could_curb_async_approach
+    curl = Curl::Easy.new('http://avsej.net/test.json')
+    parser = YAJI::Parser.new(curl)
+    object = parser.each.to_a.first
+    expected = {"foo"=>"bar", "baz"=>{"nums"=>[42, 3.1415]}}
+    assert expected, object
+  end
+
   protected
 
   def toys_json_str

data/yaji.gemspec

@@ -22,4 +22,5 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'rake-compiler'
   s.add_development_dependency 'minitest'
+  s.add_development_dependency 'curb'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: yaji
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2011-08-19 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
-  requirement: &21823240 !ruby/object:Gem::Requirement
+  requirement: &8959040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *21823240
+  version_requirements: *8959040
 - !ruby/object:Gem::Dependency
   name: minitest
-  requirement: &21822680 !ruby/object:Gem::Requirement
+  requirement: &8958360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,18 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *21822680
+  version_requirements: *8958360
+- !ruby/object:Gem::Dependency
+  name: curb
+  requirement: &8957820 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *8957820
 description: YAJI is a ruby wrapper to YAJL providing iterator interface to streaming
   JSON parser
 email: info@couchbase.com
@@ -43,6 +54,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - Gemfile
+- HISTORY.markdown
 - LICENSE
 - README.markdown
 - Rakefile