yaji 0.0.1

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+*.so
+.bundle
+Gemfile.lock
+pkg/*
+tmp/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in ijson.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,201 @@
+                              Apache License
+                        Version 2.0, January 2004
+                     http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+   "License" shall mean the terms and conditions for use, reproduction,
+   and distribution as defined by Sections 1 through 9 of this document.
+
+   "Licensor" shall mean the copyright owner or entity authorized by
+   the copyright owner that is granting the License.
+
+   "Legal Entity" shall mean the union of the acting entity and all
+   other entities that control, are controlled by, or are under common
+   control with that entity. For the purposes of this definition,
+   "control" means (i) the power, direct or indirect, to cause the
+   direction or management of such entity, whether by contract or
+   otherwise, or (ii) ownership of fifty percent (50%) or more of the
+   outstanding shares, or (iii) beneficial ownership of such entity.
+
+   "You" (or "Your") shall mean an individual or Legal Entity
+   exercising permissions granted by this License.
+
+   "Source" form shall mean the preferred form for making modifications,
+   including but not limited to software source code, documentation
+   source, and configuration files.
+
+   "Object" form shall mean any form resulting from mechanical
+   transformation or translation of a Source form, including but
+   not limited to compiled object code, generated documentation,
+   and conversions to other media types.
+
+   "Work" shall mean the work of authorship, whether in Source or
+   Object form, made available under the License, as indicated by a
+   copyright notice that is included in or attached to the work
+   (an example is provided in the Appendix below).
+
+   "Derivative Works" shall mean any work, whether in Source or Object
+   form, that is based on (or derived from) the Work and for which the
+   editorial revisions, annotations, elaborations, or other modifications
+   represent, as a whole, an original work of authorship. For the purposes
+   of this License, Derivative Works shall not include works that remain
+   separable from, or merely link (or bind by name) to the interfaces of,
+   the Work and Derivative Works thereof.
+
+   "Contribution" shall mean any work of authorship, including
+   the original version of the Work and any modifications or additions
+   to that Work or Derivative Works thereof, that is intentionally
+   submitted to Licensor for inclusion in the Work by the copyright owner
+   or by an individual or Legal Entity authorized to submit on behalf of
+   the copyright owner. For the purposes of this definition, "submitted"
+   means any form of electronic, verbal, or written communication sent
+   to the Licensor or its representatives, including but not limited to
+   communication on electronic mailing lists, source code control systems,
+   and issue tracking systems that are managed by, or on behalf of, the
+   Licensor for the purpose of discussing and improving the Work, but
+   excluding communication that is conspicuously marked or otherwise
+   designated in writing by the copyright owner as "Not a Contribution."
+
+   "Contributor" shall mean Licensor and any individual or Legal Entity
+   on behalf of whom a Contribution has been received by Licensor and
+   subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+   this License, each Contributor hereby grants to You a perpetual,
+   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+   copyright license to reproduce, prepare Derivative Works of,
+   publicly display, publicly perform, sublicense, and distribute the
+   Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+   this License, each Contributor hereby grants to You a perpetual,
+   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+   (except as stated in this section) patent license to make, have made,
+   use, offer to sell, sell, import, and otherwise transfer the Work,
+   where such license applies only to those patent claims licensable
+   by such Contributor that are necessarily infringed by their
+   Contribution(s) alone or by combination of their Contribution(s)
+   with the Work to which such Contribution(s) was submitted. If You
+   institute patent litigation against any entity (including a
+   cross-claim or counterclaim in a lawsuit) alleging that the Work
+   or a Contribution incorporated within the Work constitutes direct
+   or contributory patent infringement, then any patent licenses
+   granted to You under this License for that Work shall terminate
+   as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+   Work or Derivative Works thereof in any medium, with or without
+   modifications, and in Source or Object form, provided that You
+   meet the following conditions:
+
+   (a) You must give any other recipients of the Work or
+       Derivative Works a copy of this License; and
+
+   (b) You must cause any modified files to carry prominent notices
+       stating that You changed the files; and
+
+   (c) You must retain, in the Source form of any Derivative Works
+       that You distribute, all copyright, patent, trademark, and
+       attribution notices from the Source form of the Work,
+       excluding those notices that do not pertain to any part of
+       the Derivative Works; and
+
+   (d) If the Work includes a "NOTICE" text file as part of its
+       distribution, then any Derivative Works that You distribute must
+       include a readable copy of the attribution notices contained
+       within such NOTICE file, excluding those notices that do not
+       pertain to any part of the Derivative Works, in at least one
+       of the following places: within a NOTICE text file distributed
+       as part of the Derivative Works; within the Source form or
+       documentation, if provided along with the Derivative Works; or,
+       within a display generated by the Derivative Works, if and
+       wherever such third-party notices normally appear. The contents
+       of the NOTICE file are for informational purposes only and
+       do not modify the License. You may add Your own attribution
+       notices within Derivative Works that You distribute, alongside
+       or as an addendum to the NOTICE text from the Work, provided
+       that such additional attribution notices cannot be construed
+       as modifying the License.
+
+   You may add Your own copyright statement to Your modifications and
+   may provide additional or different license terms and conditions
+   for use, reproduction, or distribution of Your modifications, or
+   for any such Derivative Works as a whole, provided Your use,
+   reproduction, and distribution of the Work otherwise complies with
+   the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+   any Contribution intentionally submitted for inclusion in the Work
+   by You to the Licensor shall be under the terms and conditions of
+   this License, without any additional terms or conditions.
+   Notwithstanding the above, nothing herein shall supersede or modify
+   the terms of any separate license agreement you may have executed
+   with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+   names, trademarks, service marks, or product names of the Licensor,
+   except as required for reasonable and customary use in describing the
+   origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+   agreed to in writing, Licensor provides the Work (and each
+   Contributor provides its Contributions) on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+   implied, including, without limitation, any warranties or conditions
+   of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+   PARTICULAR PURPOSE. You are solely responsible for determining the
+   appropriateness of using or redistributing the Work and assume any
+   risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+   whether in tort (including negligence), contract, or otherwise,
+   unless required by applicable law (such as deliberate and grossly
+   negligent acts) or agreed to in writing, shall any Contributor be
+   liable to You for damages, including any direct, indirect, special,
+   incidental, or consequential damages of any character arising as a
+   result of this License or out of the use or inability to use the
+   Work (including but not limited to damages for loss of goodwill,
+   work stoppage, computer failure or malfunction, or any and all
+   other commercial damages or losses), even if such Contributor
+   has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+   the Work or Derivative Works thereof, You may choose to offer,
+   and charge a fee for, acceptance of support, warranty, indemnity,
+   or other liability obligations and/or rights consistent with this
+   License. However, in accepting such obligations, You may act only
+   on Your own behalf and on Your sole responsibility, not on behalf
+   of any other Contributor, and only if You agree to indemnify,
+   defend, and hold each Contributor harmless for any liability
+   incurred by, or claims asserted against, such Contributor by reason
+   of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+   To apply the Apache License to your work, attach the following
+   boilerplate notice, with the fields enclosed by brackets "[]"
+   replaced with your own identifying information. (Don't include
+   the brackets!)  The text should be enclosed in the appropriate
+   comment syntax for the file format. We also recommend that a
+   file or class name and description of purpose be included on the
+   same "printed page" as the copyright notice for easier
+   identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.markdown

@@ -0,0 +1,84 @@
+Yet another JSON iterator
+=========================
+
+YAJI is a ruby wrapper to YAJL providing iterator interface to streaming JSON parser.
+
+USAGE
+-----
+
+YAJI::Parser initializer accepts `IO` instance or `String`.
+
+    require 'yaji'
+
+    YAJI::Parser.new('{"foo":"bar"}')
+    YAJI::Parser.new(File.open('data.json'))
+
+Parser instance provides two iterators to get JSON data: event-oriented
+and object-oriented. `YAJI::Parser#parse` yields tuple `[path, event,
+value] describing some parser event. For example, this code
+
+    parser = YAJI::Parser.new('{"foo":[1, {"bar":"baz"}]}')
+    parser.parse do |path, event, value|
+      puts [path, event, value].inspect
+    end
+
+prints all parser events
+
+    ["", :start_hash, nil]
+    ["", :hash_key, "foo"]
+    ["foo", :start_array, nil]
+    ["foo/", :number, 1]
+    ["foo/", :start_hash, nil]
+    ["foo/", :hash_key, "bar"]
+    ["foo//bar", :string, "baz"]
+    ["foo/", :end_hash, nil]
+    ["foo", :end_array, nil]
+    ["", :end_hash, nil]
+
+You can call `#parse` method without block and it will return
+`Enumerator` object.
+
+The another approach is to use `YAJI::Parser#each` method to iterate
+over JSON objects. It accepts optional filter parameter if you'd like to
+iterated sub-objects. Here is the example
+
+    parser = YAJI::Parser.new('{"size":2,"items":[{"id":1}, {"id":2}]}')
+    parser.each do |obj|
+      puts obj.inspect
+    end
+
+will print only one line:
+
+    {"size"=>2, "items"=>[{"id"=>1}, {"id"=>2}]}
+
+But it might be more useful to yield items from inner array:
+
+    parser = YAJI::Parser.new('{"size":2,"items":[{"id":1}, {"id":2}]}')
+    parser.each("items/") do |obj|
+      puts obj.inspect
+    end
+
+code above will print two lines:
+
+    {"id"=>1}
+    {"id"=>2}
+
+You can use this iterator when the data is huge and you'd like to allow
+GC to collect yielded object before parser finish its job.
+
+LICENSE
+-------
+
+Copyright 2011 Couchbase, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+Dir['tasks/*.rake'].sort.each { |f| load f }
+

data/ext/yaji/extconf.rb

@@ -0,0 +1,16 @@
+# encoding: UTF-8
+require 'mkmf'
+require 'rbconfig'
+
+def define(macro, value = nil)
+  $defs.push("-D #{[macro.upcase, value].compact.join('=')}")
+end
+
+$CFLAGS << ' -Wall -funroll-loops'
+$CFLAGS << ' -Wextra -O0 -ggdb3' if ENV['DEBUG']
+
+have_library('yajl', 'yajl_parse', 'yajl/yajl_parse.h')
+define("READ_BUFSIZE", "8192")
+
+create_header("yaji_config.h")
+create_makefile("parser_ext")

data/ext/yaji/parser_ext.c

@@ -0,0 +1,370 @@
+/*
+ *  Author:: Couchbase <info@couchbase.com>
+ *  Copyright:: 2011 Couchbase, Inc.
+ *  License:: Apache License, Version 2.0
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include "parser_ext.h"
+
+#define STATUS_CONTINUE 1
+
+#define RB_P(OBJ) \
+	rb_funcall(rb_stderr, rb_intern("print"), 1, rb_funcall(OBJ, rb_intern("object_id"), 0)); \
+	rb_funcall(rb_stderr, rb_intern("puts"), 1, rb_funcall(OBJ, rb_intern("inspect"), 0));
+
+static int yaji_null(void *ctx)
+{
+	yaji_parser* p = (yaji_parser*) DATA_PTR(ctx);
+	VALUE rv = rb_ary_new3(3, p->path_str, sym_null, Qnil);
+	rb_ary_push(p->events, rv);
+	return STATUS_CONTINUE;
+}
+
+static int yaji_boolean(void *ctx, int val)
+{
+	yaji_parser* p = (yaji_parser*) DATA_PTR(ctx);
+	VALUE rv = rb_ary_new3(3, p->path_str, sym_boolean, val ? Qtrue : Qfalse);
+	rb_ary_push(p->events, rv);
+	return STATUS_CONTINUE;
+}
+
+static int yaji_number(void *ctx, const char *val, unsigned int len)
+{
+	yaji_parser* p = (yaji_parser*) DATA_PTR(ctx);
+	char buf[len+1];
+	buf[len] = 0;
+	memcpy(buf, val, len);
+	VALUE rv;
+
+	if (memchr(buf, '.', len) || memchr(buf, 'e', len) || memchr(buf, 'E', len)) {
+		rv = rb_ary_new3(3, p->path_str, sym_number, rb_float_new(strtod(buf, NULL)));
+	} else {
+		rv = rb_ary_new3(3, p->path_str, sym_number, rb_cstr2inum(buf, 10));
+	}
+	rb_ary_push(p->events, rv);
+	return STATUS_CONTINUE;
+}
+
+#ifdef HAVE_RUBY_ENCODING_H
+#define YAJI_TO_STR(val, len, str)					    \
+	str = rb_str_new((const char *)val, len);			    \
+	rb_encoding *default_internal_enc = rb_default_internal_encoding(); \
+	rb_enc_associate(str, utf8_encoding);				    \
+	if (default_internal_enc) {					    \
+		str = rb_str_export_to_enc(str, default_internal_enc);	    \
+	}
+#else
+#define YAJI_TO_STR(val, len, str)					    \
+	str = rb_str_new((const char *)val, len);
+#endif
+
+static int yaji_string(void *ctx, const unsigned char *val, unsigned int len)
+{
+	yaji_parser* p = (yaji_parser*) DATA_PTR(ctx);
+	VALUE str, rv;
+	YAJI_TO_STR((const char *)val, len, str);
+	rv = rb_ary_new3(3, p->path_str, sym_string, str);
+	rb_ary_push(p->events, rv);
+	return STATUS_CONTINUE;
+}
+
+static int yaji_hash_key(void *ctx, const unsigned char *val, unsigned int len)
+{
+	yaji_parser* p = (yaji_parser*) DATA_PTR(ctx);
+	VALUE key, rv;
+	YAJI_TO_STR((const char *)val, len, key);
+	key = p->symbolize_keys ? ID2SYM(rb_to_id(key)) : key;
+	if (p->key_in_use) {
+		rb_ary_pop(p->path);
+	} else {
+		p->key_in_use = 1;
+	}
+	p->path_str = rb_ary_join(p->path, rb_str_new_cstr("/"));
+	rb_str_freeze(p->path_str);
+	rv = rb_ary_new3(3, p->path_str, sym_hash_key, key);
+	rb_ary_push(p->events, rv);
+	rb_ary_push(p->path, key);
+	p->path_str = rb_ary_join(p->path, rb_str_new_cstr("/"));
+	rb_str_freeze(p->path_str);
+	return STATUS_CONTINUE;
+}
+
+static int yaji_start_hash(void *ctx)
+{
+	yaji_parser* p = (yaji_parser*) DATA_PTR(ctx);
+	p->key_in_use = 0;
+	VALUE rv = rb_ary_new3(3, p->path_str, sym_start_hash, Qnil);
+	rb_ary_push(p->events, rv);
+	return STATUS_CONTINUE;
+}
+
+static int yaji_end_hash(void *ctx)
+{
+	yaji_parser* p = (yaji_parser*) DATA_PTR(ctx);
+	rb_ary_pop(p->path);
+	p->path_str = rb_ary_join(p->path, rb_str_new_cstr("/"));
+	VALUE rv = rb_ary_new3(3, p->path_str, sym_end_hash, Qnil);
+	rb_ary_push(p->events, rv);
+	return STATUS_CONTINUE;
+}
+
+static int yaji_start_array(void *ctx)
+{
+	yaji_parser* p = (yaji_parser*) DATA_PTR(ctx);
+	VALUE rv = rb_ary_new3(3, p->path_str, sym_start_array, Qnil);
+	rb_ary_push(p->path, rb_str_new_cstr(""));
+	p->path_str = rb_ary_join(p->path, rb_str_new_cstr("/"));
+	rb_str_freeze(p->path_str);
+	rb_ary_push(p->events, rv);
+	return STATUS_CONTINUE;
+}
+
+static int yaji_end_array(void *ctx)
+{
+	yaji_parser* p = (yaji_parser*) DATA_PTR(ctx);
+
+	rb_ary_pop(p->path);
+	p->path_str = rb_ary_join(p->path, rb_str_new_cstr("/"));
+
+	VALUE rv = rb_ary_new3(3, p->path_str, sym_end_array, Qnil);
+	rb_ary_push(p->events, rv);
+	return STATUS_CONTINUE;
+}
+
+static VALUE rb_yaji_parser_new(int argc, VALUE *argv, VALUE klass)
+{
+	yaji_parser* p;
+	VALUE opts, obj;
+
+	obj = Data_Make_Struct(klass, yaji_parser, rb_yaji_parser_mark, rb_yaji_parser_free, p);
+	p->handle = NULL;
+	p->config.allowComments = 1;
+	p->config.checkUTF8 = 1;
+	p->symbolize_keys = 0;
+	p->rbufsize = Qnil;
+	p->input = Qnil;
+
+	rb_scan_args(argc, argv, "11", &p->input, &opts);
+	if (TYPE(p->input) == T_STRING) {
+		p->input = rb_class_new_instance(1, &p->input, c_stringio);
+	} else if (!rb_respond_to(p->input, id_read)) {
+		rb_raise(c_parse_error, "input must be a String or IO");
+	}
+	if (!NIL_P(opts)) {
+		Check_Type(opts, T_HASH);
+		if (rb_hash_aref(opts, sym_allow_comments) == Qfalse) {
+			p->config.allowComments = 0;
+		}
+		if (rb_hash_aref(opts, sym_check_utf8) == Qfalse) {
+			p->config.checkUTF8 = 0;
+		}
+		if (rb_hash_aref(opts, sym_symbolize_keys) == Qtrue) {
+			p->symbolize_keys = 1;
+		}
+		p->rbufsize = rb_hash_aref(opts, sym_read_buffer_size);
+	}
+	if (NIL_P(p->rbufsize)) {
+		p->rbufsize = INT2FIX(READ_BUFSIZE);
+	} else {
+		Check_Type(p->rbufsize, T_FIXNUM);
+	}
+	p->handle = yajl_alloc(&callbacks, &p->config, NULL, (void *)obj);
+	rb_obj_call_init(obj, 0, 0);
+	return obj;
+}
+
+static VALUE rb_yaji_parser_init(int argc, VALUE *argv, VALUE self)
+{
+	return self;
+}
+
+#define RERAISE_PARSER_ERROR(parser, chunk, len) \
+{ \
+	unsigned char* emsg = yajl_get_error(parser, 1, chunk, len); \
+	VALUE errobj = rb_exc_new2(c_parse_error, (const char*) emsg); \
+	yajl_free_error(parser, emsg); \
+	rb_exc_raise(errobj); \
+}
+
+static VALUE rb_yaji_parser_parse(int argc, VALUE* argv, VALUE self)
+{
+	yajl_status rc;
+	int i;
+	yaji_parser* p = (yaji_parser*) DATA_PTR(self);
+	VALUE rbuf, proc;
+	const char* chunk = NULL;
+	unsigned int len = 0;
+
+	rb_scan_args(argc, argv, "00&", &proc);
+	RETURN_ENUMERATOR(self, argc, argv);
+
+	rbuf = rb_str_new(NULL, 0);
+	p->path = rb_ary_new();
+	p->path_str = rb_str_new("", 0);
+	while (rb_funcall(p->input, id_read, 2, p->rbufsize, rbuf) != Qnil) {
+		chunk = RSTRING_PTR(rbuf);
+		len = RSTRING_LEN(rbuf);
+
+		p->events = rb_ary_new();
+		rc = yajl_parse(p->handle, (const unsigned char*)chunk, len);
+		if (rc == yajl_status_error) {
+			RERAISE_PARSER_ERROR(p->handle, (const unsigned char*)chunk, len);
+		}
+		for (i=0; i<RARRAY_LEN(p->events); i++) {
+			rb_funcall(proc, id_call, 1, RARRAY_PTR(p->events)[i]);
+		}
+	}
+
+	p->events = rb_ary_new();
+	rc = yajl_parse_complete(p->handle);
+	if (rc == yajl_status_insufficient_data || rc == yajl_status_error) {
+		RERAISE_PARSER_ERROR(p->handle, (const unsigned char*)chunk, len);
+	}
+	for (i=0; i<RARRAY_LEN(p->events); i++) {
+		rb_funcall(proc, id_call, 1, RARRAY_PTR(p->events)[i]);
+	}
+
+	return Qnil;
+}
+
+#define YAJI_STR_START_WITH(str, head) \
+	(RSTRING_LEN(head) <= RSTRING_LEN(str) && memcmp(RSTRING_PTR(head), RSTRING_PTR(str), RSTRING_LEN(head)) == 0)
+
+static VALUE rb_yaji_each_iter(VALUE chunk, VALUE* params_p)
+{
+	VALUE* params = (VALUE*)params_p;
+	VALUE path = rb_ary_shift(chunk);
+	VALUE event = rb_ary_shift(chunk);
+	VALUE value = rb_ary_shift(chunk);
+	VALUE proc = params[0];
+	VALUE stack = params[1];
+	VALUE query = params[2];
+	VALUE last_entry, object, container, key, hash;
+
+	if (NIL_P(query) || YAJI_STR_START_WITH(path, query)) {
+		if (event == sym_hash_key) {
+			rb_ary_push(stack, value);
+		} else if (event == sym_start_hash || event == sym_start_array) {
+			container = (event == sym_start_hash) ? rb_hash_new() : rb_ary_new();
+			last_entry = rb_ary_entry(stack, -1);
+			switch(TYPE(last_entry)) {
+			case T_STRING:
+				key = rb_ary_pop(stack);
+				hash = rb_ary_entry(stack, -1);
+				rb_hash_aset(hash, key, container);
+				break;
+			case T_ARRAY:
+				rb_ary_push(last_entry, container);
+			}
+			rb_ary_push(stack, container);
+		} else if (event == sym_end_hash || event == sym_end_array) {
+			object = rb_ary_pop(stack);
+			if (RARRAY_LEN(stack) == 0) {
+				rb_funcall(proc, id_call, 1, object);
+			}
+		} else {
+			last_entry = rb_ary_entry(stack, -1);
+			switch(TYPE(last_entry)) {
+			case T_STRING:
+				key = rb_ary_pop(stack);
+				hash = rb_ary_entry(stack, -1);
+				rb_hash_aset(hash, key, value);
+				break;
+			case T_ARRAY:
+				rb_ary_push(last_entry, value);
+				break;
+			case T_NIL:
+				rb_funcall(proc, id_call, 1, value);
+				break;
+			}
+		}
+	}
+	return Qnil;
+}
+
+static VALUE rb_yaji_parser_each(int argc, VALUE* argv, VALUE self)
+{
+	VALUE query, proc, params[3];
+	rb_scan_args(argc, argv, "01&", &query, &proc);
+	RETURN_ENUMERATOR(self, argc, argv);
+	params[0] = proc;	    // callback
+	params[1] = rb_ary_new();   // stack
+	params[2] = query;
+	rb_block_call(self, id_parse, 0, NULL, rb_yaji_each_iter, (VALUE)params);
+	return Qnil;
+}
+
+static void rb_yaji_parser_free(void *parser)
+{
+	yaji_parser* p = parser;
+	if (p) {
+		if (p->handle) {
+			yajl_free(p->handle);
+		}
+		free(p);
+	}
+}
+
+static void rb_yaji_parser_mark(void *parser)
+{
+	yaji_parser* p = parser;
+	if (p) {
+		rb_gc_mark(p->input);
+		rb_gc_mark(p->rbufsize);
+		rb_gc_mark(p->events);
+		rb_gc_mark(p->path);
+		rb_gc_mark(p->path_str);
+	}
+}
+
+/* Ruby Extension initializer */
+void Init_parser_ext() {
+	m_yaji = rb_define_module("YAJI");
+
+	c_parse_error = rb_define_class_under(m_yaji, "ParseError", rb_eStandardError);
+
+	c_yaji_parser = rb_define_class_under(m_yaji, "Parser", rb_cObject);
+	rb_define_const(c_yaji_parser, "READ_BUFFER_SIZE", INT2FIX(READ_BUFSIZE));
+	rb_define_singleton_method(c_yaji_parser, "new", rb_yaji_parser_new, -1);
+	rb_define_method(c_yaji_parser, "initialize", rb_yaji_parser_init, -1);
+	rb_define_method(c_yaji_parser, "parse", rb_yaji_parser_parse, -1);
+	rb_define_method(c_yaji_parser, "each", rb_yaji_parser_each, -1);
+
+	id_call = rb_intern("call");
+	id_read = rb_intern("read");
+	id_parse = rb_intern("parse");
+	id_rewind = rb_intern("rewind");
+
+	sym_allow_comments = ID2SYM(rb_intern("allow_comments"));
+	sym_check_utf8 = ID2SYM(rb_intern("check_utf8"));
+	sym_symbolize_keys = ID2SYM(rb_intern("symbolize_keys"));
+	sym_read_buffer_size = ID2SYM(rb_intern("read_buffer_size"));
+	sym_null = ID2SYM(rb_intern("null"));
+	sym_boolean = ID2SYM(rb_intern("boolean"));
+	sym_number = ID2SYM(rb_intern("number"));
+	sym_string = ID2SYM(rb_intern("string"));
+	sym_hash_key = ID2SYM(rb_intern("hash_key"));
+	sym_start_hash = ID2SYM(rb_intern("start_hash"));
+	sym_end_hash = ID2SYM(rb_intern("end_hash"));
+	sym_start_array = ID2SYM(rb_intern("start_array"));
+	sym_end_array = ID2SYM(rb_intern("end_array"));
+
+#ifdef HAVE_RUBY_ENCODING_H
+	utf8_encoding = rb_utf8_encoding();
+#endif
+
+	rb_require("stringio");
+	c_stringio = rb_const_get(rb_cObject, rb_intern("StringIO"));
+}

data/ext/yaji/parser_ext.h

@@ -0,0 +1,92 @@
+/*
+ *  Author:: Couchbase <info@couchbase.com>
+ *  Copyright:: 2011 Couchbase, Inc.
+ *  License:: Apache License, Version 2.0
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include <yajl/yajl_parse.h>
+
+#include <ruby.h>
+
+#ifdef HAVE_RUBY_ENCODING_H
+#include <ruby/encoding.h>
+static rb_encoding *utf8_encoding;
+#endif
+
+/* Older versions of Ruby (< 1.8.6) need these */
+#ifndef RSTRING_PTR
+#define RSTRING_PTR(s) (RSTRING(s)->ptr)
+#endif
+#ifndef RSTRING_LEN
+#define RSTRING_LEN(s) (RSTRING(s)->len)
+#endif
+#ifndef RARRAY_PTR
+#define RARRAY_PTR(s) (RARRAY(s)->ptr)
+#endif
+#ifndef RARRAY_LEN
+#define RARRAY_LEN(s) (RARRAY(s)->len)
+#endif
+
+static VALUE m_yaji, c_yaji_parser, c_parse_error, c_stringio;
+
+static ID id_call, id_read, id_parse, id_rewind;
+static ID sym_allow_comments, sym_check_utf8, sym_symbolize_keys,
+	  sym_read_buffer_size, sym_null, sym_boolean, sym_number, sym_string,
+	  sym_hash_key, sym_start_hash, sym_end_hash, sym_start_array,
+	  sym_end_array;
+
+static int yaji_null(void *ctx);
+static int yaji_boolean(void *ctx, int val);
+static int yaji_number(void *ctx, const char *val, unsigned int len);
+static int yaji_string(void *ctx, const unsigned char *val, unsigned int len);
+static int yaji_hash_key(void *ctx, const unsigned char *val, unsigned int len);
+static int yaji_start_hash(void *ctx);
+static int yaji_end_hash(void *ctx);
+static int yaji_start_array(void *ctx);
+static int yaji_end_array(void *ctx);
+
+static yajl_callbacks callbacks = {
+	yaji_null,
+	yaji_boolean,
+	NULL,
+	NULL,
+	yaji_number,
+	yaji_string,
+	yaji_start_hash,
+	yaji_hash_key,
+	yaji_end_hash,
+	yaji_start_array,
+	yaji_end_array
+};
+
+typedef struct {
+	int symbolize_keys;
+	int key_in_use;
+	VALUE input;
+	VALUE rbufsize;
+	VALUE events;
+	VALUE path;
+	VALUE path_str;
+	yajl_handle handle;
+	yajl_parser_config config;
+} yaji_parser;
+
+static VALUE rb_yaji_parser_new(int argc, VALUE *argv, VALUE klass);
+static VALUE rb_yaji_parser_init(int argc, VALUE *argv, VALUE self);
+static VALUE rb_yaji_parser_parse(int argc, VALUE *argv, VALUE self);
+static void rb_yaji_parser_free(void *parser);
+static void rb_yaji_parser_mark(void *parser);
+
+void Init_parser_ext();

data/lib/yaji.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+#
+# Author:: Couchbase <info@couchbase.com>
+# Copyright:: 2011 Couchbase, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "yaji/version"
+require "yaji/parser_ext"
+
+module YAJI
+end

data/lib/yaji/version.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+#
+# Author:: Couchbase <info@couchbase.com>
+# Copyright:: 2011 Couchbase, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module YAJI
+  VERSION = "0.0.1"
+end

data/tasks/compile.rake

@@ -0,0 +1,14 @@
+gem 'rake-compiler', '>= 0.7.5'
+require "rake/extensiontask"
+
+def gemspec
+  @clean_gemspec ||= eval(File.read(File.expand_path('../../yaji.gemspec', __FILE__)))
+end
+
+Rake::ExtensionTask.new("parser_ext", gemspec) do |ext|
+  ext.ext_dir = File.join('ext', 'yaji')
+  ext.lib_dir = File.join('lib', 'yaji')
+
+  # clean compiled extension
+  CLEAN.include "#{ext.lib_dir}/*.#{RbConfig::CONFIG['DLEXT']}"
+end

data/tasks/test.rake

@@ -0,0 +1,7 @@
+require 'rake/testtask'
+Rake::TestTask.new do |test|
+  test.libs << "test" << "."
+  test.ruby_opts << "-rruby-debug" if ENV['DEBUG']
+  test.pattern = 'test/test_*.rb'
+  test.verbose = true
+end

data/tasks/util.rake

@@ -0,0 +1,4 @@
+desc 'Start an irb session and load the library.'
+task :console do
+  exec "irb -I lib -rruby-debug -ryaji"
+end

data/test/fixture.json

@@ -0,0 +1,47 @@
+{
+  "total_rows": 4,
+  "offset": 0,
+  "rows": [
+    {
+      "id": "3e8ceb5873cf91fa418c2ab5a0000386",
+      "key": "3e8ceb5873cf91fa418c2ab5a0000386",
+      "value": {
+        "_id": "3e8ceb5873cf91fa418c2ab5a0000386",
+        "_rev": "1-967a00dff5e02add41819138abb3284d"
+      }
+    },
+    {
+      "id": "biking",
+      "key": "biking",
+      "value": {
+        "_id": "biking",
+        "_rev": "2-d4b43fd051e693a5191db2eaba89903c",
+        "title": "Biking2",
+        "body": "My biggest hobby is mountainbiking. The other day...",
+        "date": "2009/01/30 18:04:11"
+      }
+    },
+    {
+      "id": "bought-a-cat",
+      "key": "bought-a-cat",
+      "value": {
+        "_id": "bought-a-cat",
+        "_rev": "1-ccd0a7e95ade7eb2d29cc02ffa582b30",
+        "title": "Bought a Cat",
+        "body": "I went to the the pet store earlier and brought home a little kitty...",
+        "date": "2009/02/17 21:13:39"
+      }
+    },
+    {
+      "id": "hello-world",
+      "key": "hello-world",
+      "value": {
+        "_id": "hello-world",
+        "_rev": "1-97dd85b06c25328a300f3f4041def370",
+        "title": "Hello World",
+        "body": "Well hello and welcome to my new blog...",
+        "date": "2009/01/15 15:52:20"
+      }
+    }
+  ]
+}

data/test/test_parser.rb

@@ -0,0 +1,166 @@
+require 'minitest/autorun'
+require 'yaji'
+
+class TestParser < MiniTest::Unit::TestCase
+
+  def test_it_generates_events
+    events = []
+    parser = YAJI::Parser.new(toys_json_str)
+    parser.parse do |p, e, v|
+      events << [p, e, v]
+    end
+    expected = [
+      ["",                      :start_hash,  nil],
+      ["",                      :hash_key,    "total_rows"],
+      ["total_rows",            :number,      2],
+      ["",                      :hash_key,    "rows"],
+      ["rows",                  :start_array, nil],
+      ["rows/",                 :start_hash,  nil],
+      ["rows/",                 :hash_key,    "id"],
+      ["rows//id",              :string,      "buzz"],
+      ["rows/",                 :hash_key,    "props"],
+      ["rows//props",           :start_hash,  nil],
+      ["rows//props",           :hash_key,    "humanoid"],
+      ["rows//props/humanoid",  :boolean,     true],
+      ["rows//props",           :hash_key,    "armed"],
+      ["rows//props/armed",     :boolean,     true],
+      ["rows//props",           :end_hash,    nil],
+      ["rows/",                 :hash_key,    "movies"],
+      ["rows//movies",          :start_array, nil],
+      ["rows//movies/",         :number,      1],
+      ["rows//movies/",         :number,      2],
+      ["rows//movies/",         :number,      3],
+      ["rows//movies",          :end_array,   nil],
+      ["rows/",                 :end_hash,    nil],
+      ["rows/",                 :start_hash,  nil],
+      ["rows/",                 :hash_key,    "id"],
+      ["rows//id",              :string,      "barbie"],
+      ["rows/",                 :hash_key,    "props"],
+      ["rows//props",           :start_hash,  nil],
+      ["rows//props",           :hash_key,    "humanoid"],
+      ["rows//props/humanoid",  :boolean,     true],
+      ["rows//props",           :hash_key,    "armed"],
+      ["rows//props/armed",     :boolean,     false],
+      ["rows//props",           :end_hash,    nil],
+      ["rows/",                 :hash_key,    "movies"],
+      ["rows//movies",          :start_array, nil],
+      ["rows//movies/",         :number,      2],
+      ["rows//movies/",         :number,      3],
+      ["rows//movies",          :end_array,   nil],
+      ["rows/",                 :end_hash,    nil],
+      ["rows",                  :end_array,   nil],
+      ["",                      :end_hash,    nil]
+    ]
+    assert_equal expected, events
+  end
+
+  def test_it_yields_enumerator
+    parser = YAJI::Parser.new('{"hello":"world"}')
+    e = parser.parse
+    assert_equal ["", :start_hash, nil], e.next
+    assert_equal ["", :hash_key, "hello"], e.next
+    assert_equal ["hello", :string, "world"], e.next
+    assert_equal ["", :end_hash, nil], e.next
+    assert_raises(StopIteration) { e.next }
+  end
+
+  def test_it_symbolizes_keys
+    parser = YAJI::Parser.new('{"hello":"world"}', :symbolize_keys => true)
+    e = parser.parse
+    expected = [
+      ["", :start_hash, nil],
+      ["", :hash_key, :hello],
+      ["hello", :string, "world"],
+      ["", :end_hash, nil]
+    ]
+    assert_equal expected, e.to_a
+  end
+
+  def test_it_build_ruby_objects
+    parser = YAJI::Parser.new(toys_json_str)
+    objects = []
+    parser.each do |o|
+      objects << o
+    end
+    expected = [{"total_rows" => 2,
+                 "rows" => [
+                   {
+                     "id" => "buzz",
+                     "props" => { "humanoid"=> true, "armed"=> true },
+                     "movies" => [1,2,3]
+                   },
+                   {
+                     "id" => "barbie",
+                     "props" => { "humanoid"=> true, "armed"=> false },
+                     "movies" => [2,3]
+                   }
+                 ]}]
+    assert expected, objects
+  end
+
+  def test_it_yields_whole_array
+    parser = YAJI::Parser.new(toys_json_str)
+    objects = []
+    parser.each("rows") do |o|
+      objects << o
+    end
+    expected = [[{
+                   "id" => "buzz",
+                   "props" => { "humanoid"=> true, "armed"=> true },
+                   "movies" => [1,2,3]
+                 },
+                 {
+                   "id" => "barbie",
+                   "props" => { "humanoid"=> true, "armed"=> false },
+                   "movies" => [2,3]
+                 }]]
+    assert expected, objects
+  end
+
+  def test_it_yeilds_array_contents_row_by_row
+    parser = YAJI::Parser.new(toys_json_str)
+    objects = []
+    parser.each("rows/") do |o|
+      objects << o
+    end
+    expected = [{
+                  "id" => "buzz",
+                  "props" => { "humanoid"=> true, "armed"=> true },
+                  "movies" => [1,2,3]
+                },
+                {
+                  "id" => "barbie",
+                  "props" => { "humanoid"=> true, "armed"=> false },
+                  "movies" => [2,3]
+                }]
+    assert expected, objects
+  end
+
+  protected
+
+  def toys_json_str
+    <<-JSON
+      {
+        "total_rows": 2,
+        "rows": [
+          {
+            "id": "buzz",
+            "props": {
+              "humanoid": true,
+              "armed": true
+            },
+            "movies": [1,2,3]
+          },
+          {
+            "id": "barbie",
+            "props": {
+              "humanoid": true,
+              "armed": false
+            },
+            "movies": [2,3]
+          }
+        ]
+      }
+    JSON
+  end
+end

data/yaji.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "yaji/version"
+
+Gem::Specification.new do |s|
+  s.name        = "yaji"
+  s.version     = YAJI::VERSION
+  s.author      = 'Couchbase'
+  s.email       = 'info@couchbase.com'
+  s.license     = 'ASL-2'
+  s.homepage    = "https://github.com/avsej/yaji"
+  s.summary     = %q{Yet another JSON iterator}
+  s.description = %q{YAJI is a ruby wrapper to YAJL providing iterator interface to streaming JSON parser}
+
+  s.rubyforge_project = "yaji"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.extensions    = `git ls-files -- ext/**/extconf.rb`.split("\n")
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency 'rake-compiler'
+  s.add_development_dependency 'minitest'
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: yaji
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Couchbase
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-08-19 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: &21823240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *21823240
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: &21822680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *21822680
+description: YAJI is a ruby wrapper to YAJL providing iterator interface to streaming
+  JSON parser
+email: info@couchbase.com
+executables: []
+extensions:
+- ext/yaji/extconf.rb
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.markdown
+- Rakefile
+- ext/yaji/extconf.rb
+- ext/yaji/parser_ext.c
+- ext/yaji/parser_ext.h
+- lib/yaji.rb
+- lib/yaji/version.rb
+- tasks/compile.rake
+- tasks/test.rake
+- tasks/util.rake
+- test/fixture.json
+- test/test_parser.rb
+- yaji.gemspec
+homepage: https://github.com/avsej/yaji
+licenses:
+- ASL-2
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: yaji
+rubygems_version: 1.8.7
+signing_key: 
+specification_version: 3
+summary: Yet another JSON iterator
+test_files:
+- test/fixture.json
+- test/test_parser.rb