yaji 0.3.5-x64-mingw32

data/ext/yaji/yajl_parser.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2010, Lloyd Hilaiel.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ * 
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in
+ *     the documentation and/or other materials provided with the
+ *     distribution.
+ * 
+ *  3. Neither the name of Lloyd Hilaiel nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */ 
+
+#ifndef __YAJL_PARSER_H__
+#define __YAJL_PARSER_H__
+
+#include "api/yajl_parse.h"
+#include "yajl_bytestack.h"
+#include "yajl_buf.h"
+
+
+typedef enum {
+    yajl_state_start = 0,
+    yajl_state_parse_complete,
+    yajl_state_parse_error,
+    yajl_state_lexical_error,
+    yajl_state_map_start,
+    yajl_state_map_sep,    
+    yajl_state_map_need_val,
+    yajl_state_map_got_val,
+    yajl_state_map_need_key,
+    yajl_state_array_start,
+    yajl_state_array_got_val,
+    yajl_state_array_need_val
+} yajl_state;
+
+struct yajl_handle_t {
+    const yajl_callbacks * callbacks;
+    void * ctx;
+    yajl_lexer lexer;
+    const char * parseError;
+    /* the number of bytes consumed from the last client buffer,
+     * in the case of an error this will be an error offset, in the
+     * case of an error this can be used as the error offset */
+    unsigned int bytesConsumed;
+    /* temporary storage for decoded strings */
+    yajl_buf decodeBuf;
+    /* a stack of states.  access with yajl_state_XXX routines */
+    yajl_bytestack stateStack;
+    /* memory allocation routines */
+    yajl_alloc_funcs alloc;
+};
+
+YAJL_API
+yajl_status
+yajl_do_parse(yajl_handle handle, const unsigned char * jsonText,
+              unsigned int jsonTextLen);
+
+YAJL_API
+unsigned char *
+yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText,
+                         unsigned int jsonTextLen, int verbose);
+
+
+#endif

data/ext/yaji/yajl_version.c

@@ -0,0 +1,7 @@
+#include "api/yajl_version.h"
+
+int yajl_version(void)
+{
+	return YAJL_VERSION;
+}
+

data/lib/yaji.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+#
+# Author:: Couchbase <info@couchbase.com>
+# Copyright:: 2011 Couchbase, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "yaji/version"
+require "parser_ext"
+
+module YAJI
+end

data/lib/yaji/version.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+#
+# Author:: Couchbase <info@couchbase.com>
+# Copyright:: 2011 Couchbase, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module YAJI
+  VERSION = "0.3.5"
+end

data/tasks/compile.rake

@@ -0,0 +1,80 @@
+gem 'rake-compiler', '>= 0.7.5'
+require "rake/extensiontask"
+
+def gemspec
+  @clean_gemspec ||= eval(File.read(File.expand_path('../../yaji.gemspec', __FILE__)))
+end
+
+require 'rubygems/package_task'
+Gem::PackageTask.new(gemspec) do |pkg|
+  pkg.need_tar = true
+end
+
+version_router = lambda do |t|
+  File.open(t.name, 'wb') do |f|
+    f.write <<-RUBY
+      require "yaji/\#{RUBY_VERSION.sub(/\\.\\d+$/, '')}/parser_ext"
+    RUBY
+  end
+end
+
+class Platform
+  attr_reader :name, :host, :versions
+
+  def initialize(params)
+    @name = params[:name]
+    @host = params[:host]
+    @versions = params[:versions]
+  end
+
+  def each_version
+    @versions.each do |v|
+      yield(v, v[/\d\.\d\.\d/])
+    end
+  end
+
+  def short_versions
+    res = []
+    each_version do |long, short|
+      res << short
+    end
+    res
+  end
+end
+
+recent = "2.0.0-p353"
+CROSS_PLATFORMS = [
+  Platform.new(:name => 'x64-mingw32', :host => 'x86_64-w64-mingw32', :versions => %w(1.9.3-p484 2.0.0-p353 2.1.0)),
+  Platform.new(:name => 'x86-mingw32', :host => 'i686-w64-mingw32', :versions => %w(1.8.7-p374 1.9.3-p484 2.0.0-p353 2.1.0)),
+]
+Rake::ExtensionTask.new("parser_ext", gemspec) do |ext|
+  ext.ext_dir = File.join('ext', 'yaji')
+
+  ext.cross_compile = true
+  ext.cross_platform = ENV['TARGET']
+  if ENV['RUBY_CC_VERSION']
+    ext.lib_dir = "lib/yaji"
+  end
+  ext.cross_compiling do |spec|
+    spec.files.delete("lib/yaji/parser_ext.so")
+    spec.files.push("lib/parser_ext.rb", Dir["lib/yaji/*/parser_ext.so"])
+    file "#{ext.tmp_dir}/#{ext.cross_platform}/stage/lib/parser_ext.rb", &version_router
+  end
+
+  # clean compiled extension
+  CLEAN.include "#{ext.lib_dir}/*.#{RbConfig::CONFIG['DLEXT']}"
+end
+
+file "lib/parser_ext.rb", &version_router
+task :cross => "lib/parser_ext.rb"
+
+desc "Package gem for windows"
+task "package:windows" => :package do
+  CROSS_PLATFORMS.each do |platform|
+    ENV['TARGET'] = platform.name
+    platform.each_version do |long, short|
+      sh("env RUBY_CC_VERSION=#{short} RBENV_VERSION=#{long} rbenv exec rake cross compile")
+    end
+    sh("env RUBY_CC_VERSION=#{platform.short_versions.join(":")} RBENV_VERSION=#{recent} rbenv exec rake cross native gem")
+  end
+end

data/tasks/test.rake

@@ -0,0 +1,7 @@
+require 'rake/testtask'
+Rake::TestTask.new do |test|
+  test.libs << "test" << "."
+  test.ruby_opts << "-rruby-debug" if ENV['DEBUG']
+  test.pattern = 'test/test_*.rb'
+  test.verbose = true
+end

data/tasks/util.rake

@@ -0,0 +1,4 @@
+desc 'Start an irb session and load the library.'
+task :console do
+  exec "irb -I lib -rruby-debug -ryaji"
+end

data/test/fixture.json

@@ -0,0 +1,47 @@
+{
+  "total_rows": 4,
+  "offset": 0,
+  "rows": [
+    {
+      "id": "3e8ceb5873cf91fa418c2ab5a0000386",
+      "key": "3e8ceb5873cf91fa418c2ab5a0000386",
+      "value": {
+        "_id": "3e8ceb5873cf91fa418c2ab5a0000386",
+        "_rev": "1-967a00dff5e02add41819138abb3284d"
+      }
+    },
+    {
+      "id": "biking",
+      "key": "biking",
+      "value": {
+        "_id": "biking",
+        "_rev": "2-d4b43fd051e693a5191db2eaba89903c",
+        "title": "Biking2",
+        "body": "My biggest hobby is mountainbiking. The other day...",
+        "date": "2009/01/30 18:04:11"
+      }
+    },
+    {
+      "id": "bought-a-cat",
+      "key": "bought-a-cat",
+      "value": {
+        "_id": "bought-a-cat",
+        "_rev": "1-ccd0a7e95ade7eb2d29cc02ffa582b30",
+        "title": "Bought a Cat",
+        "body": "I went to the the pet store earlier and brought home a little kitty...",
+        "date": "2009/02/17 21:13:39"
+      }
+    },
+    {
+      "id": "hello-world",
+      "key": "hello-world",
+      "value": {
+        "_id": "hello-world",
+        "_rev": "1-97dd85b06c25328a300f3f4041def370",
+        "title": "Hello World",
+        "body": "Well hello and welcome to my new blog...",
+        "date": "2009/01/15 15:52:20"
+      }
+    }
+  ]
+}

data/test/test_parser.rb

@@ -0,0 +1,370 @@
+require 'minitest/autorun'
+require 'yaji'
+require 'curb'
+
+class TestParser < MiniTest::Unit::TestCase
+
+  class Generator
+    def initialize(data, options = {})
+      @callback = nil
+      @options = {:chunk_size => 10}.merge(options)
+      @chunks = case data
+                when Array
+                  data
+                when String
+                  count = (data.bytesize / @options[:chunk_size].to_f).ceil
+                  data.unpack("a#{@options[:chunk_size]}" * count)
+                end
+    end
+
+    def on_body
+      old = @callback
+      if block_given?
+        @callback = Proc.new
+      end
+      old
+    end
+
+    def perform
+      size = @options[:chunk_size]
+      if @callback
+        @chunks.each do |chunk|
+          @callback.call(chunk)
+        end
+      end
+    end
+  end
+
+  def test_it_generates_events
+    events = []
+    parser = YAJI::Parser.new(toys_json_str)
+    parser.parse do |p, e, v|
+      events << [p, e, v]
+    end
+    expected = [
+      ["",                       :start_hash,  nil],
+      ["",                       :hash_key,    "total_rows"],
+      ["/total_rows",            :number,      2],
+      ["",                       :hash_key,    "rows"],
+      ["/rows",                  :start_array, nil],
+      ["/rows/",                 :start_hash,  nil],
+      ["/rows/",                 :hash_key,    "id"],
+      ["/rows//id",              :string,      "buzz"],
+      ["/rows/",                 :hash_key,    "props"],
+      ["/rows//props",           :start_hash,  nil],
+      ["/rows//props",           :hash_key,    "humanoid"],
+      ["/rows//props/humanoid",  :boolean,     true],
+      ["/rows//props",           :hash_key,    "armed"],
+      ["/rows//props/armed",     :boolean,     true],
+      ["/rows//props",           :end_hash,    nil],
+      ["/rows/",                 :hash_key,    "movies"],
+      ["/rows//movies",          :start_array, nil],
+      ["/rows//movies/",         :number,      1],
+      ["/rows//movies/",         :number,      2],
+      ["/rows//movies/",         :number,      3],
+      ["/rows//movies",          :end_array,   nil],
+      ["/rows/",                 :end_hash,    nil],
+      ["/rows/",                 :start_hash,  nil],
+      ["/rows/",                 :hash_key,    "id"],
+      ["/rows//id",              :string,      "barbie"],
+      ["/rows/",                 :hash_key,    "props"],
+      ["/rows//props",           :start_hash,  nil],
+      ["/rows//props",           :hash_key,    "humanoid"],
+      ["/rows//props/humanoid",  :boolean,     true],
+      ["/rows//props",           :hash_key,    "armed"],
+      ["/rows//props/armed",     :boolean,     false],
+      ["/rows//props",           :end_hash,    nil],
+      ["/rows/",                 :hash_key,    "movies"],
+      ["/rows//movies",          :start_array, nil],
+      ["/rows//movies/",         :number,      2],
+      ["/rows//movies/",         :number,      3],
+      ["/rows//movies",          :end_array,   nil],
+      ["/rows/",                 :end_hash,    nil],
+      ["/rows",                  :end_array,   nil],
+      ["",                       :end_hash,    nil]
+    ]
+    assert_equal expected, events
+  end
+
+  def test_it_yields_enumerator
+    parser = YAJI::Parser.new('{"hello":"world"}')
+    e = parser.parse
+    assert_equal ["", :start_hash, nil], e.next
+    assert_equal ["", :hash_key, "hello"], e.next
+    assert_equal ["/hello", :string, "world"], e.next
+    assert_equal ["", :end_hash, nil], e.next
+    assert_raises(StopIteration) { e.next }
+  end
+
+  def test_it_symbolizes_keys
+    parser = YAJI::Parser.new('{"hello":"world"}', :symbolize_keys => true)
+    e = parser.parse
+    expected = [
+      ["", :start_hash, nil],
+      ["", :hash_key, :hello],
+      ["/hello", :string, "world"],
+      ["", :end_hash, nil]
+    ]
+    assert_equal expected, e.to_a
+  end
+
+  def test_it_build_ruby_objects
+    parser = YAJI::Parser.new(toys_json_str)
+    objects = []
+    parser.each do |o|
+      objects << o
+    end
+    expected = [{"total_rows" => 2,
+                 "rows" => [
+                   {
+                     "id" => "buzz",
+                     "props" => { "humanoid"=> true, "armed"=> true },
+                     "movies" => [1,2,3]
+                   },
+                   {
+                     "id" => "barbie",
+                     "props" => { "humanoid"=> true, "armed"=> false },
+                     "movies" => [2,3]
+                   }
+                 ]}]
+    assert_equal expected, objects
+  end
+
+  def test_it_yields_whole_array
+    parser = YAJI::Parser.new(toys_json_str)
+    objects = []
+    parser.each("/rows") do |o|
+      objects << o
+    end
+    expected = [[{
+                   "id" => "buzz",
+                   "props" => { "humanoid"=> true, "armed"=> true },
+                   "movies" => [1,2,3]
+                 },
+                 {
+                   "id" => "barbie",
+                   "props" => { "humanoid"=> true, "armed"=> false },
+                   "movies" => [2,3]
+                 }]]
+    assert_equal expected, objects
+  end
+
+  def test_it_yeilds_array_contents_row_by_row
+    parser = YAJI::Parser.new(toys_json_str)
+    objects = []
+    parser.each("/rows/") do |o|
+      objects << o
+    end
+    expected = [{
+                  "id" => "buzz",
+                  "props" => { "humanoid"=> true, "armed"=> true },
+                  "movies" => [1,2,3]
+                },
+                {
+                  "id" => "barbie",
+                  "props" => { "humanoid"=> true, "armed"=> false },
+                  "movies" => [2,3]
+                }]
+    assert_equal expected, objects
+  end
+
+  def test_it_could_curb_async_approach
+    curl = Curl::Easy.new('http://avsej.net/test.json')
+    parser = YAJI::Parser.new(curl)
+    object = parser.each.to_a.first
+    expected = {"foo"=>"bar", "baz"=>{"nums"=>[42, 3.1415]}}
+    assert_equal expected, object
+  end
+
+  def test_it_allow_several_selectors
+    parser = YAJI::Parser.new(toys_json_str)
+    objects = []
+    parser.each(["/total_rows", "/rows/"]) do |o|
+      objects << o
+    end
+    expected = [2,
+                {
+                  "id" => "buzz",
+                  "props" => { "humanoid"=> true, "armed"=> true },
+                  "movies" => [1,2,3]
+                },
+                {
+                  "id" => "barbie",
+                  "props" => { "humanoid"=> true, "armed"=> false },
+                  "movies" => [2,3]
+                }]
+    assert_equal expected, objects
+  end
+
+  def test_it_optionally_yields_object_path
+    parser = YAJI::Parser.new(toys_json_str)
+    objects = []
+    parser.each(["/total_rows", "/rows/"], :with_path => true) do |o|
+      objects << o
+    end
+    expected = [["/total_rows", 2],
+                ["/rows/", {
+                  "id" => "buzz",
+                  "props" => { "humanoid"=> true, "armed"=> true },
+                  "movies" => [1,2,3]
+                }],
+                ["/rows/", {
+                  "id" => "barbie",
+                  "props" => { "humanoid"=> true, "armed"=> false },
+                  "movies" => [2,3]
+                }]]
+    assert_equal expected, objects
+  end
+
+  def test_it_allows_to_specify_filter_and_options_at_initialization
+    parser = YAJI::Parser.new(toys_json_str,
+                              :filter => ["/total_rows", "/rows/"],
+                              :with_path => true)
+    objects = []
+    parser.each do |o|
+      objects << o
+    end
+    expected = [["/total_rows", 2],
+                ["/rows/", {
+                  "id" => "buzz",
+                  "props" => { "humanoid"=> true, "armed"=> true },
+                  "movies" => [1,2,3]
+                }],
+                ["/rows/", {
+                  "id" => "barbie",
+                  "props" => { "humanoid"=> true, "armed"=> false },
+                  "movies" => [2,3]
+                }]]
+    assert_equal expected, objects
+  end
+
+  def test_it_doesnt_raise_exception_on_empty_input
+    YAJI::Parser.new("").parse
+    YAJI::Parser.new("  ").parse
+    YAJI::Parser.new("\n").parse
+    YAJI::Parser.new(" \n\n ").parse
+  end
+
+  def test_it_allows_to_create_parser_without_input
+    YAJI::Parser.new
+    YAJI::Parser.new(:filter => 'test')
+    YAJI::Parser.new(:with_path => true)
+  end
+
+  def test_it_raises_argument_error_for_parser_without_input
+    parser = YAJI::Parser.new
+    assert_raises(ArgumentError) do
+      parser.parse
+    end
+    assert_raises(ArgumentError) do
+      parser.each{|x| }
+    end
+  end
+
+  def test_it_raises_argument_error_on_write_without_callback_set_up
+    parser = YAJI::Parser.new
+    assert_raises(ArgumentError) do
+      parser.write('{"hello":"world"}')
+    end
+  end
+
+  def test_it_allows_to_feed_the_data_on_the_fly
+    parser = YAJI::Parser.new(:filter => '/rows/')
+
+    objects = []
+    parser.on_object do |obj|
+      objects << obj
+    end
+
+    parser.write(<<-JSON)
+      {
+        "total_rows": 2,
+        "rows": [
+          {
+    JSON
+    parser.write(<<-JSON)
+            "id": "buzz",
+            "props": {
+              "humanoid": true,
+              "armed": true
+            },
+            "movies": [1,2,3]
+          },
+    JSON
+    data = <<-JSON
+          {
+            "id": "barbie",
+            "props": {
+              "humanoid": true,
+              "armed": false
+            },
+            "movies": [2,3]
+          }
+        ]
+      }
+    JSON
+    parser << data
+
+    expected = [{
+                  "id" => "buzz",
+                  "props" => { "humanoid"=> true, "armed"=> true },
+                  "movies" => [1,2,3]
+                },
+                {
+                  "id" => "barbie",
+                  "props" => { "humanoid"=> true, "armed"=> false },
+                  "movies" => [2,3]
+                }]
+    assert_equal expected, objects
+  end
+
+  def test_it_parses_chunked_data
+    generator = Generator.new(['{"total_rows":', '0,"offset":0,"rows":[]', '}'])
+    iter = YAJI::Parser.new(generator).each(["total_rows", "/rows/", "/errors/"], :with_path => true)
+    begin
+      loop do
+        iter.next
+      end
+    rescue StopIteration
+    end
+  end
+
+  def test_it_skips_empty_chunks
+    generator = Generator.new(['{"total_rows":', '0,"offset":0,"rows":[]', '}', '', nil])
+    iter = YAJI::Parser.new(generator).each(["total_rows", "/rows/", "/errors/"], :with_path => true)
+    begin
+      loop do
+        iter.next
+      end
+    rescue StopIteration
+    end
+  end
+
+  protected
+
+  def toys_json_str
+    <<-JSON
+      {
+        "total_rows": 2,
+        "rows": [
+          {
+            "id": "buzz",
+            "props": {
+              "humanoid": true,
+              "armed": true
+            },
+            "movies": [1,2,3]
+          },
+          {
+            "id": "barbie",
+            "props": {
+              "humanoid": true,
+              "armed": false
+            },
+            "movies": [2,3]
+          }
+        ]
+      }
+    JSON
+  end
+end