groonga-command-parser 1.0.6 → 1.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a75084a7badb6eaff6d7904c21749782c1565eab
-  data.tar.gz: 8e5f32b9c0ac5cbfe347e356fb8fc3843e0f52d4
+  metadata.gz: 4a650d05617980af6af0e11548cf0938e8ed631a
+  data.tar.gz: 7aceca455972e4734e8029e966d501a8ba3e14db
 SHA512:
-  metadata.gz: cc7cb7b7c40a7459bc15ac2be6aa69e8c695a330ba86c8e6df3b69516d9801533ed39ac411cb1746756ecdd9d3a1eb220f90d7e2735d04d6ee17cbbee6f76f4c
-  data.tar.gz: da59b07ea452dc7dc7be75eea93b8dfb960c85ac9ac019ccde3f9162b4d49497b7fec01056656a64920afe1735c93b58b6a9483b6cba0b8333112a782aac91ff
+  metadata.gz: a78f94997caa0239aca88c73b71fb22b9da213c40a48f60d8500e136299cf7e667af0b124d1ab5ea74a9f7ca2767a783ccb58152c6024836e4aa6b478ba7235b
+  data.tar.gz: e9a0427a34376d5e91f7f33a4aa0945457c5ac74bb1a4cc94e8abc5475a10e9ac940c3ab548e0bd9411c559411a22034479c54557bcdd76560b07b0d637fb43b

data/doc/text/news.md

@@ -1,5 +1,12 @@
 # News
 
+## 1.0.7: 2016-12-20
+
+### Improvements
+
+  * Switched to json-stream from ffi-yajl. Because ffi-yajl doesn't
+    work on Windows.
+
 ## 1.0.6: 2016-09-12
 
 ### Improvements

data/groonga-command-parser.gemspec

@@ -53,8 +53,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency("groonga-command", ">= 1.0.9")
-  spec.add_runtime_dependency("ffi")
-  spec.add_runtime_dependency("ffi-yajl")
+  spec.add_runtime_dependency("json-stream")
 
   spec.add_development_dependency("test-unit")
   spec.add_development_dependency("test-unit-notify")

data/lib/groonga/command/parser.rb

@@ -122,7 +122,6 @@ module Groonga
       def initialize
         reset
         initialize_hooks
-        initialize_load_values_parser
       end
 
       # Streaming parsing command.
@@ -262,6 +261,7 @@ module Groonga
             on_load_columns(@command, @command.columns)
           end
           if @command[:values]
+            initialize_load_values_parser
             @load_values_parser << @command[:values]
             reset
           else
@@ -271,6 +271,7 @@ module Groonga
             else
               @command.original_source << "\n"
               @loading = true
+              initialize_load_values_parser
             end
           end
         else
@@ -339,6 +340,7 @@ module Groonga
         @command = nil
         @loading = false
         @buffer = "".force_encoding("ASCII-8BIT")
+        @load_values_parser = nil
       end
 
       def initialize_hooks

data/lib/groonga/command/parser/load-values-parser.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2015  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2015-2016  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,12 +14,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-ENV["FORCE_FFI_YAJL"] = "ffi"
-require "ffi_yajl"
-
-module FFI_Yajl
-  attach_function :yajl_get_bytes_consumed, [:yajl_handle], :size_t
-end
+require "json/stream"
 
 module Groonga
   module Command
@@ -30,9 +23,7 @@ module Groonga
         attr_writer :on_value
         attr_writer :on_end
         def initialize
-          initialize_callbacks
-          @handle = nil
-          @callbacks_memory = nil
+          initialize_parser
           @on_value = nil
           @on_end = nil
           @containers = []
@@ -43,39 +34,28 @@ module Groonga
           data_size = data.bytesize
           return self if data_size.zero?
 
-          ensure_handle
-
-          status = FFI_Yajl.yajl_parse(@handle, data, data_size)
-
-          if status != :yajl_status_ok
-            consumed = FFI_Yajl.yajl_get_bytes_consumed(@handle)
-            if consumed > 0
-              consumed -= 1
-            end
-            if @containers.empty?
-              message = "there are garbages before JSON"
-            else
-              message = FFI_Yajl.yajl_get_error(@handle, 0, nil, 0).chomp
-            end
+          before_pos = @parser.pos
+          status = catch do |tag|
+            @tag = tag
             begin
-              raise Error.new(message,
+              @parser << data
+            rescue JSON::Stream::ParserError => error
+              pos = @parser.pos
+              consumed = pos - before_pos - 1
+              raise Error.new(error.message,
                               data[0, consumed],
                               data[consumed..-1])
-            ensure
-              finalize_handle
             end
+            :continue
           end
 
-          if @containers.empty?
-            consumed = FFI_Yajl.yajl_get_bytes_consumed(@handle)
-            begin
-              if consumed < data_size
-                @on_end.call(data[consumed..-1])
-              else
-                @on_end.call(nil)
-              end
-            ensure
-              finalize_handle
+          if status == :done
+            pos = @parser.pos
+            consumed = pos - before_pos
+            if consumed < data_size
+              @on_end.call(data[consumed..-1])
+            else
+              @on_end.call(nil)
             end
           end
 
@@ -83,51 +63,30 @@ module Groonga
         end
 
         private
-        def callback(*arguments)
-          FFI::Function.new(:int, [:pointer, *arguments]) do |_, *values|
-            yield(*values)
-            1
-          end
-        end
-
-        def initialize_callbacks
-          @null_callback = callback do
-            push_value(nil)
-          end
-          @boolean_callback = callback(:int) do |c_boolean|
-            push_value(c_boolean != 0)
-          end
-          @number_callback = callback(:string, :size_t) do |data, size|
-            number_data = data.slice(0, size)
-            if /[\.eE]/ =~ number_data
-              number = number_data.to_f
-            else
-              number = number_data.to_i
-            end
-            push_value(number)
-          end
-          @string_callback = callback(:string, :size_t) do |data, size|
-            string = data.slice(0, size)
-            string.force_encoding(Encoding::UTF_8)
-            push_value(string)
+        def initialize_parser
+          @parser = JSON::Stream::Parser.new
+          @parser.singleton_class.__send__(:attr_reader, :pos)
+          @parser.end_document do
+            throw(@tag, :done)
           end
-          @start_map_callback = callback do
+          @parser.start_object do
             push_container({})
           end
-          @map_key_callback = callback(:string, :size_t) do |data, size|
-            key = data.slice(0, size)
-            key.force_encoding(Encoding::UTF_8)
-            @keys.push(key)
-          end
-          @end_map_callback = callback do
+          @parser.end_object do
             pop_container
           end
-          @start_array_callback = callback do
+          @parser.start_array do
             push_container([])
           end
-          @end_array_callback = callback do
+          @parser.end_array do
             pop_container
           end
+          @parser.key do |key|
+            push_key(key)
+          end
+          @parser.value do |value|
+            push_value(value)
+          end
         end
 
         def push_container(container)
@@ -143,64 +102,19 @@ module Groonga
           end
         end
 
+        def push_key(key)
+          @keys.push(key)
+        end
+
         def push_value(value)
           container = @containers.last
           case container
-          when Hash
+          when ::Hash
             container[@keys.pop] = value
-          when Array
+          when ::Array
             container.push(value)
           end
         end
-
-        def ensure_handle
-          return if @handle
-          initialize_handle
-        end
-
-        def initialize_handle
-          @callbacks_memory = FFI::MemoryPointer.new(FFI_Yajl::YajlCallbacks)
-          callbacks = FFI_Yajl::YajlCallbacks.new(@callbacks_memory)
-          callbacks[:yajl_null] = @null_callback
-          callbacks[:yajl_boolean] = @boolean_callback
-          callbacks[:yajl_integer] = nil
-          callbacks[:yajl_double] = nil
-          callbacks[:yajl_number] = @number_callback
-          callbacks[:yajl_string] = @string_callback
-          callbacks[:yajl_start_map] = @start_map_callback
-          callbacks[:yajl_map_key] = @map_key_callback
-          callbacks[:yajl_end_map] = @end_map_callback
-          callbacks[:yajl_start_array] = @start_array_callback
-          callbacks[:yajl_end_array] = @end_array_callback
-
-          @handle = FFI_Yajl.yajl_alloc(@callbacks_memory, nil, nil)
-          FFI_Yajl.yajl_config(@handle,
-                               :yajl_allow_trailing_garbage,
-                               :int,
-                               1)
-          @finalizer = Finalizer.new(@handle)
-          ObjectSpace.define_finalizer(self, @finalizer)
-        end
-
-        def finalize_handle
-          @callbacks_memory = nil
-          @finalizer.call(object_id)
-          ObjectSpace.undefine_finalizer(self)
-          @finalizer = nil
-          @handle = nil
-        end
-
-        class Finalizer
-          def initialize(handle)
-            @handle = handle
-          end
-
-          def call(object_id)
-            return if @handle.nil?
-            FFI_Yajl.yajl_free(@handle)
-            @handle = nil
-          end
-        end
       end
     end
   end

data/lib/groonga/command/parser/version.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2013-2015  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2016  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -19,7 +17,7 @@
 module Groonga
   module Command
     class Parser
-      VERSION = "1.0.6"
+      VERSION = "1.0.7"
     end
   end
 end

data/test/run-test.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 #
-# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2016  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -36,13 +36,6 @@ end
 $LOAD_PATH.unshift(lib_dir)
 $LOAD_PATH.unshift(test_dir)
 
-# TODO: Remove me when suppress warnings patches are merged int
-# ffi_yajl.
-require "stringio"
-$VERBOSE = false
-require "ffi_yajl/ffi"
-$VERBOSE = true
-
 require "groonga-command-parser-test-utils"
 
 ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"

data/test/test-load-value-parser.rb

@@ -21,13 +21,17 @@ class LoadValuesParserTest < Test::Unit::TestCase
     @parser.on_value = lambda do |value|
       @values << value
     end
+    @parse_done = false
     @parser.on_end = lambda do |rest|
+      @parse_done = true
+      @parse_rest = rest
     end
   end
 
   def parse(data)
     data.each_line do |line|
-    @parser << line
+      @parser << line
+      break if @parse_done
     end
     @values
   end
@@ -140,10 +144,50 @@ class LoadValuesParserTest < Test::Unit::TestCase
     }
   }
 ]
+      JSON
+    end
+  end
+
+  sub_test_case "error" do
+    test "unfinished" do
+      assert_equal([
+                     {
+                       "object" => {
+                         "string" => "abc",
+                       },
+                     },
+                   ],
+                   parse(<<-JSON))
 [
-  1
-]
+  {
+    "object": {
+      "string": "abc"
+    }
+  },
+  {
+      JSON
+      assert_false(@parse_done)
+    end
+
+    test "too much" do
+      assert_equal([
+                     {
+                       "object" => {
+                         "string" => "abc",
+                       },
+                     },
+                   ],
+                   parse(<<-JSON))
+[
+  {
+    "object": {
+      "string": "abc"
+    }
+  }
+]garbage
       JSON
+      assert_equal([true, "garbage\n"],
+                   [@parse_done, @parse_rest])
     end
   end
 end

data/test/test-parser.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2011-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2016  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -374,7 +372,7 @@ EOS
           end
 
           def test_no_record_separate_comma
-            message = "parse error: after array element, I expect ',' or ']'"
+            message = "Expected comma or object or array close: char 37"
             before = <<-BEFORE
 [
 {"_key": "alice", "name": "Alice"}
@@ -394,7 +392,7 @@ EOC
           end
 
           def test_garbage_before_json
-            message = "there are garbages before JSON"
+            message = "Expected value: char 0"
             before = ""
             after = <<-AFTER
 XXX

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groonga-command-parser
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.7
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-12 00:00:00.000000000 Z
+date: 2016-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: groonga-command
@@ -25,21 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.0.9
 - !ruby/object:Gem::Dependency
-  name: ffi
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: ffi-yajl
+  name: json-stream
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -200,7 +186,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Groonga-command-parser is a Ruby library to parses [groonga](http://groonga.org/)'s
@@ -211,4 +197,3 @@ test_files:
 - test/test-load-value-parser.rb
 - test/test-command-line-splitter.rb
 - test/groonga-command-parser-test-utils.rb
-has_rdoc: