groonga-command-parser 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a842fe9087d59d4d37a94dba22f419576d192b09
-  data.tar.gz: 60a05fe87c274df64c6abd2e9d14c33162999d9d
+  metadata.gz: 653da7d4f65f66a98a4fdc5300f1c40b768d99bf
+  data.tar.gz: 3be534841fc2a4e8ee175262107ce0336d81c287
 SHA512:
-  metadata.gz: a4bee897857636e1286a8e49bd6b1a1ba49d6e1351cff6280dbb75a447855482efeea89de190ce899fdaa169020fcdd59dd6691a93ce67c0d89794d7a729d1c9
-  data.tar.gz: 79ef8b01fd136994a4782e3137c65b7a59159318e2c640dd3e6ebdf98a5ef1bb29bba036561514e665701c5558dee15144e370e46bb4fa9b739aafa875b3e7fb
+  metadata.gz: 4c12a30d319639ac368a1610080cfef25c47198565245453e56265db91508b58664fe6886e1cb80bc6436beef2252ec61734337274740ed26c2ebe1e87201c14
+  data.tar.gz: 0a541136772e374816b60ceb5052b1848c9d7a0e1f05c5526a575f17e8d16ce570084a70fcc30ad6909311c9f70b6fad517605add4423c48468f1d409228ca47

data/.yardopts

@@ -2,6 +2,8 @@
 --no-private
 --output doc/reference/en
 --title "groonga-command-parser API Reference"
+--markup markdown
+--markup-provider kramdown
 --po-dir doc/po
 -
 doc/text/*

data/README.md

@@ -1,5 +1,8 @@
 # README
 
+[![Gem Version](https://badge.fury.io/rb/groonga-command-parser.svg)](http://badge.fury.io/rb/groonga-command-parser)
+[![Build Status](https://travis-ci.org/groonga/groonga-command-parser.svg?branch=master)](https://travis-ci.org/groonga/groonga-command-parser)
+
 ## Name
 
 groonga-command-parser

data/doc/text/news.md

@@ -1,5 +1,23 @@
 # News
 
+## 1.0.4: 2015-05-23
+
+### Improvements
+
+  * Made quoted text handling Groonga compatible.
+  * Changed JSON parser to ffi-yajl from yajl-ruby.
+    * Supported JRuby by this change.
+      [GitHub#1] [Reported by Hiroyuki Sato]
+    * Improved performance for large load data.
+
+### Fixes
+
+  * Fixed encoding related parse error on Windows.
+
+### Thanks
+
+  * Hiroyuki Sato
+
 ## 1.0.3: 2014-12-12
 
 ### Improvements

data/groonga-command-parser.gemspec

@@ -1,6 +1,6 @@
 # -*- mode: ruby; coding: utf-8 -*-
 #
-# Copyright (C) 2012-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2015  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -53,7 +53,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency("groonga-command", ">= 1.0.9")
-  spec.add_runtime_dependency("yajl-ruby")
+  spec.add_runtime_dependency("ffi")
+  spec.add_runtime_dependency("ffi-yajl")
 
   spec.add_development_dependency("test-unit")
   spec.add_development_dependency("test-unit-notify")
@@ -61,6 +62,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("packnga")
   spec.add_development_dependency("yard")
-  spec.add_development_dependency("redcarpet")
+  spec.add_development_dependency("kramdown")
 end
 

data/lib/groonga/command/parser.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2011-2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2015  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,14 +17,14 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "English"
-require "shellwords"
 require "cgi"
-
-require "yajl"
+require "json"
 
 require "groonga/command"
 
 require "groonga/command/parser/error"
+require "groonga/command/parser/command-line-splitter"
+require "groonga/command/parser/load-values-parser"
 require "groonga/command/parser/version"
 
 module Groonga
@@ -91,7 +91,7 @@ module Groonga
           end
           parser.on_load_complete do |command|
             parsed_command = command
-            parsed_command[:values] ||= Yajl::Encoder.encode(values)
+            parsed_command[:values] ||= JSON.generate(values)
           end
 
           consume_data(parser, data)
@@ -117,12 +117,14 @@ module Groonga
       def initialize
         reset
         initialize_hooks
+        initialize_load_values_parser
       end
 
       # Streaming parsing command.
       # @param [String] chunk parsed chunk of command.
       def <<(chunk)
         @buffer << chunk
+        @buffer.force_encoding("ASCII-8BIT")
         consume_buffer
       end
 
@@ -214,78 +216,10 @@ module Groonga
       end
 
       def consume_load_values(tag)
-        if @in_load_values
-          json, separator, rest = @buffer.partition(/[\]},]/)
-          if @load_value_completed
-            throw(tag) if separator.empty?
-            if separator == ","
-              if /\A\s*\z/ =~ json
-                @command.original_source << json << separator
-                @buffer = rest
-                @load_value_completed = false
-                return
-              else
-                raise Error.new("record separate comma is missing",
-                                @command.original_source.lines.to_a.last,
-                                json)
-              end
-            elsif separator == "]"
-              if /\A\s*\z/ =~ json
-                @command.original_source << json << separator
-                @buffer = rest
-                on_load_complete(@command)
-                reset
-                return
-              end
-            end
-          end
-          @buffer = rest
-          parse_json(json)
-          if separator.empty?
-            throw(tag)
-          else
-            @load_value_completed = false
-            parse_json(separator)
-          end
-        else
-          spaces, start_json, rest = @buffer.partition("[")
-          unless /\A\s*\z/ =~ spaces
-            raise Error.new("there are garbages before JSON",
-                            @command.original_source.lines.to_a.last,
-                            spaces)
-          end
-          if start_json.empty?
-            @command.original_source << @buffer
-            @buffer.clear
-            throw(tag)
-          else
-            @command.original_source << spaces << start_json
-            @buffer = rest
-            @json_parser = Yajl::Parser.new
-            @json_parser.on_parse_complete = lambda do |object|
-              if object.is_a?(::Array) and @command.columns.nil?
-                @command.columns = object
-                on_load_columns(@command, object)
-              else
-                on_load_value(@command, object)
-              end
-              @load_value_completed = true
-            end
-            @in_load_values = true
-          end
-        end
-      end
-
-      def parse_json(json)
-        @command.original_source << json
-        begin
-          @json_parser << json
-        rescue Yajl::ParseError
-          before_json = @command.original_source[0..(-json.bytesize)]
-          message = "invalid JSON: #{$!.message}: <#{json}>:\n"
-          message << before_json
-          raise Error.new(message, before_json, json)
-        end
+        throw(tag) if @buffer.empty?
+        @command.original_source << @buffer
+        @load_values_parser << @buffer
+        @buffer.clear
       end
 
       def consume_line(tag)
@@ -321,16 +255,7 @@ module Groonga
             on_load_columns(@command, @command.columns)
           end
           if @command[:values]
-            values = Yajl::Parser.parse(@command[:values])
-            if @command.columns.nil? and values.first.is_a?(::Array)
-              header = values.shift
-              @command.columns = header
-              on_load_columns(@command, header)
-            end
-            values.each do |value|
-              on_load_value(@command, value)
-            end
-            on_load_complete(@command)
+            @load_values_parser << @command[:values]
             reset
           else
             @command.original_source << "\n"
@@ -377,19 +302,8 @@ module Groonga
       end
 
       def parse_command_line(command_line)
-        name, *arguments = Shellwords.shellwords(command_line)
-        arguments = arguments.collect do |argument|
-          # TODO: Groonga supports backslash escape in both single
-          # quoted argument and double quoted argument ('...' and
-          # "..."). We should not unescape except single quoted
-          # argument ('...') because Shellwords.shellwords unescape it
-          # in double quoted argument. But we can't determine whether
-          # the argument is single quoted or not after
-          # Shellwords.shellwords... And groonga supports '\n' -> new line
-          # conversion. It isn't done by Shellwords.shellwords.
-          # Should we implement Shellwords.shellwords by ourselves?
-          unescape_argument_in_command_line(argument)
-        end
+        splitter = CommandLineSplitter.new(command_line)
+        name, *arguments = splitter.split
         pair_arguments = {}
         ordered_arguments = []
         until arguments.empty?
@@ -406,31 +320,9 @@ module Groonga
         command
       end
 
-      def unescape_argument_in_command_line(argument)
-        argument.gsub(/\\(.)/) do
-          character = $1
-          case character
-          when "b"
-            "\b"
-          when "f"
-            "\f"
-          when "n"
-            "\n"
-          when "r"
-            "\r"
-          when "t"
-            "\t"
-          else
-            character
-          end
-        end
-      end
-
       def reset
         @command = nil
         @loading = false
-        @in_load_values = false
-        @load_value_completed = false
         @buffer = "".force_encoding("ASCII-8BIT")
       end
 
@@ -441,6 +333,27 @@ module Groonga
         @on_load_value_hook = nil
         @on_load_complete_hook = nil
       end
+
+      def initialize_load_values_parser
+        @load_values_parser = LoadValuesParser.new
+        @load_values_parser.on_value = lambda do |value|
+          if value.is_a?(::Array) and @command.columns.nil?
+            @command.columns = value
+            on_load_columns(@command, value)
+          else
+            on_load_value(@command, value)
+          end
+        end
+        @load_values_parser.on_end = lambda do |rest|
+          if rest
+            original_source_size = @command.original_source.size
+            @command.original_source.slice!(original_source_size - rest.size,
+                                            rest.size)
+          end
+          on_load_complete(@command)
+          reset
+        end
+      end
     end
   end
 end

data/lib/groonga/command/parser/command-line-splitter.rb

@@ -0,0 +1,91 @@
+# Copyright (C) 2015  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "English"
+require "strscan"
+
+module Groonga
+  module Command
+    class Parser
+      class CommandLineSplitter
+        def initialize(command_line)
+          @command_line = command_line
+        end
+
+        def split
+          tokens = []
+          scanner = StringScanner.new(@command_line)
+          skip_spaces(scanner)
+          start_quote = nil
+          until scanner.eos?
+            if start_quote
+              token = ""
+              loop do
+                chunk = scanner.scan_until(/#{Regexp.escape(start_quote)}/)
+                if chunk.nil?
+                  token = start_quote + token + scanner.rest
+                  scanner.terminate
+                  break
+                end
+                if chunk[-2] == "\\"
+                  token << chunk
+                else
+                  token << chunk.chomp(start_quote)
+                  break
+                end
+              end
+              tokens << unescape(token)
+              start_quote = nil
+              skip_spaces(scanner)
+            else
+              start_quote = scanner.scan(/['"]/)
+              if start_quote.nil?
+                tokens << scanner.scan_until(/\S+/)
+                skip_spaces(scanner)
+              end
+            end
+          end
+          tokens
+        end
+
+        private
+        def skip_spaces(scanner)
+          scanner.scan(/\s+/)
+        end
+
+        def unescape(token)
+          token.gsub(/\\(.)/) do
+            character = $1
+            case character
+            when "b"
+              "\b"
+            when "f"
+              "\f"
+            when "n"
+              "\n"
+            when "r"
+              "\r"
+            when "t"
+              "\t"
+            else
+              character
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/groonga/command/parser/error.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2011-2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2015  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -42,9 +42,22 @@ module Groonga
             location << " " * before.bytesize + "^"
             location << after
           else
-            location << before
-            location << after
-            location << " " * before.bytesize + "^"
+            before_lines = before.lines.to_a
+            after_lines = after.lines.to_a
+            last_before_line = before_lines.last
+            if last_before_line
+              error_offset = last_before_line.bytesize
+            else
+              error_offset = 0
+            end
+            before_lines.each do |before_line|
+              location << before_line
+            end
+            location << after_lines.first
+            location << " " * error_offset + "^\n"
+            after_lines[1..-1].each do |after_line|
+              location << after_line
+            end
           end
           location
         end

data/lib/groonga/command/parser/load-values-parser.rb

@@ -0,0 +1,209 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+# TODO: Remove me when https://github.com/chef/ffi-yajl/pull/62 is merged.
+require "stringio"
+ENV["FORCE_FFI_YAJL"] = "ffi"
+require "ffi_yajl"
+
+module FFI_Yajl
+  attach_function :yajl_get_bytes_consumed, [:yajl_handle], :size_t
+end
+
+module Groonga
+  module Command
+    class Parser
+      class LoadValuesParser
+        attr_writer :on_value
+        attr_writer :on_end
+        def initialize
+          initialize_callbacks
+          @handle = nil
+          @callbacks_memory = nil
+          @on_value = nil
+          @on_end = nil
+          @containers = []
+          @keys = []
+        end
+
+        def <<(data)
+          data_size = data.bytesize
+          return self if data_size.zero?
+
+          ensure_handle
+
+          status = FFI_Yajl.yajl_parse(@handle, data, data_size)
+
+          if status != :yajl_status_ok
+            consumed = FFI_Yajl.yajl_get_bytes_consumed(@handle)
+            if consumed > 0
+              consumed -= 1
+            end
+            if @containers.empty?
+              message = "there are garbages before JSON"
+            else
+              message = FFI_Yajl.yajl_get_error(@handle, 0, nil, 0).chomp
+            end
+            begin
+              raise Error.new(message,
+                              data[0, consumed],
+                              data[consumed..-1])
+            ensure
+              finalize_handle
+            end
+          end
+
+          if @containers.empty?
+            consumed = FFI_Yajl.yajl_get_bytes_consumed(@handle)
+            begin
+              if consumed < data_size
+                @on_end.call(data[consumed..-1])
+              else
+                @on_end.call(nil)
+              end
+            ensure
+              finalize_handle
+            end
+          end
+
+          self
+        end
+
+        private
+        def callback(*arguments)
+          FFI::Function.new(:int, [:pointer, *arguments]) do |_, *values|
+            yield(*values)
+            1
+          end
+        end
+
+        def initialize_callbacks
+          @null_callback = callback do
+            push_value(nil)
+          end
+          @boolean_callback = callback(:int) do |c_boolean|
+            push_value(c_boolean != 0)
+          end
+          @number_callback = callback(:string, :size_t) do |data, size|
+            number_data = data.slice(0, size)
+            if /[\.eE]/ =~ number_data
+              number = number_data.to_f
+            else
+              number = number_data.to_i
+            end
+            push_value(number)
+          end
+          @string_callback = callback(:string, :size_t) do |data, size|
+            string = data.slice(0, size)
+            string.force_encoding(Encoding::UTF_8)
+            push_value(string)
+          end
+          @start_map_callback = callback do
+            push_container({})
+          end
+          @map_key_callback = callback(:string, :size_t) do |data, size|
+            key = data.slice(0, size)
+            key.force_encoding(Encoding::UTF_8)
+            @keys.push(key)
+          end
+          @end_map_callback = callback do
+            pop_container
+          end
+          @start_array_callback = callback do
+            push_container([])
+          end
+          @end_array_callback = callback do
+            pop_container
+          end
+        end
+
+        def push_container(container)
+          @containers.push(container)
+        end
+
+        def pop_container
+          container = @containers.pop
+          if @containers.size == 1
+            @on_value.call(container)
+          else
+            push_value(container)
+          end
+        end
+
+        def push_value(value)
+          container = @containers.last
+          case container
+          when Hash
+            container[@keys.pop] = value
+          when Array
+            container.push(value)
+          end
+        end
+
+        def ensure_handle
+          return if @handle
+          initialize_handle
+        end
+
+        def initialize_handle
+          @callbacks_memory = FFI::MemoryPointer.new(FFI_Yajl::YajlCallbacks)
+          callbacks = FFI_Yajl::YajlCallbacks.new(@callbacks_memory)
+          callbacks[:yajl_null] = @null_callback
+          callbacks[:yajl_boolean] = @boolean_callback
+          callbacks[:yajl_integer] = nil
+          callbacks[:yajl_double] = nil
+          callbacks[:yajl_number] = @number_callback
+          callbacks[:yajl_string] = @string_callback
+          callbacks[:yajl_start_map] = @start_map_callback
+          callbacks[:yajl_map_key] = @map_key_callback
+          callbacks[:yajl_end_map] = @end_map_callback
+          callbacks[:yajl_start_array] = @start_array_callback
+          callbacks[:yajl_end_array] = @end_array_callback
+
+          @handle = FFI_Yajl.yajl_alloc(@callbacks_memory, nil, nil)
+          FFI_Yajl.yajl_config(@handle,
+                               :yajl_allow_trailing_garbage,
+                               :int,
+                               1)
+          @finalizer = Finalizer.new(@handle)
+          ObjectSpace.define_finalizer(self, @finalizer)
+        end
+
+        def finalize_handle
+          @callbacks_memory = nil
+          @finalizer.call(object_id)
+          ObjectSpace.undefine_finalizer(self)
+          @finalizer = nil
+          @handle = nil
+        end
+
+        class Finalizer
+          def initialize(handle)
+            @handle = handle
+          end
+
+          def call(object_id)
+            return if @handle.nil?
+            FFI_Yajl.yajl_free(@handle)
+            @handle = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/groonga/command/parser/version.rb

@@ -19,7 +19,7 @@
 module Groonga
   module Command
     class Parser
-      VERSION = "1.0.3"
+      VERSION = "1.0.4"
     end
   end
 end

data/test/run-test.rb

@@ -36,6 +36,13 @@ end
 $LOAD_PATH.unshift(lib_dir)
 $LOAD_PATH.unshift(test_dir)
 
+# TODO: Remove me when suppress warnings patches are merged int
+# ffi_yajl.
+require "stringio"
+$VERBOSE = false
+require "ffi_yajl/ffi"
+$VERBOSE = true
+
 require "groonga-command-parser-test-utils"
 
 ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"

data/test/test-command-line-splitter.rb

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class CommandLineSplitterTest < Test::Unit::TestCase
+  def split(command_line)
+    splitter = Groonga::Command::Parser::CommandLineSplitter.new(command_line)
+    splitter.split
+  end
+
+  test "name only" do
+    assert_equal(["status"],
+                 split("status"))
+  end
+
+  sub_test_case "arguments" do
+    sub_test_case "no quote" do
+      test "value" do
+        assert_equal(["select", "Logs"],
+                     split("select Logs"))
+      end
+
+      test "key and value" do
+        assert_equal(["select", "--table", "Logs"],
+                     split("select --table Logs"))
+      end
+
+      test "multibyte character" do
+        assert_equal(["select", "テーブル"],
+                     split("select テーブル"))
+      end
+    end
+
+    sub_test_case "single quote" do
+      test "value" do
+        assert_equal(["select", "Logs"],
+                     split("select 'Logs'"))
+      end
+
+      test "key and value" do
+        assert_equal(["select", "--table", "Logs"],
+                     split("select '--table' 'Logs'"))
+      end
+
+      test "space" do
+        assert_equal(["select", "Logs Table"],
+                     split("select 'Logs Table'"))
+      end
+
+      test "escape quote" do
+        assert_equal(["select", "Logs \' Table"],
+                     split("select 'Logs \\' Table'"))
+      end
+
+      test "new line" do
+        assert_equal(["select", "Logs \n Table"],
+                     split("select 'Logs \\n Table'"))
+      end
+    end
+
+    sub_test_case "double quote" do
+      test "value" do
+        assert_equal(["select", "Logs"],
+                     split("select \"Logs\""))
+      end
+
+      test "key and value" do
+        assert_equal(["select", "--table", "Logs"],
+                     split("select \"--table\" \"Logs\""))
+      end
+
+      test "space" do
+        assert_equal(["select", "Logs Table"],
+                     split("select \"Logs Table\""))
+      end
+
+      test "escape quote" do
+        assert_equal(["select", "Logs \" Table"],
+                     split("select \"Logs \\\" Table\""))
+      end
+
+      test "new line" do
+        assert_equal(["select", "Logs \n Table"],
+                     split("select \"Logs \\n Table\""))
+      end
+    end
+  end
+end

data/test/test-load-value-parser.rb

@@ -0,0 +1,149 @@
+# Copyright (C) 2015  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class LoadValuesParserTest < Test::Unit::TestCase
+  def setup
+    @values = []
+    @parser = Groonga::Command::Parser::LoadValuesParser.new
+    @parser.on_value = lambda do |value|
+      @values << value
+    end
+    @parser.on_end = lambda do |rest|
+    end
+  end
+
+  def parse(data)
+    data.each_line do |line|
+    @parser << line
+    end
+    @values
+  end
+
+  sub_test_case "array" do
+    test "empty" do
+      assert_equal([[]],
+                   parse("[[]]"))
+    end
+
+    test "no container" do
+      assert_equal([[1, "abc", 2.9]],
+                   parse(<<-JSON))
+[
+[1, "abc", 2.9]
+]
+      JSON
+    end
+
+    test "array" do
+      assert_equal([[[1, "abc", 2.9]]],
+                   parse(<<-JSON))
+[
+[[1, "abc", 2.9]]
+]
+      JSON
+    end
+
+    test "object" do
+      assert_equal([
+                     [
+                       {
+                         "number" => 1,
+                         "string" => "abc",
+                         "double" => 2.9,
+                       },
+                     ],
+                   ],
+                   parse(<<-JSON))
+[
+  [
+    {
+      "number": 1,
+      "string": "abc",
+      "double": 2.9
+    }
+  ]
+]
+      JSON
+    end
+  end
+
+  sub_test_case "object" do
+    test "empty" do
+      assert_equal([{}],
+                   parse("[{}]"))
+    end
+
+    test "no container" do
+      assert_equal([
+                     {
+                       "number" => 1,
+                       "string" => "abc",
+                       "double" => 2.9,
+                     },
+                   ],
+                   parse(<<-JSON))
+[
+  {
+    "number": 1,
+    "string": "abc",
+    "double": 2.9
+  }
+]
+      JSON
+    end
+
+    test "array" do
+      assert_equal([
+                     {
+                       "array" => [1, "abc", 2.9],
+                     },
+                   ],
+                   parse(<<-JSON))
+[
+  {
+    "array": [1, "abc", 2.9]
+  }
+]
+      JSON
+    end
+
+    test "object" do
+      assert_equal([
+                     {
+                       "object" => {
+                         "number" => 1,
+                         "string" => "abc",
+                         "double" => 2.9,
+                       },
+                     },
+                   ],
+                   parse(<<-JSON))
+[
+  {
+    "object": {
+      "number": 1,
+      "string": "abc",
+      "double": 2.9
+    }
+  }
+]
+[
+  1
+]
+      JSON
+    end
+  end
+end

data/test/test-parser.rb

@@ -228,10 +228,11 @@ EOC
               expected_events << [:load_columns, <<-EOC.chomp, ["_key", "name"]]
 load --table Users --columns "_key, name"
 EOC
-              expected_events << [:load_value, <<-EOC.chomp, ["alice", "Alice"]]
+              expected_events << [:load_value, <<-EOC, ["alice", "Alice"]]
 load --table Users --columns "_key, name"
 [
 ["alice", "Alice"]
+]
 EOC
               expected_events << [:load_complete, <<-EOC.chomp]
 load --table Users --columns "_key, name"
@@ -254,16 +255,19 @@ EOC
               expected_events << [:load_start, <<-EOC.chomp]
 load --table Users
 EOC
-              expected_events << [:load_columns, <<-EOC.chomp, ["_key", "name"]]
+              expected_events << [:load_columns, <<-EOC, ["_key", "name"]]
 load --table Users
 [
-["_key", "name"]
+["_key", "name"],
+["alice", "Alice"]
+]
 EOC
-              expected_events << [:load_value, <<-EOC.chomp, ["alice", "Alice"]]
+              expected_events << [:load_value, <<-EOC, ["alice", "Alice"]]
 load --table Users
 [
 ["_key", "name"],
 ["alice", "Alice"]
+]
 EOC
               expected_events << [:load_complete, <<-EOC.chomp]
 load --table Users
@@ -289,17 +293,20 @@ EOC
 load --table Users
 EOC
             value = {"_key" => "alice", "name" => "Alice"}
-            expected_events << [:load_value, <<-EOC.chomp, value]
+            expected_events << [:load_value, <<-EOC, value]
 load --table Users
 [
-{"_key": "alice", "name": "Alice"}
+{"_key": "alice", "name": "Alice"},
+{"_key": "bob",   "name": "Bob"}
+]
 EOC
             value = {"_key" => "bob", "name" => "Bob"}
-            expected_events << [:load_value, <<-EOC.chomp, value]
+            expected_events << [:load_value, <<-EOC, value]
 load --table Users
 [
 {"_key": "alice", "name": "Alice"},
 {"_key": "bob",   "name": "Bob"}
+]
 EOC
             expected_events << [:load_complete, <<-EOC.chomp]
 load --table Users
@@ -327,9 +334,14 @@ EOS
           end
 
           def test_no_record_separate_comma
-            message = "record separate comma is missing"
-            before = "{\"_key\": \"alice\", \"name\": \"Alice\"}"
-            after = "\n{\"_key\": \"bob\""
+            message = "parse error: after array element, I expect ',' or ']'"
+            before = <<-BEFORE
+[
+{"_key": "alice", "name": "Alice"}
+            BEFORE
+            after = <<-AFTER
+{"_key": "bob",   "name": "Bob"}
+            AFTER
             error = Groonga::Command::Parser::Error.new(message, before, after)
             assert_raise(error) do
               @parser << <<-EOC
@@ -343,17 +355,22 @@ EOC
 
           def test_garbage_before_json
             message = "there are garbages before JSON"
-            before = "load --table Users\n"
-            after = "XXX\n"
+            before = ""
+            after = <<-AFTER
+XXX
+[
+{"_key": "alice", "name": "Alice"}
+]
+            AFTER
             error = Groonga::Command::Parser::Error.new(message, before, after)
             assert_raise(error) do
-              @parser << <<-EOC
+              @parser << <<-JSON
 load --table Users
 XXX
 [
 {"_key": "alice", "name": "Alice"}
 ]
-EOC
+              JSON
             end
           end
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groonga-command-parser
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-12 00:00:00.000000000 Z
+date: 2015-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: groonga-command
@@ -25,7 +25,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.0.9
 - !ruby/object:Gem::Dependency
-  name: yajl-ruby
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ffi-yajl
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,7 +137,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: redcarpet
+  name: kramdown
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -153,11 +167,15 @@ files:
 - doc/text/news.md
 - groonga-command-parser.gemspec
 - lib/groonga/command/parser.rb
+- lib/groonga/command/parser/command-line-splitter.rb
 - lib/groonga/command/parser/command/groonga-command-convert-format.rb
 - lib/groonga/command/parser/error.rb
+- lib/groonga/command/parser/load-values-parser.rb
 - lib/groonga/command/parser/version.rb
 - test/groonga-command-parser-test-utils.rb
 - test/run-test.rb
+- test/test-command-line-splitter.rb
+- test/test-load-value-parser.rb
 - test/test-parser.rb
 homepage: https://github.com/groonga/groonga-command-parser
 licenses:
@@ -185,7 +203,9 @@ specification_version: 4
 summary: Groonga-command-parser is a Ruby library to parses [groonga](http://groonga.org/)'s
   command syntax. You can write a program to process groonga's command by using groonga-command-parser.
 test_files:
-- test/test-parser.rb
-- test/run-test.rb
 - test/groonga-command-parser-test-utils.rb
+- test/run-test.rb
+- test/test-command-line-splitter.rb
+- test/test-load-value-parser.rb
+- test/test-parser.rb
 has_rdoc: