groonga-command-parser 1.0.0

data/lib/groonga/command/parser/error.rb

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga/command"
+
+module Groonga
+  module Command
+    class Parser
+      class Error < Command::Error
+        attr_reader :reason, :location
+        def initialize(reason, before, after)
+          @reason = reason
+          @location = compute_location(before, after)
+          super("#{@reason}:\n#{@location}")
+        end
+
+        private
+        def compute_location(before, after)
+          location = ""
+          if before[-1] == ?\n
+            location << before
+            location << after
+            location << "^"
+          elsif after[0] == ?\n
+            location << before
+            location << "\n"
+            location << " " * before.bytesize + "^"
+            location << after
+          else
+            location << before
+            location << after
+            location << " " * before.bytesize + "^"
+          end
+          location
+        end
+      end
+    end
+  end
+end

data/lib/groonga/command/parser/version.rb

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module Groonga
+  module Command
+    class Parser
+      VERSION = "1.0.0"
+    end
+  end
+end

data/test/groonga-command-parser-test-utils.rb

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "cgi"
+require "stringio"
+
+require "groonga/command/parser"
+
+module GroongaCommandParserTestUtils
+  module CommandParser
+    private
+    def command(name, arguments)
+      Groonga::Command.find(name).new(name, arguments)
+    end
+
+    def parse_http_path(command, arguments, options={})
+      path = "/d/#{command}"
+      case options[:output_type]
+      when false
+      when nil
+        path << ".json"
+      else
+        path << ".#{options[:output_type]}"
+      end
+
+      unless arguments.empty?
+        uri_arguments = arguments.collect do |key, value|
+          [CGI.escape(key.to_s), CGI.escape(value.to_s)].join("=")
+        end
+        path << "?"
+        path << uri_arguments.join("&")
+      end
+
+      Groonga::Command::Parser.parse(path)
+    end
+
+    def parse_command_line(command, arguments, options={})
+      command_line = "#{command}"
+      case options[:output_type]
+      when false
+      when nil
+        command_line << " --output_type json"
+      else
+        command_line << " --output_type #{options[:output_type]}"
+      end
+
+      if arguments.is_a?(Hash)
+        arguments.each do |key, value|
+          escaped_value = escape_command_line_value(value)
+          command_line << " --#{key} #{escaped_value}"
+        end
+      else
+        arguments.each do |argument|
+          command_line << " #{escape_command_line_value(argument)}"
+        end
+      end
+
+      Groonga::Command::Parser.parse(command_line)
+    end
+
+    def escape_command_line_value(value)
+      if /"| / =~ value
+        '"' + value.gsub(/"/, '\"') + '"'
+      else
+        value
+      end
+    end
+  end
+
+  module HTTPCommandParser
+    include CommandParser
+
+    private
+    def parse(command, arguments={}, options={})
+      parse_http_path(command, arguments, options)
+    end
+  end
+
+  module CommandLineCommandParser
+    include CommandParser
+
+    private
+    def parse(command, arguments={}, options={})
+      parse_command_line(command, arguments, options)
+    end
+  end
+end

data/test/run-test.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+$VERBOSE = true
+
+base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+lib_dir = File.join(base_dir, "lib")
+test_dir = File.join(base_dir, "test")
+
+require "test-unit"
+require "test/unit/notify"
+
+Test::Unit::Priority.enable
+
+groonga_command_dir = File.join(base_dir, "..", "groonga-command")
+groonga_command_dir = File.expand_path(groonga_command_dir)
+if File.exist?(groonga_command_dir)
+  $LOAD_PATH.unshift(File.join(groonga_command_dir, "lib"))
+end
+
+$LOAD_PATH.unshift(lib_dir)
+$LOAD_PATH.unshift(test_dir)
+
+require "groonga-command-parser-test-utils"
+
+ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"
+
+exit Test::Unit::AutoRunner.run(true, test_dir)

data/test/test-parser.rb

@@ -0,0 +1,377 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class ParserTest < Test::Unit::TestCase
+  module ParseTests
+    def test_parameters
+      select = parse("select",
+                     :table => "Users",
+                     :filter => "age<=30")
+      assert_equal(command("select",
+                           "table" => "Users",
+                           "filter" => "age<=30",
+                           "output_type" => "json"),
+                   select)
+    end
+  end
+
+  class HTTPTest < self
+    include GroongaCommandParserTestUtils::HTTPCommandParser
+
+    def test_uri_format?
+      command = parse("status")
+      assert_predicate(command, :uri_format?)
+    end
+
+    def test_command_format?
+      command = parse("status")
+      assert_not_predicate(command, :command_format?)
+    end
+
+    def test_no_value
+      path = "/d/select?table=Users&key_only"
+      command = Groonga::Command::Parser.parse(path)
+      assert_equal({:table => "Users"}, command.arguments)
+    end
+
+    class ParseTest < self
+      include ParseTests
+    end
+  end
+
+  class CommandLineTest < self
+    include GroongaCommandParserTestUtils::CommandLineCommandParser
+
+    def test_uri_format?
+      command = parse("status")
+      assert_not_predicate(command, :uri_format?)
+    end
+
+    def test_command_format?
+      command = parse("status")
+      assert_predicate(command, :command_format?)
+    end
+
+    class ParseTest < self
+      include ParseTests
+    end
+
+    class EventTest < self
+      def setup
+        @parser = Groonga::Command::Parser.new
+      end
+
+      class CommandTest < self
+        def test_newline
+          parsed_command = nil
+          @parser.on_command do |command|
+            parsed_command = command
+          end
+
+          @parser << "status"
+          assert_nil(parsed_command)
+          @parser << "\n"
+          assert_equal("status", parsed_command.name)
+        end
+
+        def test_finish
+          parsed_command = nil
+          @parser.on_command do |command|
+            parsed_command = command
+          end
+
+          @parser << "status"
+          assert_nil(parsed_command)
+          @parser.finish
+          assert_equal("status", parsed_command.name)
+        end
+
+        def test_empty_line
+          parsed_command = nil
+          @parser.on_command do |command|
+            parsed_command = command
+          end
+
+          @parser << "\n"
+          assert_nil(parsed_command)
+
+          @parser << "status\n"
+          assert_equal("status", parsed_command.name)
+        end
+
+        def test_multi_lines
+          parsed_commands = []
+          @parser.on_command do |command|
+            parsed_commands << command
+          end
+
+          @parser << <<-COMMAND_LIST.chomp
+table_list
+status
+          COMMAND_LIST
+          assert_equal(["table_list"],
+                       parsed_commands.collect(&:name))
+
+          @parser.finish
+          assert_equal(["table_list", "status"],
+                       parsed_commands.collect(&:name))
+        end
+      end
+
+      class LoadTest < self
+        def setup
+          super
+          @events = []
+          @parser.on_load_start do |command|
+            @events << [:load_start, command.original_source.dup]
+          end
+          @parser.on_load_columns do |command, header|
+            @events << [:load_columns, command.original_source.dup, header]
+          end
+          @parser.on_load_value do |command, value|
+            @events << [:load_value, command.original_source.dup, value]
+          end
+          @parser.on_load_complete do |command|
+            @events << [:load_complete, command.original_source.dup]
+          end
+        end
+
+        class InlineTest < self
+          class BracketTest < self
+            def test_have_columns
+              command_line =
+                "load " +
+                  "--columns '_key, name' " +
+                  "--values '[[\"alice\", \"Alice\"]]' " +
+                  "--table Users"
+              @parser << command_line
+              assert_equal([], @events)
+              @parser << "\n"
+              assert_equal([
+                             [:load_start, command_line],
+                             [:load_columns, command_line, ["_key", "name"]],
+                             [:load_value, command_line, ["alice", "Alice"]],
+                             [:load_complete, command_line],
+                           ],
+                           @events)
+            end
+
+            def test_no_columns
+              command_line = "load --values '[[\"_key\"], [1]]' --table IDs"
+              @parser << command_line
+              assert_equal([], @events)
+              @parser << "\n"
+              assert_equal([
+                             [:load_start, command_line],
+                             [:load_columns, command_line, ["_key"]],
+                             [:load_value, command_line, [1]],
+                             [:load_complete, command_line],
+                           ],
+                           @events)
+            end
+          end
+
+          def test_brace
+            command_line = "load --values '[{\"_key\": 1}]' --table IDs"
+            @parser << command_line
+            assert_equal([], @events)
+            @parser << "\n"
+            assert_equal([
+                           [:load_start, command_line],
+                           [:load_value, command_line, {"_key" => 1}],
+                           [:load_complete, command_line],
+                         ],
+                         @events)
+          end
+        end
+
+        class MultiLineTest < self
+          class BracketTest < self
+            def test_have_columns
+              @parser << <<-EOC
+load --table Users --columns "_key, name"
+[
+["alice", "Alice"]
+]
+EOC
+              expected_events = []
+              expected_events << [:load_start, <<-EOC.chomp]
+load --table Users --columns "_key, name"
+EOC
+              expected_events << [:load_columns, <<-EOC.chomp, ["_key", "name"]]
+load --table Users --columns "_key, name"
+EOC
+              expected_events << [:load_value, <<-EOC.chomp, ["alice", "Alice"]]
+load --table Users --columns "_key, name"
+[
+["alice", "Alice"]
+EOC
+              expected_events << [:load_complete, <<-EOC.chomp]
+load --table Users --columns "_key, name"
+[
+["alice", "Alice"]
+]
+EOC
+              assert_equal(expected_events, @events)
+            end
+
+            def test_no_columns
+              @parser << <<-EOC
+load --table Users
+[
+["_key", "name"],
+["alice", "Alice"]
+]
+EOC
+              expected_events = []
+              expected_events << [:load_start, <<-EOC.chomp]
+load --table Users
+EOC
+              expected_events << [:load_columns, <<-EOC.chomp, ["_key", "name"]]
+load --table Users
+[
+["_key", "name"]
+EOC
+              expected_events << [:load_value, <<-EOC.chomp, ["alice", "Alice"]]
+load --table Users
+[
+["_key", "name"],
+["alice", "Alice"]
+EOC
+              expected_events << [:load_complete, <<-EOC.chomp]
+load --table Users
+[
+["_key", "name"],
+["alice", "Alice"]
+]
+EOC
+              assert_equal(expected_events, @events)
+            end
+          end
+
+          def test_brace
+            @parser << <<-EOC
+load --table Users
+[
+{"_key": "alice", "name": "Alice"},
+{"_key": "bob",   "name": "Bob"}
+]
+EOC
+            expected_events = []
+            expected_events << [:load_start, <<-EOC.chomp]
+load --table Users
+EOC
+            value = {"_key" => "alice", "name" => "Alice"}
+            expected_events << [:load_value, <<-EOC.chomp, value]
+load --table Users
+[
+{"_key": "alice", "name": "Alice"}
+EOC
+            value = {"_key" => "bob", "name" => "Bob"}
+            expected_events << [:load_value, <<-EOC.chomp, value]
+load --table Users
+[
+{"_key": "alice", "name": "Alice"},
+{"_key": "bob",   "name": "Bob"}
+EOC
+            expected_events << [:load_complete, <<-EOC.chomp]
+load --table Users
+[
+{"_key": "alice", "name": "Alice"},
+{"_key": "bob",   "name": "Bob"}
+]
+EOC
+            assert_equal(expected_events, @events)
+          end
+        end
+
+        class ErrorTest < self
+          def test_location
+            message = "record separate comma is missing"
+            before = "{\"_key\": \"alice\", \"name\": \"Alice\"}"
+            after = "\n{\"_key\": \"bob\""
+            error = Groonga::Command::Parser::Error.new(message, before, after)
+            assert_equal(<<-EOS.chomp, error.message)
+record separate comma is missing:
+{"_key": "alice", "name": "Alice"}
+                                  ^
+{"_key": "bob"
+EOS
+          end
+
+          def test_no_record_separate_comma
+            message = "record separate comma is missing"
+            before = "{\"_key\": \"alice\", \"name\": \"Alice\"}"
+            after = "\n{\"_key\": \"bob\""
+            error = Groonga::Command::Parser::Error.new(message, before, after)
+            assert_raise(error) do
+              @parser << <<-EOC
+load --table Users
+[
+{"_key": "alice", "name": "Alice"}
+{"_key": "bob",   "name": "Bob"}
+EOC
+            end
+          end
+
+          def test_garbage_before_json
+            message = "there are garbages before JSON"
+            before = "load --table Users\n"
+            after = "XXX\n"
+            error = Groonga::Command::Parser::Error.new(message, before, after)
+            assert_raise(error) do
+              @parser << <<-EOC
+load --table Users
+XXX
+[
+{"_key": "alice", "name": "Alice"}
+]
+EOC
+            end
+          end
+        end
+      end
+
+      class CommentTest < self
+        def test_newline
+          parsed_comment = nil
+          @parser.on_comment do |comment|
+            parsed_comment = comment
+          end
+
+          @parser << "# status"
+          assert_nil(parsed_comment)
+          @parser << "\n"
+          assert_equal(" status", parsed_comment)
+        end
+
+        def test_finish
+          parsed_comment = nil
+          @parser.on_comment do |comment|
+            parsed_comment = comment
+          end
+
+          @parser << "# status"
+          assert_nil(parsed_comment)
+          @parser.finish
+          assert_equal(" status", parsed_comment)
+        end
+      end
+    end
+  end
+end