rroonga 3.0.6-x86-mingw32 → 3.0.7-x86-mingw32

data/bin/groonga-index-dump

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,11 +16,11 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require 'ostruct'
-require 'optparse'
-require 'pathname'
+require "ostruct"
+require "optparse"
+require "pathname"
 
-require 'groonga'
+require "groonga"
 
 options = OpenStruct.new
 options.output_directory = "index-dump"

data/doc/text/news.textile

@@ -1,5 +1,27 @@
 h1. NEWS
 
+h2(#3-0-7). 3.0.7: 2013-09-19
+
+h3. Improvements
+
+* Added {Groonga::TableKeySupport#tokenize}. It requires groonga
+  3.0.8. Groonga 3.0.8 is not released yet.
+* Changed return object of {Groonga::Context#select} to groonga-client gem's
+  return object. It is a backward imcompatible change. Some APIs are changed.
+  For example, @drill_down@ is renamed to @drilldowns@. See
+  "Groonga::Client::Response::Select":http://rubydoc.info/gems/groonga-client/Groonga/Client/Response/Select
+  about rerturn object details.
+** Added groonga-client gem, groogna-command gem, gqtp gem dependencies.
+
+h3. Fixes
+
+* Fixed a bug that auto groonga installation is failed.
+  [GitHub#12][Patch by Keita Haga]
+
+h3. Thanks
+
+* Keita Haga
+
 h2(#3-0-6). 3.0.6: 2013-09-13
 
 h3. Improvements

data/ext/groonga/extconf.rb

@@ -232,8 +232,8 @@ end
 unless PKGConfig.have_package(package_name, major, minor, micro)
   unless have_local_groonga?(package_name, major, minor, micro)
     required_version = [major, minor, micro]
-    if (required_version <=> LatestGroongaVersion::VERSISON) > 0
-      target_version = LatestGroongaVersion::VERSISON
+    if (required_version <=> LatestGroongaVersion::VERSION) > 0
+      target_version = LatestGroongaVersion::VERSION
     else
       target_version = required_version
     end
@@ -258,6 +258,8 @@ have_header("ruby/st.h") unless have_macro("HAVE_RUBY_ST_H", "ruby.h")
 have_func("rb_errinfo", "ruby.h")
 have_type("enum ruby_value_type", "ruby.h")
 
+have_func("grn_table_tokenize", "groonga.h")
+
 checking_for(checking_message("--enable-debug-log option")) do
   enable_debug_log = enable_config("debug-log", false)
   if enable_debug_log

data/ext/groonga/rb-grn-table-key-support.c

@@ -1,6 +1,6 @@
 /* -*- coding: utf-8; mode: C; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
 /*
-  Copyright (C) 2009-2012  Kouhei Sutou <kou@clear-code.com>
+  Copyright (C) 2009-2013  Kouhei Sutou <kou@clear-code.com>
 
   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
@@ -877,6 +877,61 @@ rb_grn_table_key_support_support_key_p (VALUE self)
     return table->header.domain == GRN_ID_NIL ? Qfalse : Qtrue;
 }
 
+#ifdef HAVE_GRN_TABLE_TOKENIZE
+/*
+ * Tokenize a string using the table as lexicon.
+ *
+ * @overload tokenize(string, options={})
+ *   @param [String] string The string to be tokenized.
+ *   @param [::Hash] options
+ *   @option options [Bool] :add (true) Adds a new token to the table if true.
+ *      Returned tokens include the new token. Otherwise, a new token is
+ *      just ignored.
+ *   @return [::Array<Groonga::Record>] Tokenized tokens.
+ */
+static VALUE
+rb_grn_table_key_support_tokenize (int argc, VALUE *argv, VALUE self)
+{
+    VALUE rb_string, rb_add_p;
+    VALUE rb_options;
+    VALUE rb_tokens = Qnil;
+    grn_ctx *context;
+    grn_obj *table;
+    char *string;
+    int string_size;
+    grn_bool add_p;
+    grn_obj tokens;
+
+    rb_scan_args(argc, argv, "11", &rb_string, &rb_options);
+    rb_grn_scan_options(rb_options,
+                        "add", &rb_add_p,
+                        NULL);
+    if (NIL_P(rb_add_p)) {
+        rb_add_p = Qtrue;
+    }
+
+    rb_grn_table_key_support_deconstruct(SELF(self), &table, &context,
+                                         NULL, NULL, NULL,
+                                         NULL, NULL, NULL,
+                                         NULL);
+
+    string = StringValueCStr(rb_string);
+    string_size = RSTRING_LEN(rb_string);
+
+    add_p = RVAL2CBOOL(rb_add_p);
+
+    GRN_RECORD_INIT(&tokens, GRN_OBJ_VECTOR, grn_obj_id(context, table));
+    grn_table_tokenize(context, table, string, string_size, &tokens, add_p);
+    if (context->rc == GRN_SUCCESS) {
+        rb_tokens = GRNUVECTOR2RVAL(context, &tokens, table, self);
+    }
+    grn_obj_unlink(context, &tokens);
+    rb_grn_context_check(context, self);
+
+    return rb_tokens;
+}
+#endif
+
 void
 rb_grn_init_table_key_support (VALUE mGrn)
 {
@@ -928,4 +983,9 @@ rb_grn_init_table_key_support (VALUE mGrn)
 
     rb_define_method(rb_mGrnTableKeySupport, "support_key?",
                      rb_grn_table_key_support_support_key_p, 0);
+
+#ifdef HAVE_GRN_TABLE_TOKENIZE
+    rb_define_method(rb_mGrnTableKeySupport, "tokenize",
+                     rb_grn_table_key_support_tokenize, -1);
+#endif
 }

data/ext/groonga/rb-grn.h

@@ -84,7 +84,7 @@ RB_GRN_BEGIN_DECLS
 
 #define RB_GRN_MAJOR_VERSION 3
 #define RB_GRN_MINOR_VERSION 0
-#define RB_GRN_MICRO_VERSION 6
+#define RB_GRN_MICRO_VERSION 7
 
 #define RB_GRN_QUERY_DEFAULT_MAX_EXPRESSIONS 32
 

data/lib/groonga/context/command-executor.rb

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "groonga/command"
+require "groonga/client/response"
+
+module Groonga
+  class Context
+    class CommandExecutor
+      def initialize(context)
+        @context = context
+      end
+
+      def execute(name, parameters={})
+        parameters = normalize_parameters(name, parameters)
+        command_class = Command.find(name)
+        command = command_class.new(name, parameters)
+        request_id = @context.send(command.to_command_format)
+        loop do
+          response_id, raw_response = @context.receive
+          if request_id == response_id
+            response_class = Client::Response.find(command.name)
+            header = [0, 0, 0]
+            case command.output_type
+            when :json
+              body = JSON.parse(raw_response)
+            else
+              body = raw_response
+            end
+            response = response_class.new(command, header, body)
+            response.raw = raw_response
+            return response
+          end
+          # raise if request_id < response_id
+        end
+      end
+
+      private
+      def normalize_parameters(name, parameters)
+        case name
+        when "select"
+          normalize_select_parameters(parameters)
+        else
+          parameters
+        end
+      end
+
+      def normalize_select_parameters(parameters)
+        table = parameters[:table]
+        parameters[:table] = table.name if table.is_a?(Table)
+
+        normalize_key(parameters, :drilldown, :drill_down)
+        normalize_key(parameters,
+                      :drilldown_output_columns,
+                      :drill_down_output_columns)
+        normalize_key(parameters, :drilldown_limit, :drill_down_limit)
+
+        normalize_array_value(parameters, :output_columns)
+        normalize_array_value(parameters, :drilldown)
+        normalize_array_value(parameters, :drilldown_output_columns)
+
+        normalize_integer_value(parameters, :limit)
+        normalize_integer_value(parameters, :drilldown_limit)
+
+        parameters
+      end
+
+      def normalize_key(parameters, real_key, alias_key)
+        return unless parameters.has_key?(alias_key)
+        parameters[real_key] ||= parameters.delete(alias_key)
+      end
+
+      def normalize_array_value(parameters, key)
+        if parameters[key].is_a?(::Array)
+          parameters[key] = parameters[key].join(", ")
+        end
+      end
+
+      def normalize_integer_value(parameters, key)
+        if parameters[key].is_a?(Integer)
+          parameters[key] = parameters[key].to_s
+        end
+      end
+    end
+  end
+end

data/lib/groonga/context.rb

@@ -15,9 +15,8 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "groonga/command"
-
 require "groonga/memory-pool"
+require "groonga/context/command-executor"
 
 module Groonga
   class Context
@@ -133,8 +132,12 @@ module Groonga
     # @option options [Array] XXX TODO
     #   TODO
     def select(table, options={})
-      select = Command::Select.new(self, table, options)
-      select.exec
+      execute_command("select", {:table => table}.merge(options))
+    end
+
+    def execute_command(name, parameters={})
+      executor = CommandExecutor.new(self)
+      executor.execute(name, parameters)
     end
 
     # Restore commands dumped by "grndump" command.

data/rroonga.gemspec

@@ -1,6 +1,6 @@
 # -*- mode: ruby; coding: utf-8 -*-
 #
-# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2013  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -70,7 +70,7 @@ Gem::Specification.new do |s|
     s.extra_rdoc_files = ["README.textile"]
     s.test_files = Dir.glob("test/**/*.rb")
     Dir.chdir("bin") do
-      s.executables = Dir.glob("*")
+      s.executables = Dir.glob("*") - ["groonga-index-validate"]
     end
   end
 
@@ -82,6 +82,7 @@ Gem::Specification.new do |s|
   s.required_ruby_version = ">= 1.9.3"
 
   s.add_runtime_dependency("pkg-config")
+  s.add_runtime_dependency("groonga-client", ">= 0.0.3")
   s.add_runtime_dependency("json")
   s.add_runtime_dependency("archive-zip")
   s.add_development_dependency("test-unit", [">= 2.4.6"])

data/test/run-test.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 #
-# Copyright (C) 2009-2010  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2009-2013  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -41,18 +41,30 @@ require "test/unit/notify"
 
 Test::Unit::Priority.enable
 
+
+groonga_command_dir = File.join(base_dir, "..", "groonga-command")
+if File.exist?(groonga_command_dir)
+  $LOAD_PATH.unshift(File.join(groonga_command_dir, "lib"))
+end
+
+groonga_client_dir = File.join(base_dir, "..", "groonga-client")
+if File.exist?(groonga_client_dir)
+  $LOAD_PATH.unshift(File.join(groonga_client_dir, "lib"))
+end
+
+
 $LOAD_PATH.unshift(ext_dir)
 $LOAD_PATH.unshift(lib_dir)
 $LOAD_PATH.unshift(base_dir)
 
 $LOAD_PATH.unshift(test_dir)
-require 'groonga-test-utils'
+require "groonga-test-utils"
 
 pkg_config = File.join(base_dir, "vendor", "local", "lib", "pkgconfig")
 PKGConfig.add_path(pkg_config)
 
 Dir.glob("#{base_dir}/test/**/test{_,-}*.rb") do |file|
-  require file.sub(/\.rb$/, '')
+  require file.sub(/\.rb$/, "")
 end
 
 ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"

data/test/test-command-select.rb

@@ -76,45 +76,37 @@ class CommandSelectTest < Test::Unit::TestCase
                  [result.n_hits, result.records])
   end
 
-  def test_drill_down
+  def test_drilldowns
     result = context.select(@users,
                             :output_columns => ["_key"],
                             :drill_down => ["_key", "book"],
                             :drill_down_output_columns => "_key",
                             :drill_down_limit => 10)
-    drill_down = normalize_drill_down(result.drill_down)
-    assert_equal([4,
-                  [{"_key" => "morita"},
-                   {"_key" => "gunyara-kun"},
-                   {"_key" => "yu"},
-                   {"_key" => "ryoqun"}],
-                  {
+    drilldowns = normalize_drilldowns(result.drilldowns)
+    assert_equal({
                     "_key" => [4, [{"_key" => "morita"},
                                    {"_key" => "gunyara-kun"},
                                    {"_key" => "yu"},
                                    {"_key" => "ryoqun"}]],
                     "book" => [2, [{"_key" => "the groonga book"},
                                    {"_key" => "the groonga book (2)"}]],
-                  },
-                 ],
-                 [result.n_hits, result.records, drill_down])
+                 },
+                 drilldowns)
   end
 
-  def test_drill_down_with_no_hit
+  def test_drilldowns_with_no_hit
     result = context.select(@users,
                             :filter => "_key == \"no\\ hit\"",
                             :output_columns => ["_key"],
                             :drill_down => ["_key", "book"],
                             :drill_down_output_columns => "_key",
                             :drill_down_limit => 10)
-    drill_down = normalize_drill_down(result.drill_down)
-    assert_equal([0, [],
-                  {
-                    "_key" => [0, []],
-                    "book" => [0, []],
-                  },
-                 ],
-                 [result.n_hits, result.records, drill_down])
+    drilldowns = normalize_drilldowns(result.drilldowns)
+    assert_equal({
+                   "_key" => [0, []],
+                   "book" => [0, []],
+                 },
+                 drilldowns)
   end
 
   def test_time
@@ -201,12 +193,12 @@ class CommandSelectTest < Test::Unit::TestCase
   end
 
   private
-  def normalize_drill_down(drill_down)
-    normalized_drill_down = {}
-    drill_down.each do |key, drill|
-      normalized_drill_down[key] = [drill.n_hits, drill.records]
+  def normalize_drilldowns(drilldowns)
+    normalized_drilldowns = {}
+    drilldowns.each do |drilldown|
+      normalized_drilldowns[drilldown.key] = [drilldown.n_hits, drilldown.items]
     end
-    normalized_drill_down
+    normalized_drilldowns
   end
 
   class EscapeTest < self

data/test/test-table-key-support.rb

@@ -75,4 +75,66 @@ class TableKeySupportTest < Test::Unit::TestCase
       assert_equal(key, id.key)
     end
   end
+
+  class TokenizeTest < self
+    class TableTypeTest < self
+      def test_hash
+        Groonga::Schema.create_table("Terms",
+                                     :type => :hash,
+                                     :key_type => "ShortText",
+                                     :default_tokenizer => "TokenBigram",
+                                     :normalizer => "NormalizerAuto")
+        terms = Groonga["Terms"]
+        tokens = terms.tokenize("Hello World!")
+        assert_equal(["!", "hello", "world"],
+                     tokens.collect(&:key).sort)
+      end
+
+      def test_patricia_trie
+        Groonga::Schema.create_table("Terms",
+                                     :type => :patricia_trie,
+                                     :key_type => "ShortText",
+                                     :default_tokenizer => "TokenBigram",
+                                     :normalizer => "NormalizerAuto")
+        terms = Groonga["Terms"]
+        tokens = terms.tokenize("Hello World!")
+        assert_equal(["!", "hello", "world"],
+                     tokens.collect(&:key).sort)
+      end
+    end
+
+    class AddOptionTest < self
+      setup
+      def setup_lexicon
+        Groonga::Schema.create_table("Terms",
+                                     :type => :patricia_trie,
+                                     :key_type => "ShortText",
+                                     :default_tokenizer => "TokenBigram",
+                                     :normalizer => "NormalizerAuto")
+        @lexicon = Groonga["Terms"]
+      end
+
+      def test_default
+        tokens = @lexicon.tokenize("Hello World!")
+        assert_equal(["!", "hello", "world"],
+                     tokens.collect(&:key).sort)
+      end
+
+      def test_true
+        tokens = @lexicon.tokenize("Hello World!", :add => true)
+        assert_equal(["!", "hello", "world"],
+                     tokens.collect(&:key).sort)
+      end
+
+      def test_false
+        tokens = @lexicon.tokenize("Hello groonga!", :add => true)
+        assert_equal(["!", "groonga", "hello"],
+                     tokens.collect(&:key).sort)
+
+        tokens = @lexicon.tokenize("Hello World!", :add => false)
+        assert_equal(["!", "hello"],
+                     tokens.collect(&:key).sort)
+      end
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rroonga
 version: !ruby/object:Gem::Version
-  version: 3.0.6
+  version: 3.0.7
   prerelease: 
 platform: x86-mingw32
 authors:
@@ -13,7 +13,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-14 00:00:00.000000000 Z
+date: 2013-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pkg-config
@@ -31,6 +31,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: groonga-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.3
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -234,10 +250,10 @@ files:
 - lib/groonga/pagination.rb
 - lib/groonga/database.rb
 - lib/groonga/schema.rb
-- lib/groonga/command.rb
 - lib/groonga/posting.rb
 - lib/groonga/patricia-trie.rb
 - lib/groonga/expression-builder.rb
+- lib/groonga/context/command-executor.rb
 - lib/groonga/dumper.rb
 - lib/groonga/query-logger.rb
 - lib/groonga/expression-builder-19.rb

data/lib/groonga/command.rb

@@ -1,171 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License version 2.1 as published by the Free Software Foundation.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-module Groonga
-  module Command
-    class Builder
-      class << self
-        def escape_value(value)
-          escaped_value = value.to_s.gsub(/[\\"]/) do |matched|
-            "\\#{matched}"
-          end
-          "\"#{escaped_value}\""
-        end
-      end
-
-      attr_reader :command, :arguments
-      def initialize(command, arguments={})
-        @command = command
-        @arguments = arguments
-      end
-
-      def [](key)
-        @arguments[key]
-      end
-
-      def []=(key, value)
-        @arguments[key] = value
-      end
-
-      def build
-        query = "#{@command} "
-        @arguments.each do |key, value|
-          value = value.join(", ") if value.is_a?(::Array)
-          escaped_value = self.class.escape_value(value)
-          query << " --#{key} #{escaped_value}"
-        end
-        query
-      end
-    end
-
-    class Select
-      def initialize(context, table, options)
-        @context = context
-        @table = table
-        @options = normalize_options(options)
-      end
-
-      def exec
-        request_id = @context.send(query)
-        loop do
-          response_id, result = @context.receive
-          if request_id == response_id
-            drill_down_keys = @options["drilldown"]
-            if drill_down_keys.is_a?(String)
-              drill_down_keys = drill_down_keys.split(/(?:\s+|\s*,\s*)/)
-            end
-            return Result.parse(result, drill_down_keys)
-          end
-          # raise if request_id < response_id
-        end
-      end
-
-      private
-      def normalize_options(options)
-        normalized_options = {}
-        options.each do |key, value|
-          normalized_options[normalize_option_name(key)] = value
-        end
-        normalized_options
-      end
-
-      def normalize_option_name(name)
-        name = name.to_s.gsub(/-/, "_").gsub(/drill_down/, "drilldown")
-        name.gsub(/sort_by/, 'sortby')
-      end
-
-      def query
-        if @table.is_a?(String)
-          table_name = @table
-        else
-          table_name = @table.name
-        end
-        builder = Builder.new("select", @options.merge(:table => table_name))
-        builder.build
-      end
-
-      class Result < Struct.new(:n_hits, :columns, :values, :drill_down)
-        class << self
-          def parse(json, drill_down_keys)
-            select_result, *drill_down_results = JSON.parse(json)
-            result = new
-            n_hits, columns, values = extract_result(select_result)
-            result.n_hits = n_hits
-            result.columns = columns
-            result.values = values
-            if drill_down_results
-              result.drill_down = parse_drill_down_results(drill_down_results,
-                                                           drill_down_keys)
-            end
-            result
-          end
-
-          def create_records(columns, values)
-            records = []
-            values.each do |value|
-              record = {}
-              columns.each_with_index do |(name, type), i|
-                record[name] = convert_value(value[i], type)
-              end
-              records << record
-            end
-            records
-          end
-
-          private
-          def parse_drill_down_results(results, keys)
-            named_results = {}
-            results.each_with_index do |drill_down, i|
-              n_hits, columns, values = extract_result(drill_down)
-              drill_down_result = DrillDownResult.new
-              drill_down_result.n_hits = n_hits
-              drill_down_result.columns = columns
-              drill_down_result.values = values
-              named_results[keys[i]] = drill_down_result
-            end
-            named_results
-          end
-
-          def extract_result(result)
-            meta_data, columns, *values = result
-            n_hits, = meta_data
-            [n_hits, columns, values]
-          end
-
-          def convert_value(value, type)
-            case type
-            when "Time"
-              Time.at(value)
-            else
-              value
-            end
-          end
-        end
-
-        def records
-          @records ||= self.class.create_records(columns, values)
-        end
-
-        class DrillDownResult < Struct.new(:n_hits, :columns, :values)
-          def records
-            @records ||= Result.create_records(columns, values)
-          end
-        end
-      end
-    end
-  end
-end