rroonga 3.0.4-x86-mingw32 → 3.0.5-x86-mingw32

data/doc/text/news.textile

@@ -1,5 +1,34 @@
 h1. NEWS
 
+h2(#3-0-5). 3.0.5: 2013-07-29
+
+h3. Improvements
+
+* [dumper] supported dumping of DoubleArrayTrie.
+* Supported Int8/UInt8/Int16/UInt16 to Ruby conversion.
+  [groonga-dev,01524][Reported by Masaharu YOSHIOKA]
+* Added memory pool mechanism to reduce GC easily.
+  {Groonga::Context#push_memory_pool} and {Groonga::Context#pop_memory_pool}
+  are added.
+  You can close temporary table objects automatically:
+  <pre>
+  context.push_memory_pool do
+    # create tempoeray table objects by select, group, sort and so on...
+  end
+  # createed tempoeray table objects are closed automatically
+  </pre>
+* Supported max int32 over Fixnum
+  [Reported by xtuaok]
+
+h3. Fixes
+
+* [dumper] fixed a bug that no column table isn't dumped.
+
+h3. Thanks
+
+* Masaharu YOSHIOKA
+* xtuaok
+
 h2(#3-0-4). 3.0.4: 2013-07-04
 
 h3. Fixes

data/ext/groonga/rb-grn-context.c

@@ -470,6 +470,8 @@ rb_grn_context_initialize (int argc, VALUE *argv, VALUE self)
 	GRN_CTX_SET_ENCODING(context, encoding);
     }
 
+    rb_iv_set(self, "@memory_pools", rb_ary_new());
+
     debug("context new: %p\n", context);
 
     return Qnil;
@@ -924,6 +926,14 @@ rb_grn_context_array_reference (VALUE self, VALUE name_or_id)
     return GRNOBJECT2RVAL(Qnil, context, object, GRN_FALSE);
 }
 
+void
+rb_grn_context_object_created (VALUE rb_context, VALUE rb_object)
+{
+    ID id_object_created;
+    CONST_ID(id_object_created, "object_created");
+    rb_funcall(rb_context, id_object_created, 1, rb_object);
+}
+
 void
 rb_grn_init_context (VALUE mGrn)
 {

data/ext/groonga/rb-grn-object.c

@@ -313,6 +313,16 @@ rb_grn_object_to_ruby_object (VALUE klass, grn_ctx *context, grn_obj *object,
     rb_object = rb_obj_alloc(klass);
     rb_grn_object_assign(klass, rb_object, rb_context, context, object);
 
+    switch (object->header.type) {
+    case GRN_TABLE_NO_KEY:
+    case GRN_TABLE_HASH_KEY:
+    case GRN_TABLE_PAT_KEY:
+    case GRN_TABLE_DAT_KEY:
+        rb_grn_context_object_created(rb_context, rb_object);
+    default:
+        break;
+    }
+
     return rb_object;
 }
 

data/ext/groonga/rb-grn-utils.c

@@ -205,6 +205,18 @@ rb_grn_bulk_to_ruby_object_by_range_id (grn_ctx *context, grn_obj *bulk,
       case GRN_DB_BOOL:
         *rb_value = GRN_BOOL_VALUE(bulk) ? Qtrue : Qfalse;
         break;
+      case GRN_DB_INT8:
+        *rb_value = INT2NUM(GRN_INT8_VALUE(bulk));
+        break;
+      case GRN_DB_UINT8:
+        *rb_value = UINT2NUM(GRN_UINT8_VALUE(bulk));
+        break;
+      case GRN_DB_INT16:
+        *rb_value = INT2NUM(GRN_INT16_VALUE(bulk));
+        break;
+      case GRN_DB_UINT16:
+        *rb_value = UINT2NUM(GRN_UINT16_VALUE(bulk));
+        break;
       case GRN_DB_INT32:
         *rb_value = INT2NUM(GRN_INT32_VALUE(bulk));
         break;
@@ -373,9 +385,6 @@ rb_grn_bulk_from_ruby_object (VALUE object, grn_ctx *context, grn_obj *bulk)
         rb_grn_context_text_set(context, bulk, object);
         break;
       case T_FIXNUM:
-        grn_obj_reinit(context, bulk, GRN_DB_INT32, 0);
-        GRN_INT32_SET(context, bulk, NUM2INT(object));
-        break;
       case T_BIGNUM:
         {
             int64_t int64_value;

data/ext/groonga/rb-grn.h

@@ -76,7 +76,7 @@ RB_GRN_BEGIN_DECLS
 
 #define RB_GRN_MAJOR_VERSION 3
 #define RB_GRN_MINOR_VERSION 0
-#define RB_GRN_MICRO_VERSION 4
+#define RB_GRN_MICRO_VERSION 5
 
 #define RB_GRN_QUERY_DEFAULT_MAX_EXPRESSIONS 32
 
@@ -313,6 +313,8 @@ grn_obj       *rb_grn_context_get_backward_compatibility
                                                     (grn_ctx *context,
 						     const char *name,
 						     unsigned int name_size);
+void           rb_grn_context_object_created        (VALUE rb_context,
+						     VALUE rb_object);
 
 const char    *rb_grn_inspect                       (VALUE object);
 const char    *rb_grn_inspect_type                  (unsigned char type);

data/lib/groonga/context.rb

@@ -17,6 +17,8 @@
 
 require "groonga/command"
 
+require "groonga/memory-pool"
+
 module Groonga
   class Context
     # _path_ にある既存のデータベースを開く。ブロックを指定した場
@@ -190,5 +192,125 @@ module Groonga
         yield(buffer.dup, response) if block_given?
       end
     end
+
+    # Pushes a new memory pool to the context. Temporary objects that
+    # are created between pushing a new memory pool and popping the
+    # new memory pool are closed automatically when popping the new
+    # memory pool.
+    #
+    # It is useful for request and response style applications. These
+    # style applications can close temporary objects between a request
+    # and resopnse pair. There are some merits for closing temporary
+    # objects explicilty rather than closing implicitly by GC:
+    #
+    #   * Less memory consumption
+    #   * Faster
+    #
+    # The "less memory consumption" merit is caused by temporary
+    # objects are closed each request and response pair. The max
+    # memory consumption in these applications is the same as the max
+    # memory consumption in a request and response pair. If temporary
+    # objects are closed by GC, the max memory consumption in these
+    # applications is the same as the max memory consumption between
+    # the current GC and the next GC. These applications process many
+    # request and response pairs during two GCs.
+    #
+    # The "faster" merit is caused by reducing GC. You can reduce GC,
+    # your application run faster because GC is a heavy process. You
+    # can reduce GC because memory consumption is reduced.
+    #
+    # You can nest {#push_memory_pool} and {#pop_memory_pool} pair.
+    #
+    # @example Pushes a new memory pool with block
+    #   adults = nil
+    #   context.push_memory_pool do
+    #     users = context["Users"]
+    #     adults = users.select do |user|
+    #       user.age >= 20
+    #     end
+    #     p adults.temporary? # => true
+    #     p adults.closed?    # => false
+    #   end
+    #   p adults.closed?      # => true
+    #
+    # @example Pushes a new memory pool without block
+    #   adults = nil
+    #   context.push_memory_pool
+    #   users = context["Users"]
+    #   adults = users.select do |user|
+    #     user.age >= 20
+    #   end
+    #   p adults.temporary? # => true
+    #   p adults.closed?    # => false
+    #   context.pop_memory_pool
+    #   p adults.closed?    # => true
+    #
+    # @example Nesting push and pop pair
+    #   adults = nil
+    #   context.push_memory_pool do
+    #     users = context["Users"]
+    #     adults = users.select do |user|
+    #       user.age >= 20
+    #     end
+    #     grouped_adults = nil
+    #     context.push_memory_pool do
+    #       grouped_adults = adults.group(["hobby"])
+    #       p grouped_adults.temporary? # => true
+    #       p grouped_adults.closed?    # => false
+    #     end
+    #     p grouped_adults.closed?      # => true
+    #     p adults.temporary?           # => true
+    #     p adults.closed?              # => false
+    #   end
+    #   p adults.closed?                # => true
+    #
+    # @overload push_memory_pool
+    #   Pushes a new memory pool to the context. You need to pop the
+    #   memory pool explicitly by yourself.
+    #
+    #   @return [void]
+    #
+    # @overload push_memory_pool {}
+    #   Closes temporary objects created in the given block
+    #   automatically.
+    #
+    #   @yield []
+    #     Yields the block. Temporary objects created in the block
+    #     are closed automatically when the block is exited.
+    #   @yieldreturn [Object] It is the return value of this
+    #     method call.
+    #   @return [Object] The value returned by the block.
+    #
+    # @since 3.0.5
+    def push_memory_pool
+      memory_pool = MemoryPool.new
+      @memory_pools.push(memory_pool)
+      return unless block_given?
+
+      begin
+        yield
+      ensure
+        pop_memory_pool
+      end
+    end
+
+    # Pops the pushed memory pool.
+    #
+    # @return [void]
+    #
+    # @see push_memory_pool
+    #
+    # @since 3.0.5
+    def pop_memory_pool
+      memory_pool = @memory_pools.pop
+      memory_pool.close
+    end
+
+    # @api private
+    def object_created(object)
+      return if @memory_pools.empty?
+      memory_pool = @memory_pools.last
+      memory_pool.register(object)
+    end
   end
 end

data/lib/groonga/dumper.rb

@@ -123,6 +123,7 @@ module Groonga
     end
 
     def index_only_table?(table)
+      return false if table.columns.empty?
       table.columns.all? do |column|
         column.index?
       end
@@ -414,6 +415,8 @@ module Groonga
             parameters << ":type => :hash"
           when Groonga::PatriciaTrie
             parameters << ":type => :patricia_trie"
+          when Groonga::DoubleArrayTrie
+            parameters << ":type => :double_array_trie"
           end
           if table.domain
             parameters << ":key_type => #{table.domain.name.dump}"
@@ -547,6 +550,8 @@ module Groonga
           flags << "TABLE_HASH_KEY"
         when Groonga::PatriciaTrie
           flags << "TABLE_PAT_KEY"
+        when Groonga::DoubleArrayTrie
+          flags << "TABLE_DAT_KEY"
         end
         if table.domain
           flags << "KEY_NORMALIZE" if default_normalizer?(_normalizer_name)

data/lib/groonga/memory-pool.rb

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Groonga
+  # @api private
+  class MemoryPool
+    def initialize
+      @temporary_objects = {}
+    end
+
+    def register(object)
+      return unless object.temporary?
+      return unless object.is_a?(Groonga::Table)
+      @temporary_objects[object] = true
+    end
+
+    def close
+      @temporary_objects.each do |(object, _)|
+        object.close unless object.closed?
+      end
+      @temporary_objects.clear
+    end
+  end
+end

data/test/test-convert.rb

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+class ConvertTest < Test::Unit::TestCase
+  include GroongaTestUtils
+
+  setup :setup_database
+
+  setup
+  def setup_schema
+  end
+
+  setup
+  def setup_data
+  end
+
+  class Int64Test < self
+    def setup_schema
+      Groonga::Schema.define do |schema|
+        schema.create_table("Values") do |table|
+          table.int64("content")
+        end
+      end
+      @values = Groonga["Values"]
+    end
+
+    data("Fixnum"     => -1,
+         "Max Fixnum" => 2 ** 62 - 1,
+         "Bignum"     => 2 ** 62)
+    def test_select(value)
+      @values.add(:content => value)
+      result = @values.select do |record|
+        record.content == value
+      end
+      assert_equal([value], result.collect(&:content))
+    end
+  end
+end

data/test/test-database-dumper.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2012  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2013  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -64,6 +64,10 @@ class DatabaseDumperTest < Test::Unit::TestCase
     context["Posts"]
   end
 
+  def users
+    context["Users"]
+  end
+
   def dumped_schema
     <<-EOS
 #{dumped_schema_tables}
@@ -338,4 +342,34 @@ column_create Tags Posts_tag_text COLUMN_INDEX Posts tag_text
 COMMAND
     end
   end
+
+  class NoColumnTest < self
+    def setup_tables
+      Groonga::Schema.define do |schema|
+        schema.create_table("Users",
+                            :type => :patricia_trie,
+                            :key_type => "ShortText") do |table|
+        end
+      end
+    end
+
+    setup
+    def setup_data
+      users.add("s-yata")
+      users.add("mori")
+    end
+
+    def test_have_records
+      assert_equal(<<-EOS, dump)
+table_create Users TABLE_PAT_KEY --key_type ShortText
+
+load --table Users
+[
+[\"_key\"],
+[\"mori\"],
+[\"s-yata\"]
+]
+EOS
+    end
+  end
 end

data/test/test-memory-pool.rb

@@ -0,0 +1,88 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+class MemoryPoolTest < Test::Unit::TestCase
+  include GroongaTestUtils
+
+  def setup
+    setup_sandbox
+    setup_database
+    setup_schema
+    setup_data
+  end
+
+  def setup_schema
+    Groonga::Schema.define do |schema|
+      schema.create_table("Users",
+                          :type => :hash,
+                          :key_type => :short_text) do |table|
+        table.uint8(:age)
+        table.short_text(:hobby)
+      end
+    end
+    @users = context["Users"]
+  end
+
+  def setup_data
+    @users.add("mori",   :age => 46, :hobby => "violin")
+    @users.add("s-yata", :age => 28, :hobby => "programming")
+    @users.add("kou",    :age => 31, :hobby => "programming")
+  end
+
+  def teardown
+    teardown_sandbox
+  end
+
+  def test_block
+    adults = nil
+    context.push_memory_pool do
+      adults = @users.select do |user|
+        user.age >= 20
+      end
+      assert_true(adults.temporary?)
+      assert_false(adults.closed?)
+    end
+    assert_true(adults.closed?)
+  end
+
+  def test_not_block
+    context.push_memory_pool
+    adults = @users.select do |user|
+      user.age >= 20
+    end
+    assert_true(adults.temporary?)
+    assert_false(adults.closed?)
+    context.pop_memory_pool
+    assert_true(adults.closed?)
+  end
+
+  def test_nested
+    adults = nil
+    context.push_memory_pool do
+      adults = @users.select do |user|
+        user.age >= 20
+      end
+      grouped_adults = nil
+      context.push_memory_pool do
+        grouped_adults = adults.group(["hobby"])
+        assert_true(grouped_adults.temporary?)
+        assert_false(grouped_adults.closed?)
+      end
+      assert_true(grouped_adults.closed?)
+      assert_false(adults.closed?)
+    end
+    assert_true(adults.closed?)
+  end
+end

data/test/test-schema-dumper.rb

@@ -104,21 +104,31 @@ class SchemaDumperTest < Test::Unit::TestCase
     end
   end
 
+  def define_double_array_trie_schema
+    Groonga::Schema.define do |schema|
+      schema.create_table("Accounts",
+                          :type => :double_array_trie,
+                          :key_type => "ShortText") do |table|
+        table.short_text("name")
+      end
+    end
+  end
+
   class RubySyntaxSchemaDumperTest < SchemaDumperTest
     def test_simple
       define_simple_schema
-      assert_equal(<<-EOS, dump)
+      assert_equal(<<-SCHEMA, dump)
 create_table("Posts",
              :force => true) do |table|
   table.short_text("comments", :type => :vector)
   table.short_text("title")
 end
-EOS
+      SCHEMA
     end
 
     def test_built_in_types
       define_built_in_types_schema
-      assert_equal(<<-EOS, dump)
+      assert_equal(<<-SCHEMA, dump)
 create_table("Posts",
              :force => true) do |table|
   table.long_text("attachment")
@@ -138,12 +148,12 @@ create_table("Posts",
   table.unsigned_integer8("uint8")
   table.float("vote_average")
 end
-EOS
+      SCHEMA
     end
 
     def test_reference_table
       define_reference_table_schema
-      assert_equal(<<-EOS, dump)
+      assert_equal(<<-SCHEMA, dump)
 create_table("Terms",
              :type => :hash,
              :key_type => "ShortText",
@@ -155,12 +165,12 @@ create_table("IndexTerms",
              :key_type => "Terms",
              :force => true) do |table|
 end
-EOS
+      SCHEMA
     end
 
     def test_reference_column
       define_reference_column_schema
-      assert_equal(<<-EOS, dump)
+      assert_equal(<<-SCHEMA, dump)
 create_table("Comments",
              :force => true) do |table|
   table.text("content")
@@ -182,12 +192,12 @@ change_table("Comments") do |table|
   table.reference("children", "Items", :type => :vector)
   table.reference("item", "Items")
 end
-EOS
+      SCHEMA
     end
 
     def test_index
       define_index_schema
-      assert_equal(<<-EOS, dump)
+      assert_equal(<<-SCHEMA, dump)
 create_table("Items",
              :type => :hash,
              :key_type => "ShortText",
@@ -207,7 +217,19 @@ change_table("Terms") do |table|
   table.index("Items", "_key", :name => "Items__key")
   table.index("Items", "title", :name => "Items_title")
 end
-EOS
+      SCHEMA
+    end
+
+    def test_double_array_trie
+      define_double_array_trie_schema
+      assert_equal(<<-SCHEMA, dump)
+create_table("Accounts",
+             :type => :double_array_trie,
+             :key_type => "ShortText",
+             :force => true) do |table|
+  table.short_text("name")
+end
+      SCHEMA
     end
 
     private
@@ -219,25 +241,25 @@ EOS
   class CommandSyntaxSchemaDumperTest < SchemaDumperTest
     def test_simple
       define_simple_schema
-      assert_equal(<<-EOS, dump)
+      assert_equal(<<-SCHEMA, dump)
 table_create Posts TABLE_NO_KEY
 column_create Posts comments COLUMN_VECTOR ShortText
 column_create Posts title COLUMN_SCALAR ShortText
-EOS
+      SCHEMA
     end
 
     def test_reference_table
       define_reference_table_schema
-      assert_equal(<<-EOS, dump)
+      assert_equal(<<-SCHEMA, dump)
 table_create Terms TABLE_HASH_KEY --key_type ShortText
 
 table_create IndexTerms TABLE_HASH_KEY --key_type Terms
-EOS
+      SCHEMA
     end
 
     def test_reference_column
       define_reference_column_schema
-      assert_equal(<<-EOS, dump)
+      assert_equal(<<-SCHEMA, dump)
 table_create Comments TABLE_NO_KEY
 column_create Comments content COLUMN_SCALAR Text
 column_create Comments issued COLUMN_SCALAR Time
@@ -251,12 +273,12 @@ column_create Users name COLUMN_SCALAR ShortText
 column_create Comments author COLUMN_SCALAR Users
 column_create Comments children COLUMN_VECTOR Items
 column_create Comments item COLUMN_SCALAR Items
-EOS
+      SCHEMA
     end
 
     def test_index
       define_index_schema
-      assert_equal(<<-EOS, dump)
+      assert_equal(<<-SCHEMA, dump)
 table_create Items TABLE_HASH_KEY --key_type ShortText
 column_create Items title COLUMN_SCALAR ShortText
 
@@ -264,7 +286,15 @@ table_create Terms TABLE_PAT_KEY|KEY_NORMALIZE --key_type ShortText --default_to
 
 column_create Terms Items__key COLUMN_INDEX|WITH_POSITION Items _key
 column_create Terms Items_title COLUMN_INDEX|WITH_POSITION Items title
-EOS
+      SCHEMA
+    end
+
+    def test_double_array_trie
+      define_double_array_trie_schema
+      assert_equal(<<-SCHEMA, dump)
+table_create Accounts TABLE_DAT_KEY --key_type ShortText
+column_create Accounts name COLUMN_SCALAR ShortText
+      SCHEMA
     end
 
     private

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rroonga
 version: !ruby/object:Gem::Version
-  version: 3.0.4
+  version: 3.0.5
   prerelease: 
 platform: x86-mingw32
 authors:
@@ -13,7 +13,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-04 00:00:00.000000000 Z
+date: 2013-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pkg-config
@@ -224,6 +224,7 @@ files:
 - lib/groonga/index-column.rb
 - lib/groonga/sub-records.rb
 - lib/groonga/view-record.rb
+- lib/groonga/memory-pool.rb
 - lib/groonga/context.rb
 - lib/groonga/grntest-log.rb
 - lib/groonga/pagination.rb
@@ -295,6 +296,7 @@ files:
 - test/groonga-test-utils.rb
 - test/test-pagination.rb
 - test/test-table.rb
+- test/test-memory-pool.rb
 - test/test-variable-size-column.rb
 - test/test-plugin.rb
 - test/test-hash.rb
@@ -330,6 +332,7 @@ files:
 - test/test-schema-dumper.rb
 - test/test-schema-type.rb
 - test/test-table-select-weight.rb
+- test/test-convert.rb
 - test/test-gqtp.rb
 - test/test-context.rb
 - test/test-snippet.rb
@@ -1206,6 +1209,7 @@ test_files:
 - test/groonga-test-utils.rb
 - test/test-pagination.rb
 - test/test-table.rb
+- test/test-memory-pool.rb
 - test/test-variable-size-column.rb
 - test/test-plugin.rb
 - test/test-hash.rb
@@ -1241,6 +1245,7 @@ test_files:
 - test/test-schema-dumper.rb
 - test/test-schema-type.rb
 - test/test-table-select-weight.rb
+- test/test-convert.rb
 - test/test-gqtp.rb
 - test/test-context.rb
 - test/test-snippet.rb