rroonga 2.0.8 → 2.1.0

data/ext/groonga/rb-grn-table.c

@@ -143,7 +143,7 @@ rb_grn_table_initialize (VALUE self)
     return Qnil;
 }
 
-static VALUE
+VALUE
 rb_grn_table_inspect_content (VALUE self, VALUE inspected)
 {
     RbGrnTable *rb_grn_table;
@@ -165,25 +165,6 @@ rb_grn_table_inspect_content (VALUE self, VALUE inspected)
     if (!context)
 	return inspected;
 
-    if (table->header.type != GRN_TABLE_NO_KEY) {
-	grn_obj value;
-	grn_encoding encoding;
-
-	rb_str_cat2(inspected, ", ");
-	rb_str_cat2(inspected, "encoding: <");
-	GRN_OBJ_INIT(&value, GRN_BULK, 0, GRN_ID_NIL);
-	grn_obj_get_info(context, table, GRN_INFO_ENCODING, &value);
-	encoding = *((grn_encoding *)GRN_BULK_HEAD(&value));
-	grn_obj_unlink(context, &value);
-
-	if (context->rc == GRN_SUCCESS)
-	    rb_str_concat(inspected, rb_inspect(GRNENCODING2RVAL(encoding)));
-	else
-	    rb_str_cat2(inspected, "invalid");
-
-	rb_str_cat2(inspected, ">");
-    }
-
     rb_str_cat2(inspected, ", ");
     rb_str_cat2(inspected, "size: <");
     {
@@ -949,7 +930,6 @@ rb_grn_table_each (int argc, VALUE *argv, VALUE self)
     RbGrnTable *rb_table;
     RbGrnObject *rb_grn_object;
     grn_ctx *context = NULL;
-    grn_obj *table;
     grn_table_cursor *cursor;
     VALUE rb_cursor;
     grn_id id;
@@ -969,13 +949,8 @@ rb_grn_table_each (int argc, VALUE *argv, VALUE self)
     return Qnil;
 }
 
-/*
- * テーブルの _id_ に対応するレコードを削除する。
- *
- * @overload delete(id)
- */
 VALUE
-rb_grn_table_delete (VALUE self, VALUE rb_id)
+rb_grn_table_delete_by_id (VALUE self, VALUE rb_id)
 {
     grn_ctx *context = NULL;
     grn_obj *table;
@@ -994,6 +969,97 @@ rb_grn_table_delete (VALUE self, VALUE rb_id)
     return Qnil;
 }
 
+VALUE
+rb_grn_table_delete_by_expression (VALUE self)
+{
+    grn_ctx *context = NULL;
+    grn_obj *table;
+    VALUE rb_builder, rb_expression;
+    grn_obj *needless_records, *expression;
+    grn_operator operator = GRN_OP_OR;
+    grn_table_cursor *cursor;
+
+    rb_grn_table_deconstruct(SELF(self), &table, &context,
+			     NULL, NULL,
+			     NULL, NULL, NULL,
+			     NULL);
+
+    rb_builder = rb_grn_record_expression_builder_new(self, Qnil);
+    rb_expression = rb_grn_record_expression_builder_build(rb_builder);
+    rb_grn_object_deconstruct(RB_GRN_OBJECT(DATA_PTR(rb_expression)),
+                              &expression, NULL,
+			      NULL, NULL, NULL, NULL);
+
+    needless_records =
+	grn_table_create(context, NULL, 0, NULL,
+			 GRN_TABLE_HASH_KEY | GRN_OBJ_WITH_SUBREC,
+			 table,
+			 NULL);
+    if (!needless_records) {
+	rb_grn_context_check(context, self);
+	rb_grn_rc_check(GRN_NO_MEMORY_AVAILABLE, self);
+    }
+
+    grn_table_select(context, table, expression, needless_records, operator);
+    cursor = grn_table_cursor_open(context, needless_records,
+				   NULL, 0,
+				   NULL, 0,
+				   0, -1, 0);
+    if (cursor) {
+	while (grn_table_cursor_next(context, cursor)) {
+	    grn_id *id;
+	    grn_table_cursor_get_key(context, cursor, (void **)&id);
+	    grn_table_delete_by_id(context, table, *id);
+	}
+	grn_table_cursor_close(context, cursor);
+    }
+    grn_obj_unlink(context, needless_records);
+
+    return Qnil;
+}
+
+/*
+ * @overload delete(id)
+ *   Delete a record that has ID @id@.
+ *
+ *   @param id [Integer] The ID of delete target record.
+ *
+ *   @return void
+ *
+ * @overload delete
+ *   Delete records that are matched with the given condition
+ *   specified block.
+ *
+ *   @example Delete users that are younger than 20.
+ *     users.delete do |recod|
+ *       record.age < 20
+ *     end
+ *
+ *   @yield [record]
+ *     TODO: See #select.
+ *   @yieldparam [Groonga::RecodExpressionBuilder] record
+ *     TODO: See #select.
+ *   @yieldreturn [Groonga::ExpressionBuilder]
+ *     TODO: See #select.
+ *
+ *   @return void
+ */
+static VALUE
+rb_grn_table_delete (int argc, VALUE *argv, VALUE self)
+{
+    VALUE rb_id;
+
+    rb_scan_args(argc, argv, "01", &rb_id);
+
+    if (rb_block_given_p()) {
+	rb_grn_table_delete_by_expression(self);
+    } else {
+	rb_grn_table_delete_by_id(self, rb_id);
+    }
+
+    return Qnil;
+}
+
 /*
  * テーブルに登録されているレコードを _keys_ で指定されたルー
  * ルに従ってソートしたレコードの配列を返す。
@@ -1047,7 +1113,35 @@ rb_grn_table_delete (VALUE self, VALUE rb_id)
  *     ソートされたレコードのうち、 _:limit_ 件のみを取り出す。
  *     省略された場合または-1が指定された場合は、全件が指定され
  *     たものとみなす。
- * @return [::Array<Groonga::Record>]
+ *
+ * @return [Groonga::Array] The sorted result. You can get the
+ *   original record by {#value} method of a record in the sorted
+ *   result. Normally, you doesn't need to get the original record
+ *   because you can access via column name method:
+ *
+ *   <pre>
+ *   !!!ruby
+ *   names_recommended_access = sorted_users.collect do |sorted_user|
+ *     sorted_user.name
+ *   end
+ *   names_manually_access = sorted_users.collect do |sorted_user|
+ *     sorted_user.value.name
+ *   end
+ *   names_recommended_access == names_manually_access # => true
+ *   </pre>
+ *
+ *   If you want to access the key of the original record, you need to
+ *   get the original record.
+ *
+ * @note The return value is changed to {Groonga::Array} from
+ *   {::Array} since 2.1.0. If you want to get before 2.1.0 style
+ *   result, use the following code:
+ *
+ *   @example Describe incompatible API change
+ *     result_since_2_1_0 = table.sort(["sort_key"])
+ *     result_before_2_1_0 = result_since_2_1_0.collect do |record|
+ *       record.value
+ *     end
  */
 static VALUE
 rb_grn_table_sort (int argc, VALUE *argv, VALUE self)
@@ -1061,8 +1155,6 @@ rb_grn_table_sort (int argc, VALUE *argv, VALUE self)
     VALUE rb_keys, options;
     VALUE rb_offset, rb_limit;
     VALUE *rb_sort_keys;
-    grn_table_cursor *cursor;
-    VALUE rb_result;
     VALUE exception;
 
     rb_grn_table_deconstruct(SELF(self), &table, &context,
@@ -1150,21 +1242,7 @@ rb_grn_table_sort (int argc, VALUE *argv, VALUE self)
         rb_exc_raise(exception);
     }
 
-    rb_result = rb_ary_new();
-    cursor = grn_table_cursor_open(context, result, NULL, 0, NULL, 0,
-				   0, -1, GRN_CURSOR_ASCENDING);
-    while (grn_table_cursor_next(context, cursor) != GRN_ID_NIL) {
-	void *value;
-	grn_id *id;
-
-	grn_table_cursor_get_value(context, cursor, &value);
-	id = value;
-	rb_ary_push(rb_result, rb_grn_record_new(self, *id, Qnil));
-    }
-    grn_table_cursor_close(context, cursor);
-    grn_obj_unlink(context, result);
-
-    return rb_result;
+    return GRNOBJECT2RVAL(Qnil, context, result, GRN_TRUE);
 }
 
 /*
@@ -2149,7 +2227,7 @@ rb_grn_init_table (VALUE mGrn)
 
     rb_define_method(rb_cGrnTable, "each", rb_grn_table_each, -1);
 
-    rb_define_method(rb_cGrnTable, "delete", rb_grn_table_delete, 1);
+    rb_define_method(rb_cGrnTable, "delete", rb_grn_table_delete, -1);
 
     rb_define_method(rb_cGrnTable, "sort", rb_grn_table_sort, -1);
     rb_define_method(rb_cGrnTable, "group", rb_grn_table_group, -1);

data/ext/groonga/rb-grn.h

@@ -75,8 +75,8 @@ RB_GRN_BEGIN_DECLS
 #endif
 
 #define RB_GRN_MAJOR_VERSION 2
-#define RB_GRN_MINOR_VERSION 0
-#define RB_GRN_MICRO_VERSION 8
+#define RB_GRN_MINOR_VERSION 1
+#define RB_GRN_MICRO_VERSION 0
 
 #define RB_GRN_QUERY_DEFAULT_MAX_EXPRESSIONS 32
 
@@ -252,6 +252,7 @@ RB_GRN_VAR VALUE rb_cGrnExpression;
 RB_GRN_VAR VALUE rb_cGrnRecordExpressionBuilder;
 RB_GRN_VAR VALUE rb_cGrnColumnExpressionBuilder;
 RB_GRN_VAR VALUE rb_cGrnPlugin;
+RB_GRN_VAR VALUE rb_cGrnNormalizer;
 
 void           rb_grn_init_utils                    (VALUE mGrn);
 void           rb_grn_init_exception                (VALUE mGrn);
@@ -292,8 +293,10 @@ void           rb_grn_init_operator                 (VALUE mGrn);
 void           rb_grn_init_expression               (VALUE mGrn);
 void           rb_grn_init_expression_builder       (VALUE mGrn);
 void           rb_grn_init_logger                   (VALUE mGrn);
+void           rb_grn_init_query_logger             (VALUE mGrn);
 void           rb_grn_init_snippet                  (VALUE mGrn);
 void           rb_grn_init_plugin                   (VALUE mGrn);
+void           rb_grn_init_normalizer               (VALUE mGrn);
 
 VALUE          rb_grn_rc_to_exception               (grn_rc rc);
 const char    *rb_grn_rc_to_message                 (grn_rc rc);
@@ -360,6 +363,10 @@ VALUE          rb_grn_object_inspect_object         (VALUE inspected,
 VALUE          rb_grn_object_inspect_object_content (VALUE inspected,
 						     grn_ctx *context,
 						     grn_obj *object);
+VALUE          rb_grn_object_inspect_object_content_name
+                                                    (VALUE inspected,
+						     grn_ctx *context,
+						     grn_obj *object);
 VALUE          rb_grn_object_inspect_header         (VALUE object,
 						     VALUE inspected);
 VALUE          rb_grn_object_inspect_content        (VALUE object,
@@ -398,8 +405,9 @@ void           rb_grn_table_deconstruct             (RbGrnTable *rb_grn_table,
 						     grn_obj **range,
 						     VALUE *columns);
 
-VALUE          rb_grn_table_delete                  (VALUE self,
+VALUE          rb_grn_table_delete_by_id            (VALUE self,
 						     VALUE rb_id);
+VALUE          rb_grn_table_delete_by_expression    (VALUE self);
 VALUE          rb_grn_table_array_reference         (VALUE self,
 						     VALUE rb_id);
 VALUE          rb_grn_table_array_set               (VALUE self,
@@ -428,6 +436,8 @@ VALUE          rb_grn_table_set_column_value        (VALUE self,
 						     VALUE rb_id,
 						     VALUE rb_name,
 						     VALUE rb_value);
+VALUE          rb_grn_table_inspect_content         (VALUE object,
+						     VALUE inspected);
 
 grn_ctx       *rb_grn_table_cursor_ensure_context   (VALUE cursor,
 						     VALUE *rb_context);
@@ -596,6 +606,7 @@ VALUE          rb_grn_column_expression_builder_build
 #define RVAL2GRNOPERATOR(object)      (rb_grn_operator_from_ruby_object(object))
 
 #define RVAL2GRNLOGGER(object)        (rb_grn_logger_from_ruby_object(object))
+#define RVAL2GRNQUERYLOGGER(object)   (rb_grn_query_logger_from_ruby_object(object))
 
 #define RVAL2GRNBULK(object, context, bulk) \
     (rb_grn_bulk_from_ruby_object(object, context, bulk))

data/ext/groonga/rb-groonga.c

@@ -144,6 +144,8 @@ Init_groonga (void)
     rb_grn_init_expression(mGrn);
     rb_grn_init_expression_builder(mGrn);
     rb_grn_init_logger(mGrn);
+    rb_grn_init_query_logger(mGrn);
     rb_grn_init_snippet(mGrn);
     rb_grn_init_plugin(mGrn);
+    rb_grn_init_normalizer(mGrn);
 }

data/lib/groonga/database.rb

@@ -23,5 +23,12 @@ module Groonga
         object.is_a?(Groonga::Table)
       end
     end
+
+    def dump_index(output_directory)
+      each do |object|
+        next unless object.is_a?(Groonga::IndexColumn)
+        object.dump(output_directory)
+      end
+    end
   end
 end

data/lib/groonga/dumper.rb

@@ -340,6 +340,17 @@ module Groonga
         change_table_footer(table)
         @table_defined = true
       end
+
+      def normalizer_name(table)
+        return nil unless table.domain
+        normalizer = table.normalizer
+        return nil if normalizer.nil?
+        normalizer.name
+      end
+
+      def default_normalizer?(normalizer_name)
+        normalizer_name == "NormalizerAuto"
+      end
     end
 
     # @private
@@ -347,6 +358,7 @@ module Groonga
       private
       def create_table_header(table)
         parameters = []
+        _normalizer_name = normalizer_name(table)
         unless table.is_a?(Groonga::Array)
           case table
           when Groonga::Hash
@@ -356,7 +368,7 @@ module Groonga
           end
           if table.domain
             parameters << ":key_type => #{table.domain.name.dump}"
-            if table.normalize_key?
+            if default_normalizer?(_normalizer_name)
               parameters << ":key_normalize => true"
             end
           end
@@ -364,6 +376,9 @@ module Groonga
           if default_tokenizer
             parameters << ":default_tokenizer => #{default_tokenizer.name.dump}"
           end
+          if _normalizer_name and not default_normalizer?(_normalizer_name)
+            parameters << ":normalizer => #{_normalizer_name.dump}"
+          end
         end
         parameters << ":force => true"
         parameters.unshift("")
@@ -475,6 +490,7 @@ module Groonga
       def create_table_header(table)
         parameters = []
         flags = []
+        _normalizer_name = normalizer_name(table)
         case table
         when Groonga::Array
           flags << "TABLE_NO_KEY"
@@ -484,7 +500,7 @@ module Groonga
           flags << "TABLE_PAT_KEY"
         end
         if table.domain
-          flags << "KEY_NORMALIZE" if table.normalize_key?
+          flags << "KEY_NORMALIZE" if default_normalizer?(_normalizer_name)
           if table.is_a?(Groonga::PatriciaTrie) and table.register_key_with_sis?
             flags << "KEY_WITH_SIS"
           end
@@ -502,6 +518,9 @@ module Groonga
             parameters << "--default_tokenizer #{default_tokenizer.name}"
           end
         end
+        if _normalizer_name and not default_normalizer?(_normalizer_name)
+          parameters << "--normalizer #{_normalizer_name}"
+        end
         write("table_create #{table.name} #{parameters.join(' ')}\n")
       end
 

data/lib/groonga/index-column.rb

@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "fileutils"
+require "cgi"
+
+module Groonga
+  class IndexColumn
+    def dump(output_directory)
+      dumper = IndexColumnDumper.new(self, output_directory)
+      dumper.dump
+    end
+  end
+
+  class IndexColumnDumper
+    def initialize(column, output_directory)
+      @column = column
+      @output_directory = output_directory
+      @sources = @column.sources
+    end
+
+    def dump
+      dump_indexes
+    end
+
+    private
+    def dump_indexes
+      @column.table.open_cursor do |table_cursor|
+        @column.open_cursor(table_cursor) do |cursor|
+          postings = []
+          cursor.each do |posting|
+            if postings.empty?
+              postings << posting
+              next
+            end
+
+            current_term_posting = postings.first
+            unless same_term_posting?(current_term_posting, posting)
+              dump_postings(postings)
+              postings.clear
+            end
+
+            postings << posting
+          end
+          dump_postings(postings)
+        end
+      end
+    end
+
+    def same_term_posting?(posting1, posting2)
+      posting1.term_id == posting2.term_id
+    end
+
+    def dump_file_info(posting)
+      items = [
+        "index: #{@column.name}",
+        "term: <#{posting.term.key}>",
+        "domain: #{@column.domain.name}",
+        "range: #{@column.range.name}",
+        "have_section: #{@column.with_section?}",
+        "have_weight: #{@column.with_weight?}",
+        "have_position: #{@column.with_position?}",
+      ]
+      info = items.join("\t")
+      @output.write("#{info}\n")
+    end
+
+    def dump_posting_header
+      header_items = [
+        "weight",
+        "position",
+        "term_frequency",
+        "record",
+      ]
+      header = header_items.join("\t")
+      @output.write("  #{header}\n")
+    end
+
+    def encode_term(term)
+      CGI.escape(term.to_s)
+    end
+
+    def dump_postings(postings)
+      return if postings.empty?
+
+      distinctive_posting = postings.first
+      term = distinctive_posting.term.key
+      encoded_term = encode_term(term)
+      output_dir = File.join(@output_directory, @column.name)
+      output_path = File.join(output_dir, "#{encoded_term}.dump")
+      FileUtils.mkdir_p(output_dir)
+      File.open(output_path, "w") do |output|
+        @output = output
+        dump_file_info(distinctive_posting)
+        dump_posting_header
+        sorted_postings = postings.sort_by do |posting|
+          [source_column_name(posting), record_key(posting), posting.position]
+        end
+        sorted_postings.each do |posting|
+          dump_posting(posting)
+        end
+      end
+    end
+
+    def dump_posting(posting)
+      found_record = "#{posting.table.name}[#{posting.record.record_id}]"
+      posting_info_items = [
+        "#{posting.weight}",
+        "#{posting.position}",
+        "#{posting.term_frequency}",
+        "#{found_record}.#{source_column_name(posting)}",
+      ]
+      posting_info = posting_info_items.join("\t")
+      @output.write("  #{posting_info}\n")
+    end
+
+    def term(posting)
+      posting.term.key
+    end
+
+    def record_key(posting)
+      posting.record.key || default_key(posting)
+    end
+
+    def default_key(posting)
+      type = posting.table.domain
+      return 0 if type.is_a?(Groonga::Table)
+
+      case type.name
+      when "ShortText", "Text", "LongText"
+        ""
+      when "TokyoGeoPoint"
+        Groonga::TokyoGeoPoint.new(0, 0)
+      when "WGS84GeoPoint"
+        Groonga::WGS84GeoPoint.new(0, 0)
+      when "Bool"
+        true
+      when "Time"
+        Time.at(0)
+      else
+        0
+      end
+    end
+
+    def source_column_name(posting)
+      source = @sources[posting.section_id - 1]
+      if source.nil?
+        "<invalid section: #{posting.section_id}>"
+      elsif source.is_a?(Groonga::Table)
+        "_key"
+      else
+        source.local_name
+      end
+    end
+  end
+end

data/lib/groonga/query-logger.rb

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2012  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Groonga
+  class QueryLogger
+    module Flags
+      LABELS = {
+        COMMAND     => "command",
+        RESULT_CODE => "result_code",
+        DESTINATION => "destination",
+        CACHE       => "cache",
+        SIZE        => "size",
+        SCORE       => "score",
+      }
+
+      class << self
+        def parse(input, base_flags)
+          # TODO
+          base_flags
+        end
+
+        def label(flags)
+          labels = []
+          LABELS.each do |flag, label|
+            flags << label if (flags & flag) == flag
+          end
+          labels << "none" if labels.empty?
+          labels.join("|")
+        end
+      end
+    end
+
+    def log(flag, timestamp, info, message)
+      guard do
+        puts("#{timestamp}|#{info}#{message}")
+      end
+    end
+
+    def reopen
+    end
+
+    def fin
+    end
+
+    private
+    def guard
+      begin
+        yield
+      rescue Exception
+        $stderr.puts("#{$!.class}: #{$!.message}")
+        $stderr.puts($@)
+      end
+    end
+  end
+
+  class FileQueryLogger < QueryLogger
+    def initialize(file_name)
+      super()
+      @file = nil
+      @file_name = file_name
+    end
+
+    def reopen
+      guard do
+        return unless @file
+        @file.close
+        @file = nil
+      end
+    end
+
+    def fin
+      guard do
+        return unless @file
+        @file.close
+      end
+    end
+
+    private
+    def ensure_open
+      return if @file
+      @file = File.open(@file_name, "ab")
+    end
+
+    def puts(*arguments)
+      ensure_open
+      @file.puts(*arguments)
+      @file.flush
+    end
+  end
+
+  class CallbackQueryLogger < QueryLogger
+    def initialize(callback)
+      super()
+      @callback = callback
+    end
+
+    def log(flag, timestamp, info, message)
+      guard do
+        @callback.call(:log, flag, timestamp, info, message)
+      end
+    end
+
+    def reopen
+      guard do
+        @callback.call(:reopen)
+      end
+    end
+
+    def fin
+      guard do
+        @callback.call(:fin)
+      end
+    end
+  end
+end