rroonga 4.0.0-x86-mingw32 → 4.0.1-x86-mingw32

data/ext/groonga/rb-grn-variable-size-column.c

@@ -1,6 +1,6 @@
 /* -*- coding: utf-8; mode: C; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
 /*
-  Copyright (C) 2009-2011  Kouhei Sutou <kou@clear-code.com>
+  Copyright (C) 2009-2014  Kouhei Sutou <kou@clear-code.com>
 
   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
@@ -18,16 +18,411 @@
 
 #include "rb-grn.h"
 
-#define SELF(object) ((RbGrnColumn *)DATA_PTR(object))
+#define SELF(object) ((RbGrnVariableSizeColumn *)DATA_PTR(object))
 
 VALUE rb_cGrnVariableSizeColumn;
 
+void
+rb_grn_variable_size_column_bind (RbGrnVariableSizeColumn *rb_column,
+                                  grn_ctx *context, grn_obj *column)
+{
+    RbGrnObject *rb_grn_object;
+    int column_type;
+    unsigned char value_type;
+
+    rb_grn_object = RB_GRN_OBJECT(rb_column);
+    rb_grn_column_bind(RB_GRN_COLUMN(rb_column), context, column);
+
+    rb_column->element_value = NULL;
+    column_type = (column->header.flags & GRN_OBJ_COLUMN_TYPE_MASK);
+    if (column_type != GRN_OBJ_COLUMN_VECTOR) {
+        return;
+    }
+
+    switch (rb_grn_object->range->header.type) {
+    case GRN_TABLE_HASH_KEY:
+    case GRN_TABLE_PAT_KEY:
+    case GRN_TABLE_DAT_KEY:
+    case GRN_TABLE_NO_KEY:
+        value_type = GRN_UVECTOR;
+        break;
+    default:
+        value_type = GRN_VECTOR;
+        break;
+    }
+    if (column->header.flags & GRN_OBJ_WITH_WEIGHT) {
+        rb_column->element_value = grn_obj_open(context, value_type, 0,
+                                                rb_grn_object->range_id);
+    }
+}
+
+void
+rb_grn_variable_size_column_finalizer (grn_ctx *context, grn_obj *grn_object,
+                                       RbGrnVariableSizeColumn *rb_column)
+{
+    rb_grn_column_finalizer(context, grn_object,
+                            RB_GRN_COLUMN(rb_column));
+    if (context && rb_column->element_value)
+        grn_obj_unlink(context, rb_column->element_value);
+    rb_column->element_value = NULL;
+}
+
+static void
+rb_grn_variable_size_column_deconstruct (RbGrnVariableSizeColumn *rb_column,
+                                         grn_obj **column,
+                                         grn_ctx **context,
+                                         grn_id *domain_id,
+                                         grn_obj **domain,
+                                         grn_obj **value,
+                                         grn_obj **element_value,
+                                         grn_id *range_id,
+                                         grn_obj **range)
+{
+    RbGrnColumn *rb_grn_column;
+
+    rb_grn_column = RB_GRN_COLUMN(rb_column);
+    rb_grn_column_deconstruct(rb_grn_column, column, context,
+                              domain_id, domain, value,
+                              range_id, range);
+
+    if (element_value)
+        *element_value = rb_column->element_value;
+}
+
 /*
  * Document-class: Groonga::VariableSizeColumn < Groonga::Column
  *
- * 可変長データ用のカラム。
+ * A column for variable size data like text family types and vector
+ * column.
  */
 
+/*
+ * It gets a value of variable size column value for the record that
+ * ID is _id_.
+ *
+ * @example Gets weight vector value
+ *    Groonga::Schema.define do |schema|
+ *      schema.create_table("Products",
+ *                          :type => :patricia_trie,
+ *                          :key_type => "ShortText") do |table|
+ *        # This is weight vector.
+ *        # ":with_weight => true" is important to store weight value.
+ *        table.short_text("tags",
+ *                         :type => :vector,
+ *                         :with_weight => true)
+ *      end
+ *    end
+ *
+ *    products = Groonga["Products"]
+ *    rroonga = products.add("Rroonga")
+ *    rroonga.tags = [
+ *      {
+ *        :value  => "ruby",
+ *        :weight => 100,
+ *      },
+ *      {
+ *        :value  => "groonga",
+ *        :weight => 10,
+ *      },
+ *    ]
+ *
+ *    p rroonga.tags
+ *    # => [
+ *    #      {:value => "ruby",    :weight => 100},
+ *    #      {:value => "groonga", :weight => 10}
+ *    #    ]
+ *
+ * @overload [](id)
+ *   @param [Integer, Record] id The record ID.
+ *   @return [Array<Hash<Symbol, String>>] An array of value if the column
+ *     is a weight vector column.
+ *     Each value is a Hash like the following form:
+ *
+ *     <pre>
+ *     {
+ *       :value  => [KEY],
+ *       :weight => [WEIGHT],
+ *     }
+ *     </pre>
+ *
+ *     @[KEY]@ is the key of the table that is specified as range on
+ *     creating the weight vector.
+ *
+ *     @[WEIGHT]@ is a positive integer.
+ *
+ *   @return [::Object] See {Groonga::Object#[]} for columns except
+ *     weight vector column.
+ *
+ * @since 4.0.1.
+ */
+static VALUE
+rb_grn_variable_size_column_array_reference (VALUE self, VALUE rb_id)
+{
+    grn_ctx *context = NULL;
+    grn_obj *column, *range;
+    grn_id id;
+    grn_obj *value;
+    VALUE rb_value;
+    unsigned int i, n;
+
+    rb_grn_variable_size_column_deconstruct(SELF(self), &column, &context,
+                                            NULL, NULL, &value, NULL,
+                                            NULL, &range);
+
+    if (!(column->header.flags & GRN_OBJ_WITH_WEIGHT)) {
+        return rb_call_super(1, &rb_id);
+    }
+
+    id = RVAL2GRNID(rb_id, context, range, self);
+
+    grn_obj_reinit(context, value,
+                   value->header.domain,
+                   value->header.flags | GRN_OBJ_VECTOR);
+    grn_obj_get_value(context, column, id, value);
+    rb_grn_context_check(context, self);
+
+    n = grn_vector_size(context, value);
+    rb_value = rb_ary_new2(n);
+    for (i = 0; i < n; i++) {
+        const char *element_value;
+        unsigned int element_value_length;
+        unsigned int weight = 0;
+        grn_id domain;
+        VALUE rb_element;
+
+        element_value_length = grn_vector_get_element(context,
+                                                      value,
+                                                      i,
+                                                      &element_value,
+                                                      &weight,
+                                                      &domain);
+        rb_element = rb_hash_new();
+        rb_hash_aset(rb_element,
+                     ID2SYM(rb_intern("value")),
+                     rb_str_new(element_value, element_value_length));
+        rb_hash_aset(rb_element,
+                     ID2SYM(rb_intern("weight")),
+                     UINT2NUM(weight));
+
+        rb_ary_push(rb_value, rb_element);
+    }
+
+    return rb_value;
+}
+
+typedef struct {
+    grn_ctx *context;
+    grn_obj *vector;
+    grn_obj *element_value;
+} HashElementToVectorElementData;
+
+static int
+hash_element_to_vector_element(VALUE key, VALUE value, VALUE user_data)
+{
+    HashElementToVectorElementData *data =
+        (HashElementToVectorElementData *)user_data;
+    unsigned int weight;
+
+    GRN_BULK_REWIND(data->element_value);
+    RVAL2GRNBULK(key, data->context, data->element_value);
+
+    weight = NUM2UINT(value);
+    grn_vector_add_element(data->context, data->vector,
+                           GRN_BULK_HEAD(data->element_value),
+                           GRN_BULK_VSIZE(data->element_value),
+                           weight,
+                           data->element_value->header.domain);
+
+    return ST_CONTINUE;
+}
+
+/*
+ * It updates a value of variable size column value for the record
+ * that ID is _id_.
+ *
+ * Weight vector column is a special variable size column. This
+ * description describes only weight vector column. Other variable
+ * size column works what you think.
+ *
+ * @example Use weight vector as matrix search result weight
+ *    Groonga::Schema.define do |schema|
+ *      schema.create_table("Products",
+ *                          :type => :patricia_trie,
+ *                          :key_type => "ShortText") do |table|
+ *        # This is weight vector.
+ *        # ":with_weight => true" is important for matrix search result weight.
+ *        table.short_text("tags",
+ *                         :type => :vector,
+ *                         :with_weight => true)
+ *      end
+ *
+ *      schema.create_table("Tags",
+ *                          :type => :hash,
+ *                          :key_type => "ShortText") do |table|
+ *        # This is inverted index. It also needs ":with_weight => true".
+ *        table.index("Products.tags", :with_weight => true)
+ *      end
+ *    end
+ *
+ *    products = Groonga["Products"]
+ *    groonga = products.add("Groonga")
+ *    groonga.tags = [
+ *      {
+ *        :value  => "groonga",
+ *        :weight => 100,
+ *      },
+ *    ]
+ *    rroonga = products.add("Rroonga")
+ *    rroonga.tags = [
+ *      {
+ *        :value  => "ruby",
+ *        :weight => 100,
+ *      },
+ *      {
+ *        :value  => "groonga",
+ *        :weight => 10,
+ *      },
+ *    ]
+ *
+ *    result = products.select do |record|
+ *      # Search by "groonga"
+ *      record.match("groonga") do |match_target|
+ *        match_target.tags
+ *      end
+ *    end
+ *
+ *    result.each do |record|
+ *      p [record.key.key, record.score]
+ *    end
+ *    # Matches all records with weight.
+ *    # => ["Groonga", 101]
+ *    #    ["Rroonga", 11]
+ *
+ *    # Increases score for "ruby" 10 times
+ *    products.select(# The previous search result. Required.
+ *                    :result => result,
+ *                    # It just adds score to existing records in the result. Required.
+ *                    :operator => Groonga::Operator::ADJUST) do |record|
+ *      record.match("ruby") do |target|
+ *        target.tags * 10 # 10 times
+ *      end
+ *    end
+ *
+ *    result.each do |record|
+ *      p [record.key.key, record.score]
+ *    end
+ *    # Weight is used for increasing score.
+ *    # => ["Groonga", 101]  <- Not changed.
+ *    #    ["Rroonga", 1021] <- 1021 (= 101 * 10 + 1) increased.
+ *
+ * @overload []=(id, elements)
+ *   This description is for weight vector column.
+ *
+ *   @param [Integer, Record] id The record ID.
+ *   @param [Array<Hash<Symbol, String>>] elements An array of values
+ *     for weight vector.
+ *     Each value is a Hash like the following form:
+ *
+ *     <pre>
+ *     {
+ *       :value  => [KEY],
+ *       :weight => [WEIGHT],
+ *     }
+ *     </pre>
+ *
+ *     @[KEY]@ must be the same type of the key of the table that is
+ *     specified as range on creating the weight vector.
+ *
+ *     @[WEIGHT]@ must be an positive integer. Note that search
+ *     becomes @weight + 1@. It means that You want to get 10 as
+ *     score, you should set 9 as weight.
+ *
+ * @overload []=(id, value)
+ *   This description is for variable size columns except weight
+ *   vector column.
+ *
+ *   @param [Integer, Record] id The record ID.
+ *   @param [::Object] value A new value.
+ *   @see Groonga::Object#[]=
+ *
+ * @since 4.0.1
+ */
+static VALUE
+rb_grn_variable_size_column_array_set (VALUE self, VALUE rb_id, VALUE rb_value)
+{
+    grn_ctx *context = NULL;
+    grn_obj *column, *range;
+    grn_rc rc;
+    grn_id id;
+    grn_obj *value, *element_value;
+    int flags = GRN_OBJ_SET;
+
+    rb_grn_variable_size_column_deconstruct(SELF(self), &column, &context,
+                                            NULL, NULL, &value, &element_value,
+                                            NULL, &range);
+
+    if (!(column->header.flags & GRN_OBJ_WITH_WEIGHT)) {
+        VALUE args[2];
+        args[0] = rb_id;
+        args[1] = rb_value;
+        return rb_call_super(2, args);
+    }
+
+    id = RVAL2GRNID(rb_id, context, range, self);
+
+    grn_obj_reinit(context, value,
+                   value->header.domain,
+                   value->header.flags | GRN_OBJ_VECTOR);
+    if (RVAL2CBOOL(rb_obj_is_kind_of(rb_value, rb_cArray))) {
+        int i, n;
+        n = RARRAY_LEN(rb_value);
+        for (i = 0; i < n; i++) {
+            unsigned int weight = 0;
+            VALUE rb_element_value, rb_weight;
+
+            rb_grn_scan_options(RARRAY_PTR(rb_value)[i],
+                                "value", &rb_element_value,
+                                "weight", &rb_weight,
+                                NULL);
+
+            if (!NIL_P(rb_weight)) {
+                weight = NUM2UINT(rb_weight);
+            }
+
+            GRN_BULK_REWIND(element_value);
+            if (!NIL_P(rb_element_value)) {
+                RVAL2GRNBULK(rb_element_value, context, element_value);
+            }
+
+            grn_vector_add_element(context, value,
+                                   GRN_BULK_HEAD(element_value),
+                                   GRN_BULK_VSIZE(element_value),
+                                   weight,
+                                   element_value->header.domain);
+        }
+    } else if (RVAL2CBOOL(rb_obj_is_kind_of(rb_value, rb_cHash))) {
+        HashElementToVectorElementData data;
+        data.context = context;
+        data.vector = value;
+        data.element_value = element_value;
+        rb_hash_foreach(rb_value, hash_element_to_vector_element, (VALUE)&data);
+    } else {
+        rb_raise(rb_eArgError,
+                 "<%s>: "
+                 "weight vector value must be an array of index value or "
+                 "a hash that key is vector value and value is vector weight: "
+                 "<%s>",
+                 rb_grn_inspect(self),
+                 rb_grn_inspect(rb_value));
+    }
+
+    rc = grn_obj_set_value(context, column, id, value, flags);
+    rb_grn_context_check(context, self);
+    rb_grn_rc_check(rc, self);
+
+    return rb_value;
+}
+
 /*
  * Returns whether the column is compressed or not. If
  * @type@ is specified, it returns whether the column is
@@ -42,7 +437,7 @@ VALUE rb_cGrnVariableSizeColumn;
 static VALUE
 rb_grn_variable_size_column_compressed_p (int argc, VALUE *argv, VALUE self)
 {
-    RbGrnColumn *rb_grn_column;
+    RbGrnVariableSizeColumn *rb_grn_column;
     grn_ctx *context = NULL;
     grn_obj *column;
     grn_obj_flags flags;
@@ -112,7 +507,7 @@ rb_grn_variable_size_column_compressed_p (int argc, VALUE *argv, VALUE self)
 static VALUE
 rb_grn_variable_size_column_defrag (int argc, VALUE *argv, VALUE self)
 {
-    RbGrnColumn *rb_grn_column;
+    RbGrnVariableSizeColumn *rb_grn_column;
     grn_ctx *context = NULL;
     grn_obj *column;
     int n_segments;
@@ -143,6 +538,11 @@ rb_grn_init_variable_size_column (VALUE mGrn)
     rb_cGrnVariableSizeColumn =
         rb_define_class_under(mGrn, "VariableSizeColumn", rb_cGrnColumn);
 
+    rb_define_method(rb_cGrnVariableSizeColumn, "[]",
+                     rb_grn_variable_size_column_array_reference, 1);
+    rb_define_method(rb_cGrnVariableSizeColumn, "[]=",
+                     rb_grn_variable_size_column_array_set, 2);
+
     rb_define_method(rb_cGrnVariableSizeColumn, "compressed?",
                      rb_grn_variable_size_column_compressed_p, -1);
     rb_define_method(rb_cGrnVariableSizeColumn, "defrag",

data/ext/groonga/rb-grn.h

@@ -84,7 +84,7 @@ RB_GRN_BEGIN_DECLS
 
 #define RB_GRN_MAJOR_VERSION 4
 #define RB_GRN_MINOR_VERSION 0
-#define RB_GRN_MICRO_VERSION 0
+#define RB_GRN_MICRO_VERSION 1
 
 #define RB_GRN_QUERY_DEFAULT_MAX_EXPRESSIONS 32
 
@@ -96,6 +96,7 @@ RB_GRN_BEGIN_DECLS
 #define RB_GRN_TABLE_KEY_SUPPORT(object) ((RbGrnTableKeySupport *)(object))
 #define RB_GRN_TABLE_CURSOR(object) ((RbGrnTableCursort *)(object))
 #define RB_GRN_COLUMN(object) ((RbGrnColumn *)(object))
+#define RB_GRN_VARIABLE_SIZE_COLUMN(object) ((RbGrnVariableSizeColumn *)(object))
 #define RB_GRN_INDEX_COLUMN(object) ((RbGrnIndexColumn *)(object))
 #define RB_GRN_ACCESSOR(object) ((RbGrnAccessor *)(object))
 #define RB_GRN_EXPRESSION(object) ((RbGrnExpression *)(object))
@@ -159,11 +160,19 @@ struct _RbGrnColumn
     grn_obj *value;
 };
 
+typedef struct _RbGrnVariableSizeColumn RbGrnVariableSizeColumn;
+struct _RbGrnVariableSizeColumn
+{
+    RbGrnColumn parent;
+    grn_obj *element_value;
+};
+
 typedef struct _RbGrnIndexColumn RbGrnIndexColumn;
 struct _RbGrnIndexColumn
 {
     RbGrnColumn parent;
     grn_obj *old_value;
+    grn_obj *set_value;
     grn_obj *id_query;
     grn_obj *string_query;
 };
@@ -490,6 +499,13 @@ void           rb_grn_column_deconstruct            (RbGrnColumn *rb_grn_column,
                                                      grn_id *range_id,
                                                      grn_obj **range);
 
+void           rb_grn_variable_size_column_bind     (RbGrnVariableSizeColumn *rb_grn_column,
+                                                     grn_ctx *context,
+                                                     grn_obj *column);
+void           rb_grn_variable_size_column_finalizer(grn_ctx *context,
+                                                     grn_obj *column,
+                                                     RbGrnVariableSizeColumn *rb_grn_column);
+
 void           rb_grn_index_column_bind             (RbGrnIndexColumn *rb_grn_index_column,
                                                      grn_ctx *context,
                                                      grn_obj *object);
@@ -503,6 +519,7 @@ void           rb_grn_index_column_deconstruct      (RbGrnIndexColumn *rb_grn_in
                                                      grn_obj **domain,
                                                      grn_obj **value,
                                                      grn_obj **old_value,
+                                                     grn_obj **set_value,
                                                      grn_id *range_id,
                                                      grn_obj **range,
                                                      grn_obj **id_query,

data/lib/groonga/dumper.rb

@@ -395,10 +395,6 @@ module Groonga
         return nil if normalizer.nil?
         normalizer.name
       end
-
-      def default_normalizer?(normalizer_name)
-        normalizer_name == "NormalizerAuto"
-      end
     end
 
     # @private
@@ -418,15 +414,12 @@ module Groonga
           end
           if table.domain
             parameters << ":key_type => #{table.domain.name.dump}"
-            if default_normalizer?(_normalizer_name)
-              parameters << ":key_normalize => true"
-            end
           end
           default_tokenizer = table.default_tokenizer
           if default_tokenizer
             parameters << ":default_tokenizer => #{default_tokenizer.name.dump}"
           end
-          if _normalizer_name and not default_normalizer?(_normalizer_name)
+          if _normalizer_name
             parameters << ":normalizer => #{_normalizer_name.dump}"
           end
         end
@@ -460,9 +453,11 @@ module Groonga
         name = column.local_name
         reference = column.range
         options = column_options(column)
-        arguments = [dump_object(name),
-                     dump_object(reference.name),
-                     options].compact.join(", ")
+        arguments = [
+          dump_object(name),
+          dump_object(reference.name),
+          options,
+        ].compact.join(", ")
         write("  table.reference(#{arguments})\n")
       end
 
@@ -477,9 +472,17 @@ module Groonga
           end
           dump_object(name)
         end.join(", ")
-        arguments = [dump_object(target_table_name),
-                     sources.size == 1 ? source_names : "[#{source_names}]",
-                     ":name => #{dump_object(column.local_name)}"]
+        options = {
+          :name => column.local_name,
+        }
+        options[:with_section]  = true if column.with_section?
+        options[:with_weight]   = true if column.with_weight?
+        options[:with_position] = true if column.with_position?
+        arguments = [
+          dump_object(target_table_name),
+          sources.size == 1 ? source_names : "[#{source_names}]",
+          dump_options(options),
+        ]
         write("  table.index(#{arguments.join(', ')})\n")
       end
 
@@ -514,15 +517,19 @@ module Groonga
       def column_options(column)
         options = {}
         options[:type] = :vector if column.vector?
+        options[:with_weight] = true if column.with_weight?
         return nil if options.empty?
 
-        dumped_options = ""
-        options.each do |key, value|
+        dump_options(options)
+      end
+
+      def dump_options(options)
+        dumped_options = options.collect do |key, value|
           dumped_key = dump_object(key)
           dumped_value = dump_object(value)
-          dumped_options << "#{dumped_key} => #{dumped_value}"
+          "#{dumped_key} => #{dumped_value}"
         end
-        dumped_options
+        dumped_options.join(", ")
       end
 
       def dump_object(object)
@@ -552,7 +559,6 @@ module Groonga
           flags << "TABLE_DAT_KEY"
         end
         if table.domain
-          flags << "KEY_NORMALIZE" if default_normalizer?(_normalizer_name)
           if table.is_a?(Groonga::PatriciaTrie) and table.register_key_with_sis?
             flags << "KEY_WITH_SIS"
           end
@@ -570,7 +576,7 @@ module Groonga
             parameters << "--default_tokenizer #{default_tokenizer.name}"
           end
         end
-        if _normalizer_name and not default_normalizer?(_normalizer_name)
+        if _normalizer_name
           parameters << "--normalizer #{_normalizer_name}"
         end
         write("table_create #{table.name} #{parameters.join(' ')}\n")
@@ -595,6 +601,7 @@ module Groonga
         elsif column.vector?
           flags << "COLUMN_VECTOR"
         end
+        flags << "WITH_WEIGHT" if column.with_weight?
         # TODO: support COMPRESS_ZLIB and COMPRESS_LZO?
         parameters << "#{flags.join('|')}"
         parameters << "#{column.range.name}"
@@ -687,8 +694,12 @@ module Groonga
     def resolve_value(record, column, value)
       case value
       when ::Array
-        value.collect do |v|
-          resolve_value(record, column, v)
+        if column.vector? and column.with_weight?
+          resolve_weight_vector_value(record, column, value)
+        else
+          value.collect do |v|
+            resolve_value(record, column, v)
+          end
         end
       when Groonga::Record
         if value.support_key?
@@ -726,6 +737,19 @@ module Groonga
       end
     end
 
+    def resolve_weight_vector_value(record, column, entries)
+      resolved_weight_vector_entries = {}
+      sorted_entries = entries.sort_by do |entry|
+        entry[:value]
+      end
+      sorted_entries.each do |entry|
+        resolved_value = resolve_value(record, column, entry[:value])
+        resolved_weight = resolve_value(record, column, entry[:weight])
+        resolved_weight_vector_entries[resolved_value] = resolved_weight
+      end
+      resolved_weight_vector_entries
+    end
+
     def fix_encoding(value)
       if value.encoding == ::Encoding::ASCII_8BIT
         value.force_encoding(@table.context.ruby_encoding)

data/lib/groonga/expression-builder.rb

@@ -79,10 +79,8 @@ module Groonga
         end
       end
 
-      if builders.empty? or builders == [self]
-        expression.append_constant(1)
-        expression.append_constant(1)
-        expression.append_operation(Groonga::Operation::OR, 2)
+      if builders.empty?
+        expression.append_constant(true)
       else
         combined_builder = builders.inject(nil) do |previous, builder|
           if previous.nil?