rroonga 4.0.5 → 4.0.6

data/ext/groonga/rb-grn-table-key-support.c

@@ -1,7 +1,7 @@
 /* -*- coding: utf-8; mode: C; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
 /*
   Copyright (C) 2014  Masafumi Yokoyama <myokoym@gmail.com>
-  Copyright (C) 2009-2013  Kouhei Sutou <kou@clear-code.com>
+  Copyright (C) 2009-2014  Kouhei Sutou <kou@clear-code.com>
 
   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
@@ -144,6 +144,29 @@ rb_grn_table_key_support_inspect_content (VALUE self, VALUE inspected)
         }
     }
 
+    {
+        int i, n;
+        grn_obj token_filters;
+
+        rb_str_cat2(inspected, ", ");
+        rb_str_cat2(inspected, "token_filters: [");
+
+        GRN_PTR_INIT(&token_filters, GRN_OBJ_VECTOR, GRN_ID_NIL);
+        grn_obj_get_info(context, table,
+                         GRN_INFO_TOKEN_FILTERS,
+                         &token_filters);
+        n = GRN_BULK_VSIZE(&token_filters) / sizeof(grn_obj *);
+        for (i = 0; i < n; i++) {
+            grn_obj *token_filter = GRN_PTR_VALUE_AT(&token_filters, i);
+            if (i > 0) {
+                rb_str_cat2(inspected, ", ");
+            }
+            rb_grn_object_inspect_object_content_name(inspected, context,
+                                                      token_filter);
+        }
+        rb_str_cat2(inspected, "]");
+    }
+
     {
         grn_obj *normalizer;
 
@@ -784,6 +807,69 @@ rb_grn_table_key_support_set_default_tokenizer (VALUE self, VALUE rb_tokenizer)
     return Qnil;
 }
 
+/*
+ * Returns the token filters that are used by {Groonga::IndexColumn}.
+ *
+ * @overload token_filters
+ *   @return [::Array<Groonga::Procedure>]
+ */
+static VALUE
+rb_grn_table_key_support_get_token_filters (VALUE self)
+{
+    grn_ctx *context = NULL;
+    grn_obj *table;
+    grn_obj token_filters;
+    VALUE rb_token_filters;
+
+    rb_grn_table_key_support_deconstruct(SELF(self), &table, &context,
+                                         NULL, NULL, NULL,
+                                         NULL, NULL, NULL,
+                                         NULL);
+
+    GRN_PTR_INIT(&token_filters, GRN_VECTOR, GRN_ID_NIL);
+    grn_obj_get_info(context, table, GRN_INFO_TOKEN_FILTERS,
+                     &token_filters);
+    rb_token_filters = GRNPVECTOR2RVAL(context, &token_filters);
+    rb_grn_context_check(context, self);
+
+    return rb_token_filters;
+}
+
+/*
+ * Sets token filters that used in {Groonga::IndexColumn}.
+ *
+ * @example
+ *   # Use "TokenFilterStem" and "TokenfilterStopWord"
+ *   table.token_filters = ["TokenFilterStem", "TokenFilterStopWord"]
+ *
+ * @overload token_filters=(token_filters)
+ *   @param token_filters [::Array<String>] Token filter names.
+ */
+static VALUE
+rb_grn_table_key_support_set_token_filters (VALUE self,
+                                            VALUE rb_token_filters)
+{
+    grn_ctx *context;
+    grn_obj *table;
+    grn_obj token_filters;
+    grn_rc rc;
+
+    rb_grn_table_key_support_deconstruct(SELF(self), &table, &context,
+                                         NULL, NULL, NULL,
+                                         NULL, NULL, NULL,
+                                         NULL);
+
+    GRN_PTR_INIT(&token_filters, GRN_OBJ_VECTOR, GRN_ID_NIL);
+    RVAL2GRNPVECTOR(rb_token_filters, context, &token_filters);
+    rc = grn_obj_set_info(context, table,
+                          GRN_INFO_TOKEN_FILTERS, &token_filters);
+    grn_obj_unlink(context, &token_filters);
+    rb_grn_context_check(context, self);
+    rb_grn_rc_check(rc, self);
+
+    return Qnil;
+}
+
 /*
  * Returns the normalizer that is used by {Groonga::IndexColumn}.
  *
@@ -976,6 +1062,11 @@ rb_grn_init_table_key_support (VALUE mGrn)
     rb_define_method(rb_mGrnTableKeySupport, "default_tokenizer=",
                      rb_grn_table_key_support_set_default_tokenizer, 1);
 
+    rb_define_method(rb_mGrnTableKeySupport, "token_filters",
+                     rb_grn_table_key_support_get_token_filters, 0);
+    rb_define_method(rb_mGrnTableKeySupport, "token_filters=",
+                     rb_grn_table_key_support_set_token_filters, 1);
+
     rb_define_method(rb_mGrnTableKeySupport, "normalizer",
                      rb_grn_table_key_support_get_normalizer, 0);
     rb_define_method(rb_mGrnTableKeySupport, "normalizer=",

data/ext/groonga/rb-grn-table.c

@@ -1,6 +1,7 @@
 /* -*- coding: utf-8; mode: C; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
 /*
   Copyright (C) 2009-2014  Kouhei Sutou <kou@clear-code.com>
+  Copyright (C) 2014  Masafumi Yokoyama <myokoym@gmail.com>
 
   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
@@ -235,7 +236,7 @@ rb_grn_table_inspect (VALUE self)
  *     値の圧縮方法を指定する。省略した場合は、圧縮しない。
  *
  *     - +:zlib+ := 値をzlib圧縮して格納する。
- *     - +:lzo+ := 値をlzo圧縮して格納する。
+ *     - +:lz4+ := 値をLZ4圧縮して格納する。
  *
  * @return [Groonga::FixSizeColumn or Groonga::VariableSizeColumn]
  */
@@ -313,11 +314,14 @@ rb_grn_table_define_column (int argc, VALUE *argv, VALUE self)
     } else if (rb_grn_equal_option(rb_compress, "zlib")) {
         flags |= GRN_OBJ_COMPRESS_ZLIB;
     } else if (rb_grn_equal_option(rb_compress, "lzo")) {
-        flags |= GRN_OBJ_COMPRESS_LZO;
+        /* TODO: for backward compatibility */
+        flags |= GRN_OBJ_COMPRESS_LZ4;
+    } else if (rb_grn_equal_option(rb_compress, "lz4")) {
+        flags |= GRN_OBJ_COMPRESS_LZ4;
     } else {
         rb_raise(rb_eArgError,
                  "invalid compress type: %s: "
-                 "available types: [:zlib, :lzo, nil]",
+                 "available types: [:zlib, :lz4, nil]",
                  rb_grn_inspect(rb_compress));
     }
 
@@ -924,6 +928,50 @@ rb_grn_table_truncate (VALUE self)
     return Qnil;
 }
 
+typedef struct {
+    grn_ctx *context;
+    grn_table_cursor *cursor;
+    VALUE self;
+} EachData;
+
+static VALUE
+rb_grn_table_each_body (VALUE user_data)
+{
+    EachData *data = (EachData *)user_data;
+    grn_ctx *context = data->context;
+    grn_table_cursor *cursor = data->cursor;
+    VALUE self = data->self;
+    RbGrnObject *rb_grn_object;
+
+    rb_grn_object = RB_GRN_OBJECT(SELF(self));
+    while (GRN_TRUE) {
+        grn_id id;
+
+        if (!rb_grn_object->object) {
+            break;
+        }
+
+        id = grn_table_cursor_next(context, cursor);
+        if (id == GRN_ID_NIL) {
+            break;
+        }
+
+        rb_yield(rb_grn_record_new(self, id, Qnil));
+    }
+
+    return Qnil;
+}
+
+static VALUE
+rb_grn_table_each_ensure (VALUE user_data)
+{
+    EachData *data = (EachData *)user_data;
+
+    grn_table_cursor_close(data->context, data->cursor);
+
+    return Qnil;
+}
+
 /*
  * テーブルに登録されているレコードを順番にブロックに渡す。
  *
@@ -940,24 +988,19 @@ rb_grn_table_truncate (VALUE self)
 static VALUE
 rb_grn_table_each (int argc, VALUE *argv, VALUE self)
 {
-    RbGrnTable *rb_table;
-    RbGrnObject *rb_grn_object;
-    grn_ctx *context = NULL;
-    grn_table_cursor *cursor;
-    VALUE rb_cursor;
-    grn_id id;
+    EachData data;
 
     RETURN_ENUMERATOR(self, argc, argv);
 
-    cursor = rb_grn_table_open_grn_cursor(argc, argv, self, &context);
-    rb_cursor = GRNTABLECURSOR2RVAL(Qnil, context, cursor);
-    rb_table = SELF(self);
-    rb_grn_object = RB_GRN_OBJECT(rb_table);
-    while (rb_grn_object->object &&
-           (id = grn_table_cursor_next(context, cursor)) != GRN_ID_NIL) {
-        rb_yield(rb_grn_record_new(self, id, Qnil));
+    data.cursor = rb_grn_table_open_grn_cursor(argc, argv, self,
+                                               &(data.context));
+    if (!data.cursor) {
+        return Qnil;
     }
-    rb_grn_object_close(rb_cursor);
+
+    data.self = self;
+    rb_ensure(rb_grn_table_each_body, (VALUE)&data,
+              rb_grn_table_each_ensure, (VALUE)&data);
 
     return Qnil;
 }

data/ext/groonga/rb-grn-utils.c

@@ -624,14 +624,61 @@ rb_grn_bulk_from_ruby_object_with_type (VALUE object, grn_ctx *context,
     }
 
     if (!bulk) {
-        bulk = grn_obj_open(context, GRN_BULK, flags, GRN_ID_NIL);
+        bulk = grn_obj_open(context, GRN_BULK, flags, type_id);
         rb_grn_context_check(context, object);
     }
+    if (bulk->header.domain != type_id) {
+        grn_obj_reinit(context, bulk, type_id, 0);
+    }
     GRN_TEXT_SET(context, bulk, string, size);
 
     return bulk;
 }
 
+VALUE
+rb_grn_pvector_to_ruby_object (grn_ctx *context, grn_obj *pvector)
+{
+    VALUE array;
+    unsigned int i, n;
+
+    if (!pvector)
+        return Qnil;
+
+    n = GRN_BULK_VSIZE(pvector) / sizeof(grn_obj *);
+    array = rb_ary_new2(n);
+    for (i = 0; i < n; i++) {
+        grn_obj *object = GRN_PTR_VALUE_AT(pvector, i);
+
+        rb_ary_push(array, GRNOBJECT2RVAL(Qnil, context, object, GRN_FALSE));
+    }
+
+    return array;
+}
+
+grn_obj *
+rb_grn_pvector_from_ruby_object (VALUE object,
+                                 grn_ctx *context,
+                                 grn_obj *pvector)
+{
+    int i, n;
+    VALUE array;
+
+    if (NIL_P(object))
+        return pvector;
+
+    array = rb_grn_convert_to_array(object);
+
+    n = RARRAY_LEN(array);
+    for (i = 0; i < n; i++) {
+        VALUE rb_value = RARRAY_PTR(array)[i];
+        grn_obj *value;
+
+        value = RVAL2GRNOBJECT(rb_value, &context);
+        GRN_PTR_PUT(context, pvector, value);
+    }
+
+    return pvector;
+}
 
 VALUE
 rb_grn_vector_to_ruby_object (grn_ctx *context, grn_obj *vector)
@@ -1172,7 +1219,9 @@ rb_grn_obj_to_ruby_object (VALUE klass, grn_ctx *context,
         break;
     /* case GRN_PTR: */
     /* case GRN_UVECTOR: */
-    /* case GRN_PVECTOR: */
+    case GRN_PVECTOR:
+        return GRNPVECTOR2RVAL(context, obj);
+        break;
     case GRN_VECTOR:
         return GRNVECTOR2RVAL(context, obj);
         break;

data/ext/groonga/rb-grn-variable-size-column.c

@@ -1,6 +1,7 @@
 /* -*- coding: utf-8; mode: C; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
 /*
   Copyright (C) 2009-2014  Kouhei Sutou <kou@clear-code.com>
+  Copyright (C) 2014  Masafumi Yokoyama <myokoym@gmail.com>
 
   This library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
@@ -458,7 +459,7 @@ rb_grn_variable_size_column_array_set (VALUE self, VALUE rb_id, VALUE rb_value)
  * @overload compressed?
  *   @return [Boolean] whether the column is compressed or not.
  * @overload compressed?(type)
- *   @param [:zlib, :lzo] type (nil)
+ *   @param [:zlib, :lz4] type (nil)
  *   @return [Boolean] whether specified compressed type is used or not.
  * @since 1.3.1
  */
@@ -473,7 +474,7 @@ rb_grn_variable_size_column_compressed_p (int argc, VALUE *argv, VALUE self)
     grn_bool compressed_p = GRN_FALSE;
     grn_bool accept_any_type = GRN_FALSE;
     grn_bool need_zlib_check = GRN_FALSE;
-    grn_bool need_lzo_check = GRN_FALSE;
+    grn_bool need_lz4_check = GRN_FALSE;
 
     rb_scan_args(argc, argv, "01", &type);
 
@@ -483,10 +484,13 @@ rb_grn_variable_size_column_compressed_p (int argc, VALUE *argv, VALUE self)
         if (rb_grn_equal_option(type, "zlib")) {
             need_zlib_check = GRN_TRUE;
         } else if (rb_grn_equal_option(type, "lzo")) {
-            need_lzo_check = GRN_TRUE;
+            /* TODO: for backward compatibility */
+            need_lz4_check = GRN_TRUE;
+        } else if (rb_grn_equal_option(type, "lz4")) {
+            need_lz4_check = GRN_TRUE;
         } else {
             rb_raise(rb_eArgError,
-                     "compressed type should be <:zlib> or <:lzo>: <%s>",
+                     "compressed type should be <:zlib> or <:lz4>: <%s>",
                      rb_grn_inspect(type));
         }
     }
@@ -506,11 +510,11 @@ rb_grn_variable_size_column_compressed_p (int argc, VALUE *argv, VALUE self)
             compressed_p = GRN_BOOL_VALUE(&support_p);
         }
         break;
-      case GRN_OBJ_COMPRESS_LZO:
-        if (accept_any_type || need_lzo_check) {
+      case GRN_OBJ_COMPRESS_LZ4:
+        if (accept_any_type || need_lz4_check) {
             grn_obj support_p;
             GRN_BOOL_INIT(&support_p, 0);
-            grn_obj_get_info(context, NULL, GRN_INFO_SUPPORT_LZO, &support_p);
+            grn_obj_get_info(context, NULL, GRN_INFO_SUPPORT_LZ4, &support_p);
             compressed_p = GRN_BOOL_VALUE(&support_p);
         }
         break;

data/ext/groonga/rb-grn.h

@@ -92,7 +92,7 @@ RB_GRN_BEGIN_DECLS
 
 #define RB_GRN_MAJOR_VERSION 4
 #define RB_GRN_MINOR_VERSION 0
-#define RB_GRN_MICRO_VERSION 5
+#define RB_GRN_MICRO_VERSION 6
 
 #define RB_GRN_QUERY_DEFAULT_MAX_EXPRESSIONS 32
 
@@ -665,6 +665,11 @@ VALUE          rb_grn_column_expression_builder_build
 #define GRNBULK2RVAL(context, bulk, range, related_object)      \
     (rb_grn_bulk_to_ruby_object(context, bulk, range, related_object))
 
+#define RVAL2GRNPVECTOR(object, context, vector) \
+    (rb_grn_pvector_from_ruby_object(object, context, vector))
+#define GRNPVECTOR2RVAL(context, vector) \
+    (rb_grn_pvector_to_ruby_object(context, vector))
+
 #define RVAL2GRNVECTOR(object, context, vector) \
     (rb_grn_vector_from_ruby_object(object, context, vector))
 #define GRNVECTOR2RVAL(context, vector) \
@@ -782,6 +787,11 @@ VALUE          rb_grn_bulk_to_ruby_object           (grn_ctx *context,
                                                      grn_obj *bulk,
                                                      grn_obj *range,
                                                      VALUE related_object);
+grn_obj       *rb_grn_pvector_from_ruby_object      (VALUE object,
+                                                     grn_ctx *context,
+                                                     grn_obj *pvector);
+VALUE          rb_grn_pvector_to_ruby_object        (grn_ctx *context,
+                                                     grn_obj *pvector);
 grn_obj       *rb_grn_vector_from_ruby_object       (VALUE object,
                                                      grn_ctx *context,
                                                      grn_obj *vector);

data/lib/groonga/dumper.rb

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2011-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014  Masafumi Yokoyama <myokoym@gmail.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -419,6 +420,14 @@ module Groonga
           if default_tokenizer
             parameters << ":default_tokenizer => #{default_tokenizer.name.dump}"
           end
+          token_filters = table.token_filters
+          unless token_filters.empty?
+            dumped_token_filter_names = token_filters.collect do |token_filter|
+              token_filter.name.dump
+            end
+            dumped_token_filters = "[#{dumped_token_filter_names.join(', ')}]"
+            parameters << ":token_filters => #{dumped_token_filters}"
+          end
           if _normalizer_name
             parameters << ":normalizer => #{_normalizer_name.dump}"
           end
@@ -575,6 +584,13 @@ module Groonga
           if default_tokenizer
             parameters << "--default_tokenizer #{default_tokenizer.name}"
           end
+          token_filters = table.token_filters
+          unless token_filters.empty?
+            token_filter_names = token_filters.collect do |token_filter|
+              token_filter.name
+            end
+            parameters << "--token_filters #{token_filter_names.join(',')}"
+          end
         end
         if _normalizer_name
           parameters << "--normalizer #{_normalizer_name}"
@@ -602,7 +618,13 @@ module Groonga
           flags << "COLUMN_VECTOR"
         end
         flags << "WITH_WEIGHT" if column.with_weight?
-        # TODO: support COMPRESS_ZLIB and COMPRESS_LZO?
+        if column.is_a?(Groonga::VariableSizeColumn)
+          if column.compressed?(:zlib)
+            flags << "COMPRESS_ZLIB"
+          elsif column.compressed?(:lz4)
+            flags << "COMPRESS_LZ4"
+          end
+        end
         parameters << "#{flags.join('|')}"
         parameters << "#{column.range.name}"
         write("column_create #{parameters.join(' ')}\n")