rroonga 4.0.5-x64-mingw32 → 4.0.6-x64-mingw32

data/test/test-schema.rb

@@ -1,4 +1,5 @@
 # Copyright (C) 2009-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014  Masafumi Yokoyama <myokoym@gmail.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -90,14 +91,17 @@ class SchemaTest < Test::Unit::TestCase
     end
 
     def test_full_option
+      context.register_plugin("token_filters/stop_word")
       path = @tmp_dir + "hash.groonga"
       tokenizer = context["TokenTrigram"]
+      token_filter = context["TokenFilterStopWord"]
       Groonga::Schema.create_table("Posts",
                                    :type => :hash,
                                    :key_type => "integer",
                                    :path => path.to_s,
                                    :value_type => "UInt32",
                                    :default_tokenizer => tokenizer,
+                                   :token_filters => [token_filter],
                                    :named_path => true) do |table|
       end
       table = context["Posts"]
@@ -111,6 +115,7 @@ class SchemaTest < Test::Unit::TestCase
                    "size: <0>, " +
                    "encoding: <#{Groonga::Encoding.default.inspect}>, " +
                    "default_tokenizer: <#{tokenizer.name}>, " +
+                   "token_filters: [<#{token_filter.name}>], " +
                    "normalizer: (nil)>",
                    table.inspect)
     end
@@ -148,6 +153,7 @@ class SchemaTest < Test::Unit::TestCase
     end
 
     def test_full_option
+      context.register_plugin("token_filters/stop_word")
       path = @tmp_dir + "patricia-trie.groonga"
       Groonga::Schema.create_table("Posts",
                                    :type => :patricia_trie,
@@ -155,6 +161,7 @@ class SchemaTest < Test::Unit::TestCase
                                    :path => path.to_s,
                                    :value_type => "Float",
                                    :default_tokenizer => "TokenBigram",
+                                   :token_filters => ["TokenFilterStopWord"],
                                    :key_with_sis => true,
                                    :named_path => true,
                                    :normalizer => "NormalizerAuto") do |table|
@@ -170,6 +177,7 @@ class SchemaTest < Test::Unit::TestCase
                    "size: <0>, " +
                    "encoding: <#{Groonga::Encoding.default.inspect}>, " +
                    "default_tokenizer: <TokenBigram>, " +
+                   "token_filters: [<TokenFilterStopWord>], " +
                    "normalizer: <NormalizerAuto>>",
                    table.inspect)
     end
@@ -208,6 +216,7 @@ class SchemaTest < Test::Unit::TestCase
     end
 
     def test_full_option
+      context.register_plugin("token_filters/stop_word")
       path = @tmp_dir + "double-array-trie.groonga"
       Groonga::Schema.create_table("Posts",
                                    :type => :double_array_trie,
@@ -215,6 +224,7 @@ class SchemaTest < Test::Unit::TestCase
                                    :path => path.to_s,
                                    :value_type => "Float",
                                    :default_tokenizer => "TokenBigram",
+                                   :token_filters => ["TokenFilterStopWord"],
                                    :named_path => true,
                                    :normalizer => "NormalizerAuto") do |table|
       end
@@ -229,6 +239,7 @@ class SchemaTest < Test::Unit::TestCase
                    "size: <0>, " +
                    "encoding: <#{Groonga::Encoding.default.inspect}>, " +
                    "default_tokenizer: <TokenBigram>, " +
+                   "token_filters: [<TokenFilterStopWord>], " +
                    "normalizer: <NormalizerAuto>>",
                    table.inspect)
     end
@@ -330,7 +341,7 @@ class SchemaTest < Test::Unit::TestCase
                      :path => path.to_s,
                      :persistent => true,
                      :type => :vector,
-                     :compress => :lzo)
+                     :compress => :lz4)
       end
 
       column_name = "Posts.rate"
@@ -341,7 +352,7 @@ class SchemaTest < Test::Unit::TestCase
                    "path: <#{path}>, " +
                    "domain: <Posts>, " +
                    "range: <Niku>, " +
-                   "flags: <KEY_VAR_SIZE|COMPRESS_LZO>>",
+                   "flags: <KEY_VAR_SIZE|COMPRESS_LZ4>>",
                    column.inspect)
     end
   end

data/test/test-variable-size-column.rb

@@ -1,4 +1,5 @@
 # Copyright (C) 2009-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2014  Masafumi Yokoyama <myokoym@gmail.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -85,13 +86,13 @@ class VariableSizeColumnTest < Test::Unit::TestCase
     end
   end
 
-  def test_compressed_lzo?
+  def test_compressed_lz4?
     description = @users.define_column("description", "ShortText",
-                                       :compress => :lzo)
-    if context.support_lzo?
-      assert {description.compressed?(:lzo)}
+                                       :compress => :lz4)
+    if context.support_lz4?
+      assert {description.compressed?(:lz4)}
     else
-      assert {not description.compressed?(:lzo)}
+      assert {not description.compressed?(:lz4)}
     end
   end
 

data/vendor/local/include/groonga/groonga.h

@@ -104,7 +104,9 @@ typedef enum {
   GRN_OPERATION_NOT_SUPPORTED = -58,
   GRN_ADDRESS_IS_IN_USE = -59,
   GRN_ZLIB_ERROR = -60,
-  GRN_LZO_ERROR = -61,
+  GRN_LZ4_ERROR = -61,
+/* Just for backward compatibility. We'll remove it at 5.0.0. */
+#define GRN_LZO_ERROR GRN_LZ4_ERROR
   GRN_STACK_OVER_FLOW = -62,
   GRN_SYNTAX_ERROR = -63,
   GRN_RETRY_MAX = -64,
@@ -116,6 +118,7 @@ typedef enum {
   GRN_CAS_ERROR = -70,
   GRN_UNSUPPORTED_COMMAND_VERSION = -71,
   GRN_NORMALIZER_ERROR = -72,
+  GRN_TOKEN_FILTER_ERROR = -73,
 } grn_rc;
 
 GRN_API grn_rc grn_init(void);
@@ -299,7 +302,9 @@ typedef unsigned short int grn_obj_flags;
 #define GRN_OBJ_COMPRESS_MASK          (0x07<<4)
 #define GRN_OBJ_COMPRESS_NONE          (0x00<<4)
 #define GRN_OBJ_COMPRESS_ZLIB          (0x01<<4)
-#define GRN_OBJ_COMPRESS_LZO           (0x02<<4)
+#define GRN_OBJ_COMPRESS_LZ4           (0x02<<4)
+/* Just for backward compatibility. We'll remove it at 5.0.0. */
+#define GRN_OBJ_COMPRESS_LZO           GRN_OBJ_COMPRESS_LZ4
 
 #define GRN_OBJ_WITH_SECTION           (0x01<<7)
 #define GRN_OBJ_WITH_WEIGHT            (0x01<<8)
@@ -478,7 +483,8 @@ typedef enum {
   GRN_PROC_COMMAND,
   GRN_PROC_FUNCTION,
   GRN_PROC_HOOK,
-  GRN_PROC_NORMALIZER
+  GRN_PROC_NORMALIZER,
+  GRN_PROC_TOKEN_FILTER
 } grn_proc_type;
 
 GRN_API grn_obj *grn_proc_create(grn_ctx *ctx,
@@ -789,8 +795,11 @@ typedef enum {
   GRN_INFO_PARTIAL_MATCH_THRESHOLD,
   GRN_INFO_II_SPLIT_THRESHOLD,
   GRN_INFO_SUPPORT_ZLIB,
-  GRN_INFO_SUPPORT_LZO,
-  GRN_INFO_NORMALIZER
+  GRN_INFO_SUPPORT_LZ4,
+/* Just for backward compatibility. We'll remove it at 5.0.0. */
+#define GRN_INFO_SUPPORT_LZO GRN_INFO_SUPPORT_LZ4
+  GRN_INFO_NORMALIZER,
+  GRN_INFO_TOKEN_FILTERS
 } grn_info_type;
 
 GRN_API grn_obj *grn_obj_get_info(grn_ctx *ctx, grn_obj *obj, grn_info_type type, grn_obj *valuebuf);
@@ -1178,7 +1187,7 @@ GRN_API const char *grn_default_query_logger_get_path(void);
 #define GRN_BULK_REST(bulk) \
   (GRN_BULK_OUTP(bulk)\
    ? ((bulk)->u.b.tail - (bulk)->u.b.curr)\
-   : GRN_BULK_BUFSIZE - (bulk)->header.flags)
+   : GRN_BULK_BUFSIZE - GRN_BULK_SIZE_IN_FLAGS((bulk)->header.flags))
 #define GRN_BULK_VSIZE(bulk) \
   (GRN_BULK_OUTP(bulk)\
    ? ((bulk)->u.b.curr - (bulk)->u.b.head)\
@@ -1298,6 +1307,8 @@ GRN_API void grn_ctx_output_obj(grn_ctx *ctx,
                                 grn_obj *value, grn_obj_format *format);
 
 
+GRN_API grn_content_type grn_ctx_get_output_type(grn_ctx *ctx);
+GRN_API grn_rc grn_ctx_set_output_type(grn_ctx *ctx, grn_content_type type);
 GRN_API const char *grn_ctx_get_mime_type(grn_ctx *ctx);
 GRN_API void grn_ctx_recv_handler_set(grn_ctx *,
                                       void (*func)(grn_ctx *, int, void *),
@@ -1680,6 +1691,8 @@ GRN_API grn_obj *grn_expr_add_var(grn_ctx *ctx, grn_obj *expr,
 GRN_API grn_obj *grn_expr_get_var(grn_ctx *ctx, grn_obj *expr,
                                   const char *name, unsigned int name_size);
 GRN_API grn_obj *grn_expr_get_var_by_offset(grn_ctx *ctx, grn_obj *expr, unsigned int offset);
+GRN_API grn_rc grn_expr_clear_vars(grn_ctx *ctx, grn_obj *expr);
+
 
 GRN_API grn_obj *grn_expr_append_obj(grn_ctx *ctx, grn_obj *expr, grn_obj *obj,
                                      grn_operator op, int nargs);
@@ -1692,6 +1705,8 @@ GRN_API grn_obj *grn_expr_append_const_int(grn_ctx *ctx, grn_obj *expr, int i,
                                            grn_operator op, int nargs);
 GRN_API grn_rc grn_expr_append_op(grn_ctx *ctx, grn_obj *expr, grn_operator op, int nargs);
 
+GRN_API grn_rc grn_expr_get_keywords(grn_ctx *ctx, grn_obj *expr, grn_obj *keywords);
+
 GRN_API grn_rc grn_expr_syntax_escape(grn_ctx *ctx,
                                       const char *query, int query_size,
                                       const char *target_characters,

data/vendor/local/include/groonga/groonga/token_filter.h

@@ -0,0 +1,71 @@
+/* -*- c-basic-offset: 2 -*- */
+/*
+  Copyright(C) 2014 Brazil
+
+  This library is free software; you can redistribute it and/or
+  modify it under the terms of the GNU Lesser General Public
+  License version 2.1 as published by the Free Software Foundation.
+
+  This library is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public
+  License along with this library; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+#ifndef GRN_PLUGIN_TOKEN_FILTER_H
+#define GRN_PLUGIN_TOKEN_FILTER_H
+
+#include <stddef.h>
+
+#include <groonga/tokenizer.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cplusplus */
+
+typedef void *grn_token_filter_init_func(grn_ctx *ctx,
+                                         grn_obj *table,
+                                         grn_token_mode mode);
+
+typedef void grn_token_filter_filter_func(grn_ctx *ctx,
+                                          grn_token *current_token,
+                                          grn_token *next_token,
+                                          void *user_data);
+
+typedef void grn_token_filter_fin_func(grn_ctx *ctx,
+                                       void *user_data);
+
+
+/*
+  grn_token_filter_register() registers a plugin to the database which is
+  associated with `ctx'. `plugin_name_ptr' and `plugin_name_length' specify the
+  plugin name. Alphabetic letters ('A'-'Z' and 'a'-'z'), digits ('0'-'9') and
+  an underscore ('_') are capable characters.
+
+  `init', `filter' and `fin' specify the plugin functions.
+
+  `init' is called for initializing a token_filter for a document or
+  query.
+
+  `filter' is called for filtering tokens one by one.
+
+  `fin' is called for finalizing a token_filter.
+
+  grn_token_filter_register() returns GRN_SUCCESS on success, an error
+  code on failure.
+ */
+GRN_PLUGIN_EXPORT grn_rc grn_token_filter_register(grn_ctx *ctx,
+                                                   const char *plugin_name_ptr,
+                                                   int plugin_name_length,
+                                                   grn_token_filter_init_func *init,
+                                                   grn_token_filter_filter_func *filter,
+                                                   grn_token_filter_fin_func *fin);
+
+#ifdef __cplusplus
+}  /* extern "C" */
+#endif  /* __cplusplus */
+
+#endif  /* GRN_PLUGIN_TOKEN_FILTER_H */

data/vendor/local/include/groonga/groonga/tokenizer.h

@@ -29,6 +29,23 @@ extern "C" {
 #define GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8     "\xEF\xBF\xBE"
 #define GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8_LEN 3
 
+/*
+  grn_token_mode describes propose for tokenization.
+
+  `GRN_TOKEN_GET`: Tokenization for search.
+
+  `GRN_TOKEN_ADD`: Tokenization for adding token to index.
+
+  `GRN_TOKEN_DEL`: Tokenization for deleting token from index.
+
+  @since 4.0.7
+ */
+typedef enum {
+  GRN_TOKEN_GET = 0,
+  GRN_TOKEN_ADD,
+  GRN_TOKEN_DEL
+} grn_token_mode;
+
 /*
   grn_tokenizer_charlen() returns the length (#bytes) of the first character
   in the string specified by `str_ptr' and `str_length'. If the starting bytes
@@ -84,7 +101,7 @@ struct _grn_tokenizer_query {
   grn_encoding encoding;
   unsigned int flags;
   grn_bool have_tokenized_delimiter;
-  unsigned int token_mode;
+  grn_token_mode token_mode;
 };
 
 /*
@@ -176,6 +193,21 @@ typedef unsigned int grn_tokenizer_status;
 #define GRN_TOKENIZER_CONTINUE GRN_TOKENIZER_TOKEN_CONTINUE
 #define GRN_TOKENIZER_LAST     GRN_TOKENIZER_TOKEN_LAST
 
+typedef struct _grn_token grn_token;
+
+GRN_PLUGIN_EXPORT grn_obj *grn_token_get_data(grn_ctx *ctx,
+                                              grn_token *token);
+GRN_PLUGIN_EXPORT grn_rc grn_token_set_data(grn_ctx *ctx,
+                                            grn_token *token,
+                                            const char *str_ptr,
+                                            int str_length);
+GRN_PLUGIN_EXPORT grn_tokenizer_status grn_token_get_status(grn_ctx *ctx,
+                                                            grn_token *token);
+GRN_PLUGIN_EXPORT grn_rc grn_token_set_status(grn_ctx *ctx,
+                                              grn_token *token,
+                                              grn_tokenizer_status status);
+
+
 /*
   grn_tokenizer_token_push() pushes the next token into `token'. Note that
   grn_tokenizer_token_push() does not make a copy of the given string. This

data/vendor/local/lib/groonga/plugins/query_expanders/tsv.la

@@ -1,5 +1,5 @@
 # tsv.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.2 Debian-2.4.2-1.10
+# Generated by libtool (GNU libtool) 2.4.2 Debian-2.4.2-1.7
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.
@@ -17,7 +17,7 @@ old_library='tsv.a'
 inherited_linker_flags=''
 
 # Libraries that this one depends upon.
-dependency_libs=' /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libgroonga.la -L/home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libmsgpack.la -lws2_32'
+dependency_libs=' /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libgroonga.la -L/home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libmsgpack.la /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libonig.la -lws2_32'
 
 # Names of additional weak libraries provided by this library
 weak_library_names=''

data/vendor/local/lib/groonga/plugins/ruby/eval.la

@@ -1,5 +1,5 @@
 # eval.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.2 Debian-2.4.2-1.10
+# Generated by libtool (GNU libtool) 2.4.2 Debian-2.4.2-1.7
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.

data/vendor/local/lib/groonga/plugins/ruby/load.la

@@ -1,5 +1,5 @@
 # load.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.2 Debian-2.4.2-1.10
+# Generated by libtool (GNU libtool) 2.4.2 Debian-2.4.2-1.7
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.

data/vendor/local/lib/groonga/plugins/suggest/suggest.la

@@ -1,5 +1,5 @@
 # suggest.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.2 Debian-2.4.2-1.10
+# Generated by libtool (GNU libtool) 2.4.2 Debian-2.4.2-1.7
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.
@@ -17,7 +17,7 @@ old_library='suggest.a'
 inherited_linker_flags=''
 
 # Libraries that this one depends upon.
-dependency_libs=' /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libgroonga.la -L/home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib -lws2_32 /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libmsgpack.la'
+dependency_libs=' /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libgroonga.la -L/home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libonig.la -lws2_32 /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libmsgpack.la'
 
 # Names of additional weak libraries provided by this library
 weak_library_names=''

data/vendor/local/lib/groonga/plugins/table/table.la

@@ -1,5 +1,5 @@
 # table.la - a libtool library file
-# Generated by libtool (GNU libtool) 2.4.2 Debian-2.4.2-1.10
+# Generated by libtool (GNU libtool) 2.4.2 Debian-2.4.2-1.7
 #
 # Please DO NOT delete this file!
 # It is necessary for linking the library.
@@ -17,7 +17,7 @@ old_library='table.a'
 inherited_linker_flags=''
 
 # Libraries that this one depends upon.
-dependency_libs=' /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libgroonga.la -L/home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib -lws2_32 /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libmsgpack.la'
+dependency_libs=' /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libgroonga.la -L/home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libonig.la -lws2_32 /home/kou/work/c/groonga.clean/packages/windows/dist-x64/lib/libmsgpack.la'
 
 # Names of additional weak libraries provided by this library
 weak_library_names=''