rroonga 5.0.9-x64-mingw32 → 5.1.1-x64-mingw32

data/test/test-variable-size-column.rb

@@ -1,5 +1,5 @@
 # Copyright (C) 2009-2014  Kouhei Sutou <kou@clear-code.com>
-# Copyright (C) 2014  Masafumi Yokoyama <myokoym@gmail.com>
+# Copyright (C) 2014-2016  Masafumi Yokoyama <yokoyama@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -124,6 +124,76 @@ class VariableSizeColumnTest < Test::Unit::TestCase
     assert_equal(1, @name.defrag)
   end
 
+  def test_reindex
+    Groonga::Schema.define do |schema|
+      schema.create_table("Memos", :type => :array) do |table|
+        table.short_text("title")
+        table.text("content")
+      end
+
+      schema.create_table("BigramTerms",
+                          :type => :patricia_trie,
+                          :key_type => :short_text,
+                          :normalizer => "NormalizerAuto",
+                          :default_tokenizer => "TokenBigram") do |table|
+        table.index("Memos.title")
+        table.index("Memos.content")
+      end
+
+      schema.create_table("MeCabTerms",
+                          :type => :patricia_trie,
+                          :key_type => :short_text,
+                          :normalizer => "NormalizerAuto",
+                          :default_tokenizer => "TokenMecab") do |table|
+        table.index("Memos.title")
+        table.index("Memos.content")
+      end
+    end
+
+    memos = context["Memos"]
+    memos.add(:title => "memo1", :content => "もり")
+
+    bigram_terms = context["BigramTerms"]
+    bigram_terms.delete("memo")
+    bigram_terms.delete("り")
+
+    mecab_terms = context["MeCabTerms"]
+    mecab_terms.delete("1")
+    mecab_terms.delete("もり")
+
+    assert_equal({
+                   :bigram => [
+                     "1",
+                     "もり",
+                   ],
+                   :mecab => [
+                     "memo",
+                   ],
+                 },
+                 {
+                   :bigram => bigram_terms.collect(&:_key).sort,
+                   :mecab => mecab_terms.collect(&:_key).sort,
+                 })
+
+    context["Memos.content"].reindex
+
+    assert_equal({
+                   :bigram => [
+                     "1",
+                     "もり",
+                     "り",
+                   ],
+                   :mecab => [
+                     "memo",
+                     "もり",
+                   ],
+                 },
+                 {
+                   :bigram => bigram_terms.collect(&:_key).sort,
+                   :mecab => mecab_terms.collect(&:_key).sort,
+                 })
+  end
+
   class VectorTest < self
     class ReferenceTest < self
       def test_append

data/vendor/local/etc/groonga/httpd/groonga-httpd.conf

@@ -1,6 +1,6 @@
 worker_processes 1;
 
-# Match this to the file owner of groonga database files if groonga-httpd is
+# Match this to the file owner of Groonga database files if groonga-httpd is
 # run as root.
 user groonga groonga;
 
@@ -14,20 +14,20 @@ http {
   sendfile on;
   keepalive_timeout 65;
 
-  # The default groonga database path.
+  # The default Groonga database path.
   groonga_database /home/kou/work/c/groonga.clean/packages/windows/dist-x64/var/lib/groonga/db/db;
 
-  # Create a groonga database automatically if the groonga database doesn't
+  # Create a Groonga database automatically if the Groonga database doesn't
   # exist.
   #
   # Note that this option is danger when worker_processes is greater than 1.
-  # Because one or more worker processes may create the same groonga database
-  # at the same time. If you can create a groonga database before running
+  # Because one or more worker processes may create the same Groonga database
+  # at the same time. If you can create a Groonga database before running
   # groonga-httpd, you should do it.
   groonga_database_auto_create on;
 
-  # The default groonga cache limit. The cache limit can be set
-  # for each worker. It can't be set for each groonga database.
+  # The default Groonga cache limit. The cache limit can be set
+  # for each worker. It can't be set for each Groonga database.
   # groonga_cache_limit 100;
 
   server {
@@ -36,9 +36,9 @@ http {
 
     location /d/ {
       groonga on;
-      # You can disable log for groonga.
+      # You can disable log for Groonga.
       # groonga_log_path off;
-      # You can disable query log for groonga.
+      # You can disable query log for Groonga.
       # groonga_query_log_path off;
       # You can custom database path.
       # groonga_database /path/to/groonga/db;

data/vendor/local/include/groonga/groonga/file_reader.h

@@ -0,0 +1,40 @@
+/*
+  Copyright(C) 2015 Brazil
+
+  This library is free software; you can redistribute it and/or
+  modify it under the terms of the GNU Lesser General Public
+  License as published by the Free Software Foundation; either
+  version 2.1 of the License, or (at your option) any later version.
+
+  This library is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public
+  License along with this library; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+#ifndef GROONGA_FILE_READER_H
+#define GROONGA_FILE_READER_H
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+typedef struct _grn_file_reader grn_file_reader;
+
+GRN_API grn_file_reader *grn_file_reader_open(grn_ctx *ctx, const char *path);
+GRN_API void grn_file_reader_close(grn_ctx *ctx,
+                                   grn_file_reader *reader);
+
+GRN_API grn_rc grn_file_reader_read_line(grn_ctx *ctx,
+                                         grn_file_reader *reader,
+                                         grn_obj *buffer);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* GROONGA_FILE_READER_H */

data/vendor/local/include/groonga/groonga/groonga.h

@@ -250,6 +250,7 @@ GRN_API grn_rc grn_set_default_encoding(grn_encoding encoding);
 
 GRN_API const char *grn_get_version(void);
 GRN_API const char *grn_get_package(void);
+GRN_API const char *grn_get_package_label(void);
 
 GRN_API grn_command_version grn_get_default_command_version(void);
 GRN_API grn_rc grn_set_default_command_version(grn_command_version version);

data/vendor/local/include/groonga/groonga/obj.h

@@ -23,54 +23,17 @@
 extern "C" {
 #endif
 
+/* Just for backward compatibility. Use grn_obj_is_true() instead. */
 #define GRN_OBJ_IS_TRUE(ctx, obj, result) do {  \
-  grn_obj *obj_ = (obj);                        \
-  if (obj_) {                                   \
-    switch (obj_->header.type) {                \
-    case GRN_BULK :                             \
-      switch (obj_->header.domain) {            \
-      case GRN_DB_BOOL :                        \
-        result = GRN_BOOL_VALUE(obj_);          \
-        break;                                  \
-      case GRN_DB_INT32 :                       \
-        result = GRN_INT32_VALUE(obj_) != 0;    \
-        break;                                  \
-      case GRN_DB_UINT32 :                      \
-        result = GRN_UINT32_VALUE(obj_) != 0;   \
-        break;                                  \
-      case GRN_DB_FLOAT :                       \
-      {                                         \
-        double float_value;                     \
-        float_value = GRN_FLOAT_VALUE(obj_);    \
-        result = (float_value < -DBL_EPSILON || \
-                  DBL_EPSILON < float_value);   \
-      }                                         \
-      break;                                    \
-      case GRN_DB_SHORT_TEXT :                  \
-      case GRN_DB_TEXT :                        \
-      case GRN_DB_LONG_TEXT :                   \
-        result = GRN_TEXT_LEN(obj_) != 0;       \
-        break;                                  \
-      default :                                 \
-        result = GRN_FALSE;                     \
-        break;                                  \
-      }                                         \
-      break;                                    \
-    case GRN_VECTOR :                           \
-      result = GRN_TRUE;                        \
-      break;                                    \
-    default :                                   \
-      result = GRN_FALSE;                       \
-      break;                                    \
-    }                                           \
-  } else {                                      \
-    result = GRN_FALSE;                         \
-  }                                             \
+  result = grn_obj_is_true(ctx, obj);           \
 } while (0)
 
+GRN_API grn_bool grn_obj_is_true(grn_ctx *ctx, grn_obj *obj);
 
 GRN_API grn_bool grn_obj_is_builtin(grn_ctx *ctx, grn_obj *obj);
 GRN_API grn_bool grn_obj_is_table(grn_ctx *ctx, grn_obj *obj);
+GRN_API grn_bool grn_obj_is_accessor(grn_ctx *ctx, grn_obj *obj);
+GRN_API grn_bool grn_obj_is_key_accessor(grn_ctx *ctx, grn_obj *obj);
 GRN_API grn_bool grn_obj_is_type(grn_ctx *ctx, grn_obj *obj);
 GRN_API grn_bool grn_obj_is_proc(grn_ctx *ctx, grn_obj *obj);
 GRN_API grn_bool grn_obj_is_tokenizer_proc(grn_ctx *ctx, grn_obj *obj);
@@ -86,6 +49,8 @@ GRN_API grn_rc grn_obj_cast(grn_ctx *ctx,
                             grn_obj *dest,
                             grn_bool add_record_if_not_exist);
 
+GRN_API grn_rc grn_obj_reindex(grn_ctx *ctx, grn_obj *obj);
+
 #ifdef __cplusplus
 }
 #endif

data/vendor/local/include/groonga/groonga/portability.h

@@ -134,11 +134,11 @@
 #endif /* WIN32 */
 
 #ifdef WIN32
-# define grn_snprintf(dest, dest_size, n, format, ...)          \
-  _snprintf_s((dest), (dest_size), (n) - 1, (format), __VA_ARGS__)
+# define grn_snprintf(dest, dest_size, n, ...)                  \
+  _snprintf_s((dest), (dest_size), (n) - 1, __VA_ARGS__)
 #else /* WIN32 */
-# define grn_snprintf(dest, dest_size, n, format, ...)          \
-  snprintf((dest), (n), (format), __VA_ARGS__)
+# define grn_snprintf(dest, dest_size, n, ...)  \
+  snprintf((dest), (n), __VA_ARGS__)
 #endif /* WIN32 */
 
 #ifdef WIN32

data/vendor/local/include/groonga/groonga.h

@@ -31,5 +31,6 @@
 #include "groonga/thread.h"
 #include "groonga/windows.h"
 #include "groonga/windows_event_logger.h"
+#include "groonga/file_reader.h"
 
 #endif /* GROONGA_H */

data/vendor/local/lib/groonga/plugins/sharding/logical_range_filter.rb

@@ -224,40 +224,7 @@ module Groonga
             range_index = nil
           end
 
-          case @cover_type
-          when :all
-            filter_shard_all(range_index, expression_builder)
-          when :partial_min
-            if range_index
-              filter_by_range(range_index,
-                              @target_range.min, @target_range.min_border,
-                              nil, nil)
-            else
-              filter_table do |expression|
-                expression_builder.build_partial_min(expression)
-              end
-            end
-          when :partial_max
-            if range_index
-              filter_by_range(range_index,
-                              nil, nil,
-                              @target_range.max, @target_range.max_border)
-            else
-              filter_table do |expression|
-                expression_builder.build_partial_max(expression)
-              end
-            end
-          when :partial_min_and_max
-            if range_index
-              filter_by_range(range_index,
-                              @target_range.min, @target_range.min_border,
-                              @target_range.max, @target_range.max_border)
-            else
-              filter_table do |expression|
-                expression_builder.build_partial_min_and_max(expression)
-              end
-            end
-          end
+          execute_filter(range_index, expression_builder)
         end
 
         private
@@ -396,7 +363,7 @@ module Groonga
             relation = "<"
           end
           reason = "hit ratio "
-          reason << "(#{hit_ratio}=#{estimated_n_records}/#{max_n_records})"
+          reason << "(#{hit_ratio}=#{estimated_n_records}/#{max_n_records}) "
           reason << "#{relation} threshold (#{threshold})"
           decide_use_range_index(use_range_index_by_hit_ratio, reason,
                                  __LINE__, __method__)
@@ -444,6 +411,43 @@ module Groonga
           nil
         end
 
+        def execute_filter(range_index, expression_builder)
+          case @cover_type
+          when :all
+            filter_shard_all(range_index, expression_builder)
+          when :partial_min
+            if range_index
+              filter_by_range(range_index, expression_builder,
+                              @target_range.min, @target_range.min_border,
+                              nil, nil)
+            else
+              filter_table do |expression|
+                expression_builder.build_partial_min(expression)
+              end
+            end
+          when :partial_max
+            if range_index
+              filter_by_range(range_index, expression_builder,
+                              nil, nil,
+                              @target_range.max, @target_range.max_border)
+            else
+              filter_table do |expression|
+                expression_builder.build_partial_max(expression)
+              end
+            end
+          when :partial_min_and_max
+            if range_index
+              filter_by_range(range_index, expression_builder,
+                              @target_range.min, @target_range.min_border,
+                              @target_range.max, @target_range.max_border)
+            else
+              filter_table do |expression|
+                expression_builder.build_partial_min_and_max(expression)
+              end
+            end
+          end
+        end
+
         def filter_shard_all(range_index, expression_builder)
           table = @shard.table
           if @filter.nil?
@@ -452,7 +456,7 @@ module Groonga
               return
             end
             if range_index
-              filter_by_range(range_index,
+              filter_by_range(range_index, expression_builder,
                               nil, nil,
                               nil, nil)
             else
@@ -460,7 +464,7 @@ module Groonga
             end
           else
             if range_index
-              filter_by_range(range_index,
+              filter_by_range(range_index, expression_builder,
                               nil, nil,
                               nil, nil)
             else
@@ -480,7 +484,7 @@ module Groonga
           end
         end
 
-        def filter_by_range(range_index,
+        def filter_by_range(range_index, expression_builder,
                             min, min_border, max, max_border)
           lexicon = range_index.domain
           data_table = range_index.range
@@ -503,6 +507,10 @@ module Groonga
               else
                 options[:limit] = current_limit
               end
+              max_n_unmatched_records =
+                compute_max_n_unmatched_records(data_table.size,
+                                                options[:limit])
+              options[:max_n_unmatched_records] = max_n_unmatched_records
               if @filter
                 create_expression(data_table) do |expression|
                   expression.parse(@filter)
@@ -516,6 +524,17 @@ module Groonga
                   n_matched_records = index_cursor.select(result_set, options)
                 end
               end
+              if n_matched_records == -1
+                result_set.close
+                fallback_message =
+                  "fallback because there are too much unmatched records: "
+                fallback_message << "<#{max_n_unmatched_records}>"
+                decide_use_range_index(false,
+                                       fallback_message,
+                                       __LINE__, __method__)
+                execute_filter(nil, expression_builder)
+                return
+              end
             end
           rescue
             result_set.close
@@ -560,6 +579,19 @@ module Groonga
           flags
         end
 
+        def compute_max_n_unmatched_records(data_table_size, limit)
+          max_n_unmatched_records = limit * 100
+          max_n_sample_records = data_table_size
+          if max_n_sample_records > 10000
+            sample_ratio = 1 / (Math.log(data_table_size) ** 2)
+            max_n_sample_records = (max_n_sample_records * sample_ratio).ceil
+          end
+          if max_n_unmatched_records > max_n_sample_records
+            max_n_unmatched_records = max_n_sample_records
+          end
+          max_n_unmatched_records
+        end
+
         def filter_table
           table = @shard.table
           create_expression(table) do |expression|

data/vendor/local/lib/groonga/scripts/ruby/command_line/grndb.rb

@@ -129,6 +129,20 @@ module Groonga
           @succeeded = false
         end
 
+        open_cursor(database) do |cursor|
+          context = Context.instance
+          cursor.each do |id|
+            next if ID.builtin?(id)
+            next if context[id]
+            message =
+              "[#{cursor.key}] Can't open object. " +
+              "It's broken. " +
+              "Re-create the object or the database."
+            $stdout.puts(message)
+            @succeeded = false
+          end
+        end
+
         database.each do |object|
           case object
           when IndexColumn
@@ -164,6 +178,13 @@ module Groonga
           end
         end
       end
+
+      def open_cursor(database, &block)
+        flags =
+          TableCursorFlags::ASCENDING |
+          TableCursorFlags::BY_ID
+        TableCursor.open(database, :flags => flags, &block)
+      end
     end
   end
 end

data/vendor/local/lib/groonga/scripts/ruby/id.rb

@@ -0,0 +1,12 @@
+module Groonga
+  module ID
+    # TODO: Should we bind GRN_N_RESERVED_TYPES?
+    N_RESERVED_TYPES = 256
+
+    class << self
+      def builtin?(id)
+        id < N_RESERVED_TYPES
+      end
+    end
+  end
+end

data/vendor/local/lib/groonga/scripts/ruby/initialize/post.rb

@@ -4,6 +4,8 @@ require "context"
 
 require "writer"
 
+require "id"
+
 require "object"
 require "database"
 require "table"

data/vendor/local/lib/groonga/scripts/ruby/logger.rb

@@ -8,7 +8,9 @@ module Groonga
       else
         message = "#{error.class}: #{error.message}"
       end
-      backtrace = error.backtrace
+      # TODO: Re-enable backtrace after mruby/mruby#2917 has been merged
+      # backtrace = error.backtrace
+      backtrace = []
       last_raw_entry = backtrace.last
       if last_raw_entry
         last_entry = BacktraceEntry.parse(last_raw_entry)

data/vendor/local/lib/groonga/scripts/ruby/scan_info_data.rb

@@ -96,6 +96,8 @@ module Groonga
           match_resolve_index_expression(arg)
         when Accessor
           match_resolve_index_accessor(arg)
+        when IndexColumn
+          match_resolve_index_index_column(arg)
         when Object
           match_resolve_index_db_obj(arg)
         else
@@ -258,6 +260,10 @@ module Groonga
       put_search_index(index_info.index, index_info.section_id, expr_code.weight)
     end
 
+    def match_resolve_index_index_column(index)
+      put_search_index(index, 0, 1)
+    end
+
     def match_resolve_index_db_obj(db_obj)
       index_info = db_obj.find_index(op)
       return if index_info.nil?