isomorfeus-ferret 0.14.3 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2821c4f4967a8af0c4f6e00a4c256754b77da083fbe5dab47e0eeac705561f01
-  data.tar.gz: 79c06afc07045bbe7048fa0c570384d8ead0b8fe351dde4829a26e1a1b1bde73
+  metadata.gz: dc82aa85a233a1eb67ae971fbd6e5704b1f6b14660ed5673f9292a3021f34837
+  data.tar.gz: dea91e29a2a3c14381067d05177c455d8953b74f0187e9b7610997c4a1bc68ae
 SHA512:
-  metadata.gz: 8fc4e106796d397bdc359af7d7b42120c143cbd93e57a9cb71f1e0ca5e89f9856bc6897eb39e4358cfa0c97d77b104c6a818b4c5aaafaebdb0da2efbca00546a
-  data.tar.gz: 6002e45cd95bf3eb0f258ee209c783f4f4eb966068009cc76de35ad9b4a0f4a03a41180655d6d79d40a86151fe3e663b57ad79c604184eac134adfaa425153e8
+  metadata.gz: '064940571edfbb380795a71031274f6489010cdc28da4c26a64c769eee924593df069d72939a4f6e5e56b8b94398108e6e039c00526a250dbb5f51068a10880e'
+  data.tar.gz: 76edc3b5f07bd779498fe92d8630ec5f651e05eae82982fcedc853b9086fd7af84e827407ee0b2316456dc07bb7f508b63ded855278ff4ca38af18398ee7cc1e

data/LICENSE

@@ -28,14 +28,14 @@ Copyright (c) 2005-2006 David Balmain
 MIT License as above
 
 
-brotli_* files in ext/isomorfeus_ferret originally taken from https://github.com/google/brotli:
+brotli_* files in ext/isomorfeus_ferret_ext originally taken from https://github.com/google/brotli:
 
 Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors.
 
 MIT License as above
 
 
-bzlib* files in ext/isomorfeus_ferret originally taken from git://sourceware.org/git/bzip2.git:
+bzlib* files in ext/isomorfeus_ferret_ext originally taken from git://sourceware.org/git/bzip2.git:
 
 This program, "bzip2", the associated library "libbzip2", and all
 documentation, are copyright (C) 1996-2019 Julian R Seward.  All
@@ -76,7 +76,7 @@ Julian Seward, jseward@acm.org
 bzip2/libbzip2 version 1.0.8 of 13 July 2019
 
 
-lz4* files in ext/isomorfeus_ferret originally taken from https://github.com/lz4/lz4/tree/dev/lib:
+lz4* files in ext/isomorfeus_ferret_ext originally taken from https://github.com/lz4/lz4/tree/dev/lib:
 
 Copyright (C) 2011-2020, Yann Collet.
 
@@ -107,7 +107,7 @@ You can contact the author at:
 - LZ4 source repository : https://github.com/lz4/lz4
 
 
-stem* and libstemmer* files in ext/isomorfeus_ferret originally taken from https://snowballstem.org/:
+stem* and libstemmer* files in ext/isomorfeus_ferret_ext originally taken from https://snowballstem.org/:
 
 Copyright (c) 2001, Dr Martin Porter
 Copyright (c) 2004,2005, Richard Boulton
@@ -139,11 +139,62 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+mdbx* files in ext/isomorfeus_ferret_ext originally taken from https://gitflic.ru/project/erthink/libmdbx:
+
+The OpenLDAP Public License
+  Version 2.8, 17 August 2003
+
+Redistribution and use of this software and associated documentation
+("Software"), with or without modification, are permitted provided
+that the following conditions are met:
+
+1. Redistributions in source form must retain copyright statements
+   and notices,
+
+2. Redistributions in binary form must reproduce applicable copyright
+   statements and notices, this list of conditions, and the following
+   disclaimer in the documentation and/or other materials provided
+   with the distribution, and
+
+3. Redistributions must contain a verbatim copy of this document.
+
+The OpenLDAP Foundation may revise this license from time to time.
+Each revision is distinguished by a version number.  You may use
+this Software under terms of this license revision or under the
+terms of any subsequent revision of the license.
+
+THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS
+CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT
+SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S)
+OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+The names of the authors and copyright holders must not be used in
+advertising or otherwise to promote the sale, use or other dealing
+in this Software without specific, written prior permission.  Title
+to copyright in this Software shall at all times remain with copyright
+holders.
+
+OpenLDAP is a registered trademark of the OpenLDAP Foundation.
+
+Copyright 1999-2003 The OpenLDAP Foundation, Redwood City,
+California, USA.  All Rights Reserved.  Permission to copy and
+distribute verbatim copies of this document is granted.
+
+
 The following licenses apply to files, which are distributed within the repo
 but not distributed with the gem and not used at runtime:
 
 
-For the Reuter-21578 files in the misc/ferret_vs_others directory (corpus, etc.),
+For the Reuters-21578 files in the misc/ferret_vs_others directory (reuters-corpus, etc.),
 used for research for developing search engine technology:
 
    The copyright for the text of newswire articles and Reuters
@@ -158,7 +209,6 @@ the data set (see "Availability & Questions").
 
 Apache Lucene jars in  the misc/ferret_vs_others directory:
 
-
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/

data/ext/isomorfeus_ferret_ext/extconf.rb

@@ -1,5 +1,23 @@
 require 'mkmf'
 
-$CFLAGS << ' -O2 -Wall -Wno-sizeof-pointer-div'
+WIN_PATTERNS = [
+  /bccwin/i,
+  /cygwin/i,
+  /djgpp/i,
+  /mingw/i,
+  /mswin/i,
+  /wince/i,
+].freeze
+
+def win_platform?
+  ruby_platform = RbConfig::CONFIG['host_os']
+  !!WIN_PATTERNS.find {|r| ruby_platform =~ r }
+end
+
+$CFLAGS << ' -O2 -DMDBX_BUILD_FLAGS=\"-O2\" -Wall -Wno-sizeof-pointer-div'
+
+if win_platform?
+  $LDFLAGS << ' -lntdll '
+end
 
 create_makefile('isomorfeus_ferret_ext')

data/ext/isomorfeus_ferret_ext/frb_analysis.c

@@ -2075,10 +2075,6 @@ static void Init_RegExpAnalyzer(void) {
     rb_define_method(cRegExpAnalyzer, "token_stream", frb_re_analyzer_token_stream, 2);
 }
 
-/* rdoc hack
-extern VALUE mFerret = rb_define_module("Ferret");
-*/
-
 /*
  *  Document-module: Ferret::Analysis
  *

data/ext/isomorfeus_ferret_ext/frb_field_info.c

@@ -19,57 +19,62 @@ static VALUE sym_with_positions_offsets;
 
 extern VALUE sym_boost;
 
-void frb_fi_get_params(VALUE roptions, FrtStoreValue *store, FrtCompressionType *compression, FrtIndexValue *index, FrtTermVectorValue *term_vector, float *boost) {
+void frb_fi_get_params(VALUE roptions, unsigned int *bits, float *boost) {
     VALUE v;
     Check_Type(roptions, T_HASH);
     v = rb_hash_aref(roptions, sym_boost);
-    if (Qnil != v) {
-        *boost = (float)NUM2DBL(v);
-    } else {
-        *boost = 1.0f;
-    }
+    if (Qnil != v) *boost = (float)NUM2DBL(v);
+    else *boost = 1.0f;
+
     v = rb_hash_aref(roptions, sym_store);
     if (Qnil != v) Check_Type(v, T_SYMBOL);
     if (v == sym_no || v == sym_false || v == Qfalse) {
-        *store = FRT_STORE_NO;
+        *bits &= ~FRT_FI_IS_STORED_BM;
     } else if (v == sym_yes || v == sym_true || v == Qtrue) {
-        *store = FRT_STORE_YES;
+        *bits |= FRT_FI_IS_STORED_BM;
     } else if (v == Qnil) {
         /* leave as default */
     } else {
-        rb_raise(rb_eArgError, ":%s isn't a valid argument for :store. Please choose from [:yes, :no]",
-                 rb_id2name(SYM2ID(v)));
+        rb_raise(rb_eArgError, ":%s isn't a valid argument for :store. Please choose from [:yes, :no]", rb_id2name(SYM2ID(v)));
     }
 
     v = rb_hash_aref(roptions, sym_compression);
     if (Qnil != v) Check_Type(v, T_SYMBOL);
     if (v == sym_no || v == sym_false || v == Qfalse) {
-        *compression = FRT_COMPRESSION_NONE;
+        *bits &= ~FRT_FI_IS_COMPRESSED_BM;
+        *bits &= ~FRT_FI_COMPRESSION_BROTLI_BM;
+        *bits &= ~FRT_FI_COMPRESSION_BZ2_BM;
+        *bits &= ~FRT_FI_COMPRESSION_LZ4_BM;
     } else if (v == sym_yes || v == sym_true || v == Qtrue || v == sym_brotli) {
-        *compression = FRT_COMPRESSION_BROTLI;
+        *bits |= FRT_FI_IS_COMPRESSED_BM | FRT_FI_COMPRESSION_BROTLI_BM;
     } else if (v == sym_bz2) {
-        *compression = FRT_COMPRESSION_BZ2;
+        *bits |= FRT_FI_IS_COMPRESSED_BM | FRT_FI_COMPRESSION_BZ2_BM;
     } else if (v == sym_lz4) {
-        *compression = FRT_COMPRESSION_LZ4;
+        *bits |= FRT_FI_IS_COMPRESSED_BM | FRT_FI_COMPRESSION_LZ4_BM;
     } else if (v == Qnil) {
         /* leave as default */
     } else {
-        rb_raise(rb_eArgError, ":%s isn't a valid argument for :compression. Please choose from [:yes, :no, :brotli, :bz2, :lz4]",
-                 rb_id2name(SYM2ID(v)));
+        rb_raise(rb_eArgError, ":%s isn't a valid argument for :compression. Please choose from [:yes, :no, :brotli, :bz2, :lz4]", rb_id2name(SYM2ID(v)));
     }
 
     v = rb_hash_aref(roptions, sym_index);
     if (Qnil != v) Check_Type(v, T_SYMBOL);
     if (v == sym_no || v == sym_false || v == Qfalse) {
-        *index = FRT_INDEX_NO;
+        *bits &= ~FRT_FI_IS_INDEXED_BM;
+        *bits &= ~FRT_FI_IS_TOKENIZED_BM;
+        *bits &= ~FRT_FI_OMIT_NORMS_BM;
     } else if (v == sym_yes || v == sym_true || v == Qtrue) {
-        *index = FRT_INDEX_YES;
+        *bits |= FRT_FI_IS_INDEXED_BM | FRT_FI_IS_TOKENIZED_BM;
+        *bits &= ~FRT_FI_OMIT_NORMS_BM;
     } else if (v == sym_untokenized) {
-        *index = FRT_INDEX_UNTOKENIZED;
+        *bits |= FRT_FI_IS_INDEXED_BM;
+        *bits &= ~FRT_FI_IS_TOKENIZED_BM;
+        *bits &= ~FRT_FI_OMIT_NORMS_BM;
     } else if (v == sym_omit_norms) {
-        *index = FRT_INDEX_YES_OMIT_NORMS;
+        *bits |= FRT_FI_IS_INDEXED_BM | FRT_FI_IS_TOKENIZED_BM | FRT_FI_OMIT_NORMS_BM;
     } else if (v == sym_untokenized_omit_norms) {
-        *index = FRT_INDEX_UNTOKENIZED_OMIT_NORMS;
+        *bits |= FRT_FI_IS_INDEXED_BM | FRT_FI_OMIT_NORMS_BM;
+        *bits &= ~FRT_FI_IS_TOKENIZED_BM;
     } else if (v == Qnil) {
         /* leave as default */
     } else {
@@ -80,18 +85,28 @@ void frb_fi_get_params(VALUE roptions, FrtStoreValue *store, FrtCompressionType
     v = rb_hash_aref(roptions, sym_term_vector);
     if (Qnil != v) Check_Type(v, T_SYMBOL);
     if (v == sym_no || v == sym_false || v == Qfalse) {
-        *term_vector = FRT_TERM_VECTOR_NO;
+        *bits &= ~FRT_FI_STORE_TERM_VECTOR_BM;
+        *bits &= ~FRT_FI_STORE_POSITIONS_BM;
+        *bits &= ~FRT_FI_STORE_OFFSETS_BM;
     } else if (v == sym_yes || v == sym_true || v == Qtrue) {
-        *term_vector = FRT_TERM_VECTOR_YES;
+        *bits |= FRT_FI_STORE_TERM_VECTOR_BM;
+        *bits &= ~FRT_FI_STORE_POSITIONS_BM;
+        *bits &= ~FRT_FI_STORE_OFFSETS_BM;
     } else if (v == sym_with_positions) {
-        *term_vector = FRT_TERM_VECTOR_WITH_POSITIONS;
+        *bits |= FRT_FI_STORE_TERM_VECTOR_BM | FRT_FI_STORE_POSITIONS_BM;
+        *bits &= ~FRT_FI_STORE_OFFSETS_BM;
     } else if (v == sym_with_offsets) {
-        *term_vector = FRT_TERM_VECTOR_WITH_OFFSETS;
+        *bits |= FRT_FI_STORE_TERM_VECTOR_BM | FRT_FI_STORE_OFFSETS_BM;
+        *bits &= ~FRT_FI_STORE_POSITIONS_BM;
     } else if (v == sym_with_positions_offsets) {
-        *term_vector = FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS;
+        *bits |= FRT_FI_STORE_TERM_VECTOR_BM | FRT_FI_STORE_POSITIONS_BM | FRT_FI_STORE_OFFSETS_BM;
     } else if (v == Qnil) {
         /* leave as default */
-        if (*index == FRT_INDEX_NO) *term_vector = FRT_TERM_VECTOR_NO;
+        if ((*bits & FRT_FI_IS_INDEXED_BM) == 0) {
+            *bits &= ~FRT_FI_STORE_TERM_VECTOR_BM;
+            *bits &= ~FRT_FI_STORE_POSITIONS_BM;
+            *bits &= ~FRT_FI_STORE_OFFSETS_BM;
+        }
     } else {
         rb_raise(rb_eArgError, ":%s isn't a valid argument for :term_vector. Please choose from [:no, :yes, "
                  ":with_positions, :with_offsets, :with_positions_offsets]", rb_id2name(SYM2ID(v)));
@@ -150,17 +165,12 @@ static VALUE frb_fi_init(int argc, VALUE *argv, VALUE self) {
     VALUE roptions, rname;
     FrtFieldInfo *fi;
     TypedData_Get_Struct(self, FrtFieldInfo, &frb_field_info_t, fi);
-    FrtStoreValue store = FRT_STORE_YES;
-    FrtCompressionType compression = FRT_COMPRESSION_NONE;
-    FrtIndexValue index = FRT_INDEX_YES;
-    FrtTermVectorValue term_vector = FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS;
+    unsigned int bits = FRT_FI_DEFAULTS_BM;
     float boost = 1.0f;
 
     rb_scan_args(argc, argv, "11", &rname, &roptions);
-    if (argc > 1) {
-        frb_fi_get_params(roptions, &store, &compression, &index, &term_vector, &boost);
-    }
-    fi = frt_fi_init(fi, frb_field(rname), store, compression, index, term_vector);
+    if (argc > 1) frb_fi_get_params(roptions, &bits, &boost);
+    fi = frt_fi_init(fi, frb_field(rname), bits);
     fi->boost = boost;
     fi->rfi = self;
     return self;
@@ -185,7 +195,7 @@ static VALUE frb_fi_name(VALUE self) {
  */
 static VALUE frb_fi_is_stored(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_is_stored(fi) ? Qtrue : Qfalse;
+    return bits_is_stored(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -196,7 +206,7 @@ static VALUE frb_fi_is_stored(VALUE self) {
  */
 static VALUE frb_fi_is_compressed(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_is_compressed(fi) ? Qtrue : Qfalse;
+    return bits_is_compressed(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -207,7 +217,7 @@ static VALUE frb_fi_is_compressed(VALUE self) {
  */
 static VALUE frb_fi_is_indexed(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_is_indexed(fi) ? Qtrue : Qfalse;
+    return bits_is_indexed(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -223,7 +233,7 @@ static VALUE frb_fi_is_indexed(VALUE self) {
  */
 static VALUE frb_fi_is_tokenized(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_is_tokenized(fi) ? Qtrue : Qfalse;
+    return bits_is_tokenized(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -239,7 +249,7 @@ static VALUE frb_fi_is_tokenized(VALUE self) {
  */
 static VALUE frb_fi_omit_norms(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_omit_norms(fi) ? Qtrue : Qfalse;
+    return bits_omit_norms(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -250,7 +260,7 @@ static VALUE frb_fi_omit_norms(VALUE self) {
  */
 static VALUE frb_fi_store_term_vector(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_store_term_vector(fi) ? Qtrue : Qfalse;
+    return bits_store_term_vector(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -261,7 +271,7 @@ static VALUE frb_fi_store_term_vector(VALUE self) {
  */
 static VALUE frb_fi_store_positions(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_store_positions(fi) ? Qtrue : Qfalse;
+    return bits_store_positions(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -272,7 +282,7 @@ static VALUE frb_fi_store_positions(VALUE self) {
  */
 static VALUE frb_fi_store_offsets(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_store_offsets(fi) ? Qtrue : Qfalse;
+    return bits_store_offsets(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -285,7 +295,7 @@ static VALUE frb_fi_store_offsets(VALUE self) {
  */
 static VALUE frb_fi_has_norms(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_has_norms(fi) ? Qtrue : Qfalse;
+    return bits_has_norms(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -326,10 +336,10 @@ static VALUE frb_fi_to_h(VALUE self) {
     bool o;
 
     // :index
-    if (!fi_is_indexed(fi)) val = sym_no;
+    if (!bits_is_indexed(fi->bits)) val = sym_no;
     else {
-      bool t = fi_is_tokenized(fi);
-      o = fi_omit_norms(fi);
+      bool t = bits_is_tokenized(fi->bits);
+      o = bits_omit_norms(fi->bits);
       if (!t && o) val = sym_untokenized_omit_norms;
       else if (t && o) val = sym_omit_norms;
       else if (!t && !o) val = sym_untokenized;
@@ -338,23 +348,23 @@ static VALUE frb_fi_to_h(VALUE self) {
     rb_hash_aset(hash, sym_index, val);
 
     // :store
-    rb_hash_aset(hash, sym_store, fi_is_stored(fi) ? sym_yes : sym_no);
+    rb_hash_aset(hash, sym_store, bits_is_stored(fi->bits) ? sym_yes : sym_no);
 
     // :compress
-    if (!fi_is_compressed(fi)) val = sym_no;
+    if (!bits_is_compressed(fi->bits)) val = sym_no;
     else {
-      if (fi_is_compressed_brotli(fi)) val = sym_brotli;
-      else if (fi_is_compressed_bz2(fi)) val = sym_bz2;
-      else if (fi_is_compressed_lz4(fi)) val = sym_lz4;
+      if (bits_is_compressed_brotli(fi->bits)) val = sym_brotli;
+      else if (bits_is_compressed_bz2(fi->bits)) val = sym_bz2;
+      else if (bits_is_compressed_lz4(fi->bits)) val = sym_lz4;
       else val = sym_yes;
     }
     rb_hash_aset(hash, sym_compression, val);
 
     // :term_vector
-    if (!fi_store_term_vector(fi)) val = sym_no;
+    if (!bits_store_term_vector(fi->bits)) val = sym_no;
     else {
-      bool p = fi_store_positions(fi);
-      o = fi_store_offsets(fi);
+      bool p = bits_store_positions(fi->bits);
+      o = bits_store_offsets(fi->bits);
       if (p && o) val = sym_with_positions_offsets;
       else if (o) val = sym_with_offsets;
       else if (p) val = sym_with_positions;

data/ext/isomorfeus_ferret_ext/frb_index.c

@@ -40,7 +40,7 @@ static ID id_boost;
 
 extern VALUE sym_each;
 extern rb_encoding *utf8_encoding;
-extern void frb_fi_get_params(VALUE roptions, FrtStoreValue *store, FrtCompressionType *compression, FrtIndexValue *index, FrtTermVectorValue *term_vector, float *boost);
+extern void frb_fi_get_params(VALUE roptions, unsigned int *bits, float *boost);
 extern FrtAnalyzer *frb_get_cwrapped_analyzer(VALUE ranalyzer);
 extern VALUE frb_get_analyzer(FrtAnalyzer *a);
 extern VALUE frb_get_field_info(FrtFieldInfo *fi);
@@ -119,17 +119,12 @@ static VALUE frb_fis_init(int argc, VALUE *argv, VALUE self) {
     VALUE roptions;
     FrtFieldInfos *fis;
     TypedData_Get_Struct(self, FrtFieldInfos, &frb_field_infos_t, fis);
-    FrtStoreValue store = FRT_STORE_YES;
-    FrtCompressionType compression = FRT_COMPRESSION_NONE;
-    FrtIndexValue index = FRT_INDEX_YES;
-    FrtTermVectorValue term_vector = FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS;
+    unsigned int bits = FRT_FI_DEFAULTS_BM;
     float boost;
 
     rb_scan_args(argc, argv, "01", &roptions);
-    if (argc > 0) {
-        frb_fi_get_params(roptions, &store, &compression, &index, &term_vector, &boost);
-    }
-    fis = frt_fis_init(fis, store, compression, index, term_vector);
+    if (argc > 0) frb_fi_get_params(roptions, &bits, &boost);
+    fis = frt_fis_init(fis, bits);
     fis->rfis = self;
     return self;
 }
@@ -218,18 +213,15 @@ frb_fis_add_field(int argc, VALUE *argv, VALUE self)
 {
     FrtFieldInfos *fis = (FrtFieldInfos *)DATA_PTR(self);
     FrtFieldInfo *fi;
-    FrtStoreValue store_val = fis->store_val;
-    FrtCompressionType compression = fis->compression;
-    FrtIndexValue index = fis->index;
-    FrtTermVectorValue term_vector = fis->term_vector;
+    unsigned int bits = fis->bits;
     float boost = 1.0f;
     VALUE rname, roptions;
 
     rb_scan_args(argc, argv, "11", &rname, &roptions);
     if (argc > 1) {
-        frb_fi_get_params(roptions, &store_val, &compression, &index, &term_vector, &boost);
+        frb_fi_get_params(roptions, &bits, &boost);
     }
-    fi = frt_fi_new(frb_field(rname), store_val, compression, index, term_vector);
+    fi = frt_fi_new(frb_field(rname), bits);
     fi->boost = boost;
     frt_fis_add_field(fis, fi);
     return self;
@@ -340,7 +332,7 @@ frb_fis_get_tk_fields(VALUE self)
     VALUE rfield_names = rb_ary_new();
     int i;
     for (i = 0; i < fis->size; i++) {
-        if (!fi_is_tokenized(fis->fields[i])) continue;
+        if (!bits_is_tokenized(fis->fields[i]->bits)) continue;
         rb_ary_push(rfield_names, ID2SYM(fis->fields[i]->name));
     }
     return rfield_names;
@@ -1082,7 +1074,7 @@ static VALUE frb_iw_init(int argc, VALUE *argv, VALUE self) {
                 TypedData_Get_Struct(rval, FrtFieldInfos, &frb_field_infos_t, fis);
                 frt_index_create(store, fis);
             } else {
-                fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+                fis = frt_fis_new(FRT_FI_DEFAULTS_BM);
                 frt_index_create(store, fis);
                 frt_fis_deref(fis);
             }
@@ -1132,6 +1124,8 @@ frb_hash_to_doc_i(VALUE key, VALUE value, VALUE arg)
     if (key == Qundef) {
         return ST_CONTINUE;
     } else {
+        int ex_code = 0;
+        const char *msg = NULL;
         FrtDocument *doc = (FrtDocument *)arg;
         ID field = frb_field(key);
         VALUE val;
@@ -1162,7 +1156,15 @@ frb_hash_to_doc_i(VALUE key, VALUE value, VALUE arg)
                 frt_df_add_data_len(df, rstrdup(val), RSTRING_LEN(val), rb_enc_get(val));
                 break;
         }
-        frt_doc_add_field(doc, df);
+        FRT_TRY
+            frt_doc_add_field(doc, df);
+        FRT_XCATCHALL
+            ex_code = xcontext.excode;
+            msg = xcontext.msg;
+            FRT_HANDLED();
+        FRT_XENDTRY
+
+        if (ex_code && msg) { frb_raise(ex_code, msg); }
     }
     return ST_CONTINUE;
 }
@@ -2293,7 +2295,7 @@ frb_ir_tk_fields(VALUE self)
     VALUE rfield_names = rb_ary_new();
     int i;
     for (i = 0; i < fis->size; i++) {
-        if (!fi_is_tokenized(fis->fields[i])) continue;
+        if (!bits_is_tokenized(fis->fields[i]->bits)) continue;
         rb_ary_push(rfield_names, rb_str_new_cstr(rb_id2name(fis->fields[i]->name)));
     }
     return rfield_names;
@@ -2809,10 +2811,6 @@ void Init_IndexReader(void) {
     rb_define_method(cIndexReader, "to_enum",        frb_ir_to_enum,       0);
 }
 
-/* rdoc hack
-extern VALUE mFerret = rb_define_module("Ferret");
-*/
-
 /*
  *  Document-module: Ferret::Index
  *

data/ext/isomorfeus_ferret_ext/frb_lazy_doc.c

@@ -177,6 +177,7 @@ static VALUE frb_ld_lt(VALUE self, VALUE other) {
     rLazyDoc *other_rld;
     TypedData_Get_Struct(other, rLazyDoc, &frb_ld_t, other_rld);
     other_h = frb_ld_to_h(other);
+    (void)other_rld;
   }
   VALUE self_h = frb_ld_to_h(self);
   return rb_funcall(self_h, id_lt, 1, other_h);
@@ -190,6 +191,7 @@ static VALUE frb_ld_le(VALUE self, VALUE other) {
     rLazyDoc *other_rld;
     TypedData_Get_Struct(other, rLazyDoc, &frb_ld_t, other_rld);
     other_h = frb_ld_to_h(other);
+    (void)other_rld;
   }
   VALUE self_h = frb_ld_to_h(self);
   return rb_funcall(self_h, id_le, 1, other_h);
@@ -223,6 +225,7 @@ static VALUE frb_ld_gt(VALUE self, VALUE other) {
     rLazyDoc *other_rld;
     TypedData_Get_Struct(other, rLazyDoc, &frb_ld_t, other_rld);
     other_h = frb_ld_to_h(other);
+    (void)other_rld;
   }
   VALUE self_h = frb_ld_to_h(self);
   return rb_funcall(self_h, id_gt, 1, other_h);
@@ -236,6 +239,7 @@ static VALUE frb_ld_ge(VALUE self, VALUE other) {
     rLazyDoc *other_rld;
     TypedData_Get_Struct(other, rLazyDoc, &frb_ld_t, other_rld);
     other_h = frb_ld_to_h(other);
+    (void)other_rld;
   }
   VALUE self_h = frb_ld_to_h(self);
   return rb_funcall(self_h, id_ge, 1, other_h);

data/ext/isomorfeus_ferret_ext/frb_qparser.c

@@ -374,11 +374,6 @@ static VALUE frb_qp_set_tkz_fields(VALUE self, VALUE rfields) {
  *
  ****************************************************************************/
 
-/* rdoc hack
-extern VALUE mFerret = rb_define_module("Ferret");
-extern VALUE cQueryParser = rb_define_module_under(mFerret, "QueryParser");
-*/
-
 /*
  *  Document-class: Ferret::QueryParser::QueryParseException
  *
@@ -410,7 +405,7 @@ void Init_QueryParseException(void) {
  *
  *  If you want to use one of these characters in one of your terms you need
  *  to escape it with a \ character. \ escapes itself. The exception to this
- *  rule is within Phrases which a strings surrounded by double quotes (and
+ *  rule is within Phrases which are strings surrounded by double quotes (and
  *  will be explained further bellow in the section on PhraseQueries). In
  *  Phrases, only ", | and <> have special meaning and need to be escaped if
  *  you want the literal value. <> is escaped \<\>.

data/ext/isomorfeus_ferret_ext/frb_search.c

@@ -4593,11 +4593,6 @@ static void Init_SpanNotQuery(void) {
     rb_define_method(cSpanNotQuery, "initialize", frb_spanxq_init, 2);
 }
 
-/* rdoc hack
-extern VALUE mFerret = rb_define_module("Ferret");
-extern VALUE mSearch = rb_define_module_under(mFerret, "Search");
-*/
-
 /*
  *  Document-module: Ferret::Search::Spans
  *