isomorfeus-ferret 0.12.7 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 438b0c625088191ac9dd620da3612a34c72c6b314da3ff87722272aeff39e2fa
-  data.tar.gz: a984ec0934467dad13687f84c4e9ff06c7ee1338d7928408772e423bf54b32b2
+  metadata.gz: be4c84d556459a8ed5d2585068378c156bdb2d68507ca2786844b2c69a4e7f35
+  data.tar.gz: '0096ef29b274ea39567e876d95d8441ff80589a8d3403c5aed3801c62377cffd'
 SHA512:
-  metadata.gz: 3667dddba793f9b62215a6c7e40cc55f395d586f886853cc11f7b35a341a90ec63c542a176277720130d269ae6d86dc38cb235ea4a5ac9a0414ceaf07ff7d295
-  data.tar.gz: ea56a056a43c7041c7bb4c8729557343b9c23d7ac7d011792ed31ec152bfcc1e06327cddc8ec5953b28d8774caaba01ede0262bea13a59e57a4459f3143040f3
+  metadata.gz: 55aa9f39fd4971e1a80bbcdeba14906928140fd023a7ee3e581ef2bafc63a8a664f5b0fbaf86dec1119ac62fd441703d33a4b6ee3731f811736a1b2eedcef9a3
+  data.tar.gz: 3843bd29450fb925069733913aee9f66958dfe41aba052d7d14a18cb7f2d4df376e0756d7c97bcd01527120a6a77776d9bee2a1c0a488648fc61f500dfe6e98e

data/LICENSE

@@ -28,7 +28,86 @@ Copyright (c) 2005-2006 David Balmain
 MIT License as above
 
 
-stemmer files in ext/isomorfeus_ferret originally taken from https://snowballstem.org/:
+brotli_* files in ext/isomorfeus_ferret originally taken from https://github.com/google/brotli:
+
+Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors.
+
+MIT License as above
+
+
+bzlib* files in ext/isomorfeus_ferret originally taken from git://sourceware.org/git/bzip2.git:
+
+This program, "bzip2", the associated library "libbzip2", and all
+documentation, are copyright (C) 1996-2019 Julian R Seward.  All
+rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. The origin of this software must not be misrepresented; you must
+   not claim that you wrote the original software.  If you use this
+   software in a product, an acknowledgment in the product
+   documentation would be appreciated but is not required.
+
+3. Altered source versions must be plainly marked as such, and must
+   not be misrepresented as being the original software.
+
+4. The name of the author may not be used to endorse or promote
+   products derived from this software without specific prior written
+   permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Julian Seward, jseward@acm.org
+bzip2/libbzip2 version 1.0.8 of 13 July 2019
+
+
+lz4* files in ext/isomorfeus_ferret originally taken from https://github.com/lz4/lz4/tree/dev/lib:
+
+Copyright (C) 2011-2020, Yann Collet.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the following disclaimer
+  in the documentation and/or other materials provided with the
+  distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You can contact the author at:
+- LZ4 homepage : http://www.lz4.org
+- LZ4 source repository : https://github.com/lz4/lz4
+
+
+stem* and libstemmer* files in ext/isomorfeus_ferret originally taken from https://snowballstem.org/:
 
 Copyright (c) 2001, Dr Martin Porter
 Copyright (c) 2004,2005, Richard Boulton
@@ -37,8 +116,7 @@ Copyright (c) 2006,2007,2009,2010,2011,2014-2019, Olly Betts
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
+modification, are permitted provided that the following conditions are met:
 
   1. Redistributions of source code must retain the above copyright notice,
      this list of conditions and the following disclaimer.
@@ -68,23 +146,27 @@ Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved.
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions
 are met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
 
-THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGE.
+  1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+  2. Redistributions in binary form must reproduce the above copyright notice,
+     this list of conditions and the following disclaimer in the documentation
+     and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+The following licenses apply to files, which are distributed within the repo
+but not distributed with the gem and not used at runtime:
 
 
 For the Reuter-21578 files in the misc/ferret_vs_lucene directory (corpus, etc.),

data/README.md

@@ -16,7 +16,59 @@ During revival many things havbe been fixed, now all tests pass, no crashes and
 successfully compiles and runs with rubys >3. Its no longer a goal to have
 a c library available, but instead the usage is meant as ruby gem with a c extension only.
 
-It should work on *nixes and *nuxes and also works on Windows.
+It should work on *nixes, *nuxes, *BSDs and also works on Windows.
+
+## Improvements and Changes in Version 0.13
+
+### Breaking
+
+- For version 0.13 die index file format has changed and is no longer compatible with previous versions. Indexes of older versions must be recreated with 0.13 (export all data from and with previous version, import alls data with 0.13)
+- The :store option no longer accepts :compress, compression must now be specified by the separate :compress options (see below).
+- The ASCII-specific Tokenizers and Analyzers have been removed
+
+### Sring Encoding support
+
+#### Input strings and stored fields
+
+In versions prior 0.13 the string encoding had to match the locale string encoding.
+In 0.13 the dependency on the locale setting has been resolved, input strings are now correctly tokenized
+according to their source encoding, with positions correctly matching the input string.
+All Ruby string encodings are supported.
+When fields are stored, they are now stored with the encoding, so that when they are retrieved again, they
+retain the original encoding with positions matching the string in its original encoding.
+
+#### Tokens and Filters
+
+Tokens are internally converted to UTF-8, which may change their length compared to their original encoding,
+yet they retain position information according to the source in its original encoding.
+The benefit is, that Filters, Stemmers or anything else working with Tokens only needs to support UTF-8 encoding,
+greatly simplifying things and ensuring consistent query results.
+
+### Compression
+
+Compression semantics have changed, now Brotli, BZip2 and LZ4 compression codecs are supported.
+- BZip2: slow compression, slow decompression, high compression ratio
+- Brotli: slow compression, fast decrompression, high compression ratio, recommended for general purpose.
+- LZ4: fast compression, fast decrompression, low compression ratio
+To see performance and compression ratios `rake ferret_compression_bench` can be run from the cloned repo.
+It uses data and code within the misc/ferret_vs_lucene directory.
+
+To compress a stored field the :compression option can be used with one of: :no, :brotli, :bz2 or :lz4.
+Example:
+```ruby
+fis.add_field(:compressed_field, :store => :yes, :compression => :brotli, :term_vector => :yes)
+```
+
+### Performance
+
+The encoding support demands its toll, indexing performance dropped a bit in comparision to 0.12, but still thousands of docs per second, depending on machine/docs.
+On Windows the indexing performance is still terrible, but that may be resolved in a future project.
+
+Search performance is still excellent and multiple times faster than Lucene.
+
+Lucene achieves roughly double the indexing performance. This seems to be because of the different way strings and
+encodings are handled in Java. For example, the Java WhitespaceTokenizer code requires only one method call per character (check for whitespace), but for Ruby, to support all the different encodings, several method calls are required per character (retrieve character according to encoding, check character for whitespace).
+Ferret is internally using the standard Ruby string encoding methods.
 
 ## Documentation
 
@@ -69,6 +121,7 @@ JVM 17.0.1 (Private Build)
 ## Future
 
 Lots of things to do:
+- Improve indexing performance on Windows (WriteFile is terribly slow, maybe use mapping, see libuv)
 - Bring documentation in order in a docs directory
 - Review code (especially for memory/stack issues, typical c issues)
 - Take care of ruby GVL and threading

data/ext/isomorfeus_ferret_ext/bm_bitvector.c

@@ -9,55 +9,51 @@
 
 static FrtBitVector *bv;
 
-static void setup()
-{
+static void setup(void) {
     bv = frt_bv_new_capa(SCAN_SIZE);
 }
 
-static void teardown()
-{
+static void teardown(void) {
     frt_bv_destroy(bv);
 }
 
-static void ferret_bv_and_sparse()
-{
+static void ferret_bv_and_sparse(void) {
     FrtBitVector * _bv = frt_bv_and(bv, bv);
     free(_bv);
 }
-static void ferret_bv_or_sparse()
-{
+
+static void ferret_bv_or_sparse(void) {
     FrtBitVector * _bv = frt_bv_or(bv, bv);
     free(_bv);
 }
-static void ferret_bv_xor_sparse()
-{
+
+static void ferret_bv_xor_sparse(void) {
     FrtBitVector * _bv = frt_bv_xor(bv, bv);
     free(_bv);
 }
-static void ferret_bv_not_sparse()
-{
+
+static void ferret_bv_not_sparse(void) {
     FrtBitVector * _bv = frt_bv_not(bv);
     free(_bv);
 }
-static void ferret_bv_and_dense()
-{
+
+static void ferret_bv_and_dense(void) {
     ferret_bv_and_sparse();
 }
-static void ferret_bv_or_dense()
-{
+
+static void ferret_bv_or_dense(void) {
     ferret_bv_or_sparse();
 }
-static void ferret_bv_xor_dense()
-{
+
+static void ferret_bv_xor_dense(void) {
     ferret_bv_xor_sparse();
 }
-static void ferret_bv_not_dense()
-{
+
+static void ferret_bv_not_dense(void) {
     ferret_bv_not_sparse();
 }
 
-static void ferret_bv_set_sparse()
-{
+static void ferret_bv_set_sparse(void) {
     int i;
 
     for (i = SCAN_INC; i < SCAN_SIZE; i += SCAN_INC) {
@@ -67,8 +63,7 @@ static void ferret_bv_set_sparse()
     }
 }
 
-static void ferret_bv_scan_sparse()
-{
+static void ferret_bv_scan_sparse(void) {
     int i, j;
 
     for (i = 0; i < N; i++) {
@@ -80,8 +75,7 @@ static void ferret_bv_scan_sparse()
     }
 }
 
-static void ferret_bv_set_dense()
-{
+static void ferret_bv_set_dense(void) {
     int i;
     frt_bv_clear(bv);
     for (i = 0; i < DENSE_SCAN_SIZE; i++) {
@@ -89,8 +83,7 @@ static void ferret_bv_set_dense()
     }
 }
 
-static void ferret_bv_scan_dense()
-{
+static void ferret_bv_scan_dense(void) {
     int i, j;
 
     for (i = 0; i < N; i++) {
@@ -102,8 +95,7 @@ static void ferret_bv_scan_dense()
     }
 }
 
-BENCH(bitvector_implementations)
-{
+BENCH(bitvector_implementations) {
     BM_SETUP(setup);
 
     BM_ADD(ferret_bv_set_sparse);

data/ext/isomorfeus_ferret_ext/bm_hash.c

@@ -4,8 +4,7 @@
 
 #define N 20
 
-static void ferret_hash()
-{
+static void ferret_hash(void) {
     int i;
     void *res = NULL;
     for (i = 0; i < N; i++) {
@@ -24,13 +23,11 @@ static void ferret_hash()
     (void)res;
 }
 
-BENCH(hash_implementations)
-{
+BENCH(hash_implementations) {
     BM_ADD(ferret_hash);
 }
 
-static void standard_hash()
-{
+static void standard_hash(void) {
     int i;
     void *res = NULL;
     for (i = 0; i < N; i++) {
@@ -49,8 +46,7 @@ static void standard_hash()
 
 #define PERTURB_SHIFT 5
 static const char *dummy_key = "";
-static FrtHashEntry *h_lookup_str(FrtHash *ht, register const void *key)
-{
+static FrtHashEntry *h_lookup_str(FrtHash *ht, register const void *key) {
     register const unsigned long hash = frt_str_hash((const char *)key);
     register unsigned int perturb;
     register int mask = ht->mask;
@@ -95,8 +91,7 @@ static FrtHashEntry *h_lookup_str(FrtHash *ht, register const void *key)
     }
 }
 
-static void string_hash()
-{
+static void string_hash(void) {
     int i;
     void *res = NULL;
     for (i = 0; i < N; i++) {
@@ -114,8 +109,7 @@ static void string_hash()
     (void)res;
 }
 
-BENCH(specialized_string_hash)
-{
+BENCH(specialized_string_hash) {
     BM_ADD(standard_hash);
     BM_ADD(string_hash);
 }

data/ext/isomorfeus_ferret_ext/bm_micro_string.c

@@ -3,8 +3,7 @@
 
 #define N 10
 
-static void do_strcmp()
-{
+static void do_strcmp(void) {
     const char **word;
     char buf[100];
     int res, i;
@@ -18,8 +17,7 @@ static void do_strcmp()
     (void)res;
 }
 
-static void do_strncmp()
-{
+static void do_strncmp(void) {
     const char **word;
     char buf[100];
     int res, i;
@@ -33,8 +31,7 @@ static void do_strncmp()
     (void)res;
 }
 
-BENCH(strcmp_when_length_is_known)
-{
+BENCH(strcmp_when_length_is_known) {
     BM_COUNT(6);
     BM_DISCARD(1);
     BM_ADD(do_strcmp);

data/ext/isomorfeus_ferret_ext/bm_store.c

@@ -6,46 +6,38 @@
 #define N 10
 #define write_byte(os, b) os->buf.buf[os->buf.pos++] = (frt_uchar)b
 
-void my_os_write_voff_t(FrtOutStream *os, register off_t num)
-{
+void my_os_write_voff_t(FrtOutStream *os, register off_t num) {
     if (!(num&0x7f)) {
         if (os->buf.pos >= FRT_BUFFER_SIZE) {
             frt_os_write_byte(os, (frt_uchar)num);
-        }
-        else {
+        } else {
             write_byte(os, (frt_uchar)num);
         }
-    }
-    else if (!(num&0x3fff)) {
+    } else if (!(num&0x3fff)) {
         if (os->buf.pos >= FRT_BUFFER_SIZE - 1) {
             frt_os_write_byte(os, (frt_uchar)(0x80 | (0x3f & num))); num >>= 6;
             frt_os_write_byte(os, (frt_uchar)num);
-        }
-        else {
+        } else {
             write_byte(os, (frt_uchar)(0x80 | (0x3f & num))); num >>= 6;
             write_byte(os, (frt_uchar)num);
         }
-    }
-    else if (!(num&0x1fffff)) {
+    } else if (!(num&0x1fffff)) {
         if (os->buf.pos >= FRT_BUFFER_SIZE - 2) {
             frt_os_write_byte(os, (frt_uchar)(0xc0 | (0x1f & num))); num >>= 5;
             frt_os_write_byte(os, (frt_uchar)(0xff| num)); num >>= 8;
             frt_os_write_byte(os, (frt_uchar)num);
-        }
-        else {
+        } else {
             write_byte(os, (frt_uchar)(0xc0 | (0x1f & num))); num >>= 5;
             write_byte(os, (frt_uchar)(0xff| num)); num >>= 8;
             write_byte(os, (frt_uchar)num);
         }
-    }
-    else if (!(num&0xfffff)) {
+    } else if (!(num&0xfffff)) {
         if (os->buf.pos >= FRT_BUFFER_SIZE - 3) {
             frt_os_write_byte(os, (frt_uchar)(0xe0 | (0x0f & num))); num >>= 4;
             frt_os_write_byte(os, (frt_uchar)(0xff | num)); num >>= 8;
             frt_os_write_byte(os, (frt_uchar)(0xff | num)); num >>= 8;
             frt_os_write_byte(os, (frt_uchar)num);
-        }
-        else {
+        } else {
             write_byte(os, (frt_uchar)(0xe0 | (0x0f & num))); num >>= 4;
             write_byte(os, (frt_uchar)(0xff | num)); num >>= 8;
             write_byte(os, (frt_uchar)(0xff | num)); num >>= 8;
@@ -54,8 +46,7 @@ void my_os_write_voff_t(FrtOutStream *os, register off_t num)
     }
 }
 
-static void vint_out()
-{
+static void vint_out(void) {
     int n;
     off_t i;
     FrtOutStream *os;
@@ -70,8 +61,7 @@ static void vint_out()
 
 }
 
-static void unrolled_vint_out()
-{
+static void unrolled_vint_out(void) {
     int n;
     off_t i;
     FrtOutStream *os;
@@ -86,8 +76,7 @@ static void unrolled_vint_out()
 
 }
 
-BENCH(vint_io)
-{
+BENCH(vint_io) {
     BM_ADD(vint_out);
     BM_ADD(unrolled_vint_out);
 }

data/ext/isomorfeus_ferret_ext/brotli_common_dictionary.c

@@ -5895,7 +5895,7 @@ static BrotliDictionary kBrotliDictionary = {
 #endif
 };
 
-const BrotliDictionary* BrotliGetDictionary() {
+const BrotliDictionary* BrotliGetDictionary(void) {
   return &kBrotliDictionary;
 }
 

data/ext/isomorfeus_ferret_ext/brotli_dec_decode.c

@@ -2599,7 +2599,7 @@ const char* BrotliDecoderErrorString(BrotliDecoderErrorCode c) {
   }
 }
 
-uint32_t BrotliDecoderVersion() {
+uint32_t BrotliDecoderVersion(void) {
   return BROTLI_VERSION;
 }