isomorfeus-ferret 0.13.5 → 0.13.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f2a4a6ea097a25d4d32a958c730fffca81ded61133f37ef4c86ca5b78ecef77
-  data.tar.gz: 7be1f98b64559be5f02e39d7a74894476046fef3fdd46d1e69ee3079c74c9681
+  metadata.gz: 1e6a7a9c17cc344699bee0c8116188f86da73e11fa2a4eed02eea1c4647b6a99
+  data.tar.gz: c94c9a2735b010c02c8d5a793057b94abf8950117153415a731347c21fadf66e
 SHA512:
-  metadata.gz: d173777ee520433e8df0e1a07853f92b65aa350427668baea67a957a28a8cadc05790396d8d5167282c4c83fb1e874eec3316c226e8c961f9f3161f3bf3ecb79
-  data.tar.gz: c64a96813a4b2546e21825cb4a6310c9384a8090bafe51e9c65c49d8c2b8bc69ab3deed82780e6df09089b6a915d6b7e7418bcac30f3d9770c1358800aa2286d
+  metadata.gz: 549a2d3b09dc535278f6c9735f55a47203166df046e5a75ff55589085336bcf2cb7c25cdf491c07eca68da25682316c4bb15e5079f09df95f8a494558fe22260
+  data.tar.gz: e188f428fd881a0285ece69807f13c07dfd1eee131edcad3e38a713ffa820364992bb9068d5f94f567bd7769c7882441ce8eb67199b95b491528fecdd34503f4

data/README.md

@@ -11,12 +11,12 @@ At the [Isomorfeus Framework Project](https://isomorfeus.com)
 
 ## About this project
 
-Isomorfeus-Ferret is a revived version of the original ferret gem created by Dave Balmain.
+Isomorfeus-Ferret is a revived version of the original ferret gem created by Dave Balmain, [https://github.com/dbalmain/ferret](https://github.com/dbalmain/ferret).
 During revival many things havbe been fixed, now all tests pass, no crashes and it
 successfully compiles and runs with rubys >3. Its no longer a goal to have
 a c library available, but instead the usage is meant as ruby gem with a c extension only.
 
-It should work on *nixes, *nuxes, *BSDs and also works on Windows.
+It works on *nixes, *nuxes, *BSDs and also works on Windows and RaspberryPi.
 
 ## Improvements and Changes in Version 0.13
 
@@ -63,14 +63,8 @@ fis.add_field(:compressed_field, :store => :yes, :compression => :brotli, :term_
 
 ### Performance
 
-The encoding support demands its toll, indexing performance dropped a bit in comparision to 0.12, but still thousands of docs per second, depending on machine/docs.
-On Windows the indexing performance is still terrible, but that may be resolved in a future project.
-
-Search performance is still excellent and multiple times faster than Lucene.
-
-Lucene achieves roughly double the indexing performance. This seems to be because of the different way strings and
-encodings are handled in Java. For example, the Java WhitespaceTokenizer code requires only one method call per character (check for whitespace), but for Ruby, to support all the different encodings, several method calls are required per character (retrieve character according to encoding, check character for whitespace).
-Ferret is internally using the standard Ruby string encoding methods.
+For version 0.13.7 the performance bottle neck has been identified and removed, ferret now delivers excellent indexing perfomance on all platforms, see numbers below.
+On Windows performance is still not as good as on Linux, but that is equally true for Lucene and because of how the Windows filesystem works.
 
 ## Documentation
 
@@ -105,43 +99,41 @@ Ensure your locale is set to C.UTF-8, because the internal c tests don't know ho
 
 A recent Java JDK must be installed to compile and run lucene benchmarks.
 
-Results on Linux:
-```
-Ferret 0.13.0:
-Indexing: 9.35 secs, Docs: 19043, 2035 docs/s
-Searching took: 0.3133133s for 8000 queries
-thats 25533 q/s
-Total found: 42000
-Index size: 28Mb
-
-Lucene 9.1.0:
-Indexing: 4.20 secs, Docs: 19043, 4538 docs/s
-Searching took: 1.64s for 8000 queries
-thats 4875 q/s
-Total found: 41000
-index size: 35Mb
-
-JVM 11.0.14.1 (Ubuntu)
-```
+Results, Ferret 0.13.7 vs. Lucene 9.1.0, WhitespaceAnalyzer,
+Linux Ubuntu 20.04, FreeBSD 13.0 and Windows 10 on old Intel Core i5 from 2015,
+LinuxPi on RaspberryPi 400:
+
+| OS      | Task       | Ferret          | Lucene*        |
+|---------|------------|-----------------|----------------|
+| Linux   | Indexing   |     4905 docs/s |    4785 docs/s |
+| FreeBSD | Indexing   |     4516 docs/s |       -        |
+| Windows | Indexing   |     2361 docs/s |    2395 docs/s |
+| LinuxPi | Indexing   |     1161 docs/s |     707 docs/s |
+| Linux   | Searching  | 25664 queries/s | 4708 queries/s |
+| FreeBSD | Searching  | 25073 queries/s |    -           |
+| Windows | Searching  |  3646 queries/s |  935 queries/s |
+| LinuxPi | Searching  |  5768 queries/s |  680 queries/s |
+|         | Index Size |           28 MB |          35 MB |
+
+*Lucene 9.1.0 on JVM 11.0.14.1 (Ubuntu)
 
 ### Storing Fields with Compression, Indexing and Retrieval
 - clone repo
 - bundle install
 - rake ferret_compression_benchmark
 
-Results on Linux, 0.13.0:
+Results on Linux, 0.13.7, on old Intel Core i5 from 2015:
 
 | Compression | Index & Store | Retrieve      | Index size |
 |-------------|---------------|---------------|------------|
-| none        |   2008 docs/s | 153853 docs/s |      43 MB |
-| brotli      |   1726 docs/s |  58315 docs/s |      36 MB |
-| bzip2       |   1438 docs/s |  15382 docs/s |      38 MB |
-| lz4         |   1932 docs/s | 127100 docs/s |      41 MB |
+| none        |   4866 docs/s | 153853 docs/s |      43 MB |
+| brotli      |   3539 docs/s |  58315 docs/s |      36 MB |
+| bzip2       |   2624 docs/s |  15382 docs/s |      38 MB |
+| lz4         |   4639 docs/s | 127100 docs/s |      41 MB |
 
 ## Future
 
 Lots of things to do:
-- Improve indexing performance on Windows (WriteFile is terribly slow, maybe use mapping, see libuv)
 - Bring documentation in order in a docs directory
 - Review code (especially for memory/stack issues, typical c issues)
 - Take care of ruby GVL and threading

data/ext/isomorfeus_ferret_ext/frb_analysis.c

@@ -458,7 +458,7 @@ static VALUE frb_ts_next(VALUE self) {
 
 static void frb_tf_mark(void *p) {
     FrtTokenStream *ts = (FrtTokenStream *)p;
-    if (TkFilt(ts)->sub_ts->rts)
+    if (TkFilt(ts)->sub_ts && TkFilt(ts)->sub_ts->rts)
         rb_gc_mark(TkFilt(ts)->sub_ts->rts);
 }
 
@@ -1331,9 +1331,10 @@ static void frb_h_mark_values_i(void *key, void *value, void *arg) {
 }
 
 static void frb_pfa_mark(void *p) {
-    if (PFA(p)->default_a->ranalyzer)
+    if (PFA(p)->default_a && PFA(p)->default_a->ranalyzer)
         rb_gc_mark(PFA(p)->default_a->ranalyzer);
-    frt_h_each(PFA(p)->dict, &frb_h_mark_values_i, NULL);
+    if (PFA(p)->dict)
+        frt_h_each(PFA(p)->dict, &frb_h_mark_values_i, NULL);
 }
 
 /*** PerFieldAnalyzer ***/
@@ -1370,6 +1371,8 @@ const rb_data_type_t frb_per_field_analyzer_t = {
 
 static VALUE frb_per_field_analyzer_alloc(VALUE rclass) {
     FrtAnalyzer *a = frt_per_field_analyzer_alloc();
+    PFA(a)->default_a = NULL;
+    PFA(a)->dict = NULL;
     return TypedData_Wrap_Struct(rclass, &frb_per_field_analyzer_t, a);
 }
 
@@ -1435,7 +1438,7 @@ static VALUE frb_pfa_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstri
 /*** RegExpAnalyzer ***/
 
 static void frb_re_analyzer_mark(void *p) {
-    if (((FrtAnalyzer *)p)->current_ts->rts)
+    if (((FrtAnalyzer *)p)->current_ts && ((FrtAnalyzer *)p)->current_ts->rts)
         rb_gc_mark(((FrtAnalyzer *)p)->current_ts->rts);
 }
 

data/ext/isomorfeus_ferret_ext/frb_index.c

@@ -2,7 +2,7 @@
 #include "isomorfeus_ferret.h"
 #include <ruby.h>
 
-#undef close
+// #undef close
 
 VALUE mIndex;
 
@@ -429,6 +429,7 @@ static VALUE frb_get_field_infos(FrtFieldInfos *fis) {
 
 static VALUE frb_fis_alloc(VALUE rclass) {
     FrtFieldInfos *fis = frt_fis_alloc();
+    fis->size = 0;
     return TypedData_Wrap_Struct(rclass, &frb_field_infos_t, fis);
 }
 
@@ -674,8 +675,7 @@ frb_fis_get_tk_fields(VALUE self)
  ****************************************************************************/
 
 static void frb_te_free(void *p) {
-    FrtTermEnum *te = (FrtTermEnum *)p;
-    te->close(te);
+    ((FrtTermEnum *)p)->close((FrtTermEnum *)p);
 }
 
 static size_t frb_te_size(const void *p) {

data/ext/isomorfeus_ferret_ext/frb_qparser.c

@@ -34,7 +34,7 @@ static void frb_qp_free(void *p) {
 }
 
 static void frb_qp_mark(void *p) {
-    if (((FrtQParser *)p)->analyzer->ranalyzer)
+    if (((FrtQParser *)p)->analyzer && ((FrtQParser *)p)->analyzer->ranalyzer)
         rb_gc_mark(((FrtQParser *)p)->analyzer->ranalyzer);
 }
 
@@ -59,6 +59,7 @@ const rb_data_type_t frb_qp_t = {
 
 static VALUE frb_qp_alloc(VALUE rclass) {
     FrtQParser *qp = frt_qp_alloc();
+    qp->analyzer = NULL;
     return TypedData_Wrap_Struct(rclass, &frb_qp_t, qp);
 }
 

data/ext/isomorfeus_ferret_ext/frb_search.c

@@ -4,7 +4,7 @@
 #include "isomorfeus_ferret.h"
 #include <ruby.h>
 
-#undef close
+// #undef close
 
 VALUE mSearch;
 
@@ -713,7 +713,7 @@ static size_t frb_boolean_clause_t_size(const void *p) {
 }
 
 static void frb_bc_mark(void *p) {
-    if (((FrtBooleanClause *)p)->query->rquery)
+    if (((FrtBooleanClause *)p)->query && ((FrtBooleanClause *)p)->query->rquery)
         rb_gc_mark(((FrtBooleanClause *)p)->query->rquery);
 }
 
@@ -1837,11 +1837,11 @@ static size_t frb_filtered_query_size(const void *p) {
 }
 
 static void frb_fqq_mark(void *p) {
-    FrtFilteredQuery *fq = (FrtFilteredQuery *)p;
-    if (fq->query->rquery)
-        rb_gc_mark(fq->query->rquery);
-    if (fq->filter->rfilter)
-        rb_gc_mark(fq->filter->rfilter);
+    FrtFilteredQuery *fqq = (FrtFilteredQuery *)p;
+    if (fqq->query && fqq->query->rquery)
+        rb_gc_mark(fqq->query->rquery);
+    if (fqq->filter && fqq->filter->rfilter)
+        rb_gc_mark(fqq->filter->rfilter);
 }
 
 const rb_data_type_t frb_filtered_query_t = {
@@ -1860,6 +1860,8 @@ const rb_data_type_t frb_filtered_query_t = {
 
 static VALUE frb_fqq_alloc(VALUE rclass) {
     FrtQuery *fqq = frt_fq_alloc();
+    ((FrtFilteredQuery *)fqq)->query = NULL;
+    ((FrtFilteredQuery *)fqq)->filter = NULL;
     return TypedData_Wrap_Struct(rclass, &frb_filtered_query_t, fqq);
 }
 
@@ -2172,6 +2174,7 @@ const rb_data_type_t frb_span_near_query_t = {
 
 static VALUE frb_spannq_alloc(VALUE rclass) {
     FrtQuery *snq = frt_spannq_alloc();
+    ((FrtSpanNearQuery *)snq)->c_cnt = 0;
     return TypedData_Wrap_Struct(rclass, &frb_span_near_query_t, snq);
 }
 
@@ -2352,9 +2355,9 @@ static size_t frb_span_not_query_size(const void *p) {
 
 static void frb_spanxq_mark(void *p) {
     FrtSpanNotQuery *sxq = (FrtSpanNotQuery *)p;
-    if (sxq->inc->rquery)
+    if (sxq->inc && sxq->inc->rquery)
         rb_gc_mark(sxq->inc->rquery);
-    if (sxq->exc->rquery)
+    if (sxq->exc && sxq->exc->rquery)
         rb_gc_mark(sxq->exc->rquery);
 }
 
@@ -3557,10 +3560,12 @@ static size_t frb_index_searcher_size(const void *p) {
 
 static void frb_sea_mark(void *p) {
     FrtIndexSearcher *isea = (FrtIndexSearcher *)p;
-    if (isea->ir->rir)
-        rb_gc_mark(isea->ir->rir);
-    if (isea->ir->store->rstore)
-        rb_gc_mark(isea->ir->store->rstore);
+    if (isea->ir) {
+        if (isea->ir->rir)
+            rb_gc_mark(isea->ir->rir);
+        if (isea->ir->store && isea->ir->store->rstore)
+            rb_gc_mark(isea->ir->store->rstore);
+    }
 }
 
 const rb_data_type_t frb_index_searcher_t = {
@@ -3578,8 +3583,9 @@ const rb_data_type_t frb_index_searcher_t = {
 };
 
 static VALUE frb_sea_alloc(VALUE rclass) {
-    FrtSearcher *s = frt_isea_alloc();
-    return TypedData_Wrap_Struct(rclass, &frb_index_searcher_t, s);
+    FrtSearcher *sea = frt_isea_alloc();
+    ((FrtIndexSearcher *)sea)->ir = NULL;
+    return TypedData_Wrap_Struct(rclass, &frb_index_searcher_t, sea);
 }
 
 /*
@@ -3665,6 +3671,7 @@ const rb_data_type_t frb_multi_searcher_t = {
 
 static VALUE frb_ms_alloc(VALUE rclass) {
     FrtSearcher *s = frt_msea_alloc();
+    ((FrtMultiSearcher *)s)->s_cnt = 0;
     return TypedData_Wrap_Struct(rclass, &frb_multi_searcher_t, s);
 }
 

data/ext/isomorfeus_ferret_ext/frb_store.c

@@ -2,7 +2,7 @@
 #include "isomorfeus_ferret.h"
 #include <time.h>
 
-#undef rename
+// #undef rename
 
 static ID id_ref_cnt;
 VALUE cLock;
@@ -34,7 +34,7 @@ void frb_lock_free(void *p) {
 
 void frb_lock_mark(void *p) {
     FrtLock *lock = (FrtLock *)p;
-    if (lock->store->rstore)
+    if (lock->store && lock->store->rstore)
         rb_gc_mark(lock->store->rstore);
 }
 

data/ext/isomorfeus_ferret_ext/frt_compound_io.c

@@ -329,7 +329,7 @@ static void cw_copy_file(FrtCompoundWriter *cw, FrtCWFileEntry *src, FrtOutStrea
     frt_is_close(is);
 }
 
-void frt_cw_close(FrtCompoundWriter *cw)
+void frt_cw_close(FrtCompoundWriter *cw, FrtDeleter *dlr)
 {
     FrtOutStream *os = NULL;
     int i;
@@ -356,6 +356,9 @@ void frt_cw_close(FrtCompoundWriter *cw)
     for (i = 0; i < frt_ary_size(cw->file_entries); i++) {
         cw->file_entries[i].data_offset = frt_os_pos(os);
         cw_copy_file(cw, &cw->file_entries[i], os);
+        if (dlr) {
+            frt_deleter_queue_file(dlr, cw->file_entries[i].name);
+        }
     }
 
     /* Write the data offsets into the directory of the compound stream */

data/ext/isomorfeus_ferret_ext/frt_field_index.c

@@ -1,7 +1,7 @@
 #include <string.h>
 #include "frt_field_index.h"
 
-#undef close
+// #undef close
 
 /***************************************************************************
  *

data/ext/isomorfeus_ferret_ext/frt_fs_store.c

@@ -272,15 +272,13 @@ static const struct FrtOutStreamMethods FS_OUT_STREAM_METHODS = {
 static FrtOutStream *fs_new_output(FrtStore *store, const char *filename)
 {
     char path[FRT_MAX_FILE_PATH];
-    int fd = open(join_path(path, store->dir.path, filename),
-                  O_WRONLY | O_CREAT | O_BINARY, store->file_mode);
-    FrtOutStream *os;
+    int fd = open(join_path(path, store->dir.path, filename), O_WRONLY | O_CREAT | O_BINARY, store->file_mode);
     if (fd < 0) {
         FRT_RAISE(FRT_IO_ERROR, "couldn't create OutStream %s: <%s>",
               path, strerror(errno));
     }
 
-    os = frt_os_new();
+    FrtOutStream *os = frt_os_new();
     os->file.fd = fd;
     os->m = &FS_OUT_STREAM_METHODS;
     return os;

data/ext/isomorfeus_ferret_ext/frt_global.h

@@ -13,7 +13,7 @@
 
 #define FRT_MAX_WORD_SIZE 255
 #define FRT_MAX_FILE_PATH 1024
-#define FRT_BUFFER_SIZE 1024
+#define FRT_BUFFER_SIZE 16384
 
 typedef enum {
     FRT_COMPRESSION_NONE = 0,

data/ext/isomorfeus_ferret_ext/frt_ind.c

@@ -2,7 +2,7 @@
 #include "frt_array.h"
 #include <string.h>
 
-#undef close
+// #undef close
 
 static const char *NON_UNIQUE_KEY_ERROR_MSG =
     "Tried to use a key that was not unique";

data/ext/isomorfeus_ferret_ext/frt_index.c

@@ -11,8 +11,8 @@
 #include "bzlib.h"
 #include "lz4frame.h"
 
-#undef close
-#undef read
+// #undef close
+// #undef read
 
 extern rb_encoding *utf8_encoding;
 extern void frt_micro_sleep(const int micro_seconds);
@@ -681,15 +681,13 @@ static char *si_norm_file_name(FrtSegmentInfo *si, char *buf, int field_num)
     if (field_num >= si->norm_gens_size
         || 0 > (norm_gen = si->norm_gens[field_num])) {
         return NULL;
-    }
-    else {
+    } else {
         const char *ext = (si->use_compound_file && norm_gen > 0) ? "s" : "f";
         return fn_for_gen_field(buf, si->name, ext, norm_gen, field_num);
     }
 }
 
-static void deleter_queue_file(FrtDeleter *dlr, const char *file_name);
-#define DEL(file_name) deleter_queue_file(dlr, file_name)
+void frt_deleter_queue_file(FrtDeleter *dlr, const char *file_name);
 
 static void si_delete_files(FrtSegmentInfo *si, FrtFieldInfos *fis, FrtDeleter *dlr)
 {
@@ -700,7 +698,7 @@ static void si_delete_files(FrtSegmentInfo *si, FrtFieldInfos *fis, FrtDeleter *
 
     for (i = si->norm_gens_size - 1; i >= 0; i--) {
         if (0 <= si->norm_gens[i]) {
-            DEL(si_norm_file_name(si, file_name, fis->fields[i]->number));
+            frt_deleter_queue_file(dlr, si_norm_file_name(si, file_name, fis->fields[i]->number));
         }
     }
 
@@ -710,15 +708,14 @@ static void si_delete_files(FrtSegmentInfo *si, FrtFieldInfos *fis, FrtDeleter *
 
     if (si->use_compound_file) {
         memcpy(ext, "cfs", 4);
-        DEL(file_name);
+        frt_deleter_queue_file(dlr, file_name);
         if (0 <= si->del_gen) {
-            DEL(frt_fn_for_generation(file_name, si->name, "del", si->del_gen));
+            frt_deleter_queue_file(dlr, frt_fn_for_generation(file_name, si->name, "del", si->del_gen));
         }
-    }
-    else {
+    } else {
         for (i = FRT_NELEMS(INDEX_EXTENSIONS) - 1; i >= 0; i--) {
             memcpy(ext, INDEX_EXTENSIONS[i], 4);
-            DEL(file_name);
+            frt_deleter_queue_file(dlr, file_name);
         }
     }
 }
@@ -3695,8 +3692,7 @@ static bool file_name_filter_is_cfs_file(const char *file_name) {
  ****************************************************************************/
 
 #define DELETABLE_START_CAPA 8
-FrtDeleter *frt_deleter_new(FrtSegmentInfos *sis, FrtStore *store)
-{
+FrtDeleter *frt_deleter_new(FrtSegmentInfos *sis, FrtStore *store) {
     FrtDeleter *dlr = FRT_ALLOC(FrtDeleter);
     dlr->sis = sis;
     dlr->store = store;
@@ -3704,19 +3700,16 @@ FrtDeleter *frt_deleter_new(FrtSegmentInfos *sis, FrtStore *store)
     return dlr;
 }
 
-void frt_deleter_destroy(FrtDeleter *dlr)
-{
+void frt_deleter_destroy(FrtDeleter *dlr) {
     frt_hs_destroy(dlr->pending);
     free(dlr);
 }
 
-static void deleter_queue_file(FrtDeleter *dlr, const char *file_name)
-{
+void frt_deleter_queue_file(FrtDeleter *dlr, const char *file_name) {
     frt_hs_add(dlr->pending, frt_estrdup(file_name));
 }
 
-void frt_deleter_delete_file(FrtDeleter *dlr, char *file_name)
-{
+void frt_deleter_delete_file(FrtDeleter *dlr, char *file_name) {
     FrtStore *store = dlr->store;
     FRT_TRY
         if (store->exists(store, file_name)) {
@@ -3728,8 +3721,7 @@ void frt_deleter_delete_file(FrtDeleter *dlr, char *file_name)
     FRT_XENDTRY
 }
 
-static void deleter_commit_pending_deletions(FrtDeleter *dlr)
-{
+static void deleter_commit_pending_deletions(FrtDeleter *dlr) {
     FrtHashSetEntry *hse, *hse_next = dlr->pending->first;
     while ((hse = hse_next) != NULL) {
         hse_next = hse->next;
@@ -3737,23 +3729,13 @@ static void deleter_commit_pending_deletions(FrtDeleter *dlr)
     }
 }
 
-void frt_deleter_delete_files(FrtDeleter *dlr, char **files, int file_cnt)
-{
-    int i;
-    for (i = file_cnt - 1; i >= 0; i--) {
-        deleter_queue_file(dlr, files[i]);
-    }
-    deleter_commit_pending_deletions(dlr);
-}
-
 struct DelFilesArg {
     char  curr_seg_file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
     FrtDeleter *dlr;
     FrtHash *current;
 };
 
-static void frt_deleter_find_deletable_files_i(const char *file_name, void *arg)
-{
+static void frt_deleter_find_deletable_files_i(const char *file_name, void *arg) {
     struct DelFilesArg *dfa = (struct DelFilesArg *)arg;
     FrtDeleter *dlr = dfa->dlr;
 
@@ -3773,8 +3755,7 @@ static void frt_deleter_find_deletable_files_i(const char *file_name, void *arg)
         if (NULL != p) {
             *p = '\0';
             extension = p + 1;
-        }
-        else {
+        } else {
             extension = NULL;
         }
 
@@ -3789,16 +3770,14 @@ static void frt_deleter_find_deletable_files_i(const char *file_name, void *arg)
         if (NULL == (si = (FrtSegmentInfo *)frt_h_get(dfa->current, segment_name))) {
             /* Delete if segment is not referenced: */
             do_delete = true;
-        }
-        else {
+        } else {
             char tmp_fn[FRT_SEGMENT_NAME_MAX_LENGTH];
             /* OK, segment is referenced, but file may still be orphan'd: */
             if (file_name_filter_is_cfs_file(file_name)
                 && si->use_compound_file) {
                 /* This file is stored in a CFS file for this segment: */
                 do_delete = true;
-            }
-            else if (0 == strcmp("del", extension)) {
+            } else if (0 == strcmp("del", extension)) {
                 /* This is a _segmentName_N.del file: */
                 if (!frt_fn_for_generation(tmp_fn, segment_name, "del", si->del_gen)
                     || 0 != strcmp(file_name, tmp_fn)) {
@@ -3807,8 +3786,7 @@ static void frt_deleter_find_deletable_files_i(const char *file_name, void *arg)
                      * this segment, then delete it: */
                     do_delete = true;
                 }
-            }
-            else if (NULL != extension
+            } else if (NULL != extension
                      && ('s' == *extension || 'f' == *extension)
                      && isdigit(extension[1])) {
                 si_norm_file_name(si, tmp_fn, atoi(extension + 1));
@@ -3817,15 +3795,14 @@ static void frt_deleter_find_deletable_files_i(const char *file_name, void *arg)
                     /* This is an orphan'd norms file: */
                     do_delete = true;
                 }
-            }
-            else if (0 == strcmp("cfs", extension) && !si->use_compound_file) {
+            } else if (0 == strcmp("cfs", extension) && !si->use_compound_file) {
                 /* This is a partially written _segmentName.cfs: */
                 do_delete = true;
             }
         }
 
         if (do_delete) {
-            deleter_queue_file(dlr, file_name);
+            frt_deleter_queue_file(dlr, file_name);
         }
     }
 }
@@ -3837,8 +3814,7 @@ static void frt_deleter_find_deletable_files_i(const char *file_name, void *arg)
  * create the unused file (eg when merging segments), and we only remove from
  * deletable when a file is successfully deleted.
  */
-void frt_deleter_find_deletable_files(FrtDeleter *dlr)
-{
+void frt_deleter_find_deletable_files(FrtDeleter *dlr) {
     /* Gather all "current" segments: */
     int i;
     FrtSegmentInfos *sis = dlr->sis;
@@ -3937,9 +3913,10 @@ static FrtIndexReader *ir_setup(FrtIndexReader *ir, FrtStore *store, FrtSegmentI
         ir->store = store;
         FRT_REF(store);
     }
-    ir->sis = sis;
-    ir->fis = fis;
+    ir->sis     = sis;
+    ir->fis     = fis;
     ir->ref_cnt = 1;
+    ir->rir     = Qnil;
 
     ir->is_owner = is_owner;
     if (is_owner) {
@@ -4207,7 +4184,7 @@ static void norm_rewrite(Norm *norm, FrtStore *store, FrtDeleter *dlr,
     const int field_num = norm->field_num;
 
     if (si_norm_file_name(si, norm_file_name, field_num)) {
-        deleter_queue_file(dlr, norm_file_name);
+        frt_deleter_queue_file(dlr, norm_file_name);
     }
     frt_si_advance_norm_gen(si, field_num);
     si_norm_file_name(si, norm_file_name, field_num);
@@ -4363,7 +4340,7 @@ static void sr_commit_i(FrtIndexReader *ir)
     if (SR(ir)->undelete_all || SR(ir)->deleted_docs_dirty) {
         if (si->del_gen >= 0) {
             frt_fn_for_generation(tmp_file_name, segment, "del", si->del_gen);
-            deleter_queue_file(ir->deleter, tmp_file_name);
+            frt_deleter_queue_file(ir->deleter, tmp_file_name);
         }
         if (SR(ir)->undelete_all) {
             si->del_gen = -1;
@@ -5687,7 +5664,6 @@ static SegmentMergeInfo *smi_new(int base, FrtStore *store, FrtSegmentInfo *si)
         smi->store = frt_open_cmpd_store(store, file_name);
     }
 
-
     sprintf(file_name, "%s.fdx", segment);
     smi->doc_cnt = smi->max_doc
         = smi->store->length(smi->store, file_name) / FIELDS_IDX_PTR_SIZE;
@@ -6116,14 +6092,7 @@ int frt_iw_doc_count(FrtIndexWriter *iw)
     return doc_cnt;
 }
 
-#define MOVE_TO_COMPOUND_DIR(file_name)\
-    deleter_queue_file(dlr, file_name);\
-    frt_cw_add_file(cw, file_name)
-
-static void iw_create_compound_file(FrtStore *store, FrtFieldInfos *fis,
-                                    FrtSegmentInfo *si, char *cfs_file_name,
-                                    FrtDeleter *dlr)
-{
+static void iw_create_compound_file(FrtStore *store, FrtFieldInfos *fis, FrtSegmentInfo *si, char *cfs_file_name, FrtDeleter *dlr) {
     int i;
     FrtCompoundWriter *cw;
     char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
@@ -6137,19 +6106,18 @@ static void iw_create_compound_file(FrtStore *store, FrtFieldInfos *fis,
     cw = frt_open_cw(store, cfs_file_name);
     for (i = 0; i < FRT_NELEMS(COMPOUND_EXTENSIONS); i++) {
         memcpy(ext, COMPOUND_EXTENSIONS[i], 4);
-        MOVE_TO_COMPOUND_DIR(file_name);
+        frt_cw_add_file(cw, file_name);
     }
 
     /* Field norm file_names */
     for (i = fis->size - 1; i >= 0; i--) {
-        if (fi_has_norms(fis->fields[i])
-            && si_norm_file_name(si, file_name, i)) {
-            MOVE_TO_COMPOUND_DIR(file_name);
+        if (fi_has_norms(fis->fields[i]) && si_norm_file_name(si, file_name, i)) {
+            frt_cw_add_file(cw, file_name);
         }
     }
 
     /* Perform the merge */
-    frt_cw_close(cw);
+    frt_cw_close(cw, dlr);
 }
 
 static void iw_commit_compound_file(FrtIndexWriter *iw, FrtSegmentInfo *si)
@@ -6464,6 +6432,8 @@ static void iw_cp_fields(FrtIndexWriter *iw, FrtSegmentReader *sr, const char *s
         sprintf(file_name, "%s.del", segment);
         del_out = store_out->new_output(store_out, file_name);
         frt_is2os_copy_bytes(del_in, del_out, frt_is_length(del_in));
+        frt_os_close(del_out);
+        frt_is_close(del_in);
     }
 
     if (map) {

data/ext/isomorfeus_ferret_ext/frt_index.h

@@ -627,9 +627,9 @@ struct FrtDeleter {
 
 extern FrtDeleter *frt_deleter_new(FrtSegmentInfos *sis, FrtStore *store);
 extern void frt_deleter_destroy(FrtDeleter *dlr);
+extern void frt_deleter_queue_file(FrtDeleter *dlr, const char *file_name);
 extern void frt_deleter_delete_file(FrtDeleter *dlr, char *file_name);
 extern void frt_deleter_find_deletable_files(FrtDeleter *dlr);
-extern void frt_deleter_delete_files(FrtDeleter *dlr, char **files, int file_cnt);
 
 /****************************************************************************
  *
@@ -859,6 +859,6 @@ typedef struct FrtCompoundWriter {
 
 extern FrtCompoundWriter *frt_open_cw(FrtStore *store, char *name);
 extern void frt_cw_add_file(FrtCompoundWriter *cw, char *id);
-extern void frt_cw_close(FrtCompoundWriter *cw);
+extern void frt_cw_close(FrtCompoundWriter *cw, FrtDeleter *dlr);
 
 #endif

data/ext/isomorfeus_ferret_ext/frt_q_boolean.c

@@ -1280,7 +1280,7 @@ static int  bc_eq(FrtBooleanClause *self, FrtBooleanClause *o) {
 }
 
 FrtBooleanClause *frt_bc_alloc(void) {
-    return FRT_ALLOC(FrtBooleanClause);
+    return FRT_ALLOC_AND_ZERO(FrtBooleanClause);
 }
 
 FrtBooleanClause *frt_bc_init(FrtBooleanClause *self, FrtQuery *query, FrtBCType occur) {

data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c

@@ -2,7 +2,7 @@
 #include "frt_search.h"
 #include "frt_helper.h"
 
-#undef close
+// #undef close
 
 /****************************************************************************
  *

data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c

@@ -3,8 +3,8 @@
 #include "frt_search.h"
 #include "frt_helper.h"
 
-#undef close
-#undef read
+// #undef close
+// #undef read
 
 #define MTQ(query) ((FrtMultiTermQuery *)(query))
 

data/ext/isomorfeus_ferret_ext/frt_q_phrase.c

@@ -4,7 +4,7 @@
 #include "frt_search.h"
 #include "frt_array.h"
 
-#undef close
+// #undef close
 
 #define PhQ(query) ((FrtPhraseQuery *)(query))
 

data/ext/isomorfeus_ferret_ext/frt_q_prefix.c

@@ -2,7 +2,7 @@
 #include "frt_global.h"
 #include "frt_search.h"
 
-#undef close
+// #undef close
 
 /****************************************************************************
  *

data/ext/isomorfeus_ferret_ext/frt_q_range.c

@@ -2,7 +2,7 @@
 #include "frt_global.h"
 #include "frt_search.h"
 
-#undef close
+// #undef close
 
 /*****************************************************************************
  *

data/ext/isomorfeus_ferret_ext/frt_q_span.c

@@ -4,7 +4,7 @@
 #include "frt_search.h"
 #include "frt_hashset.h"
 
-#undef close
+// #undef close
 
 #define CLAUSE_INIT_CAPA 4
 

data/ext/isomorfeus_ferret_ext/frt_q_term.c

@@ -2,8 +2,8 @@
 #include <string.h>
 #include "frt_search.h"
 
-#undef close
-#undef read
+// #undef close
+// #undef read
 
 #define TQ(query) ((FrtTermQuery *)(query))
 #define TSc(scorer) ((TermScorer *)(scorer))

data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c

@@ -2,7 +2,7 @@
 #include "frt_global.h"
 #include "frt_search.h"
 
-#undef close
+// #undef close
 
 /****************************************************************************
  *

data/ext/isomorfeus_ferret_ext/frt_search.c

@@ -3,7 +3,7 @@
 #include "frt_search.h"
 #include "frt_array.h"
 
-#undef close
+// #undef close
 
 /***************************************************************************
  *

data/ext/isomorfeus_ferret_ext/frt_sort.c

@@ -3,7 +3,7 @@
 #include "frt_index.h"
 #include "frt_field_index.h"
 
-#undef close
+// #undef close
 
 /***************************************************************************
  *

data/ext/isomorfeus_ferret_ext/frt_store.c

@@ -149,7 +149,7 @@ void frt_os_seek(FrtOutStream *os, off_t new_pos)
  */
 void frt_os_write_byte(FrtOutStream *os, frt_uchar b)
 {
-    if (os->buf.pos >= FRT_BUFFER_SIZE) {
+    if (os->buf.pos >= (FRT_BUFFER_SIZE - 1)) {
         frt_os_flush(os);
     }
     write_byte(os, b);
@@ -157,15 +157,12 @@ void frt_os_write_byte(FrtOutStream *os, frt_uchar b)
 
 void frt_os_write_bytes(FrtOutStream *os, const frt_uchar *buf, int len)
 {
-    if (os->buf.pos > 0) {      /* flush buffer */
-        frt_os_flush(os);
-    }
-
-    if (len < FRT_BUFFER_SIZE) {
-        os->m->flush_i(os, buf, len);
-        os->buf.start += len;
+    if (len < (FRT_BUFFER_SIZE - os->buf.pos)) {
+        memcpy(os->buf.buf + os->buf.pos, buf, len);
+        os->buf.pos += len;
     }
     else {
+        frt_os_flush(os);
         int pos = 0;
         int size;
         while (pos < len) {

data/ext/isomorfeus_ferret_ext/test.c

@@ -810,7 +810,7 @@ void Init_Test(void) {
     rb_define_singleton_method(mTest, "filter",         frb_ts_filter, 0);
     rb_define_singleton_method(mTest, "fs_store",       frb_ts_fs_store, 0);
     rb_define_singleton_method(mTest, "global",         frb_ts_global, 0);
-    rb_define_singleton_method(mTest, "hash",           frb_ts_hash, 0);
+    rb_define_singleton_method(mTest, "test_hash",      frb_ts_hash, 0);
     rb_define_singleton_method(mTest, "hashset",        frb_ts_hashset, 0);
     rb_define_singleton_method(mTest, "helper",         frb_ts_helper, 0);
     rb_define_singleton_method(mTest, "highlighter",    frb_ts_highlighter, 0);

data/ext/isomorfeus_ferret_ext/test_compound_io.c

@@ -48,7 +48,7 @@ void test_compound_writer(TestCase *tc, void *data)
     cw = frt_open_cw(store, (char *)"cfile");
     frt_cw_add_file(cw, (char *)"file1");
     frt_cw_add_file(cw, (char *)"file2");
-    frt_cw_close(cw);
+    frt_cw_close(cw, NULL);
 
     is = store->open_input(store, "cfile");
     Aiequal(2, frt_is_read_vint(is));
@@ -94,7 +94,7 @@ void test_compound_io(TestCase *tc, void *data)
     frt_cw_add_file(cw, (char *)"file1");
     frt_cw_add_file(cw, (char *)"file2");
     frt_cw_add_file(cw, (char *)"file3");
-    frt_cw_close(cw);
+    frt_cw_close(cw, NULL);
 
     c_reader = frt_open_cmpd_store(store, "cfile");
     is1 = c_reader->open_input(c_reader, "file1");
@@ -137,7 +137,7 @@ void test_compound_io_many_files(TestCase *tc, void *data)
         frt_os_write_vint(os, MAGIC);
         frt_os_close(os);
     }
-    frt_cw_close(cw);
+    frt_cw_close(cw, NULL);
 
     c_reader = frt_open_cmpd_store(store, "_.cfs");
     for (i = 0; i < TEST_FILE_CNT; i++) {

data/ext/isomorfeus_ferret_ext/test_filter.c

@@ -2,7 +2,7 @@
 #include "frt_search.h"
 #include "test.h"
 
-#undef close
+// #undef close
 
 #define FILTER_DOCS_SIZE 10
 #define ARRAY_SIZE 20

data/ext/isomorfeus_ferret_ext/test_hash.c

@@ -45,6 +45,7 @@ static void test_hash_str(TestCase *tc, void *data)
 
     size_t res;
     f = temp_open();
+    Assert(f != NULL, "Could not open tempfile");
     frt_h_str_print_keys(h, f);
     fseek(f, 0, SEEK_SET);
     res = fread(buf, 1, 100, f);

data/ext/isomorfeus_ferret_ext/test_highlighter.c

@@ -3,7 +3,7 @@
 #include "testhelper.h"
 #include "test.h"
 
-#undef close
+// #undef close
 
 #define ARRAY_SIZE 100
 

data/ext/isomorfeus_ferret_ext/test_index.c

@@ -2,8 +2,8 @@
 #include "testhelper.h"
 #include "test.h"
 
-#undef close
-#undef read
+// #undef close
+// #undef read
 
 static ID body, title, text, author, year, changing_field, compressed_field_brotli, compressed_field_bzip2, compressed_field_lz4, tag;
 

data/ext/isomorfeus_ferret_ext/test_q_const_score.c

@@ -1,7 +1,7 @@
 #include "frt_search.h"
 #include "test.h"
 
-#undef close
+// #undef close
 
 static ID num;
 

data/ext/isomorfeus_ferret_ext/test_q_filtered.c

@@ -1,7 +1,7 @@
 #include "frt_search.h"
 #include "test.h"
 
-#undef close
+// #undef close
 
 static ID num, flipflop;
 

data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c

@@ -1,7 +1,7 @@
 #include "frt_search.h"
 #include "test.h"
 
-#undef close
+// #undef close
 
 #define ARRAY_SIZE 20
 

data/ext/isomorfeus_ferret_ext/test_q_span.c

@@ -1,7 +1,7 @@
 #include "frt_search.h"
 #include "test.h"
 
-#undef close
+// #undef close
 
 #define ARRAY_SIZE 20
 #define TEST_SE(query, ir, expected) do { \

data/ext/isomorfeus_ferret_ext/test_search.c

@@ -5,7 +5,7 @@
 #include "testhelper.h"
 #include "test.h"
 
-#undef close
+// #undef close
 
 #define ARRAY_SIZE 40
 

data/ext/isomorfeus_ferret_ext/test_sort.c

@@ -3,7 +3,7 @@
 #include "frt_search.h"
 #include "test.h"
 
-#undef close
+// #undef close
 
 #define ARRAY_SIZE 20
 

data/ext/isomorfeus_ferret_ext/test_store.c

@@ -3,7 +3,7 @@
 #include <limits.h>
 #include "test.h"
 
-#undef rename
+// #undef rename
 
 #define TEST_LOCK_NAME "test"
 

data/ext/isomorfeus_ferret_ext/test_term.c

@@ -1,7 +1,7 @@
 #include "frt_index.h"
 #include "test.h"
 
-#undef close
+// #undef close
 
 /***************************************************************************
  *

data/lib/isomorfeus/ferret/version.rb

@@ -1,5 +1,5 @@
 module Isomorfeus
   module Ferret
-    VERSION = '0.13.5'
+    VERSION = '0.13.8'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: isomorfeus-ferret
 version: !ruby/object:Gem::Version
-  version: 0.13.5
+  version: 0.13.8
 platform: ruby
 authors:
 - Jan Biedermann
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-20 00:00:00.000000000 Z
+date: 2022-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake