isomorfeus-ferret 0.16.2 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca79dc44a2271fca9a4f0fae7586aab07f0d2267728591da5acd93edb08805cf
-  data.tar.gz: 19aa0e83eac3c572bc50c995dfadc7820f0b45f28cd7968566c394eebe3d7426
+  metadata.gz: 74adb05c4210e599a5b55f6fc0b5c41e88fd3f50586eb57a26f49031ce1c9e07
+  data.tar.gz: '058640d7f9e2fec23eaa7094cd6c1f7cfeb50e42068567e3da2e06b795060434'
 SHA512:
-  metadata.gz: 45b3051498353bf3d957a461d1fab80ceefda0dbc2ae393fc686fd3f712f619b272d3d030b2737e372ba0e52a8cc5ce0dcf19627e6cb2bf738e9d54b6a7f8dd4
-  data.tar.gz: c9421bc05bc8f5a84f48db79b61fc6f3b41edfba3e69613a303f2057a418752f72b71775d452fea744dbb5c220ce95899c5802039ce0c23cd74b1adfb853afe3
+  metadata.gz: f6034d8bfc52b1413b8c60c24b9f6d3aa9dfa93c2048a2e0a9d17ea12c49907364ad1d2c1f93d19ab105e7981080f21ea96520d779caba1c9d7f142ef4ce79e8
+  data.tar.gz: 278e40b54014a297ad0e4c4a87a86168bf4bb6a5ddb561197c7b5ae170a0175d8414d890211ad910e06f953dd674c33d46aca3f11605f8aaa76231b01034cdfa

data/ext/isomorfeus_ferret_ext/frb_index.c

@@ -29,7 +29,6 @@ static VALUE sym_merge_factor;
 static VALUE sym_max_buffered_docs;
 static VALUE sym_max_merge_docs;
 static VALUE sym_max_field_length;
-static VALUE sym_use_compound_file;
 static VALUE sym_field_infos;
 
 static ID fsym_content;
@@ -1041,9 +1040,6 @@ static VALUE frb_iw_init(int argc, VALUE *argv, VALUE self) {
                 frb_create_dir(rval);
                 store = frt_open_mdbx_store(rs2s(rval));
             }
-            /* use_compound_file defaults to true */
-            config.use_compound_file =
-                (rb_hash_aref(roptions, sym_use_compound_file) == Qfalse) ? false : true;
 
             if ((rval = rb_hash_aref(roptions, sym_analyzer)) != Qnil) {
                 analyzer = frb_get_cwrapped_analyzer(rval);
@@ -1730,33 +1726,6 @@ frb_iw_set_max_field_length(VALUE self, VALUE rval)
     return rval;
 }
 
-/*
- *  call-seq:
- *     iw.use_compound_file -> number
- *
- *  Return the current value of use_compound_file
- */
-static VALUE
-frb_iw_get_use_compound_file(VALUE self)
-{
-    FrtIndexWriter *iw = (FrtIndexWriter *)DATA_PTR(self);
-    return iw->config.use_compound_file ? Qtrue : Qfalse;
-}
-
-/*
- *  call-seq:
- *     iw.use_compound_file = use_compound_file -> use_compound_file
- *
- *  Set the use_compound_file parameter
- */
-static VALUE
-frb_iw_set_use_compound_file(VALUE self, VALUE rval)
-{
-    FrtIndexWriter *iw = (FrtIndexWriter *)DATA_PTR(self);
-    iw->config.use_compound_file = RTEST(rval);
-    return rval;
-}
-
 /****************************************************************************
  *
  * IndexReader Methods
@@ -2789,11 +2758,6 @@ static void Init_TermVector(void) {
  *                        you want to be more thorough and you are indexing
  *                        documents from your file-system you may set this
  *                        parameter to Ferret::FIX_INT_MAX.
- *  use_compound_file::   Default: true. Uses a compound file to store the
- *                        index. This prevents an error being raised for
- *                        having too many files open at the same time. The
- *                        default is true but performance is better if this is
- *                        set to false.
  *
  *
  *  === Deleting Documents
@@ -2825,7 +2789,6 @@ void Init_IndexWriter(void) {
     sym_max_buffered_docs = ID2SYM(rb_intern("max_buffered_docs"));
     sym_max_merge_docs    = ID2SYM(rb_intern("max_merge_docs"));
     sym_max_field_length  = ID2SYM(rb_intern("max_field_length"));
-    sym_use_compound_file = ID2SYM(rb_intern("use_compound_file"));
 
     cIndexWriter = rb_define_class_under(mIndex, "IndexWriter", rb_cObject);
     rb_define_alloc_func(cIndexWriter, frb_iw_alloc);
@@ -2842,7 +2805,6 @@ void Init_IndexWriter(void) {
     rb_define_const(cIndexWriter, "DEFAULT_MAX_BUFFERED_DOCS", INT2FIX(frt_default_config.max_buffered_docs));
     rb_define_const(cIndexWriter, "DEFAULT_MAX_MERGE_DOCS", INT2FIX(frt_default_config.max_merge_docs));
     rb_define_const(cIndexWriter, "DEFAULT_MAX_FIELD_LENGTH", INT2FIX(frt_default_config.max_field_length));
-    rb_define_const(cIndexWriter, "DEFAULT_USE_COMPOUND_FILE", frt_default_config.use_compound_file ? Qtrue : Qfalse);
 
     rb_define_method(cIndexWriter, "initialize",   frb_iw_init, -1);
     rb_define_method(cIndexWriter, "doc_count",    frb_iw_get_doc_count, 0);
@@ -2881,9 +2843,6 @@ void Init_IndexWriter(void) {
 
     rb_define_method(cIndexWriter, "max_field_length",  frb_iw_get_max_field_length, 0);
     rb_define_method(cIndexWriter, "max_field_length=", frb_iw_set_max_field_length, 1);
-
-    rb_define_method(cIndexWriter, "use_compound_file",  frb_iw_get_use_compound_file, 0);
-    rb_define_method(cIndexWriter, "use_compound_file=", frb_iw_set_use_compound_file, 1);
 }
 
 /*

data/ext/isomorfeus_ferret_ext/frt_ind.c

@@ -99,7 +99,6 @@ void frt_ensure_writer_open(FrtIndex *self)
         /* make sure the analzyer isn't deleted by the FrtIndexWriter */
         FRT_REF(self->analyzer);
         self->iw = frt_iw_open(NULL, self->store, self->analyzer, false);
-        self->iw->config.use_compound_file = self->config.use_compound_file;
     }
 }
 

data/ext/isomorfeus_ferret_ext/frt_index.c

@@ -37,8 +37,7 @@ const FrtConfig frt_default_config = {
     10,             /* default merge factor */
     10000,          /* max_buffered_docs */
     INT_MAX,        /* max_merge_docs */
-    10000,          /* maximum field length (number of terms) */
-    true            /* use compound file by default */
+    10000           /* maximum field length (number of terms) */
 };
 
 static void ste_reset(FrtTermEnum *te);
@@ -56,11 +55,6 @@ static const char *INDEX_EXTENSIONS[] = {
     "frq", "prx", "fdx", "fdt", "tfx", "tix", "tis", "del", "gen", "cfs"
 };
 
-/* *** Must be three characters *** */
-static const char *COMPOUND_EXTENSIONS[] = {
-    "frq", "prx", "fdx", "fdt", "tfx", "tix", "tis"
-};
-
 static const char BASE36_DIGITMAP[] = "0123456789abcdefghijklmnopqrstuvwxyz";
 
 static char *u64_to_str36(char *buf, int buf_size, frt_u64 u)
@@ -512,7 +506,6 @@ FrtSegmentInfo *frt_si_new(char *name, int doc_cnt, FrtStore *store)
     si->norm_gens = NULL;
     si->norm_gens_size = 0;
     si->ref_cnt = 1;
-    si->use_compound_file = false;
     return si;
 }
 
@@ -534,7 +527,6 @@ static FrtSegmentInfo *si_read(FrtStore *store, FrtInStream *is)
                 si->norm_gens[i] = frt_is_read_vint(is);
             }
         }
-        si->use_compound_file = (bool)frt_is_read_byte(is);
     FRT_XCATCHALL
         frt_store_close(si->store);
         free(si->name);
@@ -555,7 +547,6 @@ static void si_write(FrtSegmentInfo *si, FrtOutStream *os)
             frt_os_write_vint(os, si->norm_gens[i]);
         }
     }
-    frt_os_write_byte(os, (frt_uchar)si->use_compound_file);
 }
 
 void frt_si_close(FrtSegmentInfo *si) {
@@ -572,17 +563,6 @@ bool frt_si_has_deletions(FrtSegmentInfo *si)
     return si->del_gen >= 0;
 }
 
-bool frt_si_has_separate_norms(FrtSegmentInfo *si)
-{
-    if (si->use_compound_file && si->norm_gens) {
-        int i;
-        for (i = si->norm_gens_size - 1; i >= 0; i--) {
-            if (si->norm_gens[i] > 0) return true;
-        }
-    }
-    return false;
-}
-
 void frt_si_advance_norm_gen(FrtSegmentInfo *si, int field_num)
 {
     if (field_num >= si->norm_gens_size) {
@@ -603,7 +583,7 @@ static char *si_norm_file_name(FrtSegmentInfo *si, char *buf, int field_num)
         || 0 > (norm_gen = si->norm_gens[field_num])) {
         return NULL;
     } else {
-        const char *ext = (si->use_compound_file && norm_gen > 0) ? "s" : "f";
+        const char *ext = "f";
         return fn_for_gen_field(buf, si->name, ext, norm_gen, field_num);
     }
 }
@@ -627,17 +607,9 @@ static void si_delete_files(FrtSegmentInfo *si, FrtFieldInfos *fis, FrtDeleter *
     file_name[seg_len] = '.';
     ext = file_name + seg_len + 1;
 
-    if (si->use_compound_file) {
-        memcpy(ext, "cfs", 4);
+    for (i = FRT_NELEMS(INDEX_EXTENSIONS) - 1; i >= 0; i--) {
+        memcpy(ext, INDEX_EXTENSIONS[i], 4);
         frt_deleter_queue_file(dlr, file_name);
-        if (0 <= si->del_gen) {
-            frt_deleter_queue_file(dlr, frt_fn_for_generation(file_name, si->name, "del", si->del_gen));
-        }
-    } else {
-        for (i = FRT_NELEMS(INDEX_EXTENSIONS) - 1; i >= 0; i--) {
-            memcpy(ext, INDEX_EXTENSIONS[i], 4);
-            frt_deleter_queue_file(dlr, file_name);
-        }
     }
 }
 
@@ -3525,30 +3497,6 @@ bool frt_file_name_filter_is_index_file(const char *file_name, bool include_lock
     return false;
 }
 
-/*
- * Returns true if this is a file that would be contained in a CFS file.  This
- * function should only be called on files that pass the above "accept" (ie,
- * are already known to be a Lucene index file).
- */
-static bool file_name_filter_is_cfs_file(const char *file_name) {
-    char *p = strrchr(file_name, '.');
-    if (NULL != p) {
-        char *extension = p + 1;
-        if (NULL != frt_h_get(fn_extensions, extension)
-            && 0 != strcmp(extension, "del")
-            && 0 != strcmp(extension, "gen")
-            && 0 != strcmp(extension, "cfs")) {
-            return true;
-        }
-        else if ('f' == *extension
-                 && '0' <= *(extension + 1)
-                 && '9' >= *(extension + 1)) {
-            return true;
-        }
-    }
-    return false;
-}
-
 /****************************************************************************
  *
  * Deleter
@@ -3639,11 +3587,7 @@ static void frt_deleter_find_deletable_files_i(const char *file_name, void *arg)
         } else {
             char tmp_fn[FRT_SEGMENT_NAME_MAX_LENGTH];
             /* OK, segment is referenced, but file may still be orphan'd: */
-            if (file_name_filter_is_cfs_file(file_name)
-                && si->use_compound_file) {
-                /* This file is stored in a CFS file for this segment: */
-                do_delete = true;
-            } else if (0 == strcmp("del", extension)) {
+            if (0 == strcmp("del", extension)) {
                 /* This is a _segmentName_N.del file: */
                 if (!frt_fn_for_generation(tmp_fn, segment_name, "del", si->del_gen)
                     || 0 != strcmp(file_name, tmp_fn)) {
@@ -3661,9 +3605,6 @@ static void frt_deleter_find_deletable_files_i(const char *file_name, void *arg)
                     /* This is an orphan'd norms file: */
                     do_delete = true;
                 }
-            } else if (0 == strcmp("cfs", extension) && !si->use_compound_file) {
-                /* This is a partially written _segmentName.cfs: */
-                do_delete = true;
             }
         }
 
@@ -4377,8 +4318,7 @@ static void sr_open_norms(FrtIndexReader *ir, FrtStore *cfs_store)
     char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
 
     for (i = si->norm_gens_size - 1; i >= 0; i--) {
-        FrtStore *store = (si->use_compound_file && si->norm_gens[i] == 0) ?
-            cfs_store : ir->store;
+        FrtStore *store = ir->store;
         if (si_norm_file_name(si, file_name, i)) {
             FrtInStream *is = store->open_input(store, file_name);
             FRT_DEREF(is);
@@ -4425,12 +4365,6 @@ static FrtIndexReader *sr_setup_i(FrtSegmentReader *sr)
     sr->cfs_store   = NULL;
 
     FRT_TRY
-        if (sr->si->use_compound_file) {
-            sprintf(file_name, "%s.cfs", sr_segment);
-            sr->cfs_store = frt_open_cmpd_store(store, file_name);
-            store = sr->cfs_store;
-        }
-
         sr->fr = frt_fr_open(store, sr_segment, ir->fis);
         sr->sfi = frt_sfi_open(store, sr_segment);
         sr->tir = frt_tir_open(store, sr->sfi, sr_segment);
@@ -5509,10 +5443,6 @@ static SegmentMergeInfo *smi_new(int base, FrtStore *store, FrtSegmentInfo *si)
     smi->si = si;
     smi->orig_store = smi->store = store;
     FRT_REF(smi->orig_store);
-    sprintf(file_name, "%s.cfs", segment);
-    if (store->exists(store, file_name)) {
-        smi->store = frt_open_cmpd_store(store, file_name);
-    }
 
     sprintf(file_name, "%s.fdx", segment);
     smi->doc_cnt = smi->max_doc
@@ -5874,8 +5804,7 @@ static void sm_merge_norms(SegmentMerger *sm)
                 if (si_norm_file_name(si, file_name, i)) {
                     const int max_doc = smi->max_doc;
                     FrtBitVector *deleted_docs =  smi->deleted_docs;
-                    store = (si->use_compound_file && si->norm_gens[i])
-                             ? smi->orig_store : smi->store;
+                    store = smi->store;
                     is = store->open_input(store, file_name);
                     if (deleted_docs) {
                         for (k = 0; k < max_doc; k++) {
@@ -5945,42 +5874,6 @@ int frt_iw_doc_count(FrtIndexWriter *iw)
     return doc_cnt;
 }
 
-static void iw_create_compound_file(FrtStore *store, FrtFieldInfos *fis, FrtSegmentInfo *si, char *cfs_file_name, FrtDeleter *dlr) {
-    int i;
-    FrtCompoundWriter *cw;
-    char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
-    char *ext;
-    int seg_len = strlen(si->name);
-
-    memcpy(file_name, si->name, seg_len);
-    file_name[seg_len] = '.';
-    ext = file_name + seg_len + 1;
-
-    cw = frt_open_cw(store, cfs_file_name);
-    for (i = 0; i < FRT_NELEMS(COMPOUND_EXTENSIONS); i++) {
-        memcpy(ext, COMPOUND_EXTENSIONS[i], 4);
-        frt_cw_add_file(cw, file_name);
-    }
-
-    /* Field norm file_names */
-    for (i = fis->size - 1; i >= 0; i--) {
-        if (bits_has_norms(fis->fields[i]->bits) && si_norm_file_name(si, file_name, i)) {
-            frt_cw_add_file(cw, file_name);
-        }
-    }
-
-    /* Perform the merge */
-    frt_cw_close(cw, dlr);
-}
-
-static void iw_commit_compound_file(FrtIndexWriter *iw, FrtSegmentInfo *si)
-{
-    char cfs_name[FRT_SEGMENT_NAME_MAX_LENGTH];
-    sprintf(cfs_name, "%s.cfs", si->name);
-
-    iw_create_compound_file(iw->store, iw->fis, si, cfs_name, iw->deleter);
-}
-
 static void iw_merge_segments(FrtIndexWriter *iw, const int min_seg, const int max_seg) {
     int i;
     FrtSegmentInfos *sis = iw->sis;
@@ -5999,11 +5892,6 @@ static void iw_merge_segments(FrtIndexWriter *iw, const int min_seg, const int m
 
     frt_sis_del_from_to(sis, min_seg, max_seg);
 
-    if (iw->config.use_compound_file) {
-        iw_commit_compound_file(iw, si);
-        si->use_compound_file = true;
-    }
-
     frt_sis_write(sis, iw->store, iw->deleter);
     deleter_commit_pending_deletions(iw->deleter);
 
@@ -6055,10 +5943,6 @@ static void iw_flush_ram_segment(FrtIndexWriter *iw) {
     si->doc_cnt = iw->dw->doc_num;
     dw_flush(iw->dw);
     pthread_mutex_lock(&iw->store->mutex);
-    if (iw->config.use_compound_file) {
-        iw_commit_compound_file(iw, si);
-        si->use_compound_file = true;
-    }
     /* commit the segments file and the fields file */
     frt_sis_write(iw->sis, iw->store, iw->deleter);
     deleter_commit_pending_deletions(iw->deleter);
@@ -6174,10 +6058,7 @@ static void iw_optimize_i(FrtIndexWriter *iw)
     while (iw->sis->size > 1
            || (iw->sis->size == 1
                && (frt_si_has_deletions(iw->sis->segs[0])
-                   || (iw->sis->segs[0]->store != iw->store)
-                   || (iw->config.use_compound_file
-                       && (!iw->sis->segs[0]->use_compound_file
-                           || frt_si_has_separate_norms(iw->sis->segs[0])))))) {
+                   || (iw->sis->segs[0]->store != iw->store)))) {
         min_segment = iw->sis->size - iw->config.merge_factor;
         iw_merge_segments_from(iw, min_segment < 0 ? 0 : min_segment);
     }
@@ -6433,9 +6314,7 @@ static void iw_cp_norms(FrtIndexWriter *iw, FrtSegmentReader *sr,
     for (i = 0; i < field_cnt; i++) {
         if (bits_has_norms(fis->fields[i]->bits)
             && si_norm_file_name(sr->si, file_name_in, i)) {
-            FrtStore *store = (sr->si->use_compound_file
-                            && sr->si->norm_gens[i] == 0) ? sr->cfs_store
-                                                          : IR(sr)->store;
+            FrtStore *store = IR(sr)->store;
             int field_num = map ? map[i] : i;
 
             norms_in = store->open_input(store, file_name_in);

data/ext/isomorfeus_ferret_ext/frt_index.h

@@ -32,7 +32,6 @@ typedef struct FrtConfig {
     int max_buffered_docs;
     int max_merge_docs;
     int max_field_length;
-    bool use_compound_file;
 } FrtConfig;
 
 extern const FrtConfig frt_default_config;
@@ -151,13 +150,11 @@ typedef struct FrtSegmentInfo {
     int         del_gen;
     int         *norm_gens;
     int         norm_gens_size;
-    bool        use_compound_file;
 } FrtSegmentInfo;
 
 extern FrtSegmentInfo *frt_si_new(char *name, int doc_cnt, FrtStore *store);
 extern void frt_si_close(FrtSegmentInfo *si);
 extern bool frt_si_has_deletions(FrtSegmentInfo *si);
-extern bool frt_si_has_separate_norms(FrtSegmentInfo *si);
 extern void frt_si_advance_norm_gen(FrtSegmentInfo *si, int field_num);
 
 /****************************************************************************

data/ext/isomorfeus_ferret_ext/frt_lang.c

@@ -13,7 +13,7 @@ void *frt_emalloc(size_t size)
     void *p = malloc(size);
 
     if (p == NULL) {
-        FRT_RAISE(FRT_MEM_ERROR, "failed to allocate %d bytes", (int)size);
+        FRT_RAISE(FRT_MEM_ERROR, "failed to allocate %llu bytes", size);
     }
 
     return p;
@@ -25,7 +25,7 @@ void *frt_ecalloc(size_t size)
     void *p = calloc(1, size);
 
     if (p == NULL) {
-        FRT_RAISE(FRT_MEM_ERROR, "failed to allocate %d bytes", (int)size);
+        FRT_RAISE(FRT_MEM_ERROR, "failed to allocate %llu bytes", size);
     }
 
     return p;
@@ -37,7 +37,7 @@ void *frt_erealloc(void *ptr, size_t size)
     void *p = realloc(ptr, size);
 
     if (p == NULL) {
-        FRT_RAISE(FRT_MEM_ERROR, "failed to reallocate %d bytes", (int)size);
+        FRT_RAISE(FRT_MEM_ERROR, "failed to reallocate %llu bytes", size);
     }
 
     return p;

data/ext/isomorfeus_ferret_ext/frt_mdbx_store.c

@@ -419,7 +419,7 @@ static void mdbxso_close_i(FrtOutStream *os) {
     key.iov_base = rf->name;
     key.iov_len  = strlen(rf->name);
 
-    val.iov_base = frt_emalloc(FRT_BUFFER_SIZE * rf->bufcnt);
+    val.iov_base = frt_emalloc(rf->len);
     val.iov_len  = rf->len;
 
     int offset = 0;

data/ext/isomorfeus_ferret_ext/frt_store.h

@@ -172,7 +172,6 @@ struct FrtStore {
     union {
         char             *path; /* for fs_store only */
         FrtHash          *ht;   /* for ram_store only */
-        FrtCompoundStore *cmpd; /* for compound_store only */
         MDBXInfo         *mdbx;  /* for mdbx store only */
     } dir;
 #if defined POSH_OS_WIN32 || defined POSH_OS_WIN64
@@ -359,17 +358,6 @@ extern FrtStore *frt_open_ram_store(FrtStore *new_store);
  */
 extern FrtStore *frt_open_ram_store_and_copy(FrtStore *store, FrtStore *from_store, bool close_store);
 
-/**
- * Open a compound store. This is basically store which is stored within a
- * single file and can in turn be stored within either a FileSystem or RAM
- * store.
- *
- * @param store the store within which this compound store will be stored
- * @param filename the name of the file in which to store the compound store
- * @return a newly allocated Compound FrtStore.
- */
-extern FrtStore *frt_open_cmpd_store(FrtStore *store, const char *filename);
-
 /*
  * == RamStore functions ==
  *

data/ext/isomorfeus_ferret_ext/test.c

@@ -724,38 +724,37 @@ static VALUE frb_ts_1710(VALUE v)         { return INT2FIX(execute_test(0)); }
 static VALUE frb_ts_analysis(VALUE v)     { return INT2FIX(execute_test(1)); }
 static VALUE frb_ts_array(VALUE v)        { return INT2FIX(execute_test(2)); }
 static VALUE frb_ts_bitvector(VALUE v)    { return INT2FIX(execute_test(3)); }
-static VALUE frb_ts_compound_io(VALUE v)  { return INT2FIX(execute_test(4)); }
-static VALUE frb_ts_document(VALUE v)     { return INT2FIX(execute_test(5)); }
-static VALUE frb_ts_except(VALUE v)       { return INT2FIX(execute_test(6)); }
-static VALUE frb_ts_fields(VALUE v)       { return INT2FIX(execute_test(7)); }
-static VALUE frb_ts_file_deleter(VALUE v) { return INT2FIX(execute_test(8)); }
-static VALUE frb_ts_filter(VALUE v)       { return INT2FIX(execute_test(9)); }
-static VALUE frb_ts_global(VALUE v)       { return INT2FIX(execute_test(10)); }
-static VALUE frb_ts_hash(VALUE v)         { return INT2FIX(execute_test(11)); }
-static VALUE frb_ts_hashset(VALUE v)      { return INT2FIX(execute_test(12)); }
-static VALUE frb_ts_helper(VALUE v)       { return INT2FIX(execute_test(13)); }
-static VALUE frb_ts_highlighter(VALUE v)  { return INT2FIX(execute_test(14)); }
-static VALUE frb_ts_index(VALUE v)        { return INT2FIX(execute_test(15)); }
-static VALUE frb_ts_lang(VALUE v)         { return INT2FIX(execute_test(16)); }
-static VALUE frb_ts_mdbx_store(VALUE v)   { return INT2FIX(execute_test(17)); }
-static VALUE frb_ts_mem_pool(VALUE v)     { return INT2FIX(execute_test(18)); }
-static VALUE frb_ts_multimapper(VALUE v)  { return INT2FIX(execute_test(19)); }
-static VALUE frb_ts_priorityqueue(VALUE v){ return INT2FIX(execute_test(20)); }
-static VALUE frb_ts_q_const_score(VALUE v){ return INT2FIX(execute_test(21)); }
-static VALUE frb_ts_q_filtered(VALUE v)   { return INT2FIX(execute_test(22)); }
-static VALUE frb_ts_q_fuzzy(VALUE v)      { return INT2FIX(execute_test(23)); }
-static VALUE frb_ts_q_parser(VALUE v)     { return INT2FIX(execute_test(24)); }
-static VALUE frb_ts_q_span(VALUE v)       { return INT2FIX(execute_test(25)); }
-static VALUE frb_ts_ram_store(VALUE v)    { return INT2FIX(execute_test(26)); }
-static VALUE frb_ts_search(VALUE v)       { return INT2FIX(execute_test(27)); }
-static VALUE frb_ts_multi_search(VALUE v) { return INT2FIX(execute_test(28)); }
-static VALUE frb_ts_segments(VALUE v)     { return INT2FIX(execute_test(29)); }
-static VALUE frb_ts_similarity(VALUE v)   { return INT2FIX(execute_test(30)); }
-static VALUE frb_ts_sort(VALUE v)         { return INT2FIX(execute_test(31)); }
-static VALUE frb_ts_term(VALUE v)         { return INT2FIX(execute_test(32)); }
-static VALUE frb_ts_term_vectors(VALUE v) { return INT2FIX(execute_test(33)); }
-static VALUE frb_ts_test(VALUE v)         { return INT2FIX(execute_test(34)); }
-static VALUE frb_ts_threading(VALUE v)    { return INT2FIX(execute_test(35)); }
+static VALUE frb_ts_document(VALUE v)     { return INT2FIX(execute_test(4)); }
+static VALUE frb_ts_except(VALUE v)       { return INT2FIX(execute_test(5)); }
+static VALUE frb_ts_fields(VALUE v)       { return INT2FIX(execute_test(6)); }
+static VALUE frb_ts_file_deleter(VALUE v) { return INT2FIX(execute_test(7)); }
+static VALUE frb_ts_filter(VALUE v)       { return INT2FIX(execute_test(8)); }
+static VALUE frb_ts_global(VALUE v)       { return INT2FIX(execute_test(9)); }
+static VALUE frb_ts_hash(VALUE v)         { return INT2FIX(execute_test(10)); }
+static VALUE frb_ts_hashset(VALUE v)      { return INT2FIX(execute_test(11)); }
+static VALUE frb_ts_helper(VALUE v)       { return INT2FIX(execute_test(12)); }
+static VALUE frb_ts_highlighter(VALUE v)  { return INT2FIX(execute_test(13)); }
+static VALUE frb_ts_index(VALUE v)        { return INT2FIX(execute_test(14)); }
+static VALUE frb_ts_lang(VALUE v)         { return INT2FIX(execute_test(15)); }
+static VALUE frb_ts_mdbx_store(VALUE v)   { return INT2FIX(execute_test(16)); }
+static VALUE frb_ts_mem_pool(VALUE v)     { return INT2FIX(execute_test(17)); }
+static VALUE frb_ts_multimapper(VALUE v)  { return INT2FIX(execute_test(18)); }
+static VALUE frb_ts_priorityqueue(VALUE v){ return INT2FIX(execute_test(19)); }
+static VALUE frb_ts_q_const_score(VALUE v){ return INT2FIX(execute_test(20)); }
+static VALUE frb_ts_q_filtered(VALUE v)   { return INT2FIX(execute_test(21)); }
+static VALUE frb_ts_q_fuzzy(VALUE v)      { return INT2FIX(execute_test(22)); }
+static VALUE frb_ts_q_parser(VALUE v)     { return INT2FIX(execute_test(23)); }
+static VALUE frb_ts_q_span(VALUE v)       { return INT2FIX(execute_test(24)); }
+static VALUE frb_ts_ram_store(VALUE v)    { return INT2FIX(execute_test(25)); }
+static VALUE frb_ts_search(VALUE v)       { return INT2FIX(execute_test(26)); }
+static VALUE frb_ts_multi_search(VALUE v) { return INT2FIX(execute_test(27)); }
+static VALUE frb_ts_segments(VALUE v)     { return INT2FIX(execute_test(28)); }
+static VALUE frb_ts_similarity(VALUE v)   { return INT2FIX(execute_test(29)); }
+static VALUE frb_ts_sort(VALUE v)         { return INT2FIX(execute_test(30)); }
+static VALUE frb_ts_term(VALUE v)         { return INT2FIX(execute_test(31)); }
+static VALUE frb_ts_term_vectors(VALUE v) { return INT2FIX(execute_test(32)); }
+static VALUE frb_ts_test(VALUE v)         { return INT2FIX(execute_test(33)); }
+static VALUE frb_ts_threading(VALUE v)    { return INT2FIX(execute_test(34)); }
 
 static VALUE frb_ts_posh(VALUE v) {
     const char *posh = POSH_GetArchString();
@@ -802,7 +801,6 @@ void Init_Test(void) {
     rb_define_singleton_method(mTest, "analysis",       frb_ts_analysis, 0);
     rb_define_singleton_method(mTest, "array",          frb_ts_array, 0);
     rb_define_singleton_method(mTest, "bitvector",      frb_ts_bitvector, 0);
-    rb_define_singleton_method(mTest, "compound_io",    frb_ts_compound_io, 0);
     rb_define_singleton_method(mTest, "document",       frb_ts_document, 0);
     rb_define_singleton_method(mTest, "except",         frb_ts_except, 0);
     rb_define_singleton_method(mTest, "fields",         frb_ts_fields, 0);

data/ext/isomorfeus_ferret_ext/test_1710.c

@@ -40,11 +40,11 @@ static void test_problem_text(TestCase *tc, void *data)
 
     frt_iw_add_doc(iw, problem_text);
     Aiequal(1, frt_iw_doc_count(iw));
-    Assert(!store->exists(store, "_0.cfs"), "data shouldn't have been written yet");
+    Assert(!store->exists(store, "_0.fdx"), "data shouldn't have been written yet");
     frt_iw_commit(iw);
-    Assert(store->exists(store, "_0.cfs"), "data should now be written");
+    Assert(store->exists(store, "_0.fdx"), "data should now be written");
     frt_iw_close(iw);
-    Assert(store->exists(store, "_0.cfs"), "data should still be there");
+    Assert(store->exists(store, "_0.fdx"), "data should still be there");
 }
 
 TestSuite *ts_1710(TestSuite *suite)

data/ext/isomorfeus_ferret_ext/test_file_deleter.c

@@ -91,22 +91,22 @@ static void test_delete_leftover_files(TestCase *tc, void *data)
     /* Create a bogus separate norms file for a
      * segment/field that actually has a separate norms file
      * already: */
-    copy_file(store, "_2_1.s0", "_2_2.s0");
+    //copy_file(store, "_2_1.s0", "_2_2.s0");
 
     /* Create a bogus separate norms file for a
      * segment/field that actually has a separate norms file
      * already, using the "not compound file" extension: */
-    copy_file(store, "_2_1.s0", "_2_2.f0");
+    //copy_file(store, "_2_1.s0", "_2_2.f0");
 
     /* Create a bogus separate norms file for a
      * segment/field that does not have a separate norms
      * file already: */
-    copy_file(store, "_2_1.s0", "_1_1.s0");
+    //copy_file(store, "_2_1.s0", "_1_1.s0");
 
     /* Create a bogus separate norms file for a
      * segment/field that does not have a separate norms
      * file already using the "not compound file" extension: */
-    copy_file(store, "_2_1.s0", "_1_1.f0");
+    //copy_file(store, "_2_1.s0", "_1_1.f0");
 
     /* Create a bogus separate del file for a
      * segment that already has a separate del file:  */
@@ -120,21 +120,6 @@ static void test_delete_leftover_files(TestCase *tc, void *data)
      * non-existent segment: */
     copy_file(store, "_0_0.del", "_188_1.del");
 
-    /* Create a bogus segment file: */
-    copy_file(store, "_0.cfs", "_188.cfs");
-
-    /* Create a bogus frq file when the CFS already exists: */
-    copy_file(store, "_0.cfs", "_0.frq");
-
-    /* Create a bogus frq file when the CFS already exists: */
-    copy_file(store, "_0.cfs", "_0.frq");
-    copy_file(store, "_0.cfs", "_0.prx");
-    copy_file(store, "_0.cfs", "_0.fdx");
-    copy_file(store, "_0.cfs", "_0.fdt");
-    copy_file(store, "_0.cfs", "_0.tfx");
-    copy_file(store, "_0.cfs", "_0.tix");
-    copy_file(store, "_0.cfs", "_0.tis");
-
     /* Create some old segments file: */
     copy_file(store, "segments_5", "segments");
     copy_file(store, "segments_5", "segments_2");

data/ext/isomorfeus_ferret_ext/test_index.c

@@ -927,24 +927,22 @@ static void test_iw_add_doc(TestCase *tc, void *data)
 
     frt_iw_add_doc(iw, docs[0]);
     Aiequal(1, frt_iw_doc_count(iw));
-    Assert(!store->exists(store, "_0.cfs"),
-           "data shouldn't have been written yet");
+    // Assert(!store->exists(store, "_0.fdx"), "data shouldn't have been written yet");
     frt_iw_commit(iw);
-    Assert(store->exists(store, "_0.cfs"), "data should now be written");
+    Assert(store->exists(store, "_0.fdx"), "data should now be written");
     frt_iw_close(iw);
-    Assert(store->exists(store, "_0.cfs"), "data should still be there");
+    Assert(store->exists(store, "_0.fdx"), "data should still be there");
 
     iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &frt_default_config);
     frt_iw_add_doc(iw, docs[1]);
     Aiequal(2, frt_iw_doc_count(iw));
-    Assert(!store->exists(store, "_1.cfs"),
-           "data shouldn't have been written yet");
-    Assert(store->exists(store, "_0.cfs"), "data should still be there");
+    // Assert(!store->exists(store, "_1.fdx"), "data shouldn't have been written yet");
+    Assert(store->exists(store, "_0.fdx"), "data should still be there");
     frt_iw_commit(iw);
-    Assert(store->exists(store, "_1.cfs"), "data should now be written");
+    Assert(store->exists(store, "_1.fdx"), "data should now be written");
     frt_iw_close(iw);
-    Assert(store->exists(store, "_1.cfs"), "data should still be there");
-    Assert(store->exists(store, "_0.cfs"), "data should still be there");
+    Assert(store->exists(store, "_1.fdx"), "data should still be there");
+    Assert(store->exists(store, "_0.fdx"), "data should still be there");
 
     destroy_docs(docs, BOOK_LIST_LENGTH);
 }
@@ -1141,8 +1139,8 @@ static void test_simulated_corrupt_index2(TestCase *tc, void *data)
     /* segment generation should be > 1 */
     Atrue(gen > 1);
 
-    Atrue(store->exists(store, "_0.cfs"));
-    store->remove(store, "_0.cfs");
+    Atrue(store->exists(store, "_0.fdx"));
+    store->remove(store, "_0.fdx");
 
     FRT_TRY
         ir = frt_ir_open(NULL, store);
@@ -1177,7 +1175,7 @@ static void test_iw_add_docs(TestCase *tc, void *data)
 
     frt_iw_close(iw);
     destroy_docs(docs, BOOK_LIST_LENGTH);
-    if (!Aiequal(3, store->count(store))) {
+    if (!Aiequal(11, store->count(store))) {
         char *buf = frt_store_to_s(store);
         Tmsg("To many files: %s\n", buf);
         free(buf);

data/ext/isomorfeus_ferret_ext/test_segments.c

@@ -53,16 +53,8 @@ static void test_si(TestCase *tc, void *data)
     si->store = store;
     FRT_REF(store);
     Asi_has_vals(si, "_2", 12, store);
-    Assert(!frt_si_has_separate_norms(si), "doesn't use compound file/have norms");
-    si->use_compound_file = true;
-    Assert(!frt_si_has_separate_norms(si), "doesn't have norms");
     frt_si_advance_norm_gen(si, 3);
-    si->use_compound_file = false;
-    Assert(!frt_si_has_separate_norms(si), "doesn't use compound file");
-    si->use_compound_file = true;
-    Assert(!frt_si_has_separate_norms(si), "has norms in compound file");
     frt_si_advance_norm_gen(si, 3);
-    Assert(frt_si_has_separate_norms(si), "has seperate norms");
     frt_si_close(si);
     frt_store_close(store);
 }

data/ext/isomorfeus_ferret_ext/tests_all.h

@@ -7,7 +7,6 @@ TestSuite *ts_1710(TestSuite *suite);
 TestSuite *ts_analysis(TestSuite *suite);
 TestSuite *ts_array(TestSuite *suite);
 TestSuite *ts_bitvector(TestSuite *suite);
-TestSuite *ts_compound_io(TestSuite *suite);
 TestSuite *ts_document(TestSuite *suite);
 TestSuite *ts_except(TestSuite *suite);
 TestSuite *ts_fields(TestSuite *suite);
@@ -49,7 +48,6 @@ const struct test_list
     {ts_analysis},
     {ts_array},
     {ts_bitvector},
-    {ts_compound_io},
     {ts_document},
     {ts_except},
     {ts_fields},

data/lib/isomorfeus/ferret/index/index.rb

@@ -720,11 +720,11 @@ module Isomorfeus
 
         # optimizes the index. This should only be called when the index will no
         # longer be updated very often, but will be read a lot.
-        def optimize()
+        def optimize
           @dir.synchronize do
-            ensure_writer_open()
-            @writer.optimize()
-            @writer.close()
+            ensure_writer_open
+            @writer.optimize
+            @writer.close
             @writer = nil
           end
         end

data/lib/isomorfeus/ferret/version.rb

@@ -1,5 +1,5 @@
 module Isomorfeus
   module Ferret
-    VERSION = '0.16.2'
+    VERSION = '0.17.0'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: isomorfeus-ferret
 version: !ruby/object:Gem::Version
-  version: 0.16.2
+  version: 0.17.0
 platform: ruby
 authors:
 - Jan Biedermann
@@ -193,7 +193,6 @@ files:
 - ext/isomorfeus_ferret_ext/frt_array.h
 - ext/isomorfeus_ferret_ext/frt_bitvector.c
 - ext/isomorfeus_ferret_ext/frt_bitvector.h
-- ext/isomorfeus_ferret_ext/frt_compound_io.c
 - ext/isomorfeus_ferret_ext/frt_config.h
 - ext/isomorfeus_ferret_ext/frt_document.c
 - ext/isomorfeus_ferret_ext/frt_document.h
@@ -333,7 +332,6 @@ files:
 - ext/isomorfeus_ferret_ext/test_analysis.c
 - ext/isomorfeus_ferret_ext/test_array.c
 - ext/isomorfeus_ferret_ext/test_bitvector.c
-- ext/isomorfeus_ferret_ext/test_compound_io.c
 - ext/isomorfeus_ferret_ext/test_document.c
 - ext/isomorfeus_ferret_ext/test_except.c
 - ext/isomorfeus_ferret_ext/test_fields.c

data/ext/isomorfeus_ferret_ext/frt_compound_io.c

@@ -1,352 +0,0 @@
-#include "frt_index.h"
-#include "frt_array.h"
-
-extern void frt_store_close(FrtStore *store);
-extern FrtInStream *frt_is_new();
-extern FrtStore *frt_store_new();
-
-/****************************************************************************
- *
- * CompoundStore
- *
- ****************************************************************************/
-
-typedef struct FileEntry {
-    frt_off_t offset;
-    frt_off_t length;
-} FileEntry;
-
-static void cmpd_touch(FrtStore *store, const char *file_name) {
-    store->dir.cmpd->store->touch(store->dir.cmpd->store, file_name);
-}
-
-static int cmpd_exists(FrtStore *store, const char *file_name) {
-    if (frt_h_get(store->dir.cmpd->entries, file_name) != NULL) {
-        return true;
-    } else {
-        return false;
-    }
-}
-
-static int cmpd_remove(FrtStore *store, const char *file_name) {
-    (void)store;
-    (void)file_name;
-    FRT_RAISE(FRT_UNSUPPORTED_ERROR, "%s", FRT_UNSUPPORTED_ERROR_MSG);
-    return 0;
-}
-
-static void cmpd_rename(FrtStore *store, const char *from, const char *to) {
-    (void)store;
-    (void)from;
-    (void)to;
-    FRT_RAISE(FRT_UNSUPPORTED_ERROR, "%s", FRT_UNSUPPORTED_ERROR_MSG);
-}
-
-static int cmpd_count(FrtStore *store) {
-    return store->dir.cmpd->entries->size;
-}
-
-static void cmpd_each(FrtStore *store, void (*func)(const char *fname, void *arg), void *arg) {
-    FrtHash *ht = store->dir.cmpd->entries;
-    int i;
-    for (i = 0; i <= ht->mask; i++) {
-        char *fn = (char *)ht->table[i].key;
-        if (fn) {
-            func(fn, arg);
-        }
-    }
-}
-
-static void cmpd_clear(FrtStore *store) {
-    (void)store;
-    FRT_RAISE(FRT_UNSUPPORTED_ERROR, "%s", FRT_UNSUPPORTED_ERROR_MSG);
-}
-
-static void cmpd_close_i(FrtStore *store) {
-    FrtCompoundStore *cmpd = store->dir.cmpd;
-    if (cmpd->stream == NULL) {
-        FRT_RAISE(FRT_IO_ERROR, "Tried to close already closed compound store");
-    }
-
-    frt_h_destroy(cmpd->entries);
-
-    frt_is_close(cmpd->stream);
-    cmpd->stream = NULL;
-    frt_store_close(cmpd->store);
-    free(store->dir.cmpd);
-}
-
-static frt_off_t cmpd_length(FrtStore *store, const char *file_name) {
-    FileEntry *fe = (FileEntry *)frt_h_get(store->dir.cmpd->entries, file_name);
-    if (fe != NULL) {
-        return fe->length;
-    } else {
-        return 0;
-    }
-}
-
-static void cmpdi_seek_i(FrtInStream *is, frt_off_t pos) {
-    (void)is;
-    (void)pos;
-}
-
-static void cmpdi_close_i(FrtInStream *is) {
-    frt_is_close(is->d.cis->sub);
-    free(is->d.cis);
-}
-
-static frt_off_t cmpdi_length_i(FrtInStream *is) {
-    return (is->d.cis->length);
-}
-
-/*
- * raises: FRT_EOF_ERROR
- */
-static void cmpdi_read_i(FrtInStream *is, frt_uchar *b, int len) {
-    FrtCompoundInStream *cis = is->d.cis;
-    frt_off_t start = frt_is_pos(is);
-
-    if ((start + len) > cis->length) {
-        FRT_RAISE(FRT_EOF_ERROR, "Tried to read past end of file. File length is "
-              "<%"FRT_OFF_T_PFX"d> and tried to read to <%"FRT_OFF_T_PFX"d>",
-              cis->length, start + len);
-    }
-
-    frt_is_seek(cis->sub, cis->offset + start);
-    frt_is_read_bytes(cis->sub, b, len);
-}
-
-static const struct FrtInStreamMethods CMPD_IN_STREAM_METHODS = {
-    cmpdi_read_i,
-    cmpdi_seek_i,
-    cmpdi_length_i,
-    cmpdi_close_i
-};
-
-static FrtInStream *cmpd_create_input(FrtInStream *sub_is, frt_off_t offset, frt_off_t length) {
-    FrtInStream *is = frt_is_new();
-    FrtCompoundInStream *cis = FRT_ALLOC(FrtCompoundInStream);
-
-    cis->sub = sub_is;
-    FRT_REF(sub_is);
-    cis->offset = offset;
-    cis->length = length;
-    is->d.cis = cis;
-    is->m = &CMPD_IN_STREAM_METHODS;
-
-    return is;
-}
-
-static FrtInStream *cmpd_open_input(FrtStore *store, const char *file_name) {
-    FileEntry *entry;
-    FrtCompoundStore *cmpd = store->dir.cmpd;
-    FrtInStream *is;
-
-    pthread_mutex_lock(&store->mutex);
-    if (cmpd->stream == NULL) {
-        pthread_mutex_unlock(&store->mutex);
-        FRT_RAISE(FRT_IO_ERROR, "Can't open compound file input stream. Parent "
-              "stream is closed.");
-    }
-
-    entry = (FileEntry *)frt_h_get(cmpd->entries, file_name);
-    if (entry == NULL) {
-        pthread_mutex_unlock(&store->mutex);
-        FRT_RAISE(FRT_IO_ERROR, "File '%s' does not exist: ", file_name);
-    }
-
-    is = cmpd_create_input(cmpd->stream, entry->offset, entry->length);
-    pthread_mutex_unlock(&store->mutex);
-
-    return is;
-}
-
-static FrtOutStream *cmpd_new_output(FrtStore *store, const char *file_name) {
-    (void)store;
-    (void)file_name;
-    FRT_RAISE(FRT_UNSUPPORTED_ERROR, "%s", FRT_UNSUPPORTED_ERROR_MSG);
-    return NULL;
-}
-
-static FrtLock *cmpd_open_lock_i(FrtStore *store, const char *lock_name) {
-    (void)store;
-    (void)lock_name;
-    FRT_RAISE(FRT_UNSUPPORTED_ERROR, "%s", FRT_UNSUPPORTED_ERROR_MSG);
-    return NULL;
-}
-
-static void cmpd_close_lock_i(FrtLock *lock) {
-    (void)lock;
-    FRT_RAISE(FRT_UNSUPPORTED_ERROR, "%s", FRT_UNSUPPORTED_ERROR_MSG);
-}
-
-FrtStore *frt_open_cmpd_store(FrtStore *store, const char *name) {
-    int count, i;
-    frt_off_t offset;
-    char *fname;
-    FileEntry *volatile entry = NULL;
-    FrtStore *new_store = NULL;
-    FrtCompoundStore *volatile cmpd = NULL;
-    FrtInStream *volatile is = NULL;
-
-    FRT_TRY
-        cmpd = FRT_ALLOC_AND_ZERO(FrtCompoundStore);
-
-        cmpd->store       = store;
-        FRT_REF(store);
-        cmpd->name        = name;
-        cmpd->entries     = frt_h_new_str(&free, &free);
-        is = cmpd->stream = store->open_input(store, cmpd->name);
-
-        /* read the directory and init files */
-        count = frt_is_read_vint(is);
-        entry = NULL;
-        for (i = 0; i < count; i++) {
-            offset = (off_t)frt_is_read_i64(is);
-            fname = frt_is_read_string(is);
-
-            if (entry != NULL) {
-                /* set length of the previous entry */
-                entry->length = offset - entry->offset;
-            }
-
-            entry = FRT_ALLOC(FileEntry);
-            entry->offset = offset;
-            frt_h_set(cmpd->entries, fname, entry);
-        }
-    FRT_XCATCHALL
-        frt_store_close(store);
-        if (is) frt_is_close(is);
-        if (cmpd->entries) frt_h_destroy(cmpd->entries);
-        free(cmpd);
-    FRT_XENDTRY
-
-    /* set the length of the final entry */
-    if (entry != NULL) {
-        entry->length = frt_is_length(is) - entry->offset;
-    }
-
-    new_store               = frt_store_new();
-    new_store->dir.cmpd     = cmpd;
-    new_store->touch        = &cmpd_touch;
-    new_store->exists       = &cmpd_exists;
-    new_store->remove       = &cmpd_remove;
-    new_store->rename       = &cmpd_rename;
-    new_store->count        = &cmpd_count;
-    new_store->clear        = &cmpd_clear;
-    new_store->length       = &cmpd_length;
-    new_store->each         = &cmpd_each;
-    new_store->close_i      = &cmpd_close_i;
-    new_store->new_output   = &cmpd_new_output;
-    new_store->open_input   = &cmpd_open_input;
-    new_store->open_lock_i  = &cmpd_open_lock_i;
-    new_store->close_lock_i = &cmpd_close_lock_i;
-
-    return new_store;
-}
-
-/****************************************************************************
- *
- * CompoundWriter
- *
- ****************************************************************************/
-
-FrtCompoundWriter *frt_open_cw(FrtStore *store, char *name) {
-    FrtCompoundWriter *cw = FRT_ALLOC(FrtCompoundWriter);
-    cw->store = store;
-    FRT_REF(store);
-    cw->name = name;
-    cw->ids = frt_hs_new_str(&free);
-    cw->file_entries = frt_ary_new_type_capa(FrtCWFileEntry, FRT_CW_INIT_CAPA);
-    return cw;
-}
-
-void frt_cw_add_file(FrtCompoundWriter *cw, char *id) {
-    id = frt_estrdup(id);
-    if (frt_hs_add(cw->ids, id) != FRT_HASH_KEY_DOES_NOT_EXIST) {
-        FRT_RAISE(FRT_IO_ERROR, "Tried to add file \"%s\" which has already been "
-              "added to the compound store", id);
-    }
-
-    frt_ary_grow(cw->file_entries);
-    frt_ary_last(cw->file_entries).name = id;
-}
-
-static void cw_copy_file(FrtCompoundWriter *cw, FrtCWFileEntry *src, FrtOutStream *os) {
-    frt_off_t start_ptr = frt_os_pos(os);
-    frt_off_t end_ptr;
-    frt_off_t remainder, length, len;
-    frt_uchar buffer[FRT_BUFFER_SIZE];
-
-    FrtInStream *is = cw->store->open_input(cw->store, src->name);
-
-    remainder = length = frt_is_length(is);
-
-    while (remainder > 0) {
-        len = FRT_MIN(remainder, FRT_BUFFER_SIZE);
-        frt_is_read_bytes(is, buffer, len);
-        frt_os_write_bytes(os, buffer, len);
-        remainder -= len;
-    }
-
-    /* Verify that remainder is 0 */
-    if (remainder != 0) {
-        FRT_RAISE(FRT_IO_ERROR, "There seems to be an error in the compound file "
-              "should have read to the end but there are <%"FRT_OFF_T_PFX"d> "
-              "bytes left", remainder);
-    }
-
-    /* Verify that the output length diff is equal to original file */
-    end_ptr = frt_os_pos(os);
-    len = end_ptr - start_ptr;
-    if (len != length) {
-        FRT_RAISE(FRT_IO_ERROR, "Difference in compound file output file offsets "
-              "<%"FRT_OFF_T_PFX"d> does not match the original file lenght "
-              "<%"FRT_OFF_T_PFX"d>", len, length);
-    }
-
-    frt_is_close(is);
-}
-
-void frt_cw_close(FrtCompoundWriter *cw, FrtDeleter *dlr) {
-    FrtOutStream *os = NULL;
-    int i;
-
-    if (cw->ids->size <= 0) {
-        FRT_RAISE(FRT_STATE_ERROR, "Tried to merge compound file with no entries");
-    }
-
-    os = cw->store->new_output(cw->store, cw->name);
-
-    frt_os_write_vint(os, frt_ary_size(cw->file_entries));
-
-    /* Write the directory with all offsets at 0.
-     * Remember the positions of directory entries so that we can adjust the
-     * offsets later */
-    for (i = 0; i < frt_ary_size(cw->file_entries); i++) {
-        cw->file_entries[i].dir_offset = frt_os_pos(os);
-        frt_os_write_u64(os, 0);  /* for now */
-        frt_os_write_string(os, cw->file_entries[i].name);
-    }
-
-    /* Open the files and copy their data into the stream.  Remember the
-     * locations of each file's data section. */
-    for (i = 0; i < frt_ary_size(cw->file_entries); i++) {
-        cw->file_entries[i].data_offset = frt_os_pos(os);
-        cw_copy_file(cw, &cw->file_entries[i], os);
-        if (dlr) {
-            frt_deleter_queue_file(dlr, cw->file_entries[i].name);
-        }
-    }
-
-    /* Write the data offsets into the directory of the compound stream */
-    for (i = 0; i < frt_ary_size(cw->file_entries); i++) {
-        frt_os_seek(os, cw->file_entries[i].dir_offset);
-        frt_os_write_u64(os, cw->file_entries[i].data_offset);
-    }
-    frt_os_close(os);
-    frt_hs_destroy(cw->ids);
-    frt_ary_free(cw->file_entries);
-    frt_store_close(cw->store);
-    free(cw);
-}

data/ext/isomorfeus_ferret_ext/test_compound_io.c

@@ -1,170 +0,0 @@
-#include "frt_store.h"
-#include "frt_index.h"
-#include "testhelper.h"
-#include "test.h"
-
-void test_compound_reader(TestCase *tc, void *data)
-{
-    FrtStore *store = (FrtStore *)data;
-    char *p;
-    FrtOutStream *os = store->new_output(store, "cfile");
-    FrtInStream *is1;
-    FrtInStream *is2;
-    FrtStore *c_reader;
-    frt_os_write_vint(os, 2);
-    frt_os_write_u64(os, 29);
-    frt_os_write_string(os, "file1");
-    frt_os_write_u64(os, 33);
-    frt_os_write_string(os, "file2");
-    frt_os_write_u32(os, 20);
-    frt_os_write_string(os, "this is file 2");
-    frt_os_close(os);
-
-    c_reader = frt_open_cmpd_store(store, "cfile");
-    Aiequal(4, c_reader->length(c_reader, "file1"));
-    Aiequal(15, c_reader->length(c_reader, "file2"));
-    is1 = c_reader->open_input(c_reader, "file1");
-    is2 = c_reader->open_input(c_reader, "file2");
-    Aiequal(20, frt_is_read_u32(is1));
-    Asequal("this is file 2", p = frt_is_read_string(is2)); free(p);
-    frt_is_close(is1);
-    frt_is_close(is2);
-    frt_store_close(c_reader);
-}
-
-void test_compound_writer(TestCase *tc, void *data)
-{
-    FrtStore *store = (FrtStore *)data;
-    char *p;
-    FrtOutStream *os1 = store->new_output(store, "file1");
-    FrtOutStream *os2 = store->new_output(store, "file2");
-    FrtCompoundWriter *cw;
-    FrtInStream *is;
-
-    frt_os_write_u32(os1, 20);
-    frt_os_write_string(os2, "this is file2");
-    frt_os_close(os1);
-    frt_os_close(os2);
-    cw = frt_open_cw(store, "cfile");
-    frt_cw_add_file(cw, "file1");
-    frt_cw_add_file(cw, "file2");
-    frt_cw_close(cw, NULL);
-
-    is = store->open_input(store, "cfile");
-    Aiequal(2, frt_is_read_vint(is));
-    Aiequal(29, frt_is_read_u64(is));
-    Asequal("file1", p = frt_is_read_string(is)); free(p);
-    Aiequal(33, frt_is_read_u64(is));
-    Asequal("file2", p = frt_is_read_string(is)); free(p);
-    Aiequal(20, frt_is_read_u32(is));
-    Asequal("this is file2", p = frt_is_read_string(is)); free(p);
-
-    frt_is_close(is);
-}
-
-void test_compound_io(TestCase *tc, void *data)
-{
-    FrtStore *c_reader;
-    FrtInStream *is1, *is2, *is3;
-    FrtStore *store = (FrtStore *)data;
-    FrtCompoundWriter *cw;
-    char *p;
-    FrtOutStream *os1 = store->new_output(store, "file1");
-    FrtOutStream *os2 = store->new_output(store, "file2");
-    FrtOutStream *os3 = store->new_output(store, "file3");
-    char long_string[10000];
-    const char *short_string = "this is a short string";
-    int slen = (int)strlen(short_string);
-    int i;
-
-    for (i = 0; i < 20; i++) {
-        frt_os_write_u32(os1, rand()%10000);
-    }
-
-    for (i = 0; i < 10000 - slen; i += slen) {
-        sprintf(long_string + i, "%s", short_string);
-    }
-    long_string[i] = 0;
-    frt_os_write_string(os2, long_string);
-    frt_os_write_string(os3, short_string);
-    frt_os_close(os1);
-    frt_os_close(os2);
-    frt_os_close(os3);
-    cw = frt_open_cw(store, (char *)"cfile");
-    frt_cw_add_file(cw, (char *)"file1");
-    frt_cw_add_file(cw, (char *)"file2");
-    frt_cw_add_file(cw, (char *)"file3");
-    frt_cw_close(cw, NULL);
-
-    c_reader = frt_open_cmpd_store(store, "cfile");
-    is1 = c_reader->open_input(c_reader, "file1");
-    for (i = 0; i < 20; i++) {
-        Assert(frt_is_read_u32(is1) < 10000, "should be a rand between 0 and 10000");
-    }
-    frt_is_close(is1);
-    is2 = c_reader->open_input(c_reader, "file2");
-    Asequal(long_string, p = frt_is_read_string(is2)); free(p);
-    frt_is_close(is2);
-    is3 = c_reader->open_input(c_reader, "file3");
-    Asequal(short_string, p = frt_is_read_string(is3)); free(p);
-    frt_is_close(is3);
-
-    frt_store_close(c_reader);
-}
-
-#define MAX_TEST_WORDS 50
-#define TEST_FILE_CNT 100
-
-void test_compound_io_many_files(TestCase *tc, void *data)
-{
-    static const int MAGIC = 250777;
-
-    FrtStore *store = (FrtStore *)data;
-    char buf[MAX_TEST_WORDS * (TEST_WORD_LIST_MAX_LEN + 1)];
-    char *str;
-    int i;
-    FrtOutStream *os;
-    FrtInStream *is;
-    FrtCompoundWriter *cw;
-    FrtStore *c_reader;
-
-    cw = frt_open_cw(store, (char *)"_.cfs");
-    for (i = 0; i < TEST_FILE_CNT; i++) {
-        sprintf(buf, "_%d.txt", i);
-        frt_cw_add_file(cw, buf);
-        os = store->new_output(store, buf);
-        frt_os_write_string(os, make_random_string(buf, MAX_TEST_WORDS));
-        frt_os_write_vint(os, MAGIC);
-        frt_os_close(os);
-    }
-    frt_cw_close(cw, NULL);
-
-    c_reader = frt_open_cmpd_store(store, "_.cfs");
-    for (i = 0; i < TEST_FILE_CNT; i++) {
-        sprintf(buf, "_%d.txt", i);
-        is = c_reader->open_input(c_reader, buf);
-        str = frt_is_read_string(is);
-
-        free(str);
-        Aiequal(MAGIC, frt_is_read_vint(is));
-        Aiequal(frt_is_length(is), frt_is_pos(is));
-        frt_is_close(is);
-    }
-    frt_store_close(c_reader);
-}
-
-TestSuite *ts_compound_io(TestSuite *suite)
-{
-    FrtStore *store = frt_open_ram_store(NULL);
-
-    suite = ADD_SUITE(suite);
-
-    tst_run_test(suite, test_compound_reader, store);
-    tst_run_test(suite, test_compound_writer, store);
-    tst_run_test(suite, test_compound_io, store);
-    tst_run_test(suite, test_compound_io_many_files, store);
-
-    frt_store_close(store);
-
-    return suite;
-}